def run(self):
interface = PluginManagementInterface.find_correct_plugin_manager()
jobs = []
if self.create_jobs:
jobs = create_jobs_for_execution(self.project,
self.plugin_executions)
interface.execute_plugins(self.project, jobs, self.plugin_executions)
def job_output(request, id, type):
if not request.user.is_authenticated() or not request.user.has_perm(
'smartshark.job_output'):
messages.error(request,
'You are not authorized to perform this action.')
return HttpResponseRedirect('/admin/smartshark/project')
job = get_object_or_404(Job, pk=id)
interface = PluginManagementInterface.find_correct_plugin_manager()
if type == 'output':
output = interface.get_output_log(job)
elif type == 'error':
output = interface.get_error_log(job)
elif type == 'arguments':
return render(
request, 'smartshark/job/execution_arguments.html', {
'exe_arguments':
job.plugin_execution.executionhistory_set.all().order_by(
'execution_argument__position'),
'cmd':
interface.get_sent_bash_command(job),
})
return render(request, 'smartshark/job/output.html', {
'output': '\n'.join(output),
'job': job,
})
def set_job_stati(self, request, queryset):
interface = PluginManagementInterface.find_correct_plugin_manager()
job_stati = interface.get_job_stati(queryset)
i = 0
for job in queryset:
job.status = job_stati[i]
job.save()
i += 1
messages.info(request, 'Job stati set from backend.')
def plugin_execution_status(request, id):
if not request.user.is_authenticated() or not request.user.has_perm(
'smartshark.plugin_execution_status'):
messages.error(request,
'You are not authorized to perform this action.')
return HttpResponseRedirect('/admin/smartshark/project')
interface = PluginManagementInterface.find_correct_plugin_manager()
plugin_execution = get_object_or_404(PluginExecution, pk=id)
# Get all jobs from all plugin_executions which did not terminate yet
# Update the job stati for these jobs
jobs = Job.objects.filter(plugin_execution=plugin_execution,
status='WAIT').all()
job_stati = interface.get_job_stati(jobs)
i = 0
for job in jobs:
job.status = job_stati[i]
job.save()
i += 1
job_filter = JobExecutionFilter(
request.GET,
queryset=Job.objects.all().filter(plugin_execution=plugin_execution))
rev = [
exitjob.revision_hash if exitjob.revision_hash else ''
for exitjob in job_filter.qs.filter(status='EXIT')
]
exit_job_revisions = ''
if rev:
exit_job_revisions = ','.join(rev)
# Set up pagination
paginator = Paginator(job_filter.qs, 10)
page = request.GET.get('page')
try:
jobs = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
jobs = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
jobs = paginator.page(paginator.num_pages)
return render(
request, 'smartshark/project/plugin_execution_status.html', {
'plugin_execution': plugin_execution,
'filter': job_filter,
'jobs': jobs,
'overall': len(job_filter.qs),
'queried_status': job_filter.data.get('status', None),
'done_jobs': len(job_filter.qs.filter(status='DONE')),
'exit_jobs': len(rev),
'waiting_jobs': len(job_filter.qs.filter(status='WAIT')),
'exit_job_revisions': exit_job_revisions
})
def recheck_coast_parse_error(self, request, queryset):
# die on multiple projects!
self._die_on_multiple_projects(queryset)
interface = PluginManagementInterface.find_correct_plugin_manager()
# we need to get the most current job for each commit (because of repetitions for coastSHARK runs)
jobs = {}
for pe in PluginExecution.objects.filter(
plugin__name__startswith='coastSHARK',
project=queryset[0].project).order_by('submitted_at'):
for obj in queryset:
try:
jobs[obj.commit] = Job.objects.get(
plugin_execution=pe, revision_hash=obj.commit)
except Job.DoesNotExist:
pass
modified = 0
not_modified = []
for obj in queryset:
# split of file for coastSHARK
tmp = obj.text
collect_state = False
coast_files = []
for line in tmp.split('\n'):
if collect_state and not line.strip().startswith(
'+++ mecoSHARK +++'):
coast_files.append(line.strip()[1:])
if line.strip().startswith('+++ coastSHARK +++'):
collect_state = True
if line.strip().startswith('+++ mecoSHARK +++'):
collect_state = False
job = jobs[obj.commit]
stdout = interface.get_output_log(job)
new_lines = []
parse_error_files = []
for file in coast_files:
for line in stdout:
if file in line and line.startswith(
('Parser Error in file', 'Lexer Error in file')):
new_lines.append(file + ' ({})'.format(line))
parse_error_files.append(file)
if set(parse_error_files) == set(coast_files):
modified += 1
else:
not_modified.append(obj.commit)
logger.info(
'no change to True on following commits: {}'.format(not_modified))
messages.info(
request,
'Would fix coastSHARK verification on {} of {} commits.'.format(
modified, len(queryset)))
def handle(self, *args, **options):
tmp = options['plugin_name'].split('=')
if len(tmp) > 1:
plugin_name = tmp[0]
plugin_version = tmp[1]
plugin = Plugin.objects.get(name__icontains=plugin_name,
version=plugin_version)
else:
plugin = Plugin.objects.get(name__icontains=options['plugin_name'])
project = Project.objects.get(name__icontains=options['project_name'])
pe = PluginExecution.objects.filter(
plugin=plugin, project=project).order_by('submitted_at')[0]
self.stdout.write('looking in pluginexecution from: {}'.format(
pe.submitted_at))
jobs = Job.objects.filter(plugin_execution=pe,
status=options['filter_state'].upper())
if not options['execute']:
self.stdout.write(
'not executing, to execute operation run with --execute')
h = 'Searching for {} in {} logs of {} jobs with state {} for plugin {} on project {}'.format(
options['filter_string'], options['filter_log_type'], len(jobs),
options['filter_state'], plugin.name, project.name)
self.stdout.write(h)
interface = PluginManagementInterface.find_correct_plugin_manager()
if options['execute']:
found_revs = []
notfound_revs = []
for job in jobs:
output = []
if options['filter_log_type'] == 'error':
output = interface.get_error_log(job)
if options['filter_log_type'] == 'output':
output = interface.get_output_log(job)
if output:
output = '\n'.join(output)
if options['filter_string'] in output:
found_revs.append(job.revision_hash)
else:
notfound_revs.append(job.revision_hash)
self.stdout.write('String found in {} of {} jobs'.format(
len(found_revs), len(jobs)))
if len(notfound_revs) < len(found_revs):
self.stdout.write('not found: {}'.format(
','.join(notfound_revs)))
else:
self.stdout.write('found: {}'.format(','.join(found_revs)))
def check_coast_parse_error(self, request, queryset):
# die on multiple projects!
self._die_on_multiple_projects(queryset)
interface = PluginManagementInterface.find_correct_plugin_manager()
# we need to get the most current job for each commit (because of repetitions for coastSHARK runs)
jobs = {}
for pe in PluginExecution.objects.filter(plugin__name__startswith='coastSHARK', project=queryset[0].project).order_by('submitted_at'):
for obj in queryset:
try:
jobs[obj.commit] = Job.objects.get(plugin_execution=pe, revision_hash=obj.commit)
except Job.DoesNotExist:
pass
except Job.MultipleObjectsReturned:
job_ids = [str(job.id) for job in Job.objects.filter(plugin_execution=pe, revision_hash=obj.commit)]
messages.warning(request, 'Commit: {} has more than one Job, ignoring it, job_ids ({})'.format(obj.commit, ','.join(job_ids)))
modified = 0
for obj in queryset:
# split of file for coastSHARK
tmp = obj.text
collect_state = False
coast_files = []
for line in tmp.split('\n'):
if collect_state and not line.strip().startswith('+++ mecoSHARK +++'):
coast_files.append(line.strip()[1:])
if line.strip().startswith('+++ coastSHARK +++'):
collect_state = True
if line.strip().startswith('+++ mecoSHARK +++'):
collect_state = False
job = jobs[obj.commit]
stdout = interface.get_output_log(job)
new_lines = []
parse_error_files = []
for file in coast_files:
for line in stdout:
if file in line and line.startswith(('Parser Error in file', 'Lexer Error in file')):
new_lines.append(file + ' ({})'.format(line))
parse_error_files.append(file)
if set(parse_error_files) == set(coast_files):
obj.coastSHARK = True
modified += 1
if new_lines:
obj.text = '\n'.join(new_lines) + '\n----\n' + obj.text
obj.save()
messages.info(request, 'Changed coastSHARK verification to True on {} of {} commits.'.format(modified, len(queryset)))
def install(request):
plugins = []
if not request.user.is_authenticated() or not request.user.has_perm(
'smartshark.install_plugin'):
messages.error(request,
'You are not authorized to perform this action.')
return HttpResponseRedirect('/admin/smartshark/plugin')
if not request.GET.get('ids'):
messages.error(request, 'No plugin ids were given to install.')
return HttpResponseRedirect('/admin/smartshark/plugin')
for plugin_id in request.GET.get('ids', '').split(','):
plugin = get_object_or_404(Plugin, pk=plugin_id)
plugins.append(plugin)
if request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect('/admin/smartshark/plugin')
# create a form instance and populate it with data from the request:
form = get_form(plugins, request.POST, 'install')
# check whether it's valid:
if form.is_valid():
# Parse the fields and set the corresponding values of the install arguments in the database
set_argument_values(form.cleaned_data)
# Install plugins
installations = PluginManagementInterface.find_correct_plugin_manager(
).install_plugins(plugins)
# Check if plugins successfully installed
append_success_messages_to_req(installations, plugins, request)
return HttpResponseRedirect('/admin/smartshark/plugin')
# if a GET (or any other method) we'll create a blank form
else:
form = get_form(plugins, request.POST or None, 'install')
return render(
request, 'smartshark/plugin/install.html', {
'form': form,
'plugins': plugins,
'substitutions': create_substitutions_for_display()
})
def handle(self, *args, **options):
tmp = options['plugin_name'].split('=')
if len(tmp) > 1:
plugin_name = tmp[0]
plugin_version = Decimal(tmp[1])
plugin = Plugin.objects.get(name__icontains=plugin_name,
version=plugin_version)
else:
plugin = Plugin.objects.get(name__icontains=options['plugin_name'])
project = Project.objects.get(name__icontains=options['project_name'])
pe = PluginExecution.objects.get(plugin=plugin, project=project)
jobs = Job.objects.filter(plugin_execution=pe,
status=options['filter_state'].upper())
if not options['execute']:
self.stdout.write(
'not executing, to execute operation run with --execute')
h = 'Searching for {} in {} logs of {} jobs with state {} for plugin {} on project {}'.format(
options['filter_string'], options['filter_log_type'], len(jobs),
options['filter_state'], plugin.name, project.name)
self.stdout.write(h)
interface = PluginManagementInterface.find_correct_plugin_manager()
if options['execute']:
found = 0
for job in jobs:
output = []
if options['filter_log_type'] == 'error':
output = interface.get_error_log(job)
if options['filter_log_type'] == 'output':
output = interface.get_output_log(job)
if output:
output = '\n'.join(output)
if options['filter_string'] in output:
found += 1
self.stdout.write('String found in {} of {} jobs'.format(
found, len(jobs)))
def start_collection(request):
ak = request.POST.get('ak', None)
if not ak:
return HttpResponse('Unauthorized', status=401)
if ak != settings.API_KEY:
return HttpResponse('Unauthorized', status=401)
project_mongo_ids = request.POST.get('project_mongo_ids', None)
plugin_ids = request.POST.get('plugin_ids', None)
if not project_mongo_ids:
return HttpResponse('project mongo ids required', status=400)
if not plugin_ids:
return HttpResponse('no plugins selected', status=400)
interface = PluginManagementInterface.find_correct_plugin_manager()
plugins = []
projects = []
for mongo_id in project_mongo_ids.split(','):
project = Project.objects.get(mongo_id=mongo_id)
projects.append(project)
for plugin_id in plugin_ids.split(','):
plugin = Plugin.objects.get(pk=plugin_id, active=True, installed=True)
plugins.append(plugin)
for project in projects:
# check plugin requirements
for req_plugin in plugin.requires.all():
if not _check_if_at_least_one_execution_was_successful(req_plugin, project):
return HttpResponse('not all requirements for plugin {} are met. Plugin {} was not executed successfully for project {} before'.format(plugin, req_plugin, project), status=400)
# check if plugin alredy runs
plugin_executions = PluginExecution.objects.all().filter(plugin=plugin, project=project)
# Get all jobs from all plugin_executions which did not terminate yet
jobs = []
for plugin_execution in plugin_executions:
jobs.extend(Job.objects.filter(plugin_execution=plugin_execution, status='WAIT').all())
# Update the job stati for these jobs
job_stati = interface.get_job_stati(jobs)
i = 0
for job in jobs:
job.status = job_stati[i]
job.save()
# check if some plugin has unfinished jobs
has_unfinished_jobs = False
for plugin_execution in plugin_executions:
if plugin_execution.has_unfinished_jobs():
has_unfinished_jobs = True
if has_unfinished_jobs:
return HttpResponse('Plugin {} has unfinished jobs in project {}'.format(plugin, project), status=500)
form = get_form(plugins, request.POST, 'execute')
if form.is_valid():
execution_type = form.cleaned_data.get('execution', None)
revisions = form.cleaned_data.get('revisions', None)
repository_url = form.cleaned_data.get('repository_url', None)
sorted_plugins = order_plugins(plugins)
for project in projects:
plugin_executions = []
for plugin in sorted_plugins:
# Create Plugin Execution Objects
plugin_execution = PluginExecution(project=project, plugin=plugin)
if plugin.plugin_type == 'repo' or plugin.plugin_type == 'rev':
plugin_execution.repository_url = repository_url
if plugin.plugin_type == 'rev':
plugin_execution.execution_type = execution_type
plugin_execution.revisions = revisions
plugin_execution.save()
plugin_executions.append(plugin_execution)
# Set execution history with execution values for the plugin execution
set_argument_execution_values(form.cleaned_data, plugin_executions)
# Create jobs and execute them in a separate thread
thread = JobSubmissionThread(project, plugin_executions)
thread.start()
return HttpResponse(status=202)
else:
print(form.errors)
return HttpResponse('Form Invalid', status=400)
def handle(self, *args, **options):
if not options['project_name']:
self.stdout.write('need project name')
interface = PluginManagementInterface.find_correct_plugin_manager()
project = Project.objects.get(name__iexact=options['project_name'])
# get failed commits from CommitVerification where coastSHARK failed
commits = CommitVerification.objects.filter(project=project,
coastSHARK=False)
# we need to get the most current job for each commit (because of repetitions for coastSHARK runs)
jobs = {}
for pe in PluginExecution.objects.filter(
plugin__name__startswith='coastSHARK',
project=project).order_by('submitted_at'):
for obj in commits:
try:
jobs[obj.commit] = Job.objects.get(
plugin_execution=pe, revision_hash=obj.commit)
except Job.DoesNotExist:
pass
except Job.MultipleObjectsReturned:
jobs[obj.commit] = Job.objects.filter(
plugin_execution=pe, revision_hash=obj.commit).last()
modified = 0
for obj in commits:
# we have not found a previous job with this commit, we can not check for parse error
if obj.commit not in jobs.keys():
self.stdout.write(
'could not find revision {} in previous jobs, skipping'.
format(obj.commit))
continue
# split of file for coastSHARK
tmp = obj.text
collect_state = False
coast_files = []
for line in tmp.split('\n'):
if collect_state and not line.strip().startswith(
'+++ mecoSHARK +++'):
coast_files.append(line.strip()[1:])
if line.strip().startswith('+++ coastSHARK +++'):
collect_state = True
if line.strip().startswith('+++ mecoSHARK +++'):
collect_state = False
# get job from our precalculated dict and fetch its stdout log
job = jobs[obj.commit]
stdout = interface.get_output_log(job)
new_lines = []
parse_error_files = []
for file in coast_files:
for line in stdout:
if file in line and line.startswith(
('Parser Error in file', 'Lexer Error in file')):
new_lines.append(file + ' ({})'.format(line))
parse_error_files.append(file)
if set(parse_error_files) == set(coast_files):
obj.coastSHARK = True
modified += 1
if new_lines:
obj.text = '\n'.join(new_lines) + '\n----\n' + obj.text
obj.save()
self.stdout.write(
'Changed coastSHARK verification to True on {} of {} commits.'.
format(modified, len(commits)))
import os
from smartshark.datacollection.pluginmanagementinterface import PluginManagementInterface
from smartshark.models import SmartsharkUser, Plugin, Project, PluginExecution
from django.db.models.signals import post_save, pre_save, m2m_changed, post_delete, pre_delete
from django.dispatch import receiver
import tarfile
from smartshark.mongohandler import handler
interface = PluginManagementInterface.find_correct_plugin_manager()
@receiver(m2m_changed, sender=SmartsharkUser.roles.through)
def add_roles(sender, **kwargs):
if kwargs['action'] == 'post_remove' or kwargs['action'] == 'post_add':
user = kwargs["instance"]
roles = [role.name for role in user.roles.all()]
handler.update_roles(user.user.username, roles)
@receiver(post_delete, sender=Plugin)
def delete_archive(sender, **kwargs):
plugin = kwargs["instance"]
try:
os.remove(plugin.get_full_path_to_archive())
except FileNotFoundError:
pass
# delete plugin on bash system
interface.delete_plugins([plugin])
def choose_plugins(request):
projects = []
if not request.user.is_authenticated() or not request.user.has_perm(
'smartshark.start_collection'):
messages.error(request,
'You are not authorized to perform this action.')
return HttpResponseRedirect('/admin/smartshark/project')
if request.GET.get('ids'):
for project_id in request.GET.get('ids', '').split(','):
projects.append(get_object_or_404(Project, pk=project_id))
else:
messages.error(request, 'No project ids were given.')
return HttpResponseRedirect('/admin/smartshark/project')
# if this is a POST request we need to process the form data
if request.method == 'POST':
if 'cancel' in request.POST:
return HttpResponseRedirect('/admin/smartshark/project')
# create a form instance and populate it with data from the request:
form = ProjectForm(request.POST)
# check whether it's valid:
if form.is_valid():
plugin_ids = []
interface = PluginManagementInterface.find_correct_plugin_manager()
# check requirements
for plugin in form.cleaned_data['plugins']:
plugin_ids.append(str(plugin.id))
# check for each plugin if required plugin is set
'''
missing_plugins = []
for req_plugin in plugin.requires.all():
if req_plugin not in form.cleaned_data['plugins']:
missing_plugins.append(str(req_plugin))
if missing_plugins:
messages.error(request, 'Not all requirements for plugin %s are met. Plugin(s) %s is/are required!'
% (str(plugin), ', '.join(missing_plugins)))
return HttpResponseRedirect(request.get_full_path())
'''
# check if schema problems exist between plugins
# TODO
# if plugin with this project is in plugin execution and has status != finished | error -> problem
for project in projects:
for req_plugin in plugin.requires.all():
logger.debug("Looking at required plugin %s" %
str(req_plugin))
# todo: implement check for plugins taking into account the plugin version e.g., if vcsshark-0.10 is required
# also allow vcsshark-0.11 or newer (if info.json allows that (>=))
#if not _check_if_at_least_one_execution_was_successful(req_plugin, project):
# messages.error(request,
# 'Not all requirements for plugin %s are met. Plugin %s was not executed '
# 'successfully for project %s before!'
# % (str(plugin), str(req_plugin), str(project)))
# return HttpResponseRedirect(request.get_full_path())
logger.debug(
"At least one plugin execution for plugin %s was successful."
% str(req_plugin))
# Update job information
plugin_executions = PluginExecution.objects.all().filter(
plugin=plugin, project=project)
# Get all jobs from all plugin_executions which did not terminate yet
jobs = []
for plugin_execution in plugin_executions:
jobs.extend(
Job.objects.filter(
plugin_execution=plugin_execution,
status='WAIT').all())
# Update the job stati for these jobs
job_stati = interface.get_job_stati(jobs)
i = 0
for job in jobs:
job.status = job_stati[i]
job.save()
# check if some plugin has unfinished jobs
has_unfinished_jobs = False
for plugin_execution in plugin_executions:
if plugin_execution.has_unfinished_jobs():
has_unfinished_jobs = True
if has_unfinished_jobs:
messages.error(
request,
'Plugin %s is already scheduled for project %s.' %
(str(plugin), project))
return HttpResponseRedirect(request.get_full_path())
return HttpResponseRedirect(
'/smartshark/project/collection/start?plugins=%s&projects=%s' %
(','.join(plugin_ids), request.GET.get('ids')))
# if a GET (or any other method) we'll create a blank form
else:
form = ProjectForm()
return render(request, 'smartshark/project/action_collection.html', {
'form': form,
'projects': projects,
})
