def run(self): interface = PluginManagementInterface.find_correct_plugin_manager() jobs = [] if self.create_jobs: jobs = create_jobs_for_execution(self.project, self.plugin_executions) interface.execute_plugins(self.project, jobs, self.plugin_executions)
def job_output(request, id, type): if not request.user.is_authenticated() or not request.user.has_perm( 'smartshark.job_output'): messages.error(request, 'You are not authorized to perform this action.') return HttpResponseRedirect('/admin/smartshark/project') job = get_object_or_404(Job, pk=id) interface = PluginManagementInterface.find_correct_plugin_manager() if type == 'output': output = interface.get_output_log(job) elif type == 'error': output = interface.get_error_log(job) elif type == 'arguments': return render( request, 'smartshark/job/execution_arguments.html', { 'exe_arguments': job.plugin_execution.executionhistory_set.all().order_by( 'execution_argument__position'), 'cmd': interface.get_sent_bash_command(job), }) return render(request, 'smartshark/job/output.html', { 'output': '\n'.join(output), 'job': job, })
def set_job_stati(self, request, queryset): interface = PluginManagementInterface.find_correct_plugin_manager() job_stati = interface.get_job_stati(queryset) i = 0 for job in queryset: job.status = job_stati[i] job.save() i += 1 messages.info(request, 'Job stati set from backend.')
def plugin_execution_status(request, id): if not request.user.is_authenticated() or not request.user.has_perm( 'smartshark.plugin_execution_status'): messages.error(request, 'You are not authorized to perform this action.') return HttpResponseRedirect('/admin/smartshark/project') interface = PluginManagementInterface.find_correct_plugin_manager() plugin_execution = get_object_or_404(PluginExecution, pk=id) # Get all jobs from all plugin_executions which did not terminate yet # Update the job stati for these jobs jobs = Job.objects.filter(plugin_execution=plugin_execution, status='WAIT').all() job_stati = interface.get_job_stati(jobs) i = 0 for job in jobs: job.status = job_stati[i] job.save() i += 1 job_filter = JobExecutionFilter( request.GET, queryset=Job.objects.all().filter(plugin_execution=plugin_execution)) rev = [ exitjob.revision_hash if exitjob.revision_hash else '' for exitjob in job_filter.qs.filter(status='EXIT') ] exit_job_revisions = '' if rev: exit_job_revisions = ','.join(rev) # Set up pagination paginator = Paginator(job_filter.qs, 10) page = request.GET.get('page') try: jobs = paginator.page(page) except PageNotAnInteger: # If page is not an integer, deliver first page. jobs = paginator.page(1) except EmptyPage: # If page is out of range (e.g. 9999), deliver last page of results. jobs = paginator.page(paginator.num_pages) return render( request, 'smartshark/project/plugin_execution_status.html', { 'plugin_execution': plugin_execution, 'filter': job_filter, 'jobs': jobs, 'overall': len(job_filter.qs), 'queried_status': job_filter.data.get('status', None), 'done_jobs': len(job_filter.qs.filter(status='DONE')), 'exit_jobs': len(rev), 'waiting_jobs': len(job_filter.qs.filter(status='WAIT')), 'exit_job_revisions': exit_job_revisions })
def recheck_coast_parse_error(self, request, queryset): # die on multiple projects! self._die_on_multiple_projects(queryset) interface = PluginManagementInterface.find_correct_plugin_manager() # we need to get the most current job for each commit (because of repetitions for coastSHARK runs) jobs = {} for pe in PluginExecution.objects.filter( plugin__name__startswith='coastSHARK', project=queryset[0].project).order_by('submitted_at'): for obj in queryset: try: jobs[obj.commit] = Job.objects.get( plugin_execution=pe, revision_hash=obj.commit) except Job.DoesNotExist: pass modified = 0 not_modified = [] for obj in queryset: # split of file for coastSHARK tmp = obj.text collect_state = False coast_files = [] for line in tmp.split('\n'): if collect_state and not line.strip().startswith( '+++ mecoSHARK +++'): coast_files.append(line.strip()[1:]) if line.strip().startswith('+++ coastSHARK +++'): collect_state = True if line.strip().startswith('+++ mecoSHARK +++'): collect_state = False job = jobs[obj.commit] stdout = interface.get_output_log(job) new_lines = [] parse_error_files = [] for file in coast_files: for line in stdout: if file in line and line.startswith( ('Parser Error in file', 'Lexer Error in file')): new_lines.append(file + ' ({})'.format(line)) parse_error_files.append(file) if set(parse_error_files) == set(coast_files): modified += 1 else: not_modified.append(obj.commit) logger.info( 'no change to True on following commits: {}'.format(not_modified)) messages.info( request, 'Would fix coastSHARK verification on {} of {} commits.'.format( modified, len(queryset)))
def handle(self, *args, **options): tmp = options['plugin_name'].split('=') if len(tmp) > 1: plugin_name = tmp[0] plugin_version = tmp[1] plugin = Plugin.objects.get(name__icontains=plugin_name, version=plugin_version) else: plugin = Plugin.objects.get(name__icontains=options['plugin_name']) project = Project.objects.get(name__icontains=options['project_name']) pe = PluginExecution.objects.filter( plugin=plugin, project=project).order_by('submitted_at')[0] self.stdout.write('looking in pluginexecution from: {}'.format( pe.submitted_at)) jobs = Job.objects.filter(plugin_execution=pe, status=options['filter_state'].upper()) if not options['execute']: self.stdout.write( 'not executing, to execute operation run with --execute') h = 'Searching for {} in {} logs of {} jobs with state {} for plugin {} on project {}'.format( options['filter_string'], options['filter_log_type'], len(jobs), options['filter_state'], plugin.name, project.name) self.stdout.write(h) interface = PluginManagementInterface.find_correct_plugin_manager() if options['execute']: found_revs = [] notfound_revs = [] for job in jobs: output = [] if options['filter_log_type'] == 'error': output = interface.get_error_log(job) if options['filter_log_type'] == 'output': output = interface.get_output_log(job) if output: output = '\n'.join(output) if options['filter_string'] in output: found_revs.append(job.revision_hash) else: notfound_revs.append(job.revision_hash) self.stdout.write('String found in {} of {} jobs'.format( len(found_revs), len(jobs))) if len(notfound_revs) < len(found_revs): self.stdout.write('not found: {}'.format( ','.join(notfound_revs))) else: self.stdout.write('found: {}'.format(','.join(found_revs)))
def check_coast_parse_error(self, request, queryset): # die on multiple projects! self._die_on_multiple_projects(queryset) interface = PluginManagementInterface.find_correct_plugin_manager() # we need to get the most current job for each commit (because of repetitions for coastSHARK runs) jobs = {} for pe in PluginExecution.objects.filter(plugin__name__startswith='coastSHARK', project=queryset[0].project).order_by('submitted_at'): for obj in queryset: try: jobs[obj.commit] = Job.objects.get(plugin_execution=pe, revision_hash=obj.commit) except Job.DoesNotExist: pass except Job.MultipleObjectsReturned: job_ids = [str(job.id) for job in Job.objects.filter(plugin_execution=pe, revision_hash=obj.commit)] messages.warning(request, 'Commit: {} has more than one Job, ignoring it, job_ids ({})'.format(obj.commit, ','.join(job_ids))) modified = 0 for obj in queryset: # split of file for coastSHARK tmp = obj.text collect_state = False coast_files = [] for line in tmp.split('\n'): if collect_state and not line.strip().startswith('+++ mecoSHARK +++'): coast_files.append(line.strip()[1:]) if line.strip().startswith('+++ coastSHARK +++'): collect_state = True if line.strip().startswith('+++ mecoSHARK +++'): collect_state = False job = jobs[obj.commit] stdout = interface.get_output_log(job) new_lines = [] parse_error_files = [] for file in coast_files: for line in stdout: if file in line and line.startswith(('Parser Error in file', 'Lexer Error in file')): new_lines.append(file + ' ({})'.format(line)) parse_error_files.append(file) if set(parse_error_files) == set(coast_files): obj.coastSHARK = True modified += 1 if new_lines: obj.text = '\n'.join(new_lines) + '\n----\n' + obj.text obj.save() messages.info(request, 'Changed coastSHARK verification to True on {} of {} commits.'.format(modified, len(queryset)))
def install(request): plugins = [] if not request.user.is_authenticated() or not request.user.has_perm( 'smartshark.install_plugin'): messages.error(request, 'You are not authorized to perform this action.') return HttpResponseRedirect('/admin/smartshark/plugin') if not request.GET.get('ids'): messages.error(request, 'No plugin ids were given to install.') return HttpResponseRedirect('/admin/smartshark/plugin') for plugin_id in request.GET.get('ids', '').split(','): plugin = get_object_or_404(Plugin, pk=plugin_id) plugins.append(plugin) if request.method == 'POST': if 'cancel' in request.POST: return HttpResponseRedirect('/admin/smartshark/plugin') # create a form instance and populate it with data from the request: form = get_form(plugins, request.POST, 'install') # check whether it's valid: if form.is_valid(): # Parse the fields and set the corresponding values of the install arguments in the database set_argument_values(form.cleaned_data) # Install plugins installations = PluginManagementInterface.find_correct_plugin_manager( ).install_plugins(plugins) # Check if plugins successfully installed append_success_messages_to_req(installations, plugins, request) return HttpResponseRedirect('/admin/smartshark/plugin') # if a GET (or any other method) we'll create a blank form else: form = get_form(plugins, request.POST or None, 'install') return render( request, 'smartshark/plugin/install.html', { 'form': form, 'plugins': plugins, 'substitutions': create_substitutions_for_display() })
def handle(self, *args, **options): tmp = options['plugin_name'].split('=') if len(tmp) > 1: plugin_name = tmp[0] plugin_version = Decimal(tmp[1]) plugin = Plugin.objects.get(name__icontains=plugin_name, version=plugin_version) else: plugin = Plugin.objects.get(name__icontains=options['plugin_name']) project = Project.objects.get(name__icontains=options['project_name']) pe = PluginExecution.objects.get(plugin=plugin, project=project) jobs = Job.objects.filter(plugin_execution=pe, status=options['filter_state'].upper()) if not options['execute']: self.stdout.write( 'not executing, to execute operation run with --execute') h = 'Searching for {} in {} logs of {} jobs with state {} for plugin {} on project {}'.format( options['filter_string'], options['filter_log_type'], len(jobs), options['filter_state'], plugin.name, project.name) self.stdout.write(h) interface = PluginManagementInterface.find_correct_plugin_manager() if options['execute']: found = 0 for job in jobs: output = [] if options['filter_log_type'] == 'error': output = interface.get_error_log(job) if options['filter_log_type'] == 'output': output = interface.get_output_log(job) if output: output = '\n'.join(output) if options['filter_string'] in output: found += 1 self.stdout.write('String found in {} of {} jobs'.format( found, len(jobs)))
def start_collection(request): ak = request.POST.get('ak', None) if not ak: return HttpResponse('Unauthorized', status=401) if ak != settings.API_KEY: return HttpResponse('Unauthorized', status=401) project_mongo_ids = request.POST.get('project_mongo_ids', None) plugin_ids = request.POST.get('plugin_ids', None) if not project_mongo_ids: return HttpResponse('project mongo ids required', status=400) if not plugin_ids: return HttpResponse('no plugins selected', status=400) interface = PluginManagementInterface.find_correct_plugin_manager() plugins = [] projects = [] for mongo_id in project_mongo_ids.split(','): project = Project.objects.get(mongo_id=mongo_id) projects.append(project) for plugin_id in plugin_ids.split(','): plugin = Plugin.objects.get(pk=plugin_id, active=True, installed=True) plugins.append(plugin) for project in projects: # check plugin requirements for req_plugin in plugin.requires.all(): if not _check_if_at_least_one_execution_was_successful(req_plugin, project): return HttpResponse('not all requirements for plugin {} are met. Plugin {} was not executed successfully for project {} before'.format(plugin, req_plugin, project), status=400) # check if plugin alredy runs plugin_executions = PluginExecution.objects.all().filter(plugin=plugin, project=project) # Get all jobs from all plugin_executions which did not terminate yet jobs = [] for plugin_execution in plugin_executions: jobs.extend(Job.objects.filter(plugin_execution=plugin_execution, status='WAIT').all()) # Update the job stati for these jobs job_stati = interface.get_job_stati(jobs) i = 0 for job in jobs: job.status = job_stati[i] job.save() # check if some plugin has unfinished jobs has_unfinished_jobs = False for plugin_execution in plugin_executions: if plugin_execution.has_unfinished_jobs(): has_unfinished_jobs = True if has_unfinished_jobs: return HttpResponse('Plugin {} has unfinished jobs in project {}'.format(plugin, project), status=500) form = get_form(plugins, request.POST, 'execute') if form.is_valid(): execution_type = form.cleaned_data.get('execution', None) revisions = form.cleaned_data.get('revisions', None) repository_url = form.cleaned_data.get('repository_url', None) sorted_plugins = order_plugins(plugins) for project in projects: plugin_executions = [] for plugin in sorted_plugins: # Create Plugin Execution Objects plugin_execution = PluginExecution(project=project, plugin=plugin) if plugin.plugin_type == 'repo' or plugin.plugin_type == 'rev': plugin_execution.repository_url = repository_url if plugin.plugin_type == 'rev': plugin_execution.execution_type = execution_type plugin_execution.revisions = revisions plugin_execution.save() plugin_executions.append(plugin_execution) # Set execution history with execution values for the plugin execution set_argument_execution_values(form.cleaned_data, plugin_executions) # Create jobs and execute them in a separate thread thread = JobSubmissionThread(project, plugin_executions) thread.start() return HttpResponse(status=202) else: print(form.errors) return HttpResponse('Form Invalid', status=400)
def handle(self, *args, **options): if not options['project_name']: self.stdout.write('need project name') interface = PluginManagementInterface.find_correct_plugin_manager() project = Project.objects.get(name__iexact=options['project_name']) # get failed commits from CommitVerification where coastSHARK failed commits = CommitVerification.objects.filter(project=project, coastSHARK=False) # we need to get the most current job for each commit (because of repetitions for coastSHARK runs) jobs = {} for pe in PluginExecution.objects.filter( plugin__name__startswith='coastSHARK', project=project).order_by('submitted_at'): for obj in commits: try: jobs[obj.commit] = Job.objects.get( plugin_execution=pe, revision_hash=obj.commit) except Job.DoesNotExist: pass except Job.MultipleObjectsReturned: jobs[obj.commit] = Job.objects.filter( plugin_execution=pe, revision_hash=obj.commit).last() modified = 0 for obj in commits: # we have not found a previous job with this commit, we can not check for parse error if obj.commit not in jobs.keys(): self.stdout.write( 'could not find revision {} in previous jobs, skipping'. format(obj.commit)) continue # split of file for coastSHARK tmp = obj.text collect_state = False coast_files = [] for line in tmp.split('\n'): if collect_state and not line.strip().startswith( '+++ mecoSHARK +++'): coast_files.append(line.strip()[1:]) if line.strip().startswith('+++ coastSHARK +++'): collect_state = True if line.strip().startswith('+++ mecoSHARK +++'): collect_state = False # get job from our precalculated dict and fetch its stdout log job = jobs[obj.commit] stdout = interface.get_output_log(job) new_lines = [] parse_error_files = [] for file in coast_files: for line in stdout: if file in line and line.startswith( ('Parser Error in file', 'Lexer Error in file')): new_lines.append(file + ' ({})'.format(line)) parse_error_files.append(file) if set(parse_error_files) == set(coast_files): obj.coastSHARK = True modified += 1 if new_lines: obj.text = '\n'.join(new_lines) + '\n----\n' + obj.text obj.save() self.stdout.write( 'Changed coastSHARK verification to True on {} of {} commits.'. format(modified, len(commits)))
import os from smartshark.datacollection.pluginmanagementinterface import PluginManagementInterface from smartshark.models import SmartsharkUser, Plugin, Project, PluginExecution from django.db.models.signals import post_save, pre_save, m2m_changed, post_delete, pre_delete from django.dispatch import receiver import tarfile from smartshark.mongohandler import handler interface = PluginManagementInterface.find_correct_plugin_manager() @receiver(m2m_changed, sender=SmartsharkUser.roles.through) def add_roles(sender, **kwargs): if kwargs['action'] == 'post_remove' or kwargs['action'] == 'post_add': user = kwargs["instance"] roles = [role.name for role in user.roles.all()] handler.update_roles(user.user.username, roles) @receiver(post_delete, sender=Plugin) def delete_archive(sender, **kwargs): plugin = kwargs["instance"] try: os.remove(plugin.get_full_path_to_archive()) except FileNotFoundError: pass # delete plugin on bash system interface.delete_plugins([plugin])
def choose_plugins(request): projects = [] if not request.user.is_authenticated() or not request.user.has_perm( 'smartshark.start_collection'): messages.error(request, 'You are not authorized to perform this action.') return HttpResponseRedirect('/admin/smartshark/project') if request.GET.get('ids'): for project_id in request.GET.get('ids', '').split(','): projects.append(get_object_or_404(Project, pk=project_id)) else: messages.error(request, 'No project ids were given.') return HttpResponseRedirect('/admin/smartshark/project') # if this is a POST request we need to process the form data if request.method == 'POST': if 'cancel' in request.POST: return HttpResponseRedirect('/admin/smartshark/project') # create a form instance and populate it with data from the request: form = ProjectForm(request.POST) # check whether it's valid: if form.is_valid(): plugin_ids = [] interface = PluginManagementInterface.find_correct_plugin_manager() # check requirements for plugin in form.cleaned_data['plugins']: plugin_ids.append(str(plugin.id)) # check for each plugin if required plugin is set ''' missing_plugins = [] for req_plugin in plugin.requires.all(): if req_plugin not in form.cleaned_data['plugins']: missing_plugins.append(str(req_plugin)) if missing_plugins: messages.error(request, 'Not all requirements for plugin %s are met. Plugin(s) %s is/are required!' % (str(plugin), ', '.join(missing_plugins))) return HttpResponseRedirect(request.get_full_path()) ''' # check if schema problems exist between plugins # TODO # if plugin with this project is in plugin execution and has status != finished | error -> problem for project in projects: for req_plugin in plugin.requires.all(): logger.debug("Looking at required plugin %s" % str(req_plugin)) # todo: implement check for plugins taking into account the plugin version e.g., if vcsshark-0.10 is required # also allow vcsshark-0.11 or newer (if info.json allows that (>=)) #if not _check_if_at_least_one_execution_was_successful(req_plugin, project): # messages.error(request, # 'Not all requirements for plugin %s are met. Plugin %s was not executed ' # 'successfully for project %s before!' # % (str(plugin), str(req_plugin), str(project))) # return HttpResponseRedirect(request.get_full_path()) logger.debug( "At least one plugin execution for plugin %s was successful." % str(req_plugin)) # Update job information plugin_executions = PluginExecution.objects.all().filter( plugin=plugin, project=project) # Get all jobs from all plugin_executions which did not terminate yet jobs = [] for plugin_execution in plugin_executions: jobs.extend( Job.objects.filter( plugin_execution=plugin_execution, status='WAIT').all()) # Update the job stati for these jobs job_stati = interface.get_job_stati(jobs) i = 0 for job in jobs: job.status = job_stati[i] job.save() # check if some plugin has unfinished jobs has_unfinished_jobs = False for plugin_execution in plugin_executions: if plugin_execution.has_unfinished_jobs(): has_unfinished_jobs = True if has_unfinished_jobs: messages.error( request, 'Plugin %s is already scheduled for project %s.' % (str(plugin), project)) return HttpResponseRedirect(request.get_full_path()) return HttpResponseRedirect( '/smartshark/project/collection/start?plugins=%s&projects=%s' % (','.join(plugin_ids), request.GET.get('ids'))) # if a GET (or any other method) we'll create a blank form else: form = ProjectForm() return render(request, 'smartshark/project/action_collection.html', { 'form': form, 'projects': projects, })