def __init__(self, source, highlights=None, filename=None, line=None, references_to=None, references_from=None, references_declarations=None): self._source = source self._source_len = len(source) try: self._highlights = self.__get_highlights(highlights) except Exception as e: logger.error('Source highlights error ({}:{}): {}'.format( filename, line, e)) self._highlights = OrderedDict([((0, self._source_len), None)]) try: self.references_data = [] self.declarations = {} self._references = self.__get_references(references_to, references_from, references_declarations) except Exception as e: logger.error('Source references error ({}:{}): {}'.format( filename, line, e)) self.references_data = [] self.declarations = {} self._references = [] self.html_code = self.__to_html()
def get_time(): while True: try: return time.time() except Exception as e: logger.error(e) pass
def __get_tree(self): if self.type == 'safe': tags_qs = SafeTag.objects elif self.type == 'unsafe': tags_qs = UnsafeTag.objects else: raise BridgeException() if isinstance(self._tags_ids, (list, set)): tags_qs = tags_qs.filter(id__in=self._tags_ids) data = {} for tag in tags_qs.select_related('author').order_by('id'): data[tag.pk] = { 'object': tag, 'parent': tag.parent_id, 'children': [] } for t_id in data: if not data[t_id]['parent']: continue if data[t_id]['parent'] not in data: logger.error('Tag {} does not have parent'.format(data[t_id]['object'].name)) continue data[data[t_id]['parent']]['children'].append(data[t_id]['object'].id) return data
def update_decision(self, decision, new_status, old_status=None): status_map = { TASK_STATUS[0][0]: 'tasks_pending', TASK_STATUS[1][0]: 'tasks_processing', TASK_STATUS[2][0]: 'tasks_finished', TASK_STATUS[3][0]: 'tasks_error', TASK_STATUS[4][0]: 'tasks_cancelled' } if old_status: # Decrement counter for old status decr_field = status_map[old_status] old_num = getattr(decision, decr_field) if old_num > 0: setattr(decision, decr_field, old_num - 1) else: logger.error( 'Something wrong with Decision: number of {} tasks is 0, ' 'but there is at least one such task in the system'.format( old_status)) else: # Task was created decision.tasks_total += 1 # Increment counter for new status incr_field = status_map[new_status] new_num = getattr(decision, incr_field) setattr(decision, incr_field, new_num + 1) decision.save()
def copy_files_with_replace(request, decision_id, files_qs): new_job_files = [] files_to_replace = {} # Upload provided files first if request.data.get('files'): if isinstance(request.data['files'], str): try: files_map = json.loads(request.data['files']) except Exception as e: logger.error("Can't decode files data: {}".format( request.data['files'])) raise BridgeException("Can't decode files data: {}".format(e)) elif isinstance(request.data['files'], dict): files_map = request.data['files'] else: raise BridgeException('Wrong files data: "{}" ({})'.format( request.data['files'], type(request.data['files']))) for fname, fkey in files_map.items(): if fkey in request.FILES: files_to_replace[fname] = request.FILES[fkey] new_file = file_get_or_create(files_to_replace[fname], fname, JobFile) new_job_files.append( FileSystem(decision_id=decision_id, file_id=new_file.id, name=fname)) # Copy other files for f_id, f_name in files_qs: if f_name in files_to_replace: continue new_job_files.append( FileSystem(decision_id=decision_id, file_id=f_id, name=f_name)) FileSystem.objects.bulk_create(new_job_files)
def __recalc(self): if self.type == 'leaves': RecalculateLeaves(self._roots) elif self.type == 'unsafe': UnsafeUtils.RecalculateConnections(self._roots) elif self.type == 'safe': SafeUtils.RecalculateConnections(self._roots) elif self.type == 'unknown': UnknownUtils.RecalculateConnections(self._roots) elif self.type == 'resources': RecalculateResources(self._roots) elif self.type == 'compinst': RecalculateComponentInstances(self._roots) elif self.type == 'coverage': RecalculateCoverageCache(self._roots) elif self.type == 'all': RecalculateLeaves(self._roots) UnsafeUtils.RecalculateConnections(self._roots) SafeUtils.RecalculateConnections(self._roots) UnknownUtils.RecalculateConnections(self._roots) RecalculateResources(self._roots) RecalculateComponentInstances(self._roots) RecalculateCoverageCache(self._roots) elif self.type == 'for_uploaded': RecalculateLeaves(self._roots) UnsafeUtils.RecalculateConnections(self._roots) SafeUtils.RecalculateConnections(self._roots) UnknownUtils.RecalculateConnections(self._roots) RecalculateComponentInstances(self._roots) RecalculateCoverageCache(self._roots) else: logger.error('Wrong type of recalculation') raise BridgeException()
def __recalc(self): if self.type == 'leaves': RecalculateLeaves(self._decisions) elif self.type == 'safe_links': recalculate_safe_links(self._decisions) elif self.type == 'unsafe_links': recalculate_unsafe_links(self._decisions) elif self.type == 'unknown_links': recalculate_unknown_links(self._decisions) elif self.type == 'safe_reports': reports_ids = list(ReportSafe.objects.filter(decision_id__in=self._decision_ids) .values_list('id', flat=True)) RecalculateSafeCache(reports_ids) elif self.type == 'unsafe_reports': reports_ids = list(ReportUnsafe.objects.filter(decision_id__in=self._decision_ids) .values_list('id', flat=True)) RecalculateUnsafeCache(reports_ids) elif self.type == 'unknown_reports': reports_ids = list(ReportUnknown.objects.filter(decision_id__in=self._decision_ids) .values_list('id', flat=True)) RecalculateUnknownCache(reports_ids) elif self.type == 'decision_cache': RecalculateDecisionCache(self._decisions) elif self.type == 'coverage': RecalculateCoverage(self._decisions) elif self.type == 'all': RecalculateLeaves(self._decisions) recalculate_safe_links(self._decisions) recalculate_unsafe_links(self._decisions) recalculate_unknown_links(self._decisions) RecalculateDecisionCache(self._decisions) RecalculateCoverage(self._decisions) else: logger.error('Wrong type of recalculation') raise BridgeException()
def __finish_report_component(self, data): self._logger.log("F0", data.get('identifier')) data['log'] = self.__get_archive(data.get('log')) data['attr_data'] = self.__upload_attrs_files(self.__get_archive(data.get('attr_data'))) # Update report atomicly with transaction.atomic(): report = self.__get_report_for_update(data.get('identifier')) self._logger.log("F1", report.pk, report.identifier) serializer = ReportComponentSerializer( instance=report, data=data, decision=self.decision, fields={'wall_time', 'cpu_time', 'memory', 'attrs', 'data', 'log'} ) serializer.is_valid(raise_exception=True) report = serializer.save(finish_date=now()) self._logger.log("F2", report.pk) self.__update_decision_cache( report.component, finished=True, cpu_time=report.cpu_time, wall_time=report.wall_time, memory=report.memory ) self._logger.log("F3", report.pk) # Get report ancestors before report might be deleted if report.parent and self.decision.is_lightweight: # WARNING: If report has children it will be deleted! # It means verification reports (children) are not finished yet. self._logger.log("F4", report.pk) if not self.__remove_report(report.pk): logger.error('Report deletion is failed!')
def __get_status(self, status): if status not in set(x[0] for x in DECISION_STATUS): raise ValueError('Unsupported status: %s' % status) if status == DECISION_STATUS[3][0]: if self.decision.status != DECISION_STATUS[2][0]: self.error = "Only processing decisions can be finished" return DECISION_STATUS[5][0] unfinished_reports = list( ReportComponent.objects.filter(decision=self.decision, finish_date=None).values_list( 'identifier', flat=True)) if len(unfinished_reports) > 0: self.error = 'There are unfinished reports ({}): {}'.format( len(unfinished_reports), unfinished_reports) logger.error(self.error) if len(self.error) > 1024: self.error = 'There are unfinished reports ({})'.format( len(unfinished_reports)) return DECISION_STATUS[5][0] try: core_r = ReportComponent.objects.get(parent=None, decision=self.decision) except ReportComponent.DoesNotExist: self.error = "The decision doesn't have Core report" return DECISION_STATUS[5][0] if ReportUnknown.objects.filter(parent=core_r, component=core_r.component, decision=self.decision).exists(): return DECISION_STATUS[4][0] try: self.__check_progress() except ServiceError as e: self.error = str(e) return DECISION_STATUS[5][0] except Exception as e: logger.exception(e) self.error = 'Unknown error while checking progress' return DECISION_STATUS[5][0] elif status == DECISION_STATUS[4][0]: try: core_r = ReportComponent.objects.get(parent=None, decision=self.decision) except ReportComponent.DoesNotExist: pass else: unfinished_components = ReportComponent.objects.filter( decision=self.decision, finish_date=None) core_unknowns = ReportUnknown.objects.filter( parent=core_r, component=core_r.component, decision=self.decision) if unfinished_components.exists( ) or not core_unknowns.exists(): status = DECISION_STATUS[5][0] if self.error is None: self.error = "The scheduler hasn't given an error description" return status
def __get_unknown_desc(self, report): try: return ArchiveFileContent(report, 'problem_description', PROBLEM_DESC_FILE).content.decode('utf8') except Exception as e: logger.error("Can't get problem description for unknown '%s': %s" % (report.id, e)) return None
def upload_all(self, reports): # Check that all archives are valid ZIP files for report in reports: try: self.__upload(report) except Exception as e: if str(e).__contains__('report_decision_id_identifier'): logger.error('UniqueError') logger.exception(e) self.__process_exception(e)
def get_objects(self): if self._objects is None: try: self.__generate_sql() except EmptyQuery as e: logger.error(e) self._objects = [] else: self._objects = self.sql.execute() return self._objects
def __connect_unknown_mark(self): reports_filter = {'component': self.mark.component} if len(self._mark_attrs) > 0: reports_filter['id__in'] = set() for unknown_id in self._unknowns_attrs: if self._mark_attrs.issubset(self._unknowns_attrs[unknown_id]): reports_filter['id__in'].add(unknown_id) new_markreports = [] problems = {} for unknown in ReportUnknown.objects.filter(**reports_filter): try: problem_description = ArchiveFileContent(unknown, 'problem_description', PROBLEM_DESC_FILE)\ .content.decode('utf8') except Exception as e: logger.exception( "Can't get problem description for unknown '%s': %s" % (unknown.id, e)) return problem = MatchUnknown(problem_description, self.mark.function, self.mark.problem_pattern, self.mark.is_regexp).problem if problem is None: continue elif len(problem) > 20: problem = 'Too long!' logger.error("Problem '%s' for mark %s is too long" % (problem, self.mark.identifier), stack_info=True) if problem not in problems: problems[problem] = UnknownProblem.objects.get_or_create( name=problem)[0] ass_type = ASSOCIATION_TYPE[0][0] if self._prime_id == unknown.id: ass_type = ASSOCIATION_TYPE[1][0] new_markreports.append( MarkUnknownReport(mark=self.mark, report=unknown, problem=problems[problem], type=ass_type, author=self.mark.author)) if unknown in self.changes: self.changes[unknown]['kind'] = '=' else: self.changes[unknown] = {'kind': '+', 'problems': {}} if problems[problem].id not in self.changes[unknown]['problems']: self.changes[unknown]['problems'][problems[problem].id] = [ 0, 0 ] self.changes[unknown]['problems'][problems[problem].id][1] += 1 MarkUnknownReport.objects.bulk_create(new_markreports) update_unknowns_cache(list(self.changes))
def __update_resources(self, newdata): d = newdata while d is not None: if d['parent'] is not None and d['parent'] not in self._data: logger.error('Updating resources failed', stack_info=True) return self._resources.update(d['id'], newdata) if d['parent'] is not None: d = self._data[d['parent']] else: d = None
def get_context_data(self, instance, **kwargs): if 'line' not in self.request.GET: raise exceptions.APIException('File line was not provided') if 'file_name' not in self.request.GET: raise exceptions.APIException('File name was not provided') context = super().get_context_data(instance, **kwargs) context['data'] = GetCoverageData( instance, self.request.query_params['line'], self.request.query_params['file_name']).data if not context['data']: logger.error('Coverage data was not found') raise exceptions.APIException('Coverage data was not found') return context
def post(self, request): if not MarkAccess(request.user).can_upload: raise exceptions.PermissionDenied( _("You don't have an access to create new marks")) marks_links = [] failed_mark_uploads = 0 marks_uploader = MarksUploader(request.user) for f in self.request.FILES.getlist('file'): with zipfile.ZipFile(f, 'r') as zfp: if all( file_name.endswith('.zip') for file_name in zfp.namelist()): marks_dir = extract_archive(f) for arch_name in os.listdir(marks_dir.name): with open(os.path.join(marks_dir.name, arch_name), mode='rb') as fp: try: marks_links.append( marks_uploader.upload_mark( File(fp, name=arch_name))[1]) except Exception as e: logger.exception(e) logger.error( 'Uploading of mark "{}" has failed.'. format(arch_name)) failed_mark_uploads += 1 else: marks_links.append(marks_uploader.upload_mark(f)[1]) if len(marks_links) == 1: return Response({'url': marks_links[0]}) if failed_mark_uploads: return Response({ 'message': _('Number of created marks: %(number)s.' ' Number of marks which uploading failed: %(failed_number)s.' ' See logs for details.') % { 'number': len(marks_links), 'failed_number': failed_mark_uploads } }) else: return Response({ 'message': _('Number of created marks: %(number)s') % { 'number': len(marks_links) } })
def __init__(self, description, func, pattern, is_regexp): self.description = description self.function = func self.pattern = pattern if is_regexp: self.problem = self.__match_desc_regexp() else: self.problem = self.__match_desc() if isinstance(self.problem, str) and len(self.problem) == 0: self.problem = None if isinstance(self.problem, str) and len(self.problem) > MAX_PROBLEM_LEN: logger.error("Generated problem '%s' is too long" % self.problem) self.problem = 'Too long!'
def __clear_files_with_ref(self, model, files_dir): objects_without_relations(model).delete() files_in_the_system = set() to_delete = set() for instance in model.objects.all(): # file_path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, f.file.name)) for file_path in self.__files_paths(instance): files_in_the_system.add(file_path) if not (os.path.exists(file_path) and os.path.isfile(file_path)): logger.error('Deleted from DB (file not exists): {}'.format( os.path.relpath(file_path, settings.MEDIA_ROOT) ), stack_info=True) to_delete.add(instance.pk) model.objects.filter(id__in=to_delete).delete() self.__clear_unused_files(files_dir, files_in_the_system)
def __upload_all(self, marks_dir): upload_result = {'safe': 0, 'unsafe': 0, 'unknown': 0, 'fail': 0} for file_name in os.listdir(marks_dir): mark_path = os.path.join(marks_dir, file_name) if os.path.isfile(mark_path): with open(mark_path, mode='rb') as fp: try: mark_type = self._uploader.upload_mark(fp)[0] except Exception as e: logger.exception(e) logger.error( 'Uploading of mark "{}" has failed.'.format( file_name)) mark_type = 'fail' upload_result.setdefault(mark_type, 0) upload_result[mark_type] += 1 return upload_result
def save_notifications(request): activate(request.user.extended.language) if request.method == 'POST': try: new_ntf = request.user.notifications except ObjectDoesNotExist: new_ntf = Notifications() new_ntf.user = request.user try: new_ntf.self_ntf = json.loads(request.POST.get( 'self_ntf', 'false')) except Exception as e: logger.error("Can't parse json: %s" % e, stack_info=True) return JsonResponse({'error': str(UNKNOWN_ERROR)}) new_ntf.settings = request.POST.get('notifications', '[]') new_ntf.save() return JsonResponse({'message': _('Saved')}) return JsonResponse({'error': str(UNKNOWN_ERROR)})
def create_job(kwargs): if 'name' not in kwargs or len(kwargs['name']) == 0: logger.error('The job name was not got') raise BridgeException() try: Job.objects.get(name=kwargs['name']) except ObjectDoesNotExist: pass else: raise BridgeException(_('The job name is already used')) if 'author' not in kwargs or not isinstance(kwargs['author'], User): logger.error('The job author was not got') raise BridgeException() newjob = Job(name=kwargs['name'], change_date=now(), change_author=kwargs['author'], parent=kwargs.get('parent')) if 'identifier' in kwargs and kwargs['identifier'] is not None: if Job.objects.filter(identifier=kwargs['identifier']).count() > 0: # This exception will be occurred only on jobs population (if for preset jobs identifier would be set) # or jobs uploading raise BridgeException(_('The job with specified identifier already exists')) newjob.identifier = kwargs['identifier'] else: time_encoded = now().strftime("%Y%m%d%H%M%S%f%z").encode('utf-8') newjob.identifier = hashlib.md5(time_encoded).hexdigest() newjob.save() new_version = create_version(newjob, kwargs) if 'filedata' in kwargs: try: SaveFileData(kwargs['filedata'], new_version) except Exception as e: logger.exception(e) newjob.delete() raise BridgeException() if 'absolute_url' in kwargs: # newjob_url = reverse('jobs:job', args=[newjob.pk]) # Notify(newjob, 0, {'absurl': kwargs['absolute_url'] + newjob_url}) pass else: # Notify(newjob, 0) pass return newjob
def get_context_data(self, **kwargs): job_access = jobs.utils.JobAccess(self.request.user, self.object) if not job_access.can_view(): raise BridgeException(code=400) versions = jobs.utils.JobVersionsData(self.object, self.request.user) if versions.first_version is None: logger.error("There is a job without versions") raise BridgeException() report = ReportComponent.objects.filter(root__job=self.object, parent=None).first() return { 'job': self.object, 'job_access': job_access, 'created_by': versions.first_version.change_author, 'versions': versions.versions, 'last_version': versions.last_version, 'roles': role_info(versions.last_version, self.request.user), 'parents': jobs.utils.get_job_parents(self.request.user, self.object), 'children': jobs.utils.get_job_children(self.request.user, self.object), 'progress': GetJobsProgresses(self.request.user, [self.object.id]).data[self.object.id], 'reportdata': ViewJobData(self.request.user, self.get_view(VIEW_TYPES[2]), report) }
def __finish_verification_report(self, data): update_data = {'finish_date': now()} self._logger.log("FV0", data.get('identifier')) # Get verification report by identifier if not data.get('identifier'): raise exceptions.ValidationError(detail={'identifier': "Required"}) try: report = ReportComponent.objects.only('id', 'component', *MPTT_FIELDS)\ .get(decision=self.decision, identifier=data['identifier'], verification=True) except ReportComponent.DoesNotExist: raise exceptions.ValidationError( detail={'identifier': "The report wasn't found"}) self._logger.log("FV1", report.pk, report.identifier) if self.decision.is_lightweight: if report.is_leaf_node(): # Remove verification report if it doesn't have children for lightweight decisions # But before update decision caches self.__update_decision_cache(report.component, finished=True) self._logger.log("FV2", report.pk) if not self.__remove_report(report.pk): logger.error('Report deletion is failed!') return # Set parent to Core for lightweight decisions that will be preserved update_data['parent'] = ReportComponent.objects.only('id').get( decision=self.decision, parent=None) self._logger.log("FV3", report.pk, update_data['parent'].id) # Save report with new data with transaction.atomic(): report = ReportComponent.objects.select_for_update().get( id=report.id) for k, v in update_data.items(): setattr(report, k, v) report.save() self._logger.log("FV4", report.pk) self.__update_decision_cache(report.component, finished=True) self._logger.log("FV5", report.pk, report.parent_id)
def __get_tree(self): tags_qs = Tag.objects if isinstance(self._tags_ids, (list, set)): tags_qs = tags_qs.filter(id__in=self._tags_ids) data = OrderedDict() for tag in tags_qs.select_related('author').order_by('id'): data[tag.pk] = { 'object': tag, 'parent': tag.parent_id, 'children': [] } for t_id in data: if not data[t_id]['parent']: continue if data[t_id]['parent'] not in data: logger.error('Tag {} does not have parent'.format(data[t_id]['object'].name)) continue data[data[t_id]['parent']]['children'].append(data[t_id]['object'].id) return data
def __upload(self): actions = { 'start': self.__create_report_component, 'finish': self.__finish_report_component, 'attrs': self.__update_attrs, 'verification': self.__create_verification_report, 'verification finish': self.__finish_verification_report, 'unsafe': self.__create_unsafe_reports, 'safe': self.__create_report_safe, 'unknown': self.__create_report_unknown, 'data': self.__update_report_data, 'job coverage': self.__upload_job_coverage } identifier = self.job.identifier + self.data['id'] actions[self.data['type']](identifier) if len(self.ordered_attrs) != len(set(self.ordered_attrs)): logger.error( "Attributes were redefined. List of attributes that should be unique: %s" % self.ordered_attrs) raise ValueError("attributes were redefined")
def analize(self, max_cnt=None): with open(self._log_file, mode='r', encoding='utf-8') as fp: cnt = 0 for line in fp: m = re.match(r'^\[.*?]\s\((.*?)\)\s(.*)$', line) if m: try: exec_time = float(m.group(1)) except ValueError: logger.error('Call log ValueError: {}, {}, {}'.format( m.group(1), line, m.groups())) raise query_sql = m.group(2) if query_sql: query_sql = self.parse_sql(query_sql) if not query_sql: query_sql = 'UPDATE report TREE' self.__save_data(query_sql, exec_time) cnt += 1 if max_cnt and cnt > max_cnt: break
def __connect(self): self.report.markreport_set.all().delete() unknown_attrs = set( a_id for a_id, in self.report.attrs.values_list('attr_id')) try: problem_desc = ArchiveFileContent(self.report, 'problem_description', PROBLEM_DESC_FILE)\ .content.decode('utf8') except Exception as e: logger.exception("Can't get problem desc for unknown '%s': %s" % (self.report.id, e)) return new_markreports = [] problems = {} for mark in MarkUnknown.objects.filter( component=self.report.component): if mark.id in self._marks_attrs and not self._marks_attrs[ mark.id].issubset(unknown_attrs): continue problem = MatchUnknown(problem_desc, mark.function, mark.problem_pattern, mark.is_regexp).problem if problem is None: continue elif len(problem) > 20: problem = 'Too long!' logger.error("Generated problem '%s' for mark %s is too long" % (problem, mark.identifier), stack_info=True) if problem not in problems: problems[problem] = UnknownProblem.objects.get_or_create( name=problem)[0] new_markreports.append( MarkUnknownReport(mark=mark, report=self.report, problem=problems[problem])) MarkUnknownReport.objects.bulk_create(new_markreports) if self._update_cache: update_unknowns_cache([self.report])
def __clear_files_with_ref(self, table, files_dir): self.__is_not_used() objects_without_relations(table).delete() files_in_the_system = set() files_to_delete = set() for f in table.objects.all(): file_path = os.path.abspath( os.path.join(settings.MEDIA_ROOT, f.file.name)) files_in_the_system.add(file_path) if not (os.path.exists(file_path) and os.path.isfile(file_path)): logger.error('Deleted from DB (file not exists): %s' % f.file.name, stack_info=True) files_to_delete.add(f.pk) table.objects.filter(id__in=files_to_delete).delete() files_directory = os.path.join(settings.MEDIA_ROOT, files_dir) if os.path.exists(files_directory): files_on_disk = set( os.path.abspath(os.path.join(files_directory, x)) for x in os.listdir(files_directory)) for f in files_on_disk - files_in_the_system: os.remove(f)
def __init__(self, decision, status, error=None): self.decision = decision self.error = error self.status = self.__get_status(status) try: self.__remove_tasks() except ServiceError as e: logger.exception(e) self.decision.error = str(e) self.status = DECISION_STATUS[5][0] if self.error is not None: if len(self.error) > 1024: logger.error( "The decision '{}' finished with large error: {}".format( self.decision.identifier, self.error)) self.error = "Length of error for decision '{}' is large (1024 characters is maximum)"\ .format(self.decision.identifier) self.status = DECISION_STATUS[8][0] self.decision.error = self.error self.decision.status = self.status self.decision.finish_date = now() self.decision.save() decision_status_changed(self.decision)
def populate_preset(self, preset_job: PresetJob): if not preset_job: return False if len(self._presets_data['jobs']) == 0: return False # Find preset data preset_data = self.__find_preset_data(str(preset_job.identifier), self._presets_data['jobs']) if not preset_data: logger.error( "Preset job with identifier '{}' was not found".format( preset_job.identifier)) return False # Update just name and files if 'children' in preset_data: return self.__update_preset_dir(preset_job, name=preset_data['name']) elif 'directory' in preset_data: return self.__update_preset_leaf(preset_job, preset_data['directory'], name=preset_data['name']) return False