class SeveralMarksGenerator: def __init__(self, marks): self.marks = marks self.stream = ZipStream() self.name = 'KleverMarks.zip' def generate_mark(self, markgen): buf = b'' for data in self.stream.compress_stream(markgen.name, markgen): buf += data if len(buf) > CHUNK_SIZE: yield buf buf = b'' if len(buf) > 0: yield buf def __iter__(self): for mark in self.marks: if isinstance(mark, MarkSafe): markgen = SafeMarkGenerator(mark) elif isinstance(mark, MarkUnsafe): markgen = UnsafeMarkGenerator(mark) elif isinstance(mark, MarkUnknown): markgen = UnknownMarkGenerator(mark) else: continue yield from self.generate_mark(markgen) yield self.stream.close_stream()
class AllMarksGenerator: def __init__(self): curr_time = now() self.name = 'Marks--%s-%s-%s.zip' % (curr_time.day, curr_time.month, curr_time.year) self.stream = ZipStream() def generators(self): for mark in MarkSafe.objects.all(): yield SafeMarkGenerator(mark) for mark in MarkUnsafe.objects.all(): yield UnsafeMarkGenerator(mark) for mark in MarkUnknown.objects.all(): yield UnknownMarkGenerator(mark) def __iter__(self): for markgen in self.generators(): buf = b'' for data in self.stream.compress_stream(markgen.name, markgen): buf += data if len(buf) > CHUNK_SIZE: yield buf buf = b'' if len(buf) > 0: yield buf yield self.stream.close_stream()
class KleverCoreArchiveGen: def __init__(self, decision): self.decision = decision self.arcname = 'VJ__{}.zip'.format(decision.identifier) self.stream = ZipStream() def __iter__(self): for file_inst in FileSystem.objects.filter( decision=self.decision).select_related('file'): arch_name = '/'.join(['root', file_inst.name]) file_src = '/'.join( [settings.MEDIA_ROOT, file_inst.file.file.name]) for data in self.stream.compress_file(file_src, arch_name): yield data yield self.stream.close_stream()
class AllMarksGen(object): def __init__(self): curr_time = now() self.name = 'Marks--%s-%s-%s.zip' % (curr_time.day, curr_time.month, curr_time.year) self.stream = ZipStream() def __iter__(self): for table in [MarkSafe, MarkUnsafe, MarkUnknown]: for mark in table.objects.filter(~Q(version=0)): markgen = MarkArchiveGenerator(mark) buf = b'' for data in self.stream.compress_stream(markgen.name, markgen): buf += data if len(buf) > CHUNK_SIZE: yield buf buf = b'' if len(buf) > 0: yield buf yield self.stream.close_stream()
class JobsArchivesGen: def __init__(self, jobs_to_download): self.jobs = jobs_to_download self.stream = ZipStream() self.name = 'KleverJobs.zip' def generate_job(self, jobgen): buf = b'' for data in self.stream.compress_stream(jobgen.name, jobgen): buf += data if len(buf) > CHUNK_SIZE: yield buf buf = b'' if len(buf) > 0: yield buf def __iter__(self): for job_id in self.jobs: jobgen = JobArchiveGenerator( self.jobs[job_id]['instance'], decisions_ids=self.jobs[job_id]['decisions']) yield from self.generate_job(jobgen) yield self.stream.close_stream()
class MarkGeneratorBase: type = None attrs_model = None tags_model = None def __init__(self, mark): assert self.type is not None, 'Wrong usage' self.mark = mark self.name = 'Mark-{}-{}.zip'.format(self.type, self.mark.identifier) self.stream = ZipStream() def common_data(self): return { 'type': self.type, 'identifier': str(self.mark.identifier), 'is_modifiable': self.mark.is_modifiable } def version_data(self, version): data = { 'comment': version.comment, 'description': version.description, 'attrs': self.attrs.get(version.id, []), } if self.tags is not None: data['tags'] = self.tags.get(version.id, []) return data @cached_property def attrs(self): assert self.attrs_model is not None, 'Wrong usage' mark_attrs = {} for mattr in self.attrs_model.objects.filter( mark_version__mark=self.mark).order_by('id'): mark_attrs.setdefault(mattr.mark_version_id, []) mark_attrs[mattr.mark_version_id].append({ 'name': mattr.name, 'value': mattr.value, 'is_compare': mattr.is_compare }) return mark_attrs @cached_property def tags(self): if not self.tags_model: return None all_tags = {} for version_id, tag_name in self.tags_model.objects.filter(mark_version__mark=self.mark) \ .values_list('mark_version_id', 'tag__name'): all_tags.setdefault(version_id, []) all_tags[version_id].append(tag_name) return all_tags def versions_queryset(self): return self.mark.versions.all() def __iter__(self): # Add main mark data content = json.dumps(self.common_data(), ensure_ascii=False, sort_keys=True, indent=4) for data in self.stream.compress_string('mark.json', content): yield data # Add versions data for markversion in self.versions_queryset(): content = json.dumps(self.version_data(markversion), ensure_ascii=False, sort_keys=True, indent=4) for data in self.stream.compress_string( 'version-{}.json'.format(markversion.version), content): yield data yield self.stream.close_stream()
class MarkArchiveGenerator: def __init__(self, mark): self.mark = mark if isinstance(self.mark, MarkUnsafe): self.type = 'unsafe' elif isinstance(self.mark, MarkSafe): self.type = 'safe' elif isinstance(self.mark, MarkUnknown): self.type = 'unknown' else: return self.name = 'Mark-%s-%s.zip' % (self.type, self.mark.identifier[:10]) self.stream = ZipStream() def __iter__(self): for markversion in self.mark.versions.all(): version_data = { 'status': markversion.status, 'comment': markversion.comment, 'description': markversion.description } if self.type == 'unknown': version_data['function'] = markversion.function version_data['problem'] = markversion.problem_pattern version_data['is_regexp'] = markversion.is_regexp if markversion.link is not None: version_data['link'] = markversion.link else: version_data['attrs'] = [] for aname, aval, compare in markversion.attrs.order_by('id')\ .values_list('attr__name__name', 'attr__value', 'is_compare'): version_data['attrs'].append({ 'attr': aname, 'value': aval, 'is_compare': compare }) version_data['tags'] = list( tag for tag, in markversion.tags.values_list('tag__tag')) version_data['verdict'] = markversion.verdict if self.type == 'unsafe': version_data['function'] = markversion.function.name with markversion.error_trace.file.file as fp: version_data['error_trace'] = fp.read().decode('utf8') content = json.dumps(version_data, ensure_ascii=False, sort_keys=True, indent=4) for data in self.stream.compress_string( 'version-%s' % markversion.version, content): yield data common_data = { 'is_modifiable': self.mark.is_modifiable, 'mark_type': self.type, 'format': self.mark.format, 'identifier': self.mark.identifier } if self.type == 'unknown': common_data['component'] = self.mark.component.name content = json.dumps(common_data, ensure_ascii=False, sort_keys=True, indent=4) for data in self.stream.compress_string('markdata', content): yield data yield self.stream.close_stream()
class FilesForCompetitionArchive: obj_attr = 'Program fragment' requirement_attr = 'Requirements specification' def __init__(self, decision, filters): self.decision = decision self._attrs = self.__get_attrs() self._archives = self.__get_archives() self._archives_to_upload = [] self.__get_archives_to_upload(filters) self.stream = ZipStream() self.name = 'svcomp.zip' def __iter__(self): cnt = 0 names_in_use = set() for arch_path, name_pattern in self._archives_to_upload: if name_pattern in names_in_use: cnt += 1 arch_name = '%s_%s.zip' % (name_pattern, cnt) else: arch_name = '%s.zip' % name_pattern names_in_use.add(name_pattern) for data in self.stream.compress_file(arch_path, arch_name): yield data yield self.stream.close_stream() def __get_archives(self): archives = {} for report in ReportComponent.objects.filter(decision=self.decision, verification=True)\ .exclude(verifier_files='').only('id', 'verifier_files'): archives[report.id] = report.verifier_files.path return archives def __get_attrs(self): # Select attributes for all safes, unsafes and unknowns attrs = {} for report_id, a_name, a_value in ReportAttr.objects\ .filter(report__decision=self.decision, name__in=[self.obj_attr, self.requirement_attr]) \ .exclude(report__reportunsafe=None, report__reportsafe=None, report__reportunknown=None) \ .values_list('report_id', 'name', 'value'): if report_id not in attrs: attrs[report_id] = {} attrs[report_id][a_name] = a_value return attrs def __add_archive(self, r_type, r_id, p_id): if p_id in self._archives and r_id in self._attrs \ and self.obj_attr in self._attrs[r_id] \ and self.requirement_attr in self._attrs[r_id]: ver_obj = self._attrs[r_id][self.obj_attr].replace('~', 'HOME').replace('/', '---') ver_requirement = self._attrs[r_id][self.requirement_attr].replace(':', '-') dirname = 'Unknowns' if r_type == 'f' else 'Unsafes' if r_type == 'u' else 'Safes' self._archives_to_upload.append( (self._archives[p_id], '{0}/{1}__{2}__{3}'.format(dirname, r_type, ver_requirement, ver_obj)) ) def __get_archives_to_upload(self, filters): common_filters = {'decision': self.decision, 'parent__reportcomponent__verification': True} if filters.get('safes'): for r_id, p_id in ReportSafe.objects.filter(**common_filters).values_list('id', 'parent_id'): self.__add_archive('s', r_id, p_id) if filters.get('unsafes'): for r_id, p_id in ReportUnsafe.objects.filter(**common_filters).values_list('id', 'parent_id'): self.__add_archive('u', r_id, p_id) if filters.get('problems'): for problem_data in filters['problems']: if problem_data.get('component') and problem_data.get('problem'): unknowns_qs = ReportUnknown.objects.filter( markreport_set__problem=problem_data['problem'], component=problem_data['component'], **common_filters ) else: unknowns_qs = ReportUnknown.objects.filter(cache__marks_total=0, **common_filters) for r_id, p_id in unknowns_qs.values_list('id', 'parent_id'): self.__add_archive('f', r_id, p_id) elif filters.get('unknowns'): for r_id, p_id in ReportUnknown.objects.filter(**common_filters).values_list('id', 'parent_id'): self.__add_archive('f', r_id, p_id)
class JobArchiveGenerator: def __init__(self, job, decisions_ids=None): self.job = job self._decisions_ids = list(map( int, decisions_ids)) if decisions_ids else None self.name = 'Job-{}.zip'.format(self.job.identifier) self._arch_files = set() self.stream = ZipStream() def __iter__(self): # Job data yield from self.stream.compress_string('job.json', self.__get_job_data()) yield from self.stream.compress_string( '{}.json'.format(Decision.__name__), self.__add_decisions_data()) yield from self.stream.compress_string( '{}.json'.format(DecisionCache.__name__), self.__get_decision_cache()) yield from self.stream.compress_string( '{}.json'.format(OriginalSources.__name__), self.__get_original_src()) yield from self.stream.compress_string( '{}.json'.format(ReportComponent.__name__), self.__get_reports_data()) yield from self.stream.compress_string( '{}.json'.format(ReportSafe.__name__), self.__get_safes_data()) yield from self.stream.compress_string( '{}.json'.format(ReportUnsafe.__name__), self.__get_unsafes_data()) yield from self.stream.compress_string( '{}.json'.format(ReportUnknown.__name__), self.__get_unknowns_data()) yield from self.stream.compress_string( '{}.json'.format(ReportAttr.__name__), self.__get_attrs_data()) yield from self.stream.compress_string( '{}.json'.format(CoverageArchive.__name__), self.__get_coverage_data()) self.__add_job_files() self.__add_additional_sources() for file_path, arcname in self._arch_files: yield from self.stream.compress_file(file_path, arcname) yield self.stream.close_stream() @cached_property def _decision_filter(self): if self._decisions_ids: return Q(decision_id__in=self._decisions_ids) return Q(decision__job_id=self.job.id) def __get_job_data(self): return self.__get_json(DownloadJobSerializer(instance=self.job).data) def __add_job_files(self): job_files = {} for fs in FileSystem.objects.filter( decision__job=self.job).select_related('file'): job_files[fs.file.hash_sum] = (fs.file.file.path, fs.file.file.name) for f_path, arcname in job_files.values(): self._arch_files.add((f_path, arcname)) def __add_decisions_data(self): if self._decisions_ids: qs_filter = Q(id__in=self._decisions_ids) else: qs_filter = Q(job_id=self.job.id) decisions_list = [] for decision in Decision.objects.filter(qs_filter).select_related( 'scheduler', 'configuration'): decisions_list.append( DownloadDecisionSerializer(instance=decision).data) self._arch_files.add((decision.configuration.file.path, decision.configuration.file.name)) return self.__get_json(decisions_list) def __get_decision_cache(self): return self.__get_json( DecisionCacheSerializer(instance=DecisionCache.objects.filter( self._decision_filter), many=True).data) def __get_original_src(self): if self._decisions_ids: qs_filter = Q(reportcomponent__decision_id__in=self._decisions_ids) else: qs_filter = Q(reportcomponent__decision__job_id=self.job.id) sources = {} for src_arch in OriginalSources.objects.filter(qs_filter): sources[src_arch.identifier] = src_arch.archive.name self._arch_files.add( (src_arch.archive.path, src_arch.archive.name)) return self.__get_json(sources) def __get_reports_data(self): reports = [] for report in ReportComponent.objects.filter(self._decision_filter)\ .select_related('parent', 'computer', 'original_sources', 'additional_sources').order_by('level'): report_data = DownloadReportComponentSerializer( instance=report).data # Add report files if report_data['log']: self._arch_files.add((report.log.path, report_data['log'])) if report_data['verifier_files']: self._arch_files.add((report.verifier_files.path, report_data['verifier_files'])) reports.append(report_data) return self.__get_json(reports) def __get_safes_data(self): safes_queryset = ReportSafe.objects.filter( self._decision_filter).select_related('parent').order_by('id') return self.__get_json( DownloadReportSafeSerializer(instance=safes_queryset, many=True).data) def __get_unsafes_data(self): reports = [] for report in ReportUnsafe.objects.filter( self._decision_filter).select_related('parent').order_by('id'): report_data = DownloadReportUnsafeSerializer(instance=report).data if report_data['error_trace']: self._arch_files.add( (report.error_trace.path, report_data['error_trace'])) reports.append(report_data) return self.__get_json(reports) def __get_unknowns_data(self): reports = [] for report in ReportUnknown.objects.filter( self._decision_filter).select_related('parent').order_by('id'): report_data = DownloadReportUnknownSerializer(instance=report).data if report_data['problem_description']: self._arch_files.add((report.problem_description.path, report_data['problem_description'])) reports.append(report_data) return self.__get_json(reports) def __get_attrs_data(self): if self._decisions_ids: qs_filter = Q(report__decision_id__in=self._decisions_ids) else: qs_filter = Q(report__decision__job_id=self.job.id) attrs_data = {} for ra in ReportAttr.objects.filter(qs_filter).select_related( 'data', 'report').order_by('id'): data = DownloadReportAttrSerializer(instance=ra).data if data['data_file']: self._arch_files.add((ra.data.file.path, data['data_file'])) attrs_data.setdefault(ra.report.decision_id, {}) attrs_data[ra.report.decision_id].setdefault( ra.report.identifier, []) attrs_data[ra.report.decision_id][ra.report.identifier].append( data) return self.__get_json(attrs_data) def __get_coverage_data(self): if self._decisions_ids: qs_filter = Q(report__decision_id__in=self._decisions_ids) else: qs_filter = Q(report__decision__job_id=self.job.id) coverage_data = [] for carch in CoverageArchive.objects.filter(qs_filter).select_related( 'report').order_by('id'): coverage_data.append({ 'decision': carch.report.decision_id, 'report': carch.report.identifier, 'identifier': carch.identifier, 'archive': carch.archive.name, 'name': carch.name }) self._arch_files.add((carch.archive.path, carch.archive.name)) return self.__get_json(coverage_data) def __add_additional_sources(self): for src_arch in AdditionalSources.objects.filter( self._decision_filter): self._arch_files.add( (src_arch.archive.path, src_arch.archive.name)) def __get_json(self, data): return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
class FilesForCompetitionArchive: obj_attr = 'Program fragment' requirement_attr = 'Requirement' def __init__(self, job, filters): try: self.root = ReportRoot.objects.get(job=job) except ObjectDoesNotExist: raise BridgeException(_('The job is not decided')) self._attrs = self.__get_attrs() self._archives = self.__get_archives() self.filters = filters self._archives_to_upload = [] self.__get_archives_to_upload() self.stream = ZipStream() def __iter__(self): cnt = 0 names_in_use = set() for arch_path, name_pattern in self._archives_to_upload: # TODO: original extension (currently it's supposed that verification files are zip archives) if name_pattern in names_in_use: cnt += 1 arch_name = '%s_%s.zip' % (name_pattern, cnt) else: arch_name = '%s.zip' % name_pattern for data in self.stream.compress_file(arch_path, arch_name): yield data yield self.stream.close_stream() def __get_archives(self): archives = {} for c in ReportComponent.objects.filter(root=self.root, verification=True).exclude(verifier_input='')\ .only('id', 'verifier_input'): if c.verifier_input: archives[c.id] = c.verifier_input.path return archives def __get_attrs(self): names = {} for a_name in AttrName.objects.filter(name__in=[self.obj_attr, self.requirement_attr]): names[a_name.id] = a_name.name attrs = {} # Select attributes for all safes, unsafes and unknowns for r_id, n_id, a_val in ReportAttr.objects.filter(report__root=self.root, attr__name_id__in=names)\ .exclude(report__reportunsafe=None, report__reportsafe=None, report__reportunknown=None)\ .values_list('report_id', 'attr__name_id', 'attr__value'): if r_id not in attrs: attrs[r_id] = {} attrs[r_id][names[n_id]] = a_val return attrs def __add_archive(self, r_type, r_id, p_id): if p_id in self._archives and r_id in self._attrs \ and self.obj_attr in self._attrs[r_id] \ and self.requirement_attr in self._attrs[r_id]: ver_obj = self._attrs[r_id][self.obj_attr].replace('~', 'HOME').replace('/', '---') ver_requirement = self._attrs[r_id][self.requirement_attr].replace(':', '-') dirname = 'Unknowns' if r_type == 'f' else 'Unsafes' if r_type == 'u' else 'Safes' self._archives_to_upload.append( (self._archives[p_id], '{0}/{1}__{2}__{3}'.format(dirname, r_type, ver_requirement, ver_obj)) ) def __get_archives_to_upload(self): for f_t in self.filters: if isinstance(f_t, list) and f_t: for problem in f_t: comp_id, problem_id = problem.split('_')[0:2] if comp_id == problem_id == '0': queryset = ReportUnknown.objects.annotate(mr_len=Count('markreport_set'))\ .filter(root=self.root, mr_len=0).exclude(parent__parent=None)\ .values_list('id', 'parent_id') else: queryset = ReportUnknown.objects \ .filter(root=self.root, markreport_set__problem_id=problem_id, component_id=comp_id)\ .exclude(parent__parent=None).values_list('id', 'parent_id') for args in queryset: self.__add_archive('f', *args) else: model = ReportUnsafe if f_t == 'u' else ReportSafe if f_t == 's' else ReportUnknown for args in model.objects.filter(root=self.root).exclude(parent__parent=None)\ .values_list('id', 'parent_id'): self.__add_archive('f' if isinstance(f_t, list) else f_t, *args)