class MarkArchiveGenerator:
def __init__(self, mark):
self.mark = mark
if isinstance(self.mark, MarkUnsafe):
self.type = 'unsafe'
elif isinstance(self.mark, MarkSafe):
self.type = 'safe'
elif isinstance(self.mark, MarkUnknown):
self.type = 'unknown'
else:
return
self.name = 'Mark-%s-%s.zip' % (self.type, self.mark.identifier[:10])
self.stream = ZipStream()
def __iter__(self):
for markversion in self.mark.versions.all():
version_data = {
'status': markversion.status,
'comment': markversion.comment,
'description': markversion.description
}
if self.type == 'unknown':
version_data['function'] = markversion.function
version_data['problem'] = markversion.problem_pattern
version_data['is_regexp'] = markversion.is_regexp
if markversion.link is not None:
version_data['link'] = markversion.link
else:
version_data['attrs'] = []
for aname, aval, compare in markversion.attrs.order_by('id')\
.values_list('attr__name__name', 'attr__value', 'is_compare'):
version_data['attrs'].append({
'attr': aname,
'value': aval,
'is_compare': compare
})
version_data['tags'] = list(
tag for tag, in markversion.tags.values_list('tag__tag'))
version_data['verdict'] = markversion.verdict
if self.type == 'unsafe':
version_data['function'] = markversion.function.name
with markversion.error_trace.file.file as fp:
version_data['error_trace'] = fp.read().decode('utf8')
content = json.dumps(version_data,
ensure_ascii=False,
sort_keys=True,
indent=4)
for data in self.stream.compress_string(
'version-%s' % markversion.version, content):
yield data
common_data = {
'is_modifiable': self.mark.is_modifiable,
'mark_type': self.type,
'format': self.mark.format,
'identifier': self.mark.identifier
}
if self.type == 'unknown':
common_data['component'] = self.mark.component.name
content = json.dumps(common_data,
ensure_ascii=False,
sort_keys=True,
indent=4)
for data in self.stream.compress_string('markdata', content):
yield data
yield self.stream.close_stream()
class MarkGeneratorBase:
type = None
attrs_model = None
tags_model = None
def __init__(self, mark):
assert self.type is not None, 'Wrong usage'
self.mark = mark
self.name = 'Mark-{}-{}.zip'.format(self.type, self.mark.identifier)
self.stream = ZipStream()
def common_data(self):
return {
'type': self.type,
'identifier': str(self.mark.identifier),
'is_modifiable': self.mark.is_modifiable
}
def version_data(self, version):
data = {
'comment': version.comment,
'description': version.description,
'attrs': self.attrs.get(version.id, []),
}
if self.tags is not None:
data['tags'] = self.tags.get(version.id, [])
return data
@cached_property
def attrs(self):
assert self.attrs_model is not None, 'Wrong usage'
mark_attrs = {}
for mattr in self.attrs_model.objects.filter(
mark_version__mark=self.mark).order_by('id'):
mark_attrs.setdefault(mattr.mark_version_id, [])
mark_attrs[mattr.mark_version_id].append({
'name':
mattr.name,
'value':
mattr.value,
'is_compare':
mattr.is_compare
})
return mark_attrs
@cached_property
def tags(self):
if not self.tags_model:
return None
all_tags = {}
for version_id, tag_name in self.tags_model.objects.filter(mark_version__mark=self.mark) \
.values_list('mark_version_id', 'tag__name'):
all_tags.setdefault(version_id, [])
all_tags[version_id].append(tag_name)
return all_tags
def versions_queryset(self):
return self.mark.versions.all()
def __iter__(self):
# Add main mark data
content = json.dumps(self.common_data(),
ensure_ascii=False,
sort_keys=True,
indent=4)
for data in self.stream.compress_string('mark.json', content):
yield data
# Add versions data
for markversion in self.versions_queryset():
content = json.dumps(self.version_data(markversion),
ensure_ascii=False,
sort_keys=True,
indent=4)
for data in self.stream.compress_string(
'version-{}.json'.format(markversion.version), content):
yield data
yield self.stream.close_stream()
class JobArchiveGenerator:
def __init__(self, job, decisions_ids=None):
self.job = job
self._decisions_ids = list(map(
int, decisions_ids)) if decisions_ids else None
self.name = 'Job-{}.zip'.format(self.job.identifier)
self._arch_files = set()
self.stream = ZipStream()
def __iter__(self):
# Job data
yield from self.stream.compress_string('job.json',
self.__get_job_data())
yield from self.stream.compress_string(
'{}.json'.format(Decision.__name__), self.__add_decisions_data())
yield from self.stream.compress_string(
'{}.json'.format(DecisionCache.__name__),
self.__get_decision_cache())
yield from self.stream.compress_string(
'{}.json'.format(OriginalSources.__name__),
self.__get_original_src())
yield from self.stream.compress_string(
'{}.json'.format(ReportComponent.__name__),
self.__get_reports_data())
yield from self.stream.compress_string(
'{}.json'.format(ReportSafe.__name__), self.__get_safes_data())
yield from self.stream.compress_string(
'{}.json'.format(ReportUnsafe.__name__), self.__get_unsafes_data())
yield from self.stream.compress_string(
'{}.json'.format(ReportUnknown.__name__),
self.__get_unknowns_data())
yield from self.stream.compress_string(
'{}.json'.format(ReportAttr.__name__), self.__get_attrs_data())
yield from self.stream.compress_string(
'{}.json'.format(CoverageArchive.__name__),
self.__get_coverage_data())
self.__add_job_files()
self.__add_additional_sources()
for file_path, arcname in self._arch_files:
yield from self.stream.compress_file(file_path, arcname)
yield self.stream.close_stream()
@cached_property
def _decision_filter(self):
if self._decisions_ids:
return Q(decision_id__in=self._decisions_ids)
return Q(decision__job_id=self.job.id)
def __get_job_data(self):
return self.__get_json(DownloadJobSerializer(instance=self.job).data)
def __add_job_files(self):
job_files = {}
for fs in FileSystem.objects.filter(
decision__job=self.job).select_related('file'):
job_files[fs.file.hash_sum] = (fs.file.file.path,
fs.file.file.name)
for f_path, arcname in job_files.values():
self._arch_files.add((f_path, arcname))
def __add_decisions_data(self):
if self._decisions_ids:
qs_filter = Q(id__in=self._decisions_ids)
else:
qs_filter = Q(job_id=self.job.id)
decisions_list = []
for decision in Decision.objects.filter(qs_filter).select_related(
'scheduler', 'configuration'):
decisions_list.append(
DownloadDecisionSerializer(instance=decision).data)
self._arch_files.add((decision.configuration.file.path,
decision.configuration.file.name))
return self.__get_json(decisions_list)
def __get_decision_cache(self):
return self.__get_json(
DecisionCacheSerializer(instance=DecisionCache.objects.filter(
self._decision_filter),
many=True).data)
def __get_original_src(self):
if self._decisions_ids:
qs_filter = Q(reportcomponent__decision_id__in=self._decisions_ids)
else:
qs_filter = Q(reportcomponent__decision__job_id=self.job.id)
sources = {}
for src_arch in OriginalSources.objects.filter(qs_filter):
sources[src_arch.identifier] = src_arch.archive.name
self._arch_files.add(
(src_arch.archive.path, src_arch.archive.name))
return self.__get_json(sources)
def __get_reports_data(self):
reports = []
for report in ReportComponent.objects.filter(self._decision_filter)\
.select_related('parent', 'computer', 'original_sources', 'additional_sources').order_by('level'):
report_data = DownloadReportComponentSerializer(
instance=report).data
# Add report files
if report_data['log']:
self._arch_files.add((report.log.path, report_data['log']))
if report_data['verifier_files']:
self._arch_files.add((report.verifier_files.path,
report_data['verifier_files']))
reports.append(report_data)
return self.__get_json(reports)
def __get_safes_data(self):
safes_queryset = ReportSafe.objects.filter(
self._decision_filter).select_related('parent').order_by('id')
return self.__get_json(
DownloadReportSafeSerializer(instance=safes_queryset,
many=True).data)
def __get_unsafes_data(self):
reports = []
for report in ReportUnsafe.objects.filter(
self._decision_filter).select_related('parent').order_by('id'):
report_data = DownloadReportUnsafeSerializer(instance=report).data
if report_data['error_trace']:
self._arch_files.add(
(report.error_trace.path, report_data['error_trace']))
reports.append(report_data)
return self.__get_json(reports)
def __get_unknowns_data(self):
reports = []
for report in ReportUnknown.objects.filter(
self._decision_filter).select_related('parent').order_by('id'):
report_data = DownloadReportUnknownSerializer(instance=report).data
if report_data['problem_description']:
self._arch_files.add((report.problem_description.path,
report_data['problem_description']))
reports.append(report_data)
return self.__get_json(reports)
def __get_attrs_data(self):
if self._decisions_ids:
qs_filter = Q(report__decision_id__in=self._decisions_ids)
else:
qs_filter = Q(report__decision__job_id=self.job.id)
attrs_data = {}
for ra in ReportAttr.objects.filter(qs_filter).select_related(
'data', 'report').order_by('id'):
data = DownloadReportAttrSerializer(instance=ra).data
if data['data_file']:
self._arch_files.add((ra.data.file.path, data['data_file']))
attrs_data.setdefault(ra.report.decision_id, {})
attrs_data[ra.report.decision_id].setdefault(
ra.report.identifier, [])
attrs_data[ra.report.decision_id][ra.report.identifier].append(
data)
return self.__get_json(attrs_data)
def __get_coverage_data(self):
if self._decisions_ids:
qs_filter = Q(report__decision_id__in=self._decisions_ids)
else:
qs_filter = Q(report__decision__job_id=self.job.id)
coverage_data = []
for carch in CoverageArchive.objects.filter(qs_filter).select_related(
'report').order_by('id'):
coverage_data.append({
'decision': carch.report.decision_id,
'report': carch.report.identifier,
'identifier': carch.identifier,
'archive': carch.archive.name,
'name': carch.name
})
self._arch_files.add((carch.archive.path, carch.archive.name))
return self.__get_json(coverage_data)
def __add_additional_sources(self):
for src_arch in AdditionalSources.objects.filter(
self._decision_filter):
self._arch_files.add(
(src_arch.archive.path, src_arch.archive.name))
def __get_json(self, data):
return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
