def __upload_attrs(self):
attrs_data = self.__read_json_file('{}.json'.format(ReportAttr.__name__), required=True)
attrs_cache = {}
new_attrs = []
new_attr_files = {}
cnt = 0
self._logger.start(JOB_UPLOAD_STATUS[10][0], total=100)
for old_d_id in attrs_data:
decision_id = self.__get_decision_id(int(old_d_id))
for r_id in attrs_data[old_d_id]:
for adata in attrs_data[old_d_id][r_id]:
report_id = self.saved_reports[(decision_id, r_id)]
data_file = adata.pop('data_file', None)
serializer = DownloadReportAttrSerializer(data=adata)
serializer.is_valid(raise_exception=True)
validated_data = serializer.validated_data
new_attrs.append(ReportAttr(report_id=report_id, **validated_data))
if data_file is not None:
file_key = (decision_id, data_file)
new_attr_files.setdefault(file_key, [])
new_attr_files[file_key].append(cnt)
cnt += 1
if report_id in self._leaves_ids:
attrs_cache.setdefault(report_id, {'attrs': {}})
attrs_cache[report_id]['attrs'][validated_data['name']] = validated_data['value']
self._logger.update(10)
attr_file_number = len(new_attr_files)
if attr_file_number > 0:
# Upload attributes' files
with transaction.atomic():
step_percent_number = int(attr_file_number / 70)
attr_file_cnt = 0
for decision_id, file_path in new_attr_files:
attr_file_obj = AttrFile(decision_id=decision_id)
with open(self.__full_path(file_path), mode='rb') as fp:
attr_file_obj.file.save(os.path.basename(file_path), File(fp), save=True)
for i in new_attr_files[(decision_id, file_path)]:
# Add link to file for attributes that have it
new_attrs[i].data_id = attr_file_obj.id
attr_file_cnt += 1
if attr_file_cnt > step_percent_number:
self._logger.update(int(70 * attr_file_cnt / attr_file_number))
attr_file_cnt = 0
else:
self._logger.update(70)
ReportAttr.objects.bulk_create(new_attrs)
self._logger.update(10)
decisions_ids = list(self._decisions.values())
update_cache_atomic(ReportSafeCache.objects.filter(report__decision_id__in=decisions_ids), attrs_cache)
update_cache_atomic(ReportUnsafeCache.objects.filter(report__decision_id__in=decisions_ids), attrs_cache)
update_cache_atomic(ReportUnknownCache.objects.filter(report__decision_id__in=decisions_ids), attrs_cache)
self._logger.update(10)
self._logger.end()
def __get_attr_file_id(self, rel_path, decision_id):
if rel_path is None:
return None
if rel_path not in self._attr_files:
instance = AttrFile(decision_id=decision_id)
with open(self.__full_path(rel_path), mode='rb') as fp:
instance.file.save(os.path.basename(rel_path), File(fp), save=True)
self._attr_files[rel_path] = instance.id
return self._attr_files[rel_path]
def __get_files(self, archive):
archive.seek(0)
try:
files_dir = extract_archive(archive)
except Exception as e:
logger.exception("Archive extraction failed: %s" % e, stack_info=True)
raise ValueError('Archive "%s" with attributes data is corrupted' % archive.name)
for dir_path, dir_names, file_names in os.walk(files_dir.name):
for file_name in file_names:
full_path = os.path.join(dir_path, file_name)
rel_path = os.path.relpath(full_path, files_dir.name).replace('\\', '/')
newfile = AttrFile(root_id=self._root_id)
with open(full_path, mode='rb') as fp:
newfile.file.save(os.path.basename(rel_path), File(fp), True)
self._files[rel_path] = newfile.id
def __upload_attrs_files(self, archive):
if not archive:
return {}
try:
files_dir = extract_archive(archive)
except Exception as e:
logger.exception("Archive extraction failed: %s" % e)
raise exceptions.ValidationError(detail={'attr_data': 'Archive "{}" is corrupted'.format(archive.name)})
db_files = {}
for dir_path, dir_names, file_names in os.walk(files_dir.name):
for file_name in file_names:
full_path = os.path.join(dir_path, file_name)
rel_path = os.path.relpath(full_path, files_dir.name).replace('\\', '/')
newfile = AttrFile(decision=self.decision)
with open(full_path, mode='rb') as fp:
newfile.file.save(os.path.basename(rel_path), File(fp), save=True)
db_files[rel_path] = newfile.pk
return db_files