def validate_error_trace(self, archive):
try:
with zipfile.ZipFile(archive, mode='r') as zfp:
error_trace = json.loads(
zfp.read(ERROR_TRACE_FILE).decode('utf8'))
except Exception as e:
logger.exception(e)
self.fail('wrong_format',
detail='file does not exist or it is wrong JSON')
archive.seek(0)
if not isinstance(error_trace, dict):
self.fail('wrong_format', detail='error trace is not a dictionary')
if not isinstance(error_trace.get('files'), list):
self.fail(
'wrong_format',
detail='error trace does not have files or it is not a list')
if 'trace' not in error_trace:
self.fail('wrong_format',
detail='error trace does not have "trace"')
if error_trace['trace']:
self.__check_node(error_trace['trace'])
if error_trace['trace']['type'] != 'thread':
self.fail(
'wrong_format',
detail='root error trace node type should be a "thread"')
return archive
def upload_solution(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'You are not signing in'})
if request.user.extended.role not in [USER_ROLES[2][0], USER_ROLES[4][0]]:
return JsonResponse({'error': 'No access'})
if request.method != 'POST':
return JsonResponse({'error': 'Just POST requests are supported'})
if 'task id' not in request.POST:
return JsonResponse({'error': 'Task identifier is not specified'})
if 'description' not in request.POST:
return JsonResponse({'error': 'Description is not specified'})
archive = None
for f in request.FILES.getlist('file'):
archive = f
if archive is None:
return JsonResponse({'error': 'The solution archive was not got'})
try:
service.utils.SaveSolution(request.POST['task id'], archive,
request.POST['description'])
except service.utils.NotAnError as e:
logger.info(str(e))
return JsonResponse({'error': str(e)})
except Exception as e:
logger.exception(e)
return JsonResponse({'error': str(e)})
return JsonResponse({})
def __get_response(self, *args, **kwargs):
self.__is_not_used(*args, **kwargs)
try:
generator = self.get_generator()
except Exception as e:
if not isinstance(e, BridgeException):
logger.exception(e)
raise BridgeException()
raise
if generator is None:
raise BridgeException()
file_name = getattr(generator, 'name', None) or self.get_filename()
if not isinstance(file_name, str) or len(file_name) == 0:
raise BridgeException()
file_size = getattr(generator, 'size', None)
mimetype = mimetypes.guess_type(os.path.basename(file_name))[0]
response = StreamingHttpResponse(generator, content_type=mimetype)
if file_size is not None:
response['Content-Length'] = file_size
response['Content-Disposition'] = 'attachment; filename="{}"'.format(
file_name)
return response
def validate_configuration(user, conf_str):
validated_data = {}
# Get configuration
if conf_str:
try:
configuration = GetConfiguration(
user_conf=json.loads(conf_str)).for_json()
except Exception as e:
logger.exception(e)
raise exceptions.ValidationError(
{'configuration': _('The configuration has wrong format')})
else:
configuration = get_default_configuration(user).for_json()
validated_data['priority'] = configuration['priority']
validated_data['weight'] = configuration['weight']
# Validate task scheduler
try:
validated_data['scheduler'] = validate_scheduler(
type=configuration['task scheduler'])
except BridgeException as e:
raise exceptions.ValidationError({'scheduler': str(e)})
# Save configuration file
conf_db = file_get_or_create(
json.dumps(configuration, indent=2, sort_keys=True,
ensure_ascii=False), 'configuration.json', JobFile)
validated_data['configuration_id'] = conf_db.id
return validated_data
def download_solution(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'You are not signing in'})
if request.user.extended.role not in [USER_ROLES[2][0], USER_ROLES[4][0]]:
return JsonResponse({'error': 'No access'})
if request.method != 'POST':
return JsonResponse({'error': 'Just POST requests are supported'})
if 'task id' not in request.POST:
return JsonResponse({'error': 'Task identifier is not specified'})
try:
res = service.utils.GetSolution(request.POST['task id'])
except service.utils.NotAnError as e:
logger.info(str(e))
return JsonResponse({'error': str(e)})
except Exception as e:
logger.exception(e)
return JsonResponse({'error': str(e)})
if res.task.status == TASK_STATUS[3][0]:
return JsonResponse({'task error': res.task.error})
mimetype = mimetypes.guess_type(os.path.basename(res.solution.archname))[0]
response = StreamingHttpResponse(FileWrapper(res.solution.archive, 8192),
content_type=mimetype)
response['Content-Length'] = len(res.solution.archive)
response[
'Content-Disposition'] = 'attachment; filename="%s"' % res.solution.archname
return response
def get(self, *args, **kwargs):
self.__is_not_used(*args, **kwargs)
try:
self.generator = self.get_generator()
except Exception as e:
if not isinstance(e, BridgeException):
logger.exception(e)
raise BridgeException()
raise
if self.generator is None:
raise BridgeException()
self.file_name = self.get_filename()
if not isinstance(self.file_name, str) or len(self.file_name) == 0:
raise BridgeException()
mimetype = mimetypes.guess_type(os.path.basename(self.file_name))[0]
response = StreamingHttpResponse(self.generator, content_type=mimetype)
if self.file_size is not None:
response['Content-Length'] = self.file_size
response[
'Content-Disposition'] = 'attachment; filename="%s"' % self.file_name
return response
def update_progress(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'You are not signing in'})
if request.user.extended.role not in [USER_ROLES[2][0], USER_ROLES[4][0]]:
return JsonResponse({'error': 'No access'})
if request.method != 'POST':
return JsonResponse({'error': 'Only POST requests are supported'})
if 'job id' not in request.session:
return JsonResponse({'error': 'The job id was not found in session'})
if 'progress' not in request.POST:
return JsonResponse({'error': 'Job progress data is required'})
try:
job = Job.objects.get(id=request.session['job id'])
except ObjectDoesNotExist:
return JsonResponse({'error': 'The job was not found'})
if job.status != JOB_STATUS[2][0]:
return JsonResponse({'error': 'The job is not solving'})
try:
service.utils.JobProgressData(job).update(request.POST['progress'])
except service.utils.ServiceError as e:
return JsonResponse({'error': str(e)})
except Exception as e:
logger.exception(e)
return JsonResponse({'error': 'Unknown error'})
return JsonResponse({})
def post(self, *args, **kwargs):
self.is_not_used(*args, **kwargs)
self.object = self.get_object()
if self.kwargs['action'] == 'create' \
and not mutils.MarkAccess(self.request.user, report=self.object).can_create():
raise BridgeException(
_("You don't have an access to create new marks"),
response_type='json')
elif self.kwargs['action'] == 'edit' \
and not mutils.MarkAccess(self.request.user, mark=self.object).can_edit():
raise BridgeException(
_("You don't have an access to edit this mark"),
response_type='json')
try:
res = mutils.NewMark(self.request.user, self.object,
json.loads(self.request.POST['data']))
if self.kwargs['action'] == 'edit':
res.change_mark()
else:
res.create_mark()
cache_id = MarkChangesTable(self.request.user, res.mark,
res.changes).cache_id
except BridgeException as e:
raise BridgeException(str(e), response_type='json')
except Exception as e:
logger.exception(e)
raise BridgeException(response_type='json')
return JsonResponse({'cache_id': cache_id})
def __get_reports_cache(self, reports_qs):
reports_ids = list(r.id for r in reports_qs)
convert_function = COMPARE_FUNCTIONS[self._mark.function]['convert']
new_cache = []
reports_cache = {}
for conv in UnsafeConvertionCache.objects\
.filter(unsafe_id__in=reports_ids, converted__function=convert_function)\
.select_related('converted'):
reports_cache[conv.unsafe_id] = set(
conv.converted.trace_cache['forest'])
for report in reports_qs:
if report.id in reports_cache:
continue
try:
error_trace = get_report_trace(report)
conv = convert_error_trace(error_trace, convert_function)
except Exception as e:
logger.exception(e)
reports_cache[report.id] = None
else:
reports_cache[report.id] = set(conv.trace_cache['forest'])
new_cache.append(
UnsafeConvertionCache(unsafe_id=report.id,
converted_id=conv.id))
if new_cache:
UnsafeConvertionCache.objects.bulk_create(new_cache)
return reports_cache
def get_report_trace(report):
try:
error_trace_str = ArchiveFileContent(report, 'error_trace', ERROR_TRACE_FILE).content.decode('utf8')
except Exception as e:
logger.exception(e, stack_info=True)
raise BridgeException("Can't exctract error trace for unsafe '%s' from archive" % report.pk)
return json.loads(error_trace_str)
def get_context_data(self, **kwargs):
if not JobAccess(self.request.user, self.object.root.job).can_view():
raise BridgeException(code=400)
try:
etv = GetETV(
ArchiveFileContent(self.object, 'error_trace',
ERROR_TRACE_FILE).content.decode('utf8'),
self.request.user)
except Exception as e:
logger.exception(e, stack_info=True)
etv = None
return {
'report':
self.object,
'report_type':
'unsafe',
'parents':
reports.utils.get_parents(self.object),
'SelfAttrsData':
reports.utils.report_attibutes(self.object),
'MarkTable':
ReportMarkTable(self.request.user, self.object,
self.get_view(VIEW_TYPES[10])),
'etv':
etv,
'include_assumptions':
self.request.user.extended.assumptions,
'include_jquery_ui':
True,
'resources':
reports.utils.get_leaf_resources(self.request.user, self.object)
}
def population(request):
try:
activate(request.user.extended.language)
except ObjectDoesNotExist:
activate(request.LANGUAGE_CODE)
if not request.user.extended or request.user.extended.role != USER_ROLES[2][0]:
return BridgeErrorResponse(_("You don't have an access to this page"))
need_service = (len(Extended.objects.filter(role=USER_ROLES[4][0])) == 0)
if request.method == 'POST':
service_username = request.POST.get('service_username', '')
if len(service_username) == 0:
service_username = None
if need_service and service_username is None:
return BridgeErrorResponse(_("Can't populate without Manager and service user"))
try:
changes = Population(
user=request.user, service=(service_username, request.POST.get('service_password'))
).changes
except BridgeException as e:
return render(request, 'Population.html', {'error': str(e)})
except Exception as e:
logger.exception(e)
return render(request, 'Population.html', {'error': str(UNKNOWN_ERROR)})
return render(request, 'Population.html', {'changes': changes})
return render(request, 'Population.html', {'need_service': need_service})
def schedule_task(request):
if not request.user.is_authenticated:
return JsonResponse({'error': 'You are not signing in'})
if request.user.extended.role not in [USER_ROLES[2][0], USER_ROLES[4][0]]:
return JsonResponse({'error': 'No access'})
if 'job id' not in request.session:
return JsonResponse({'error': 'Session does not have job id'})
if request.method != 'POST':
return JsonResponse({'error': 'Just POST requests are supported'})
if 'description' not in request.POST:
return JsonResponse({'error': 'Task description is not specified'})
archive = None
for f in request.FILES.getlist('file'):
archive = f
if archive is None:
return JsonResponse({'error': 'The task archive was not got'})
try:
res = service.utils.ScheduleTask(request.session['job id'],
request.POST['description'], archive)
except service.utils.NotAnError as e:
logger.info(str(e))
return JsonResponse({'error': str(e)})
except Exception as e:
logger.exception(e)
return JsonResponse({'error': str(e)})
return JsonResponse({'task id': res.task_id})
def __get_status(self, status):
if status not in set(x[0] for x in DECISION_STATUS):
raise ValueError('Unsupported status: %s' % status)
if status == DECISION_STATUS[3][0]:
if self.decision.status != DECISION_STATUS[2][0]:
self.error = "Only processing decisions can be finished"
return DECISION_STATUS[5][0]
unfinished_reports = list(
ReportComponent.objects.filter(decision=self.decision,
finish_date=None).values_list(
'identifier', flat=True))
if len(unfinished_reports) > 0:
self.error = 'There are unfinished reports ({}): {}'.format(
len(unfinished_reports), unfinished_reports)
logger.error(self.error)
if len(self.error) > 1024:
self.error = 'There are unfinished reports ({})'.format(
len(unfinished_reports))
return DECISION_STATUS[5][0]
try:
core_r = ReportComponent.objects.get(parent=None,
decision=self.decision)
except ReportComponent.DoesNotExist:
self.error = "The decision doesn't have Core report"
return DECISION_STATUS[5][0]
if ReportUnknown.objects.filter(parent=core_r,
component=core_r.component,
decision=self.decision).exists():
return DECISION_STATUS[4][0]
try:
self.__check_progress()
except ServiceError as e:
self.error = str(e)
return DECISION_STATUS[5][0]
except Exception as e:
logger.exception(e)
self.error = 'Unknown error while checking progress'
return DECISION_STATUS[5][0]
elif status == DECISION_STATUS[4][0]:
try:
core_r = ReportComponent.objects.get(parent=None,
decision=self.decision)
except ReportComponent.DoesNotExist:
pass
else:
unfinished_components = ReportComponent.objects.filter(
decision=self.decision, finish_date=None)
core_unknowns = ReportUnknown.objects.filter(
parent=core_r,
component=core_r.component,
decision=self.decision)
if unfinished_components.exists(
) or not core_unknowns.exists():
status = DECISION_STATUS[5][0]
if self.error is None:
self.error = "The scheduler hasn't given an error description"
return status
def get_context_data(self, **kwargs):
context = {'versions': [], 'action': self.kwargs['action']}
if self.kwargs['action'] == 'edit':
access = mutils.MarkAccess(self.request.user, mark=self.object)
if not access.can_edit():
raise BridgeException(
_("You don't have an access to edit this mark"))
context['mark'] = self.object
context['selected_version'] = int(
self.request.GET.get('version', self.object.version))
for m in self.object.versions.order_by('-version'):
if m.version == self.object.version:
title = _("Current version")
else:
change_time = m.change_date.astimezone(
pytz.timezone(self.request.user.extended.timezone))
title = change_time.strftime("%d.%m.%Y %H:%M:%S")
if m.author is not None:
title += " (%s)" % m.author.get_full_name()
if len(m.comment) > 0:
title += ': ' + m.comment
context['versions'].append({
'version': m.version,
'title': title
})
if context['selected_version'] == m.version:
context['markdata'] = MarkData(self.kwargs['type'],
mark_version=m)
if 'markdata' not in context:
raise BridgeException(_('The mark version was not found'))
context['cancel_url'] = reverse(
'marks:mark', args=[self.kwargs['type'], self.object.id])
else:
if self.kwargs['type'] == 'unknown':
try:
context['problem_description'] = \
ArchiveFileContent(self.object, 'problem_description', PROBLEM_DESC_FILE).content.decode('utf8')
except Exception as e:
logger.exception(
"Can't get problem description for unknown '%s': %s" %
(self.object.id, e))
raise BridgeException()
access = mutils.MarkAccess(self.request.user, report=self.object)
if not access.can_create():
raise BridgeException(
_("You don't have an access to create new marks"))
context['report'] = self.object
context['markdata'] = MarkData(self.kwargs['type'],
report=self.object)
context['cancel_url'] = reverse(
'reports:{0}'.format(self.kwargs['type']),
args=[
self.object.trace_id
if self.kwargs['type'] == 'unsafe' else self.object.id
])
context['access'] = access
return context
def post(self, request):
try:
Recalculation(request.data['type'], request.data['decisions'])
except BridgeException as e:
raise exceptions.ValidationError({'error': str(e)})
except Exception as e:
logger.exception(e)
raise exceptions.APIException({'error': str(UNKNOWN_ERROR)})
return Response({})
def to_internal_value(self, data):
try:
if isinstance(data, str):
data = json.loads(data)
return JSTreeConverter().parse_tree(data)
except BridgeException as e:
raise exceptions.ValidationError(str(e))
except Exception as e:
logger.exception(e)
self.fail('wrong_format')
def post(self, *args, **kwargs):
self.is_not_used(*args, **kwargs)
try:
return JsonResponse({'job_id': JobForm(self.request.user, self.get_object(),
self.kwargs['action']).save(self.request.POST).id})
except BridgeException as e:
raise BridgeException(str(e), response_type='json')
except Exception as e:
logger.exception(e)
raise BridgeException(response_type='json')
def __init__(self, compare_func, error_trace):
self._func = compare_func
self._trace = error_trace
if self._func in {'callback_call_forests', 'thread_call_forests'}:
try:
self.__check_forests()
except Exception as e:
logger.exception(e)
raise BridgeException(
_('The converted error trace has wrong format'))
def post(self, *args, **kwargs):
self.is_not_used(*args, **kwargs)
self.object = self.get_object()
try:
return self.render_to_response(self.get_context_data(object=self.object))
except BridgeException as e:
raise BridgeException(e, back=reverse('jobs:prepare_run', args=[self.object.pk]))
except Exception as e:
logger.exception(e)
raise BridgeException(back=reverse('jobs:prepare_run', args=[self.object.pk]))
def upload_all(self, reports):
# Check that all archives are valid ZIP files
for report in reports:
try:
self.__upload(report)
except Exception as e:
if str(e).__contains__('report_decision_id_identifier'):
logger.error('UniqueError')
logger.exception(e)
self.__process_exception(e)
def __connect_unknown_mark(self):
reports_filter = {'component': self.mark.component}
if len(self._mark_attrs) > 0:
reports_filter['id__in'] = set()
for unknown_id in self._unknowns_attrs:
if self._mark_attrs.issubset(self._unknowns_attrs[unknown_id]):
reports_filter['id__in'].add(unknown_id)
new_markreports = []
problems = {}
for unknown in ReportUnknown.objects.filter(**reports_filter):
try:
problem_description = ArchiveFileContent(unknown, 'problem_description', PROBLEM_DESC_FILE)\
.content.decode('utf8')
except Exception as e:
logger.exception(
"Can't get problem description for unknown '%s': %s" %
(unknown.id, e))
return
problem = MatchUnknown(problem_description, self.mark.function,
self.mark.problem_pattern,
self.mark.is_regexp).problem
if problem is None:
continue
elif len(problem) > 20:
problem = 'Too long!'
logger.error("Problem '%s' for mark %s is too long" %
(problem, self.mark.identifier),
stack_info=True)
if problem not in problems:
problems[problem] = UnknownProblem.objects.get_or_create(
name=problem)[0]
ass_type = ASSOCIATION_TYPE[0][0]
if self._prime_id == unknown.id:
ass_type = ASSOCIATION_TYPE[1][0]
new_markreports.append(
MarkUnknownReport(mark=self.mark,
report=unknown,
problem=problems[problem],
type=ass_type,
author=self.mark.author))
if unknown in self.changes:
self.changes[unknown]['kind'] = '='
else:
self.changes[unknown] = {'kind': '+', 'problems': {}}
if problems[problem].id not in self.changes[unknown]['problems']:
self.changes[unknown]['problems'][problems[problem].id] = [
0, 0
]
self.changes[unknown]['problems'][problems[problem].id][1] += 1
MarkUnknownReport.objects.bulk_create(new_markreports)
update_unknowns_cache(list(self.changes))
def bridge_exception_handler(exc, context):
# Switch from PDFRenderer to JSONRenderer for exceptions
context['request'].accepted_renderer = JSONRenderer()
if isinstance(exc, BridgeException):
exc = exceptions.APIException(str(exc))
if not isinstance(exc, (Http404, PermissionDenied, exceptions.APIException)):
logger.exception(exc)
# Always return API exception for DRF requests
exc = exceptions.APIException(str(UNKNOWN_ERROR))
return exception_handler(exc, context)
def __remove_report(self, report_id):
cnt = 0
while cnt < 5:
try:
report = ReportComponent.objects.select_for_update().get(id=report_id)
report.delete()
return True
except Exception as e:
logger.exception(e)
time.sleep(0.1)
cnt += 1
return False
def __match_desc_regexp(self):
try:
m = re.search(self.function, self.description, re.MULTILINE)
except Exception as e:
logger.exception("Regexp error: %s" % e, stack_info=True)
return None
if m is None:
return None
try:
return self.pattern.format(*m.groups())
except IndexError:
return self.pattern
def get_context_data(self, **kwargs):
if not jobs.utils.JobAccess(self.request.user, self.object).can_decide():
raise BridgeException(_("You don't have an access to upload reports for this job"))
try:
reports_dir = extract_archive(self.request.FILES['archive'])
except Exception as e:
logger.exception(e)
raise BridgeException(_('Extraction of the archive has failed'))
UploadReportsWithoutDecision(self.object, self.request.user, reports_dir.name)
return {}
def __match_desc_regexp(self):
try:
m = re.search(self._func, self._desc, re.MULTILINE)
except Exception as e:
logger.exception("Regexp error: %s" % e, stack_info=True)
return None, str(e)
if m is not None:
try:
return self._pattern.format(
*m.groups()), self.__get_matched_text(*m.span())
except IndexError:
return self._pattern, self.__get_matched_text(*m.span())
return None, ''
def __process_exception(self, exc):
if isinstance(exc, exceptions.ValidationError):
err_detail = exc.detail
else:
logger.exception(exc)
err_detail = 'Unknown error: {}'.format(exc)
try:
FinishDecision(self.decision, DECISION_STATUS[5][0],
self.__collapse_detail(err_detail))
except Exception as e:
logger.exception(e)
err_detail = 'Error while finishing decision with error'
raise exceptions.ValidationError(detail=err_detail)
def validate(self, attrs):
res = super().validate(attrs)
if res.get('is_regexp'):
try:
re.search(res['function'], '')
except Exception as e:
logger.exception(e)
raise exceptions.ValidationError(
detail={
'function':
_("The pattern is wrong, please refer to documentation on the standard "
"Python library for processing reqular expressions")
})
return res
def post(self, request):
if not MarkAccess(request.user).can_upload:
raise exceptions.PermissionDenied(
_("You don't have an access to create new marks"))
marks_links = []
failed_mark_uploads = 0
marks_uploader = MarksUploader(request.user)
for f in self.request.FILES.getlist('file'):
with zipfile.ZipFile(f, 'r') as zfp:
if all(
file_name.endswith('.zip')
for file_name in zfp.namelist()):
marks_dir = extract_archive(f)
for arch_name in os.listdir(marks_dir.name):
with open(os.path.join(marks_dir.name, arch_name),
mode='rb') as fp:
try:
marks_links.append(
marks_uploader.upload_mark(
File(fp, name=arch_name))[1])
except Exception as e:
logger.exception(e)
logger.error(
'Uploading of mark "{}" has failed.'.
format(arch_name))
failed_mark_uploads += 1
else:
marks_links.append(marks_uploader.upload_mark(f)[1])
if len(marks_links) == 1:
return Response({'url': marks_links[0]})
if failed_mark_uploads:
return Response({
'message':
_('Number of created marks: %(number)s.'
' Number of marks which uploading failed: %(failed_number)s.'
' See logs for details.') % {
'number': len(marks_links),
'failed_number': failed_mark_uploads
}
})
else:
return Response({
'message': _('Number of created marks: %(number)s') % {
'number': len(marks_links)
}
})
