def _update(self, data, user, many=True):
ids = []
error_line_ids = set()
classification_ids = set()
bug_number_classifications = {}
for item in data:
line_id = item.get("id")
if line_id is None:
return "No text log error id provided", HTTP_400_BAD_REQUEST
try:
line_id = int(line_id)
except ValueError:
return "Text log error id was not an integer", HTTP_400_BAD_REQUEST
error_line_ids.add(line_id)
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
bug_number = item.get("bug_number")
if (not classification_id and
bug_number is not None and
bug_number not in bug_number_classifications):
bug_number_classifications[bug_number], _ = (
ClassifiedFailure.objects.get_or_create(bug_number=bug_number))
ids.append((line_id, classification_id, bug_number))
error_lines = as_dict(
TextLogError.objects
.prefetch_related('classified_failures')
.filter(id__in=error_line_ids), "id")
if len(error_lines) != len(error_line_ids):
missing = error_line_ids - set(error_lines.keys())
return ("No text log error with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return ("No classification with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
for line_id, classification_id, bug_number in ids:
logger.debug("line_id: %s, classification_id: %s, bug_number: %s" %
(line_id, classification_id, bug_number))
error_line = error_lines[line_id]
if classification_id is not None:
logger.debug("Using classification id")
classification = classifications[classification_id]
if bug_number is not None and bug_number != classification.bug_number:
logger.debug("Updating classification bug number")
classification = classification.set_bug(bug_number)
elif bug_number is not None:
logger.debug("Using bug number")
classification = bug_number_classifications[bug_number]
else:
logger.debug("Using null classification")
classification = None
error_line.mark_best_classification_verified(classification)
error_line.step.job.update_after_verification(user)
# Force failure line to be reloaded, including .classified_failures
rv = (TextLogError.objects
.prefetch_related('classified_failures')
.filter(id__in=error_line_ids))
if not many:
rv = rv[0]
return (serializers.TextLogErrorSerializer(rv, many=many).data,
HTTP_200_OK)
def _update(self, data, user, many=True):
ids = []
error_line_ids = set()
classification_ids = set()
bug_number_classifications = {}
for item in data:
line_id = item.get("id")
if line_id is None:
return "No text log error id provided", HTTP_400_BAD_REQUEST
try:
line_id = int(line_id)
except ValueError:
return "Text log error id was not an integer", HTTP_400_BAD_REQUEST
error_line_ids.add(line_id)
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
bug_number = item.get("bug_number")
if (not classification_id and
bug_number is not None and
bug_number not in bug_number_classifications):
bug_number_classifications[bug_number], _ = (
ClassifiedFailure.objects.get_or_create(bug_number=bug_number))
ids.append((line_id, classification_id, bug_number))
error_lines = as_dict(
TextLogError.objects
.prefetch_related('classified_failures')
.filter(id__in=error_line_ids), "id")
if len(error_lines) != len(error_line_ids):
missing = error_line_ids - set(error_lines.keys())
return ("No text log error with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return ("No classification with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
jobs = set()
for line_id, classification_id, bug_number in ids:
logger.debug("line_id: %s, classification_id: %s, bug_number: %s",
line_id, classification_id, bug_number)
error_line = error_lines[line_id]
if classification_id is not None:
logger.debug("Using classification id")
classification = classifications[classification_id]
if bug_number is not None and bug_number != classification.bug_number:
logger.debug("Updating classification bug number")
classification = classification.set_bug(bug_number)
elif bug_number is not None:
logger.debug("Using bug number")
classification = bug_number_classifications[bug_number]
else:
logger.debug("Using null classification")
classification = None
jobs.add(error_line.step.job)
error_line.mark_best_classification_verified(classification)
for job in jobs:
job.update_after_verification(user)
# Force failure line to be reloaded, including .classified_failures
rv = (TextLogError.objects
.prefetch_related('classified_failures')
.filter(id__in=error_line_ids))
if not many:
rv = rv[0]
return (serializers.TextLogErrorSerializer(rv, many=many).data,
HTTP_200_OK)
def _update(self, data, many=False):
bug_numbers = {}
for item in data:
classified_failure_id = int(item.get("id"))
if classified_failure_id is None:
return "No id provided", 400
bug_number = item.get('bug_number')
if bug_number is None:
return "No bug number provided", 400
bug_numbers[classified_failure_id] = int(bug_number)
classified_failures = as_dict(
ClassifiedFailure.objects.filter(id__in=bug_numbers.keys()), "id")
if len(classified_failures) != len(bug_numbers):
missing = set(bug_numbers.keys()) - set(classified_failures.keys())
return "No classified failures with id: {0}".format(", ".join(missing)), 404
merges = {}
existing = ClassifiedFailure.objects.filter(bug_number__in=bug_numbers.values()).all()
# Look for other classified failures with the same bug as one being updated, since
# we don't want duplicate bugs
existing = [item for item in existing
if item.id not in bug_numbers or item.bug_number != bug_numbers[item.id]]
if existing:
if any(item.id in bug_numbers for item in existing):
return "Cannot swap classified failure bug numbers in a single operation", 400
classified_failures.update(as_dict(existing, "id"))
bug_to_classified_failure = defaultdict(list)
for id, bug in bug_numbers.iteritems():
bug_to_classified_failure[bug].append(classified_failures[id])
# Merge the ClassifiedFailure being updated into the existing ClassifiedFailure
# with the same bug number
for retain in existing:
for remove in bug_to_classified_failure[retain.bug_number]:
removed_id = remove.id
remove.replace_with(retain)
merges[removed_id] = retain
# Ensure that the return value is ordered in the same way as the request
rv = []
for item in data:
classification_id = int(item.get("id"))
bug_number = bug_numbers[classification_id]
if classification_id in merges:
classification = merges[classification_id]
else:
classification = classified_failures[int(item.get("id"))]
classification.bug_number = bug_number
classification.save()
rv.append(classification)
if not many:
rv = rv[0]
return self.serializer_class(rv, many=many).data, 200
def _update(self, data, many=False):
bug_numbers = {}
for item in data:
classified_failure_id = int(item.get("id"))
if classified_failure_id is None:
return "No id provided", HTTP_400_BAD_REQUEST
bug_number = item.get('bug_number')
if bug_number is None:
return "No bug number provided", HTTP_400_BAD_REQUEST
bug_numbers[classified_failure_id] = int(bug_number)
classified_failures = as_dict(
ClassifiedFailure.objects.filter(id__in=bug_numbers.keys()), "id")
existing = (ClassifiedFailure.objects
.filter(bug_number__in=bug_numbers.values())
.all())
# Look for other classified failures with the same bug as one being updated, since
# we don't want duplicate bugs
existing = as_dict((item for item in existing
if item.id not in bug_numbers or
item.bug_number != bug_numbers[item.id]), "bug_number")
# We don't count classified failures as missing if there is an existing failure
# with the same bug number, even if it has a different id. This is because classification
# of other jobs may have caused the classified failure to have been deleted and replaced
# by another, without updating all the ids in the client
missing = set(bug_numbers.keys()) - set(classified_failures.keys())
replacements = {}
for missing_id in missing:
if bug_numbers[missing_id] in existing:
existing_cf = existing[bug_numbers[missing_id]]
replacements[missing_id] = existing_cf
missing -= set(replacements.keys())
if missing:
return ("No classified failures with id: {0}"
.format(", ".join(str(item) for item in missing)),
HTTP_404_NOT_FOUND)
if any(item.id in bug_numbers for item in existing.values()):
return ("Cannot swap classified failure bug numbers in a single operation",
HTTP_400_BAD_REQUEST)
classified_failures.update(as_dict(existing.values(), "id"))
for old_id, replacement in iteritems(replacements):
bug_numbers[replacement.id] = replacement
del bug_numbers[old_id]
merges = {}
if existing:
bug_to_classified_failure = defaultdict(list)
for id, bug_number in iteritems(bug_numbers):
bug_to_classified_failure[bug_number].append(classified_failures[id])
# Merge the ClassifiedFailure being updated into the existing ClassifiedFailure
# with the same bug number
for bug_number, retain in iteritems(existing):
for remove in bug_to_classified_failure[bug_number]:
removed_id = remove.id
remove.replace_with(retain)
merges[removed_id] = retain
# Ensure that the return value is ordered in the same way as the request
rv = []
for item in data:
classification_id = int(item.get("id"))
if classification_id in merges:
classification = merges[classification_id]
elif classification_id in replacements:
classification = replacements[classification_id]
else:
bug_number = bug_numbers[classification_id]
classification = classified_failures[classification_id].set_bug(bug_number)
rv.append(classification)
if not many:
rv = rv[0]
return self.serializer_class(rv, many=many).data, HTTP_200_OK
def _update(self, data, user, many=True):
by_project = defaultdict(list)
ids = []
failure_line_ids = set()
classification_ids = set()
for item in data:
line_id = int(item.get("id"))
if line_id is None:
return "No failure line id provided", HTTP_400_BAD_REQUEST
failure_line_ids.add(line_id)
if "best_classification" not in item:
return "No classification id provided", HTTP_400_BAD_REQUEST
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
ids.append((line_id, classification_id))
failure_lines = as_dict(
FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids), "id")
if len(failure_lines) != len(failure_line_ids):
missing = failure_line_ids - set(failure_lines.keys())
return ("No failure line with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return ("No classification with id: {0}".format(
", ".join(missing)), HTTP_404_NOT_FOUND)
for line_id, classification_id in ids:
failure_line = failure_lines[line_id]
if classification_id is not None:
classification = classifications[classification_id]
else:
classification = None
by_project[failure_line.repository.name].append(
failure_line.job_guid)
failure_line.mark_best_classification_verified(classification)
for project, job_guids in iteritems(by_project):
for job in Job.objects.filter(guid__in=job_guids):
job.update_after_verification(user)
# Force failure line to be reloaded, including .classified_failures
rv = FailureLine.objects.prefetch_related(
'classified_failures').filter(id__in=failure_line_ids)
if not many:
rv = rv[0]
return (serializers.FailureLineNoStackSerializer(rv, many=many).data,
HTTP_200_OK)
def _update(self, data, user, many=True):
by_project = defaultdict(list)
ids = []
failure_line_ids = set()
classification_ids = set()
for item in data:
line_id = int(item.get("id"))
if line_id is None:
return "No failure line id provided", HTTP_400_BAD_REQUEST
failure_line_ids.add(line_id)
if "best_classification" not in item:
return "No classification id provided", HTTP_400_BAD_REQUEST
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
ids.append((line_id, classification_id))
failure_lines = as_dict(
FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids), "id")
if len(failure_lines) != len(failure_line_ids):
missing = failure_line_ids - set(failure_lines.keys())
return ("No failure line with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return ("No classification with id: {0}".format(", ".join(missing)),
HTTP_404_NOT_FOUND)
for line_id, classification_id in ids:
failure_line = failure_lines[line_id]
if classification_id is not None:
classification = classifications[classification_id]
else:
classification = None
by_project[failure_line.repository.name].append(failure_line.job_guid)
failure_line.mark_best_classification_verified(classification)
for project, job_guids in by_project.iteritems():
with JobsModel(project) as jm:
jobs = jm.get_job_ids_by_guid(job_guids)
for job in jobs.values():
jm.update_after_verification(job["id"], user)
# Force failure line to be reloaded, including .classified_failures
rv = FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids)
if not many:
rv = rv[0]
return (serializers.FailureLineNoStackSerializer(rv, many=many).data,
HTTP_200_OK)
def _update(self, data, many=False):
bug_numbers = {}
for item in data:
classified_failure_id = int(item.get("id"))
if classified_failure_id is None:
return "No id provided", HTTP_400_BAD_REQUEST
bug_number = item.get('bug_number')
if bug_number is None:
return "No bug number provided", HTTP_400_BAD_REQUEST
bug_numbers[classified_failure_id] = int(bug_number)
classified_failures = as_dict(
ClassifiedFailure.objects.filter(id__in=bug_numbers.keys()), "id")
existing = (ClassifiedFailure.objects.filter(
bug_number__in=bug_numbers.values()).all())
# Look for other classified failures with the same bug as one being updated, since
# we don't want duplicate bugs
existing = as_dict(
(item for item in existing if item.id not in bug_numbers
or item.bug_number != bug_numbers[item.id]), "bug_number")
# We don't count classified failures as missing if there is an existing failure
# with the same bug number, even if it has a different id. This is because classification
# of other jobs may have caused the classified failure to have been deleted and replaced
# by another, without updating all the ids in the client
missing = set(bug_numbers.keys()) - set(classified_failures.keys())
replacements = {}
for missing_id in missing:
if bug_numbers[missing_id] in existing:
existing_cf = existing[bug_numbers[missing_id]]
replacements[missing_id] = existing_cf
missing -= set(replacements.keys())
if missing:
return ("No classified failures with id: {0}".format(", ".join(
str(item) for item in missing)), HTTP_404_NOT_FOUND)
if any(item.id in bug_numbers for item in existing.values()):
return (
"Cannot swap classified failure bug numbers in a single operation",
HTTP_400_BAD_REQUEST)
classified_failures.update(as_dict(existing.values(), "id"))
for old_id, replacement in iteritems(replacements):
bug_numbers[replacement.id] = replacement
del bug_numbers[old_id]
merges = {}
if existing:
bug_to_classified_failure = defaultdict(list)
for id, bug_number in iteritems(bug_numbers):
bug_to_classified_failure[bug_number].append(
classified_failures[id])
# Merge the ClassifiedFailure being updated into the existing ClassifiedFailure
# with the same bug number
for bug_number, retain in iteritems(existing):
for remove in bug_to_classified_failure[bug_number]:
removed_id = remove.id
remove.replace_with(retain)
merges[removed_id] = retain
# Ensure that the return value is ordered in the same way as the request
rv = []
for item in data:
classification_id = int(item.get("id"))
if classification_id in merges:
classification = merges[classification_id]
elif classification_id in replacements:
classification = replacements[classification_id]
else:
bug_number = bug_numbers[classification_id]
classification = classified_failures[
classification_id].set_bug(bug_number)
rv.append(classification)
if not many:
rv = rv[0]
return self.serializer_class(rv, many=many).data, HTTP_200_OK
def _update(self, data, email, many=True):
by_project = defaultdict(list)
ids = []
failure_line_ids = set()
classification_ids = set()
for item in data:
line_id = int(item.get("id"))
if line_id is None:
return "No failure line id provided", 400
failure_line_ids.add(line_id)
if "best_classification" not in item:
return "No classification id provided", 400
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
ids.append((line_id, classification_id))
failure_lines = as_dict(
FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids), "id")
if len(failure_lines) != len(failure_line_ids):
missing = failure_line_ids - set(failure_lines.keys())
return "No failure line with id: {0}".format(
", ".join(missing)), 404
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return "No classification with id: {0}".format(
", ".join(missing)), 404
for line_id, classification_id in ids:
failure_line = failure_lines[line_id]
if classification_id is not None:
classification = classifications[classification_id]
else:
classification = None
by_project[failure_line.repository.name].append(
failure_line.job_guid)
failure_line.best_classification = classification
failure_line.best_is_verified = True
failure_line.save()
if (classification is not None and classification
not in failure_line.classified_failures.all()):
manual_detector = Matcher.objects.get(name="ManualDetector")
match = FailureMatch(failure_line=failure_line,
classified_failure=classification,
matcher=manual_detector,
score=1.0)
match.save()
for project, job_guids in by_project.iteritems():
with JobsModel(project) as jm:
jobs = jm.get_job_ids_by_guid(job_guids)
for job in jobs.values():
jm.update_after_verification(job["id"], email)
# Force failure line to be reloaded, including .classified_failures
rv = FailureLine.objects.prefetch_related(
'classified_failures').filter(id__in=failure_line_ids)
if not many:
rv = rv[0]
return serializers.FailureLineNoStackSerializer(rv,
many=many).data, 200
def _update(self, data, many=False):
bug_numbers = {}
for item in data:
classified_failure_id = int(item.get("id"))
if classified_failure_id is None:
return "No id provided", 400
bug_number = item.get('bug_number')
if bug_number is None:
return "No bug number provided", 400
bug_numbers[classified_failure_id] = int(bug_number)
classified_failures = as_dict(
ClassifiedFailure.objects.filter(id__in=bug_numbers.keys()), "id")
if len(classified_failures) != len(bug_numbers):
missing = set(bug_numbers.keys()) - set(classified_failures.keys())
return "No classified failures with id: {0}".format(
", ".join(missing)), 404
merges = {}
existing = ClassifiedFailure.objects.filter(
bug_number__in=bug_numbers.values()).all()
# Look for other classified failures with the same bug as one being updated, since
# we don't want duplicate bugs
existing = [
item for item in existing if item.id not in bug_numbers
or item.bug_number != bug_numbers[item.id]
]
if existing:
if any(item.id in bug_numbers for item in existing):
return "Cannot swap classified failure bug numbers in a single operation", 400
classified_failures.update(as_dict(existing, "id"))
bug_to_classified_failure = defaultdict(list)
for id, bug in bug_numbers.iteritems():
bug_to_classified_failure[bug].append(classified_failures[id])
# Merge the ClassifiedFailure being updated into the existing ClassifiedFailure
# with the same bug number
for retain in existing:
for remove in bug_to_classified_failure[retain.bug_number]:
removed_id = remove.id
remove.replace_with(retain)
merges[removed_id] = retain
# Ensure that the return value is ordered in the same way as the request
rv = []
for item in data:
classification_id = int(item.get("id"))
bug_number = bug_numbers[classification_id]
if classification_id in merges:
classification = merges[classification_id]
else:
classification = classified_failures[int(item.get("id"))]
classification.bug_number = bug_number
classification.save()
rv.append(classification)
if not many:
rv = rv[0]
return self.serializer_class(rv, many=many).data, 200
def _update(self, data, email, many=True):
by_project = defaultdict(list)
ids = []
failure_line_ids = set()
classification_ids = set()
for item in data:
line_id = int(item.get("id"))
if line_id is None:
return "No failure line id provided", 400
failure_line_ids.add(line_id)
if "best_classification" not in item:
return "No classification id provided", 400
classification_id = item.get("best_classification")
if classification_id is not None:
classification_ids.add(classification_id)
ids.append((line_id, classification_id))
failure_lines = as_dict(
FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids), "id")
if len(failure_lines) != len(failure_line_ids):
missing = failure_line_ids - set(failure_lines.keys())
return "No failure line with id: {0}".format(", ".join(missing)), 404
classifications = as_dict(
ClassifiedFailure.objects.filter(id__in=classification_ids), "id")
if len(classifications) != len(classification_ids):
missing = classification_ids - set(classifications.keys())
return "No classification with id: {0}".format(", ".join(missing)), 404
for line_id, classification_id in ids:
failure_line = failure_lines[line_id]
if classification_id is not None:
classification = classifications[classification_id]
else:
classification = None
by_project[failure_line.repository.name].append(failure_line.job_guid)
failure_line.best_classification = classification
failure_line.best_is_verified = True
failure_line.save()
if (classification is not None and
classification not in failure_line.classified_failures.all()):
manual_detector = Matcher.objects.get(name="ManualDetector")
match = FailureMatch(failure_line=failure_line,
classified_failure=classification,
matcher=manual_detector,
score=1.0)
match.save()
for project, job_guids in by_project.iteritems():
with JobsModel(project) as jm:
jobs = jm.get_job_ids_by_guid(job_guids)
for job in jobs.values():
jm.update_after_verification(job["id"], email)
# Force failure line to be reloaded, including .classified_failures
rv = FailureLine.objects.prefetch_related('classified_failures').filter(
id__in=failure_line_ids)
if not many:
rv = rv[0]
return serializers.FailureLineNoStackSerializer(rv, many=many).data, 200