def __resubmit_job__(self, test_job, metadata):
infra_messages_re_list = []
project_settings = self.__resolve_settings__(test_job)
error_messages_settings = project_settings.get(
'CI_LAVA_INFRA_ERROR_MESSAGES', [])
for message_re in error_messages_settings:
try:
r = re.compile(message_re, re.I)
infra_messages_re_list.append(r)
except re.error:
# ignore incorrect expressions
self.log_debug("'%s' is not a valid regex" % message_re)
for regex in infra_messages_re_list:
error_msg = metadata.get('error_msg')
if error_msg and regex.search(error_msg) is not None and \
test_job.resubmitted_count < 3:
resubmitted_job = self.resubmit(test_job)
if project_settings.get('CI_LAVA_SEND_ADMIN_EMAIL', True):
# delay sending email by 15 seconds to allow the database object to be saved
send_testjob_resubmit_admin_email.apply_async(
args=[test_job.pk, resubmitted_job.pk], countdown=15)
# re-submit the job only once
# even if there are more matches
break
def __resubmit_job__(self, test_job, metadata):
infra_messages_re_list = []
for message_re in self.settings.get('CI_LAVA_INFRA_ERROR_MESSAGES', []):
try:
r = re.compile(message_re, re.I)
infra_messages_re_list.append(r)
except re.error:
# ignore incorrect expressions
self.log_debug("'%s' is not a valid regex" % message_re)
for regex in infra_messages_re_list:
if regex.search(metadata['error_msg']) is not None and \
test_job.resubmitted_count < 3:
resubmitted_job = self.resubmit(test_job)
if self.settings.get('CI_LAVA_SEND_ADMIN_EMAIL', True):
# delay sending email by 15 seconds to allow the database object to be saved
send_testjob_resubmit_admin_email.apply_async(args=[test_job.pk, resubmitted_job.pk], countdown=15)
# re-submit the job only once
# even if there are more matches
break
def __parse_results__(self, data, test_job):
lava_infra_error_messages = []
if self.settings is not None:
lava_infra_error_messages = self.settings.get(
'CI_LAVA_INFRA_ERROR_MESSAGES', [])
if data['is_pipeline'] is False:
# in case of v1 job, return empty data
return (data['status'], {}, {}, {})
definition = yaml.load(data['definition'])
if data['multinode_definition']:
definition = yaml.load(data['multinode_definition'])
test_job.name = definition['job_name'][:255]
job_metadata = definition.get('metadata', {})
suite_versions = {}
for key, value in job_metadata.items():
if key.endswith('__version'):
suite_versions[key.replace('__version', '')] = value
if suite_versions:
job_metadata['suite_versions'] = suite_versions
results = {}
metrics = {}
completed = True
if data['status'] == 'Canceled':
# consider all canceled jobs as incomplete and discard any results
completed = False
else:
for result in data['results']:
if result['suite'] != 'lava':
suite = result['suite'].split("_", 1)[-1]
res_name = "%s/%s" % (suite, result['name'])
# YAML from LAVA has all values serialized to strings
if result['measurement'] == 'None':
res_value = result['result']
results.update({res_name: res_value})
else:
res_value = result['measurement']
metrics.update({res_name: float(res_value)})
else:
# add artificial 'boot' test result for each test job
if result['name'] == 'auto-login-action':
# by default the boot test is named after the device_type
res_name = "boot/%s" % (definition['device_type'])
res_time_name = "boot/time-%s" % (
definition['device_type'])
if 'testsuite' in job_metadata.keys():
# If 'testsuite' metadata key is present in the job
# it's appended to the test name. This way regressions can
# be found with more granularity
res_name = "%s-%s" % (res_name,
job_metadata['testsuite'])
results.update({res_name: result['result']})
metrics.update(
{res_time_name: float(result['measurement'])})
if result['name'] == 'job' and result['result'] == 'fail':
metadata = result['metadata']
test_job.failure = str(metadata)
test_job.save()
# detect jobs failed because of infrastructure issues
if metadata['error_type'] in [
'Infrastructure', 'Lava', 'Job'
]:
completed = False
# automatically resubmit in some cases
if metadata['error_type'] in ['Infrastructure', 'Job'] and \
any(substring.lower() in metadata['error_msg'].lower() for substring in lava_infra_error_messages):
if test_job.resubmitted_count < 3:
resubmitted_job = self.resubmit(test_job)
if self.settings.get(
'CI_LAVA_SEND_ADMIN_EMAIL', True):
# delay sending email by 15 seconds to allow the database object to be saved
send_testjob_resubmit_admin_email.apply_async(
args=[test_job.pk, resubmitted_job.pk],
countdown=15)
# don't send admin_email
continue
return (data['status'], completed, job_metadata, results, metrics)
