def __resubmit_job__(self, test_job, metadata): infra_messages_re_list = [] project_settings = self.__resolve_settings__(test_job) error_messages_settings = project_settings.get( 'CI_LAVA_INFRA_ERROR_MESSAGES', []) for message_re in error_messages_settings: try: r = re.compile(message_re, re.I) infra_messages_re_list.append(r) except re.error: # ignore incorrect expressions self.log_debug("'%s' is not a valid regex" % message_re) for regex in infra_messages_re_list: error_msg = metadata.get('error_msg') if error_msg and regex.search(error_msg) is not None and \ test_job.resubmitted_count < 3: resubmitted_job = self.resubmit(test_job) if project_settings.get('CI_LAVA_SEND_ADMIN_EMAIL', True): # delay sending email by 15 seconds to allow the database object to be saved send_testjob_resubmit_admin_email.apply_async( args=[test_job.pk, resubmitted_job.pk], countdown=15) # re-submit the job only once # even if there are more matches break
def __resubmit_job__(self, test_job, metadata): infra_messages_re_list = [] for message_re in self.settings.get('CI_LAVA_INFRA_ERROR_MESSAGES', []): try: r = re.compile(message_re, re.I) infra_messages_re_list.append(r) except re.error: # ignore incorrect expressions self.log_debug("'%s' is not a valid regex" % message_re) for regex in infra_messages_re_list: if regex.search(metadata['error_msg']) is not None and \ test_job.resubmitted_count < 3: resubmitted_job = self.resubmit(test_job) if self.settings.get('CI_LAVA_SEND_ADMIN_EMAIL', True): # delay sending email by 15 seconds to allow the database object to be saved send_testjob_resubmit_admin_email.apply_async(args=[test_job.pk, resubmitted_job.pk], countdown=15) # re-submit the job only once # even if there are more matches break
def __parse_results__(self, data, test_job): lava_infra_error_messages = [] if self.settings is not None: lava_infra_error_messages = self.settings.get( 'CI_LAVA_INFRA_ERROR_MESSAGES', []) if data['is_pipeline'] is False: # in case of v1 job, return empty data return (data['status'], {}, {}, {}) definition = yaml.load(data['definition']) if data['multinode_definition']: definition = yaml.load(data['multinode_definition']) test_job.name = definition['job_name'][:255] job_metadata = definition.get('metadata', {}) suite_versions = {} for key, value in job_metadata.items(): if key.endswith('__version'): suite_versions[key.replace('__version', '')] = value if suite_versions: job_metadata['suite_versions'] = suite_versions results = {} metrics = {} completed = True if data['status'] == 'Canceled': # consider all canceled jobs as incomplete and discard any results completed = False else: for result in data['results']: if result['suite'] != 'lava': suite = result['suite'].split("_", 1)[-1] res_name = "%s/%s" % (suite, result['name']) # YAML from LAVA has all values serialized to strings if result['measurement'] == 'None': res_value = result['result'] results.update({res_name: res_value}) else: res_value = result['measurement'] metrics.update({res_name: float(res_value)}) else: # add artificial 'boot' test result for each test job if result['name'] == 'auto-login-action': # by default the boot test is named after the device_type res_name = "boot/%s" % (definition['device_type']) res_time_name = "boot/time-%s" % ( definition['device_type']) if 'testsuite' in job_metadata.keys(): # If 'testsuite' metadata key is present in the job # it's appended to the test name. This way regressions can # be found with more granularity res_name = "%s-%s" % (res_name, job_metadata['testsuite']) results.update({res_name: result['result']}) metrics.update( {res_time_name: float(result['measurement'])}) if result['name'] == 'job' and result['result'] == 'fail': metadata = result['metadata'] test_job.failure = str(metadata) test_job.save() # detect jobs failed because of infrastructure issues if metadata['error_type'] in [ 'Infrastructure', 'Lava', 'Job' ]: completed = False # automatically resubmit in some cases if metadata['error_type'] in ['Infrastructure', 'Job'] and \ any(substring.lower() in metadata['error_msg'].lower() for substring in lava_infra_error_messages): if test_job.resubmitted_count < 3: resubmitted_job = self.resubmit(test_job) if self.settings.get( 'CI_LAVA_SEND_ADMIN_EMAIL', True): # delay sending email by 15 seconds to allow the database object to be saved send_testjob_resubmit_admin_email.apply_async( args=[test_job.pk, resubmitted_job.pk], countdown=15) # don't send admin_email continue return (data['status'], completed, job_metadata, results, metrics)