def fetch(self, test_job):
try:
data = self.__get_job_details__(test_job.job_id)
if data['status'] in self.complete_statuses:
data['results'] = self.__get_testjob_results_yaml__(
test_job.job_id)
# fetch logs
logs = ""
try:
logs = self.__get_job_logs__(test_job.job_id)
except Exception:
self.log_warn(("Logs for job %s are not available" %
test_job.job_id) + "\n" +
traceback.format_exc())
return self.__parse_results__(data, test_job) + (logs, )
except xmlrpc.client.ProtocolError as error:
raise TemporaryFetchIssue(str(error))
except xmlrpc.client.Fault as fault:
if fault.faultCode // 100 == 5:
# assume HTTP errors 5xx are temporary issues
raise TemporaryFetchIssue(str(fault))
else:
raise FetchIssue(str(fault))
except ssl.SSLError as fault:
raise FetchIssue(str(fault))
def fetch(self, test_job):
try:
data = self.__get_job_details__(test_job.job_id)
status_key = 'status'
if not self.use_xml_rpc:
status_key = 'state'
if data[status_key] in self.complete_statuses:
# fill in start and end datetime for the job
start_time = data.get('start_time', None)
end_time = data.get('end_time', None)
# convert to datetime
if type(start_time) == str:
try:
start_time = isoparse(start_time)
except ValueError:
start_time = None
if type(end_time) == str:
try:
end_time = isoparse(end_time)
except ValueError:
end_time = None
test_job.started_at = start_time
test_job.ended_at = end_time
test_job.failure = None
test_job.save()
data['results'] = self.__get_testjob_results_yaml__(
test_job.job_id)
# fetch logs
raw_logs = BytesIO()
try:
raw_logs = BytesIO(
self.__download_full_log__(test_job.job_id))
except Exception:
self.log_warn(("Logs for job %s are not available" %
test_job.job_id) + "\n" +
traceback.format_exc())
return self.__parse_results__(data, test_job, raw_logs)
except xmlrpc.client.ProtocolError as error:
raise TemporaryFetchIssue(self.url_remove_token(str(error)))
except xmlrpc.client.Fault as fault:
if fault.faultCode // 100 == 5:
# assume HTTP errors 5xx are temporary issues
raise TemporaryFetchIssue(self.url_remove_token(str(fault)))
else:
raise FetchIssue(self.url_remove_token(str(fault)))
except ssl.SSLError as fault:
raise FetchIssue(self.url_remove_token(str(fault)))
except requests.exceptions.RequestException as fault:
if isinstance(fault, requests.exceptions.Timeout):
# assume HTTP errors 5xx are temporary issues
raise TemporaryFetchIssue(self.url_remove_token(str(fault)))
else:
raise FetchIssue(self.url_remove_token(str(fault)))
def test_exception_when_fetching(self, fetch_method):
fetch_method.side_effect = FetchIssue("ERROR")
fetch.apply(args=[self.test_job.id])
self.test_job.refresh_from_db()
self.assertEqual("ERROR", self.test_job.failure)
self.assertTrue(self.test_job.fetched)
def __check_required_keys__(self, required_keys, results):
missing_keys = []
for k in required_keys:
if k not in results:
missing_keys.append(k)
if len(missing_keys):
keys = ', '.join(missing_keys)
results_json = json.dumps(results)
raise FetchIssue(f'{keys} are required and missing from {results_json}')
def __get_job_details__(self, job_id):
if self.use_xml_rpc:
return self.proxy.scheduler.job_details(job_id)
response = requests.get(urljoin(self.api_url_base,
"jobs/%s" % (job_id)),
headers=self.authentication,
timeout=self.settings.get(
timeout_variable_name, DEFAULT_TIMEOUT))
if response.status_code == 200:
return response.json()
raise FetchIssue(response.text)
def parse_job_id(self, job_id):
"""
Parsing the job id means getting back specific TuxSuite information
from job_id. Ex:
Given a job_id = "BUILD:linaro@anders#1yPYGaOEPNwr2pCqBgONY43zORq",
the return value should be a tuple like
('BUILD', 'linaro@anders', '1yPYGaOEPNwr2pCqBgONY43zORq')
"""
regex = r'^(BUILD|TEST):([0-9a-z_\-]+@[0-9a-z_\-]+)#([a-zA-Z0-9]+)$'
matches = re.findall(regex, job_id)
if len(matches) == 0:
raise FetchIssue(f'Job id "{job_id}" does not match "{regex}"')
# The regex below is supposed to find only one match
return matches[0]
def parse_build_results(self, test_job, job_url, results, settings):
required_keys = ['build_status', 'warnings_count', 'download_url', 'retry']
self.__check_required_keys__(required_keys, results)
# Generate generic test/metric name
test_name = results.get('build_name') or self.generate_test_name(results)
test_job.name = test_name
build_status = results['build_status']
if build_status == 'error' and results['retry'] < 2:
# SQUAD should retry fetching the build until retry == 2
raise TemporaryFetchIssue(results.get('status_message', 'TuxSuite Error'))
# Make metadata
metadata_keys = settings.get('BUILD_METADATA_KEYS', [])
metadata = {k: results.get(k) for k in metadata_keys}
metadata['job_url'] = job_url
metadata['config'] = urljoin(results.get('download_url') + '/', 'config')
# Create tests and metrics
tests = {}
metrics = {}
completed = True
if results['retry'] >= 2:
# This indicates that TuxSuite gave up trying to work on this build
status = 'Incomplete'
tests[f'build/{test_name}'] = 'skip'
logs = ''
else:
status = 'Complete'
tests[f'build/{test_name}'] = build_status
metrics[f'build/{test_name}-warnings'] = results['warnings_count']
logs = self.fetch_url(results['download_url'], 'build.log').text
try:
metrics[f'build/{test_name}-duration'] = results['tuxmake_metadata']['results']['duration']['build']
except KeyError:
raise FetchIssue('Missing duration from build results')
return status, completed, metadata, tests, metrics, logs
