def download_coverage_artifacts(self, build_task_id):
try:
os.mkdir('ccov-artifacts')
except OSError as e:
if e.errno != errno.EEXIST:
raise e
task_data = taskcluster.get_task_details(build_task_id)
artifacts = taskcluster.get_task_artifacts(build_task_id)
for artifact in artifacts:
if 'target.code-coverage-gcno.zip' in artifact['name']:
taskcluster.download_artifact(build_task_id, '', artifact)
all_suites = set()
tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId'])
test_tasks = [t for t in tasks if taskcluster.is_coverage_task(t)]
for test_task in test_tasks:
suite_name = taskcluster.get_suite_name(test_task)
all_suites.add(suite_name)
test_task_id = test_task['status']['taskId']
artifacts = taskcluster.get_task_artifacts(test_task_id)
for artifact in artifacts:
if any(n in artifact['name'] for n in
['code-coverage-gcda.zip', 'code-coverage-jsvm.zip']):
taskcluster.download_artifact(test_task_id, suite_name,
artifact)
self.suites = list(all_suites)
self.suites.sort()
def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([
taskcluster.get_task_details(build_task_id)['taskGroupId']
for build_task_id in self.task_ids.values()
])
test_tasks = [
task for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task)
]
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in test_tasks:
status = test_task['status']['state']
assert status in ALL_STATUSES
chunk_name = taskcluster.get_chunk(
test_task['task']['metadata']['name'])
platform_name = taskcluster.get_platform(
test_task['task']['metadata']['name'])
# Ignore awsy and talos as they aren't actually suites of tests.
if any(to_ignore in chunk_name
for to_ignore in self.suites_to_ignore):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status']
['state']]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in test_tasks:
executor.submit(self.download, test_task)
logger.info('Code coverage artifacts downloaded')
def download_coverage_artifacts(self, build_task_id):
try:
os.mkdir('ccov-artifacts')
except OSError as e:
if e.errno != errno.EEXIST:
raise e
task_data = taskcluster.get_task_details(build_task_id)
all_suites = set()
def rewriting_task(path):
return lambda: self.rewrite_jsvm_lcov(path)
tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId'])
test_tasks = [t for t in tasks if taskcluster.is_coverage_task(t)]
with ThreadPoolExecutorResult() as executor:
for test_task in test_tasks:
suite_name = taskcluster.get_suite_name(test_task)
# Ignore awsy and talos as they aren't actually suites of tests.
if any(to_ignore in suite_name
for to_ignore in ['awsy', 'talos']):
continue
all_suites.add(suite_name)
test_task_id = test_task['status']['taskId']
for artifact in taskcluster.get_task_artifacts(test_task_id):
if not any(
n in artifact['name'] for n in
['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']):
continue
artifact_path = taskcluster.download_artifact(
test_task_id, suite_name, artifact)
if 'code-coverage-jsvm.zip' in artifact['name']:
executor.submit(rewriting_task(artifact_path))
self.suites = list(all_suites)
self.suites.sort()
logger.info('Code coverage artifacts downloaded')
def download_coverage_artifacts(build_task_id):
try:
os.mkdir('ccov-artifacts')
except:
pass
task_data = taskcluster.get_task_details(build_task_id)
artifacts = taskcluster.get_task_artifacts(build_task_id)
for artifact in artifacts:
if 'target.code-coverage-gcno.zip' in artifact['name']:
taskcluster.download_artifact(build_task_id, artifact)
tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId'])
test_tasks = [t for t in tasks if is_coverage_task(t)]
for test_task in test_tasks:
test_task_id = test_task['status']['taskId']
artifacts = taskcluster.get_task_artifacts(test_task_id)
for artifact in artifacts:
if 'code-coverage-gcda.zip' in artifact['name']:
taskcluster.download_artifact(test_task_id, artifact)
def disable_test_get_tasks_in_group():
task_id = taskcluster.get_last_task('linux')
task_data = taskcluster.get_task_details(task_id)
tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId'])
assert len(tasks) > 0
def download_coverage_artifacts(self):
mkdir('ccov-artifacts')
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids])
test_tasks = [
task
for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task)
]
FINISHED_STATUSES = ['completed', 'failed', 'exception']
ALL_STATUSES = FINISHED_STATUSES + ['unscheduled', 'pending', 'running']
downloaded_tasks = {}
downloaded_tasks_lock = Lock()
def should_download(status, chunk_name, platform_name):
with downloaded_tasks_lock:
if (chunk_name, platform_name) not in downloaded_tasks:
return True
other_status = downloaded_tasks[(chunk_name, platform_name)]
if (status == 'failed' and other_status == 'exception') or (status == 'completed' and other_status != 'completed'):
downloaded_tasks[(chunk_name, platform_name)] = status
return True
else:
return False
def download_artifact(test_task):
status = test_task['status']['state']
assert status in ALL_STATUSES
while status not in FINISHED_STATUSES:
time.sleep(60)
status = taskcluster.get_task_status(test_task['status']['taskId'])['status']['state']
assert status in ALL_STATUSES
chunk_name = taskcluster.get_chunk_name(test_task)
platform_name = taskcluster.get_platform_name(test_task)
# Ignore awsy and talos as they aren't actually suites of tests.
if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore):
return
# If we have already downloaded this chunk from another task, check if the
# other task has a better status than this one.
if not should_download(status, chunk_name, platform_name):
return
test_task_id = test_task['status']['taskId']
for artifact in taskcluster.get_task_artifacts(test_task_id):
if not any(n in artifact['name'] for n in ['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']):
continue
artifact_path = taskcluster.download_artifact(test_task_id, chunk_name, platform_name, artifact)
logger.info('%s artifact downloaded' % artifact_path)
if 'code-coverage-jsvm.zip' in artifact['name']:
self.rewrite_jsvm_lcov(artifact_path)
logger.info('%s artifact rewritten' % artifact_path)
def download_artifact_task(test_task):
return lambda: download_artifact(test_task)
with ThreadPoolExecutorResult() as executor:
for test_task in test_tasks:
executor.submit(download_artifact_task(test_task))
logger.info('Code coverage artifacts downloaded')
def test_get_tasks_in_group(GROUP_TASKS_1, GROUP_TASKS_2):
responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?limit=200', json=GROUP_TASKS_1, status=200, match_querystring=True) # noqa
responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?continuationToken=1%2132%21YVB0OUZiSWRRd21od0RJUERZTHVhdw--~1%2132%21ZnJVcGRRT0VTalN0Nm9Ua1Ztcy04UQ--&limit=200', json=GROUP_TASKS_2, status=200, match_querystring=True) # noqa
assert taskcluster.get_tasks_in_group('aPt9FbIdQwmhwDIPDYLuaw') == GROUP_TASKS_1['tasks'] + GROUP_TASKS_2['tasks']
