def download_coverage_artifacts(self, build_task_id): try: os.mkdir('ccov-artifacts') except OSError as e: if e.errno != errno.EEXIST: raise e task_data = taskcluster.get_task_details(build_task_id) artifacts = taskcluster.get_task_artifacts(build_task_id) for artifact in artifacts: if 'target.code-coverage-gcno.zip' in artifact['name']: taskcluster.download_artifact(build_task_id, '', artifact) all_suites = set() tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId']) test_tasks = [t for t in tasks if taskcluster.is_coverage_task(t)] for test_task in test_tasks: suite_name = taskcluster.get_suite_name(test_task) all_suites.add(suite_name) test_task_id = test_task['status']['taskId'] artifacts = taskcluster.get_task_artifacts(test_task_id) for artifact in artifacts: if any(n in artifact['name'] for n in ['code-coverage-gcda.zip', 'code-coverage-jsvm.zip']): taskcluster.download_artifact(test_task_id, suite_name, artifact) self.suites = list(all_suites) self.suites.sort()
def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids.values() ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task) ] # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task['status']['state'] assert status in ALL_STATUSES chunk_name = taskcluster.get_chunk( test_task['task']['metadata']['name']) platform_name = taskcluster.get_platform( test_task['task']['metadata']['name']) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status'] ['state']]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in test_tasks: executor.submit(self.download, test_task) logger.info('Code coverage artifacts downloaded')
def download_coverage_artifacts(self, build_task_id): try: os.mkdir('ccov-artifacts') except OSError as e: if e.errno != errno.EEXIST: raise e task_data = taskcluster.get_task_details(build_task_id) all_suites = set() def rewriting_task(path): return lambda: self.rewrite_jsvm_lcov(path) tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId']) test_tasks = [t for t in tasks if taskcluster.is_coverage_task(t)] with ThreadPoolExecutorResult() as executor: for test_task in test_tasks: suite_name = taskcluster.get_suite_name(test_task) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in suite_name for to_ignore in ['awsy', 'talos']): continue all_suites.add(suite_name) test_task_id = test_task['status']['taskId'] for artifact in taskcluster.get_task_artifacts(test_task_id): if not any( n in artifact['name'] for n in ['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']): continue artifact_path = taskcluster.download_artifact( test_task_id, suite_name, artifact) if 'code-coverage-jsvm.zip' in artifact['name']: executor.submit(rewriting_task(artifact_path)) self.suites = list(all_suites) self.suites.sort() logger.info('Code coverage artifacts downloaded')
def download_coverage_artifacts(build_task_id): try: os.mkdir('ccov-artifacts') except: pass task_data = taskcluster.get_task_details(build_task_id) artifacts = taskcluster.get_task_artifacts(build_task_id) for artifact in artifacts: if 'target.code-coverage-gcno.zip' in artifact['name']: taskcluster.download_artifact(build_task_id, artifact) tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId']) test_tasks = [t for t in tasks if is_coverage_task(t)] for test_task in test_tasks: test_task_id = test_task['status']['taskId'] artifacts = taskcluster.get_task_artifacts(test_task_id) for artifact in artifacts: if 'code-coverage-gcda.zip' in artifact['name']: taskcluster.download_artifact(test_task_id, artifact)
def disable_test_get_tasks_in_group(): task_id = taskcluster.get_last_task('linux') task_data = taskcluster.get_task_details(task_id) tasks = taskcluster.get_tasks_in_group(task_data['taskGroupId']) assert len(tasks) > 0
def download_coverage_artifacts(self): mkdir('ccov-artifacts') # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task) ] FINISHED_STATUSES = ['completed', 'failed', 'exception'] ALL_STATUSES = FINISHED_STATUSES + ['unscheduled', 'pending', 'running'] downloaded_tasks = {} downloaded_tasks_lock = Lock() def should_download(status, chunk_name, platform_name): with downloaded_tasks_lock: if (chunk_name, platform_name) not in downloaded_tasks: return True other_status = downloaded_tasks[(chunk_name, platform_name)] if (status == 'failed' and other_status == 'exception') or (status == 'completed' and other_status != 'completed'): downloaded_tasks[(chunk_name, platform_name)] = status return True else: return False def download_artifact(test_task): status = test_task['status']['state'] assert status in ALL_STATUSES while status not in FINISHED_STATUSES: time.sleep(60) status = taskcluster.get_task_status(test_task['status']['taskId'])['status']['state'] assert status in ALL_STATUSES chunk_name = taskcluster.get_chunk_name(test_task) platform_name = taskcluster.get_platform_name(test_task) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore): return # If we have already downloaded this chunk from another task, check if the # other task has a better status than this one. if not should_download(status, chunk_name, platform_name): return test_task_id = test_task['status']['taskId'] for artifact in taskcluster.get_task_artifacts(test_task_id): if not any(n in artifact['name'] for n in ['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']): continue artifact_path = taskcluster.download_artifact(test_task_id, chunk_name, platform_name, artifact) logger.info('%s artifact downloaded' % artifact_path) if 'code-coverage-jsvm.zip' in artifact['name']: self.rewrite_jsvm_lcov(artifact_path) logger.info('%s artifact rewritten' % artifact_path) def download_artifact_task(test_task): return lambda: download_artifact(test_task) with ThreadPoolExecutorResult() as executor: for test_task in test_tasks: executor.submit(download_artifact_task(test_task)) logger.info('Code coverage artifacts downloaded')
def test_get_tasks_in_group(GROUP_TASKS_1, GROUP_TASKS_2): responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?limit=200', json=GROUP_TASKS_1, status=200, match_querystring=True) # noqa responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?continuationToken=1%2132%21YVB0OUZiSWRRd21od0RJUERZTHVhdw--~1%2132%21ZnJVcGRRT0VTalN0Nm9Ua1Ztcy04UQ--&limit=200', json=GROUP_TASKS_2, status=200, match_querystring=True) # noqa assert taskcluster.get_tasks_in_group('aPt9FbIdQwmhwDIPDYLuaw') == GROUP_TASKS_1['tasks'] + GROUP_TASKS_2['tasks']