def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids.values() ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task) ] for test_task in test_tasks: status = test_task['status']['state'] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status) logger.info('Waiting for task {} to finish...'.format( test_task['status']['taskId'])) time.sleep(60) status = taskcluster.get_task_status( test_task['status']['taskId']) # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task['status']['state'] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status) chunk_name = taskcluster.get_chunk( test_task['task']['metadata']['name']) platform_name = taskcluster.get_platform( test_task['task']['metadata']['name']) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status'] ['state']]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info('Code coverage artifacts downloaded')
def __init__( self, repository, revision, task_name_filter, cache_root, working_dir, required_platforms=[], ): os.makedirs(working_dir, exist_ok=True) self.artifacts_dir = os.path.join(working_dir, "ccov-artifacts") self.reports_dir = os.path.join(working_dir, "ccov-reports") logger.info( "Local storage initialized.", artifacts=self.artifacts_dir, reports=self.reports_dir, ) self.repository = repository self.revision = revision assert (self.revision is not None and self.repository is not None), "Missing repo/revision" logger.info("Mercurial setup", repository=self.repository, revision=self.revision) if cache_root is not None: assert os.path.isdir( cache_root), f"Cache root {cache_root} is not a dir." self.repo_dir = os.path.join(cache_root, self.branch) # Load coverage tasks for all platforms decision_task_id = taskcluster.get_decision_task( self.branch, self.revision) assert decision_task_id is not None, "The decision task couldn't be found" group = taskcluster.get_task_details(decision_task_id)["taskGroupId"] test_tasks = [ task for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task["task"]) ] # Check the required platforms are present platforms = set( taskcluster.get_platform(test_task["task"]) for test_task in test_tasks) for platform in required_platforms: assert platform in platforms, f"{platform} missing in the task group." self.artifactsHandler = ArtifactsHandler(test_tasks, self.artifacts_dir, task_name_filter)
def test_get_tasks_in_group(GROUP_TASKS_1, GROUP_TASKS_2): responses.add( responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?limit=200', json=GROUP_TASKS_1, status=200, match_querystring=True) # noqa responses.add( responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?continuationToken=1%2132%21YVB0OUZiSWRRd21od0RJUERZTHVhdw--~1%2132%21ZnJVcGRRT0VTalN0Nm9Ua1Ztcy04UQ--&limit=200', json=GROUP_TASKS_2, status=200, match_querystring=True) # noqa assert taskcluster.get_tasks_in_group( 'aPt9FbIdQwmhwDIPDYLuaw' ) == GROUP_TASKS_1['tasks'] + GROUP_TASKS_2['tasks']
def test_get_tasks_in_group(mock_taskcluster, GROUP_TASKS_1, GROUP_TASKS_2): responses.add( responses.GET, "http://taskcluster.test/api/queue/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?limit=200", json=GROUP_TASKS_1, status=200, match_querystring=True, ) # noqa responses.add( responses.GET, "http://taskcluster.test/api/queue/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?continuationToken=1%2132%21YVB0OUZiSWRRd21od0RJUERZTHVhdw--~1%2132%21ZnJVcGRRT0VTalN0Nm9Ua1Ztcy04UQ--&limit=200", # noqa json=GROUP_TASKS_2, status=200, match_querystring=True, ) # noqa assert (list(taskcluster.get_tasks_in_group("aPt9FbIdQwmhwDIPDYLuaw")) == GROUP_TASKS_1["tasks"] + GROUP_TASKS_2["tasks"])
def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)["taskGroupId"] for build_task_id in self.task_ids.values() if build_task_id is not None ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task["task"]) and not self.is_filtered_task(task) ] logger.info("Downloading artifacts from {} tasks".format( len(test_tasks))) for test_task in test_tasks: status = test_task["status"]["state"] task_id = test_task["status"]["taskId"] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status) logger.info(f"Waiting for task {task_id} to finish...") time.sleep(60) task_status = taskcluster.get_task_status(task_id) status = task_status["status"]["state"] # Update the task status, as we will use it to compare statuses later. test_task["status"]["state"] = status # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task["status"]["state"] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status) chunk_name = taskcluster.get_chunk(test_task["task"]) platform_name = taskcluster.get_platform(test_task["task"]) if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"] ["state"]]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info("Code coverage artifacts downloaded")
def trigger_missing(server_address: str, out_dir: str = ".") -> None: triggered_revisions_path = os.path.join(out_dir, "triggered_revisions.zst") url = f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.{secrets[secrets.APP_CHANNEL]}.cron.latest/artifacts/public/triggered_revisions.zst" # noqa r = requests.head(url, allow_redirects=True) if r.status_code != 404: utils.download_file(url, triggered_revisions_path) try: dctx = zstandard.ZstdDecompressor() with open(triggered_revisions_path, "rb") as zf: with dctx.stream_reader(zf) as reader: with io.TextIOWrapper(reader, encoding="ascii") as f: triggered_revisions = set(rev for rev in f.read().splitlines()) except FileNotFoundError: triggered_revisions = set() # Get all mozilla-central revisions from the past year. days = 365 if secrets[secrets.APP_CHANNEL] == "production" else 30 a_year_ago = datetime.utcnow() - timedelta(days=days) with hgmo.HGMO(server_address=server_address) as hgmo_server: data = hgmo_server.get_pushes( startDate=a_year_ago.strftime("%Y-%m-%d"), full=False, tipsonly=True) revisions = [(push_data["changesets"][0], int(push_data["date"])) for push_data in data["pushes"].values()] logger.info(f"{len(revisions)} pushes in the past year") assert (secrets[secrets.GOOGLE_CLOUD_STORAGE] is not None), "Missing GOOGLE_CLOUD_STORAGE secret" bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE]) missing_revisions = [] for revision, timestamp in revisions: # Skip revisions that have already been triggered. If they are still missing, # it means there is a problem that is preventing us from ingesting them. if revision in triggered_revisions: continue # If the revision was already ingested, we don't need to trigger ingestion for it again. if uploader.gcp_covdir_exists(bucket, "mozilla-central", revision, "all", "all"): triggered_revisions.add(revision) continue missing_revisions.append((revision, timestamp)) logger.info(f"{len(missing_revisions)} missing pushes in the past year") yesterday = int(datetime.timestamp(datetime.utcnow() - timedelta(days=1))) task_group_id = slugId() logger.info(f"Triggering tasks in the {task_group_id} group") triggered = 0 for revision, timestamp in reversed(missing_revisions): # If it's older than yesterday, we assume the group finished. # If it is newer than yesterday, we load the group and check if all tasks in it finished. if timestamp > yesterday: decision_task_id = taskcluster.get_decision_task( "mozilla-central", revision) if decision_task_id is None: continue group = taskcluster.get_task_details( decision_task_id)["taskGroupId"] if not all(task["status"]["state"] in taskcluster.FINISHED_STATUSES for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task["task"])): continue trigger_task(task_group_id, revision) triggered_revisions.add(revision) triggered += 1 if triggered == MAXIMUM_TRIGGERS: break cctx = zstandard.ZstdCompressor(threads=-1) with open(triggered_revisions_path, "wb") as zf: with cctx.stream_writer(zf) as compressor: with io.TextIOWrapper(compressor, encoding="ascii") as f: f.write("\n".join(triggered_revisions))
def test_get_tasks_in_group(GROUP_TASKS_1, GROUP_TASKS_2): responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?limit=200', json=GROUP_TASKS_1, status=200, match_querystring=True) # noqa responses.add(responses.GET, 'https://queue.taskcluster.net/v1/task-group/aPt9FbIdQwmhwDIPDYLuaw/list?continuationToken=1%2132%21YVB0OUZiSWRRd21od0RJUERZTHVhdw--~1%2132%21ZnJVcGRRT0VTalN0Nm9Ua1Ztcy04UQ--&limit=200', json=GROUP_TASKS_2, status=200, match_querystring=True) # noqa assert taskcluster.get_tasks_in_group('aPt9FbIdQwmhwDIPDYLuaw') == GROUP_TASKS_1['tasks'] + GROUP_TASKS_2['tasks']