def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)['taskGroupId'] for build_task_id in self.task_ids.values() ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task) ] for test_task in test_tasks: status = test_task['status']['state'] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status) logger.info('Waiting for task {} to finish...'.format( test_task['status']['taskId'])) time.sleep(60) status = taskcluster.get_task_status( test_task['status']['taskId']) # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task['status']['state'] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status) chunk_name = taskcluster.get_chunk( test_task['task']['metadata']['name']) platform_name = taskcluster.get_platform( test_task['task']['metadata']['name']) # Ignore awsy and talos as they aren't actually suites of tests. if any(to_ignore in chunk_name for to_ignore in self.suites_to_ignore): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status'] ['state']]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info('Code coverage artifacts downloaded')
def download(self, test_task): chunk_name = taskcluster.get_chunk(test_task['task']['metadata']['name']) platform_name = taskcluster.get_platform(test_task['task']['metadata']['name']) test_task_id = test_task['status']['taskId'] for artifact in taskcluster.get_task_artifacts(test_task_id): if not any(n in artifact['name'] for n in ['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']): continue artifact_path = self.generate_path(platform_name, chunk_name, artifact) taskcluster.download_artifact(artifact_path, test_task_id, artifact['name']) logger.info('%s artifact downloaded' % artifact_path)
def test_get_chunk(): tests = [ ('test-linux64-ccov/debug-mochitest-1', 'mochitest-1'), ('test-linux64-ccov/debug-mochitest-e10s-7', 'mochitest-7'), ('test-linux64-ccov/debug-cppunit', 'cppunit'), ('test-linux64-ccov/debug-firefox-ui-functional-remote-e10s', 'firefox-ui-functional-remote'), ('test-windows10-64-ccov/debug-mochitest-1', 'mochitest-1'), ('test-windows10-64-ccov/debug-mochitest-e10s-7', 'mochitest-7'), ('test-windows10-64-ccov/debug-cppunit', 'cppunit'), ] for (name, chunk) in tests: assert taskcluster.get_chunk(name) == chunk
def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) logger.info("Downloading artifacts from {} tasks".format(len(self.test_tasks))) for test_task in self.test_tasks: status = test_task["status"]["state"] task_id = test_task["status"]["taskId"] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status ) logger.info(f"Waiting for task {task_id} to finish...") time.sleep(60) task_status = taskcluster.get_task_status(task_id) status = task_status["status"]["state"] # Update the task status, as we will use it to compare statuses later. test_task["status"]["state"] = status # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in self.test_tasks: status = test_task["status"]["state"] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status ) chunk_name = taskcluster.get_chunk(test_task["task"]) platform_name = taskcluster.get_platform(test_task["task"]) if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"]["state"]]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info("Code coverage artifacts downloaded")
def test_get_chunk(): tests = [ ('test-linux64-ccov/debug-mochitest-1', 'mochitest-1'), ('test-linux64-ccov/debug-mochitest-e10s-7', 'mochitest-7'), ('test-linux64-ccov/debug-cppunit', 'cppunit'), ('test-linux64-ccov/debug-firefox-ui-functional-remote-e10s', 'firefox-ui-functional-remote'), ('test-windows10-64-ccov/debug-mochitest-1', 'mochitest-1'), ('test-windows10-64-ccov/debug-mochitest-e10s-7', 'mochitest-7'), ('test-windows10-64-ccov/debug-cppunit', 'cppunit'), ('build-linux64-ccov/debug', 'build'), ('build-android-test-ccov/opt', 'build'), ('build-win64-ccov/debug', 'build'), ] for (name, chunk) in tests: assert taskcluster.get_chunk(name) == chunk
def download(self, test_task): chunk_name = taskcluster.get_chunk( test_task["task"]["metadata"]["name"]) platform_name = taskcluster.get_platform( test_task["task"]["metadata"]["name"]) test_task_id = test_task["status"]["taskId"] for artifact in taskcluster.get_task_artifacts(test_task_id): if not any(n in artifact["name"] for n in ["code-coverage-grcov.zip", "code-coverage-jsvm.zip"]): continue artifact_path = self.generate_path(platform_name, chunk_name, artifact) taskcluster.download_artifact(artifact_path, test_task_id, artifact["name"]) logger.info("%s artifact downloaded" % artifact_path)
def test_get_chunk(): tests = [ ("test-linux64-ccov/debug-mochitest-1", "mochitest-1"), ("test-linux64-ccov/debug-mochitest-e10s-7", "mochitest-7"), ("test-linux64-ccov/debug-cppunit", "cppunit"), ( "test-linux64-ccov/debug-firefox-ui-functional-remote-e10s", "firefox-ui-functional-remote", ), ("test-windows10-64-ccov/debug-mochitest-1", "mochitest-1"), ("test-windows10-64-ccov/debug-mochitest-e10s-7", "mochitest-7"), ("test-windows10-64-ccov/debug-cppunit", "cppunit"), ("build-linux64-ccov/debug", "build"), ("build-android-test-ccov/opt", "build"), ("build-win64-ccov/debug", "build"), ] for (name, chunk) in tests: assert taskcluster.get_chunk(name) == chunk
def download_all(self): os.makedirs(self.parent_dir, exist_ok=True) # The test tasks for the Linux and Windows builds are in the same group, # but the following code is generic and supports build tasks split in # separate groups. groups = set([ taskcluster.get_task_details(build_task_id)["taskGroupId"] for build_task_id in self.task_ids.values() if build_task_id is not None ]) test_tasks = [ task for group in groups for task in taskcluster.get_tasks_in_group(group) if taskcluster.is_coverage_task(task["task"]) and not self.is_filtered_task(task) ] logger.info("Downloading artifacts from {} tasks".format( len(test_tasks))) for test_task in test_tasks: status = test_task["status"]["state"] task_id = test_task["status"]["taskId"] while status not in FINISHED_STATUSES: assert status in ALL_STATUSES, "State '{}' not recognized".format( status) logger.info(f"Waiting for task {task_id} to finish...") time.sleep(60) task_status = taskcluster.get_task_status(task_id) status = task_status["status"]["state"] # Update the task status, as we will use it to compare statuses later. test_task["status"]["state"] = status # Choose best tasks to download (e.g. 'completed' is better than 'failed') download_tasks = {} for test_task in test_tasks: status = test_task["status"]["state"] assert status in FINISHED_STATUSES, "State '{}' not recognized".format( status) chunk_name = taskcluster.get_chunk(test_task["task"]) platform_name = taskcluster.get_platform(test_task["task"]) if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE): continue if (chunk_name, platform_name) not in download_tasks: # If the chunk hasn't been downloaded before, this is obviously the best task # to download it from. download_tasks[(chunk_name, platform_name)] = test_task else: # Otherwise, compare the status of this task with the previously selected task. prev_task = download_tasks[(chunk_name, platform_name)] if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"] ["state"]]: download_tasks[(chunk_name, platform_name)] = test_task with ThreadPoolExecutorResult() as executor: for test_task in download_tasks.values(): executor.submit(self.download, test_task) logger.info("Code coverage artifacts downloaded")
def generate(repo_dir, revision, artifactsHandler, out_dir='.'): logger.info('Generating chunk mapping...') sqlite_file = os.path.join(out_dir, 'chunk_mapping.sqlite') tarxz_file = os.path.join(out_dir, 'chunk_mapping.tar.xz') with sqlite3.connect(sqlite_file) as conn: logger.info('Creating tables.') c = conn.cursor() c.execute('CREATE TABLE file_to_chunk (path text, platform text, chunk text)') c.execute('CREATE TABLE chunk_to_test (platform text, chunk text, path text)') c.execute('CREATE TABLE file_to_test (source text, test text)') logger.info('Populating file_to_test table.') test_coverage_suites = get_test_coverage_suites() logger.info('Found {} test suites.'.format(len(test_coverage_suites))) for suites in group_by_20k(test_coverage_suites): test_coverage_tests = get_test_coverage_tests(suites) for tests in group_by_20k(test_coverage_tests): tests_files_data = get_test_coverage_files(tests) source_names = tests_files_data['source.file.name'] test_iter = enumerate(tests_files_data['test.name']) source_test_iter = ((source_names[i], test) for i, test in test_iter) c.executemany('INSERT INTO file_to_test VALUES (?,?)', source_test_iter) with ThreadPoolExecutor(max_workers=4) as executor: futures = {} for platform in PLATFORMS: logger.info('Reading chunk coverage artifacts for {}.'.format(platform)) for chunk in artifactsHandler.get_chunks(platform): suite = taskcluster.get_suite(chunk) if not is_chunk_only_suite(suite): continue assert chunk.strip() != '', 'chunk can not be an empty string' artifacts = artifactsHandler.get(platform=platform, chunk=chunk) assert len(artifacts) > 0, 'There should be at least one artifact' future = executor.submit(grcov.files_list, artifacts, source_dir=repo_dir) futures[future] = (platform, chunk) logger.info('Populating chunk_to_test table for {}.'.format(platform)) for suite in get_suites(revision): if not is_chunk_only_suite(suite): continue tests_data = get_tests_chunks(revision, platform, suite) if len(tests_data) == 0: logger.warn('No tests found for platform {} and suite {}.'.format(platform, suite)) continue logger.info('Adding tests for platform {} and suite {}'.format(platform, suite)) task_names = tests_data['run.key'] test_iter = enumerate(tests_data['result.test']) chunk_test_iter = ((platform, taskcluster.get_chunk(task_names[i]), test) for i, test in test_iter) c.executemany('INSERT INTO chunk_to_test VALUES (?,?,?)', chunk_test_iter) logger.info('Populating file_to_chunk table.') for future in concurrent.futures.as_completed(futures): (platform, chunk) = futures[future] files = future.result() c.executemany('INSERT INTO file_to_chunk VALUES (?,?,?)', ((f, platform, chunk) for f in files)) logger.info('Writing the chunk mapping archive at {}.'.format(tarxz_file)) with tarfile.open(tarxz_file, 'w:xz') as tar: tar.add(sqlite_file, os.path.basename(sqlite_file))
def generate(repo_dir, revision, artifactsHandler, out_dir='.'): logger.info('Generating chunk mapping...') sqlite_file = os.path.join(out_dir, 'chunk_mapping.sqlite') tarxz_file = os.path.join(out_dir, 'chunk_mapping.tar.xz') with sqlite3.connect(sqlite_file) as conn: logger.info('Creating tables.') c = conn.cursor() c.execute('CREATE TABLE file_to_chunk (path text, platform text, chunk text)') c.execute('CREATE TABLE chunk_to_test (platform text, chunk text, path text)') c.execute('CREATE TABLE file_to_test (source text, test text)') logger.info('Populating file_to_test table.') test_coverage_suites = get_test_coverage_suites() logger.info('Found {} test suites.'.format(len(test_coverage_suites))) for suites in group_by_20k(test_coverage_suites): test_coverage_tests = get_test_coverage_tests(suites) for tests in group_by_20k(test_coverage_tests): tests_files_data = get_test_coverage_files(tests) source_names = tests_files_data['source.file.name'] test_iter = enumerate(tests_files_data['test.name']) source_test_iter = ((source_names[i], test) for i, test in test_iter) c.executemany('INSERT INTO file_to_test VALUES (?,?)', source_test_iter) with ThreadPoolExecutor(max_workers=4) as executor: futures = {} for platform in PLATFORMS: logger.info('Reading chunk coverage artifacts for {}.'.format(platform)) for chunk in artifactsHandler.get_chunks(): suite = taskcluster.get_suite(chunk) if not is_chunk_only_suite(suite): continue future = executor.submit(grcov.files_list, artifactsHandler.get(platform=platform, chunk=chunk), source_dir=repo_dir) futures[future] = (platform, chunk) logger.info('Populating chunk_to_test table for {}.'.format(platform)) for suite in get_suites(revision): if not is_chunk_only_suite(suite): continue tests_data = get_tests_chunks(revision, platform, suite) if len(tests_data) == 0: logger.warn('No tests found for platform {} and suite {}.'.format(platform, suite)) continue logger.info('Adding tests for platform {} and suite {}'.format(platform, suite)) task_names = tests_data['run.key'] test_iter = enumerate(tests_data['result.test']) chunk_test_iter = ((platform, taskcluster.get_chunk(task_names[i]), test) for i, test in test_iter) c.executemany('INSERT INTO chunk_to_test VALUES (?,?,?)', chunk_test_iter) logger.info('Populating file_to_chunk table.') for future in concurrent.futures.as_completed(futures): (platform, chunk) = futures[future] files = future.result() c.executemany('INSERT INTO file_to_chunk VALUES (?,?,?)', ((f, platform, chunk) for f in files)) logger.info('Writing the chunk mapping archive at {}.'.format(tarxz_file)) with tarfile.open(tarxz_file, 'w:xz') as tar: tar.add(sqlite_file, os.path.basename(sqlite_file))
def test_get_chunk(task_name, expected): task = json.load(open(os.path.join(FIXTURES_DIR, f"{task_name}.json"))) assert taskcluster.get_chunk(task) == expected
def chunk_test_iter(): test_iter = enumerate(tests_data["result.test"]) return ((platform, taskcluster.get_chunk(task_names[i]), test) for i, test in test_iter)