def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([
taskcluster.get_task_details(build_task_id)['taskGroupId']
for build_task_id in self.task_ids.values()
])
test_tasks = [
task for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task)
]
for test_task in test_tasks:
status = test_task['status']['state']
while status not in FINISHED_STATUSES:
assert status in ALL_STATUSES, "State '{}' not recognized".format(
status)
logger.info('Waiting for task {} to finish...'.format(
test_task['status']['taskId']))
time.sleep(60)
status = taskcluster.get_task_status(
test_task['status']['taskId'])
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in test_tasks:
status = test_task['status']['state']
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
status)
chunk_name = taskcluster.get_chunk(
test_task['task']['metadata']['name'])
platform_name = taskcluster.get_platform(
test_task['task']['metadata']['name'])
# Ignore awsy and talos as they aren't actually suites of tests.
if any(to_ignore in chunk_name
for to_ignore in self.suites_to_ignore):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status']
['state']]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in download_tasks.values():
executor.submit(self.download, test_task)
logger.info('Code coverage artifacts downloaded')
def download(self, test_task):
chunk_name = taskcluster.get_chunk(test_task['task']['metadata']['name'])
platform_name = taskcluster.get_platform(test_task['task']['metadata']['name'])
test_task_id = test_task['status']['taskId']
for artifact in taskcluster.get_task_artifacts(test_task_id):
if not any(n in artifact['name'] for n in ['code-coverage-grcov.zip', 'code-coverage-jsvm.zip']):
continue
artifact_path = self.generate_path(platform_name, chunk_name, artifact)
taskcluster.download_artifact(artifact_path, test_task_id, artifact['name'])
logger.info('%s artifact downloaded' % artifact_path)
def test_get_chunk():
tests = [
('test-linux64-ccov/debug-mochitest-1', 'mochitest-1'),
('test-linux64-ccov/debug-mochitest-e10s-7', 'mochitest-7'),
('test-linux64-ccov/debug-cppunit', 'cppunit'),
('test-linux64-ccov/debug-firefox-ui-functional-remote-e10s',
'firefox-ui-functional-remote'),
('test-windows10-64-ccov/debug-mochitest-1', 'mochitest-1'),
('test-windows10-64-ccov/debug-mochitest-e10s-7', 'mochitest-7'),
('test-windows10-64-ccov/debug-cppunit', 'cppunit'),
]
for (name, chunk) in tests:
assert taskcluster.get_chunk(name) == chunk
def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
logger.info("Downloading artifacts from {} tasks".format(len(self.test_tasks)))
for test_task in self.test_tasks:
status = test_task["status"]["state"]
task_id = test_task["status"]["taskId"]
while status not in FINISHED_STATUSES:
assert status in ALL_STATUSES, "State '{}' not recognized".format(
status
)
logger.info(f"Waiting for task {task_id} to finish...")
time.sleep(60)
task_status = taskcluster.get_task_status(task_id)
status = task_status["status"]["state"]
# Update the task status, as we will use it to compare statuses later.
test_task["status"]["state"] = status
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in self.test_tasks:
status = test_task["status"]["state"]
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
status
)
chunk_name = taskcluster.get_chunk(test_task["task"])
platform_name = taskcluster.get_platform(test_task["task"])
if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"]["state"]]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in download_tasks.values():
executor.submit(self.download, test_task)
logger.info("Code coverage artifacts downloaded")
def test_get_chunk():
tests = [
('test-linux64-ccov/debug-mochitest-1', 'mochitest-1'),
('test-linux64-ccov/debug-mochitest-e10s-7', 'mochitest-7'),
('test-linux64-ccov/debug-cppunit', 'cppunit'),
('test-linux64-ccov/debug-firefox-ui-functional-remote-e10s', 'firefox-ui-functional-remote'),
('test-windows10-64-ccov/debug-mochitest-1', 'mochitest-1'),
('test-windows10-64-ccov/debug-mochitest-e10s-7', 'mochitest-7'),
('test-windows10-64-ccov/debug-cppunit', 'cppunit'),
('build-linux64-ccov/debug', 'build'),
('build-android-test-ccov/opt', 'build'),
('build-win64-ccov/debug', 'build'),
]
for (name, chunk) in tests:
assert taskcluster.get_chunk(name) == chunk
def download(self, test_task):
chunk_name = taskcluster.get_chunk(
test_task["task"]["metadata"]["name"])
platform_name = taskcluster.get_platform(
test_task["task"]["metadata"]["name"])
test_task_id = test_task["status"]["taskId"]
for artifact in taskcluster.get_task_artifacts(test_task_id):
if not any(n in artifact["name"] for n in
["code-coverage-grcov.zip", "code-coverage-jsvm.zip"]):
continue
artifact_path = self.generate_path(platform_name, chunk_name,
artifact)
taskcluster.download_artifact(artifact_path, test_task_id,
artifact["name"])
logger.info("%s artifact downloaded" % artifact_path)
def test_get_chunk():
tests = [
("test-linux64-ccov/debug-mochitest-1", "mochitest-1"),
("test-linux64-ccov/debug-mochitest-e10s-7", "mochitest-7"),
("test-linux64-ccov/debug-cppunit", "cppunit"),
(
"test-linux64-ccov/debug-firefox-ui-functional-remote-e10s",
"firefox-ui-functional-remote",
),
("test-windows10-64-ccov/debug-mochitest-1", "mochitest-1"),
("test-windows10-64-ccov/debug-mochitest-e10s-7", "mochitest-7"),
("test-windows10-64-ccov/debug-cppunit", "cppunit"),
("build-linux64-ccov/debug", "build"),
("build-android-test-ccov/opt", "build"),
("build-win64-ccov/debug", "build"),
]
for (name, chunk) in tests:
assert taskcluster.get_chunk(name) == chunk
def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([
taskcluster.get_task_details(build_task_id)["taskGroupId"]
for build_task_id in self.task_ids.values()
if build_task_id is not None
])
test_tasks = [
task for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task["task"])
and not self.is_filtered_task(task)
]
logger.info("Downloading artifacts from {} tasks".format(
len(test_tasks)))
for test_task in test_tasks:
status = test_task["status"]["state"]
task_id = test_task["status"]["taskId"]
while status not in FINISHED_STATUSES:
assert status in ALL_STATUSES, "State '{}' not recognized".format(
status)
logger.info(f"Waiting for task {task_id} to finish...")
time.sleep(60)
task_status = taskcluster.get_task_status(task_id)
status = task_status["status"]["state"]
# Update the task status, as we will use it to compare statuses later.
test_task["status"]["state"] = status
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in test_tasks:
status = test_task["status"]["state"]
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
status)
chunk_name = taskcluster.get_chunk(test_task["task"])
platform_name = taskcluster.get_platform(test_task["task"])
if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"]
["state"]]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in download_tasks.values():
executor.submit(self.download, test_task)
logger.info("Code coverage artifacts downloaded")
def generate(repo_dir, revision, artifactsHandler, out_dir='.'):
logger.info('Generating chunk mapping...')
sqlite_file = os.path.join(out_dir, 'chunk_mapping.sqlite')
tarxz_file = os.path.join(out_dir, 'chunk_mapping.tar.xz')
with sqlite3.connect(sqlite_file) as conn:
logger.info('Creating tables.')
c = conn.cursor()
c.execute('CREATE TABLE file_to_chunk (path text, platform text, chunk text)')
c.execute('CREATE TABLE chunk_to_test (platform text, chunk text, path text)')
c.execute('CREATE TABLE file_to_test (source text, test text)')
logger.info('Populating file_to_test table.')
test_coverage_suites = get_test_coverage_suites()
logger.info('Found {} test suites.'.format(len(test_coverage_suites)))
for suites in group_by_20k(test_coverage_suites):
test_coverage_tests = get_test_coverage_tests(suites)
for tests in group_by_20k(test_coverage_tests):
tests_files_data = get_test_coverage_files(tests)
source_names = tests_files_data['source.file.name']
test_iter = enumerate(tests_files_data['test.name'])
source_test_iter = ((source_names[i], test) for i, test in test_iter)
c.executemany('INSERT INTO file_to_test VALUES (?,?)', source_test_iter)
with ThreadPoolExecutor(max_workers=4) as executor:
futures = {}
for platform in PLATFORMS:
logger.info('Reading chunk coverage artifacts for {}.'.format(platform))
for chunk in artifactsHandler.get_chunks(platform):
suite = taskcluster.get_suite(chunk)
if not is_chunk_only_suite(suite):
continue
assert chunk.strip() != '', 'chunk can not be an empty string'
artifacts = artifactsHandler.get(platform=platform, chunk=chunk)
assert len(artifacts) > 0, 'There should be at least one artifact'
future = executor.submit(grcov.files_list, artifacts, source_dir=repo_dir)
futures[future] = (platform, chunk)
logger.info('Populating chunk_to_test table for {}.'.format(platform))
for suite in get_suites(revision):
if not is_chunk_only_suite(suite):
continue
tests_data = get_tests_chunks(revision, platform, suite)
if len(tests_data) == 0:
logger.warn('No tests found for platform {} and suite {}.'.format(platform, suite))
continue
logger.info('Adding tests for platform {} and suite {}'.format(platform, suite))
task_names = tests_data['run.key']
test_iter = enumerate(tests_data['result.test'])
chunk_test_iter = ((platform, taskcluster.get_chunk(task_names[i]), test) for i, test in test_iter)
c.executemany('INSERT INTO chunk_to_test VALUES (?,?,?)', chunk_test_iter)
logger.info('Populating file_to_chunk table.')
for future in concurrent.futures.as_completed(futures):
(platform, chunk) = futures[future]
files = future.result()
c.executemany('INSERT INTO file_to_chunk VALUES (?,?,?)', ((f, platform, chunk) for f in files))
logger.info('Writing the chunk mapping archive at {}.'.format(tarxz_file))
with tarfile.open(tarxz_file, 'w:xz') as tar:
tar.add(sqlite_file, os.path.basename(sqlite_file))
def generate(repo_dir, revision, artifactsHandler, out_dir='.'):
logger.info('Generating chunk mapping...')
sqlite_file = os.path.join(out_dir, 'chunk_mapping.sqlite')
tarxz_file = os.path.join(out_dir, 'chunk_mapping.tar.xz')
with sqlite3.connect(sqlite_file) as conn:
logger.info('Creating tables.')
c = conn.cursor()
c.execute('CREATE TABLE file_to_chunk (path text, platform text, chunk text)')
c.execute('CREATE TABLE chunk_to_test (platform text, chunk text, path text)')
c.execute('CREATE TABLE file_to_test (source text, test text)')
logger.info('Populating file_to_test table.')
test_coverage_suites = get_test_coverage_suites()
logger.info('Found {} test suites.'.format(len(test_coverage_suites)))
for suites in group_by_20k(test_coverage_suites):
test_coverage_tests = get_test_coverage_tests(suites)
for tests in group_by_20k(test_coverage_tests):
tests_files_data = get_test_coverage_files(tests)
source_names = tests_files_data['source.file.name']
test_iter = enumerate(tests_files_data['test.name'])
source_test_iter = ((source_names[i], test) for i, test in test_iter)
c.executemany('INSERT INTO file_to_test VALUES (?,?)', source_test_iter)
with ThreadPoolExecutor(max_workers=4) as executor:
futures = {}
for platform in PLATFORMS:
logger.info('Reading chunk coverage artifacts for {}.'.format(platform))
for chunk in artifactsHandler.get_chunks():
suite = taskcluster.get_suite(chunk)
if not is_chunk_only_suite(suite):
continue
future = executor.submit(grcov.files_list, artifactsHandler.get(platform=platform, chunk=chunk), source_dir=repo_dir)
futures[future] = (platform, chunk)
logger.info('Populating chunk_to_test table for {}.'.format(platform))
for suite in get_suites(revision):
if not is_chunk_only_suite(suite):
continue
tests_data = get_tests_chunks(revision, platform, suite)
if len(tests_data) == 0:
logger.warn('No tests found for platform {} and suite {}.'.format(platform, suite))
continue
logger.info('Adding tests for platform {} and suite {}'.format(platform, suite))
task_names = tests_data['run.key']
test_iter = enumerate(tests_data['result.test'])
chunk_test_iter = ((platform, taskcluster.get_chunk(task_names[i]), test) for i, test in test_iter)
c.executemany('INSERT INTO chunk_to_test VALUES (?,?,?)', chunk_test_iter)
logger.info('Populating file_to_chunk table.')
for future in concurrent.futures.as_completed(futures):
(platform, chunk) = futures[future]
files = future.result()
c.executemany('INSERT INTO file_to_chunk VALUES (?,?,?)', ((f, platform, chunk) for f in files))
logger.info('Writing the chunk mapping archive at {}.'.format(tarxz_file))
with tarfile.open(tarxz_file, 'w:xz') as tar:
tar.add(sqlite_file, os.path.basename(sqlite_file))
def test_get_chunk(task_name, expected):
task = json.load(open(os.path.join(FIXTURES_DIR, f"{task_name}.json")))
assert taskcluster.get_chunk(task) == expected
def chunk_test_iter():
test_iter = enumerate(tests_data["result.test"])
return ((platform, taskcluster.get_chunk(task_names[i]), test)
for i, test in test_iter)
