def test_is_coverage_task():
assert taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "test-linux64-ccov/debug-mochitest-1"}}}
)
assert not taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "test-linux64/debug-mochitest-1"}}}
)
assert taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "test-windows10-64-ccov/debug-cppunit"}}}
)
assert not taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "test-windows10-64/debug-cppunit"}}}
)
assert taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "build-win64-ccov/debug"}}}
)
assert not taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "build-win64/debug"}}}
)
assert taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "build-linux64-ccov/debug"}}}
)
assert not taskcluster.is_coverage_task(
{"task": {"metadata": {"name": "build-linux64/debug"}}}
)
def test_is_coverage_task():
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-linux64-ccov/debug-mochitest-1'
}
}
})
assert not taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'test-linux64/debug-mochitest-1'
}
}})
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-windows10-64-ccov/debug-cppunit'
}
}
})
assert not taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'test-windows10-64/debug-cppunit'
}
}})
assert taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'build-win64-ccov/debug'
}
}})
assert not taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'build-win64/debug'
}
}})
assert taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'build-linux64-ccov/debug'
}
}})
assert not taskcluster.is_coverage_task(
{'task': {
'metadata': {
'name': 'build-linux64/debug'
}
}})
def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([
taskcluster.get_task_details(build_task_id)['taskGroupId']
for build_task_id in self.task_ids.values()
])
test_tasks = [
task for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task)
]
for test_task in test_tasks:
status = test_task['status']['state']
while status not in FINISHED_STATUSES:
assert status in ALL_STATUSES, "State '{}' not recognized".format(
status)
logger.info('Waiting for task {} to finish...'.format(
test_task['status']['taskId']))
time.sleep(60)
status = taskcluster.get_task_status(
test_task['status']['taskId'])
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in test_tasks:
status = test_task['status']['state']
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
status)
chunk_name = taskcluster.get_chunk(
test_task['task']['metadata']['name'])
platform_name = taskcluster.get_platform(
test_task['task']['metadata']['name'])
# Ignore awsy and talos as they aren't actually suites of tests.
if any(to_ignore in chunk_name
for to_ignore in self.suites_to_ignore):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task['status']
['state']]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in download_tasks.values():
executor.submit(self.download, test_task)
logger.info('Code coverage artifacts downloaded')
def __init__(
self,
repository,
revision,
task_name_filter,
cache_root,
working_dir,
required_platforms=[],
):
os.makedirs(working_dir, exist_ok=True)
self.artifacts_dir = os.path.join(working_dir, "ccov-artifacts")
self.reports_dir = os.path.join(working_dir, "ccov-reports")
logger.info(
"Local storage initialized.",
artifacts=self.artifacts_dir,
reports=self.reports_dir,
)
self.repository = repository
self.revision = revision
assert (self.revision is not None
and self.repository is not None), "Missing repo/revision"
logger.info("Mercurial setup",
repository=self.repository,
revision=self.revision)
if cache_root is not None:
assert os.path.isdir(
cache_root), f"Cache root {cache_root} is not a dir."
self.repo_dir = os.path.join(cache_root, self.branch)
# Load coverage tasks for all platforms
decision_task_id = taskcluster.get_decision_task(
self.branch, self.revision)
assert decision_task_id is not None, "The decision task couldn't be found"
group = taskcluster.get_task_details(decision_task_id)["taskGroupId"]
test_tasks = [
task for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task["task"])
]
# Check the required platforms are present
platforms = set(
taskcluster.get_platform(test_task["task"])
for test_task in test_tasks)
for platform in required_platforms:
assert platform in platforms, f"{platform} missing in the task group."
self.artifactsHandler = ArtifactsHandler(test_tasks,
self.artifacts_dir,
task_name_filter)
def test_is_coverage_task():
cov_task = {
'task': {
'metadata': {
'name': 'test-linux64-ccov/debug-mochitest-1'
}
}
}
assert taskcluster.is_coverage_task(cov_task)
nocov_task = {
'task': {
'metadata': {
'name': 'test-linux64/debug-mochitest-1'
}
}
}
assert not taskcluster.is_coverage_task(nocov_task)
cov_task = {
'task': {
'metadata': {
'name': 'test-windows10-64-ccov/debug-cppunit'
}
}
}
assert taskcluster.is_coverage_task(cov_task)
nocov_task = {
'task': {
'metadata': {
'name': 'test-windows10-64/debug-cppunit'
}
}
}
assert not taskcluster.is_coverage_task(nocov_task)
def download_all(self):
os.makedirs(self.parent_dir, exist_ok=True)
# The test tasks for the Linux and Windows builds are in the same group,
# but the following code is generic and supports build tasks split in
# separate groups.
groups = set([
taskcluster.get_task_details(build_task_id)["taskGroupId"]
for build_task_id in self.task_ids.values()
if build_task_id is not None
])
test_tasks = [
task for group in groups
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task["task"])
and not self.is_filtered_task(task)
]
logger.info("Downloading artifacts from {} tasks".format(
len(test_tasks)))
for test_task in test_tasks:
status = test_task["status"]["state"]
task_id = test_task["status"]["taskId"]
while status not in FINISHED_STATUSES:
assert status in ALL_STATUSES, "State '{}' not recognized".format(
status)
logger.info(f"Waiting for task {task_id} to finish...")
time.sleep(60)
task_status = taskcluster.get_task_status(task_id)
status = task_status["status"]["state"]
# Update the task status, as we will use it to compare statuses later.
test_task["status"]["state"] = status
# Choose best tasks to download (e.g. 'completed' is better than 'failed')
download_tasks = {}
for test_task in test_tasks:
status = test_task["status"]["state"]
assert status in FINISHED_STATUSES, "State '{}' not recognized".format(
status)
chunk_name = taskcluster.get_chunk(test_task["task"])
platform_name = taskcluster.get_platform(test_task["task"])
if any(to_ignore in chunk_name for to_ignore in SUITES_TO_IGNORE):
continue
if (chunk_name, platform_name) not in download_tasks:
# If the chunk hasn't been downloaded before, this is obviously the best task
# to download it from.
download_tasks[(chunk_name, platform_name)] = test_task
else:
# Otherwise, compare the status of this task with the previously selected task.
prev_task = download_tasks[(chunk_name, platform_name)]
if STATUS_VALUE[status] > STATUS_VALUE[prev_task["status"]
["state"]]:
download_tasks[(chunk_name, platform_name)] = test_task
with ThreadPoolExecutorResult() as executor:
for test_task in download_tasks.values():
executor.submit(self.download, test_task)
logger.info("Code coverage artifacts downloaded")
def trigger_missing(server_address: str, out_dir: str = ".") -> None:
triggered_revisions_path = os.path.join(out_dir, "triggered_revisions.zst")
url = f"https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.relman.code-coverage.{secrets[secrets.APP_CHANNEL]}.cron.latest/artifacts/public/triggered_revisions.zst" # noqa
r = requests.head(url, allow_redirects=True)
if r.status_code != 404:
utils.download_file(url, triggered_revisions_path)
try:
dctx = zstandard.ZstdDecompressor()
with open(triggered_revisions_path, "rb") as zf:
with dctx.stream_reader(zf) as reader:
with io.TextIOWrapper(reader, encoding="ascii") as f:
triggered_revisions = set(rev
for rev in f.read().splitlines())
except FileNotFoundError:
triggered_revisions = set()
# Get all mozilla-central revisions from the past year.
days = 365 if secrets[secrets.APP_CHANNEL] == "production" else 30
a_year_ago = datetime.utcnow() - timedelta(days=days)
with hgmo.HGMO(server_address=server_address) as hgmo_server:
data = hgmo_server.get_pushes(
startDate=a_year_ago.strftime("%Y-%m-%d"),
full=False,
tipsonly=True)
revisions = [(push_data["changesets"][0], int(push_data["date"]))
for push_data in data["pushes"].values()]
logger.info(f"{len(revisions)} pushes in the past year")
assert (secrets[secrets.GOOGLE_CLOUD_STORAGE]
is not None), "Missing GOOGLE_CLOUD_STORAGE secret"
bucket = get_bucket(secrets[secrets.GOOGLE_CLOUD_STORAGE])
missing_revisions = []
for revision, timestamp in revisions:
# Skip revisions that have already been triggered. If they are still missing,
# it means there is a problem that is preventing us from ingesting them.
if revision in triggered_revisions:
continue
# If the revision was already ingested, we don't need to trigger ingestion for it again.
if uploader.gcp_covdir_exists(bucket, "mozilla-central", revision,
"all", "all"):
triggered_revisions.add(revision)
continue
missing_revisions.append((revision, timestamp))
logger.info(f"{len(missing_revisions)} missing pushes in the past year")
yesterday = int(datetime.timestamp(datetime.utcnow() - timedelta(days=1)))
task_group_id = slugId()
logger.info(f"Triggering tasks in the {task_group_id} group")
triggered = 0
for revision, timestamp in reversed(missing_revisions):
# If it's older than yesterday, we assume the group finished.
# If it is newer than yesterday, we load the group and check if all tasks in it finished.
if timestamp > yesterday:
decision_task_id = taskcluster.get_decision_task(
"mozilla-central", revision)
if decision_task_id is None:
continue
group = taskcluster.get_task_details(
decision_task_id)["taskGroupId"]
if not all(task["status"]["state"] in taskcluster.FINISHED_STATUSES
for task in taskcluster.get_tasks_in_group(group)
if taskcluster.is_coverage_task(task["task"])):
continue
trigger_task(task_group_id, revision)
triggered_revisions.add(revision)
triggered += 1
if triggered == MAXIMUM_TRIGGERS:
break
cctx = zstandard.ZstdCompressor(threads=-1)
with open(triggered_revisions_path, "wb") as zf:
with cctx.stream_writer(zf) as compressor:
with io.TextIOWrapper(compressor, encoding="ascii") as f:
f.write("\n".join(triggered_revisions))
def test_is_coverage_task(task_name, expected):
task = json.load(open(os.path.join(FIXTURES_DIR, f"{task_name}.json")))
assert taskcluster.is_coverage_task(task) is expected
def test_is_coverage_task():
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-linux64-ccov/debug-mochitest-1'
}
}
})
assert not taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-linux64/debug-mochitest-1'
}
}
})
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-windows10-64-ccov/debug-cppunit'
}
}
})
assert not taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'test-windows10-64/debug-cppunit'
}
}
})
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'build-win64-ccov/debug'
}
}
})
assert not taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'build-win64/debug'
}
}
})
assert taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'build-linux64-ccov/debug'
}
}
})
assert not taskcluster.is_coverage_task({
'task': {
'metadata': {
'name': 'build-linux64/debug'
}
}
})
