def get_task_metrics_from_date(model, date, output_directory):
options = get_taskcluster_options()
index = taskcluster.Index(options)
index.ping()
# Split the date
from_date = date.split(".")
namespaces = []
# Start at the root level
# We need an empty list in order to append namespaces part to it
namespaces.append([])
# Recursively list all namespaces greater or equals than the given date
while namespaces:
current_ns = namespaces.pop()
# Handle version level namespaces
if not current_ns:
ns_uri = ROOT_URI.format(model)
else:
current_ns_date = ".".join(current_ns)
ns_uri = DATE_URI.format(model, current_ns_date)
ns_full_uri = NAMESPACE_URI.format(ns_uri)
tasks = index.listTasks(ns_full_uri)
for task in tasks["tasks"]:
task_uri = task["namespace"]
r = get_task_metrics_from_uri(task_uri)
# Write the file on disk
file_name = f"metric_{'_'.join(task_uri.split('.'))}.json"
file_path = abspath(join(output_directory, file_name))
with open(file_path, "w") as metric_file:
metric_file.write(r.text)
LOGGER.info(f"Metrics saved to {file_path!r}")
for namespace in get_namespaces(index, ns_full_uri):
new_ns = current_ns.copy()
new_ns.append(namespace["name"])
if not is_later_or_equal(new_ns, from_date):
LOGGER.debug("NEW namespace %s is before %s", new_ns,
from_date)
continue
# Might not be efficient but size of `namespaces` shouldn't be too
# big as we are doing a depth-first traversal
if new_ns not in namespaces:
namespaces.append(new_ns)
def get_model_names(task_id: str) -> List[str]:
options = get_taskcluster_options()
queue = taskcluster.Queue(options)
task = queue.task(task_id)
model_names = []
for i, task_id in enumerate(task["dependencies"]):
LOGGER.info("Loading task dependencies {}/{} {}".format(
i + 1, len(task["dependencies"]), task_id))
model_name = get_model_name(queue, task_id)
LOGGER.info("Adding model %r to download list", model_name)
model_names.append(model_name)
return model_names