def fetch_graph_and_labels(parameters, graph_config): decision_task_id = find_decision_task(parameters, graph_config) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # fetch everything in parallel; this avoids serializing any delay in downloading # each artifact (such as waiting for the artifact to be mirrored locally) with futures.ThreadPoolExecutor(CONCURRENCY) as e: fetches = [] # fetch any modifications made by action tasks and swap out new tasks # for old ones def fetch_action(task_id): logger.info(f"fetching label-to-taskid.json for action task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") namespace = "{}.v2.{}.pushlog-id.{}.actions".format( graph_config["trust-domain"], parameters["project"], parameters["pushlog_id"], ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_action, task_id)) # Similarly for cron tasks.. def fetch_cron(task_id): logger.info(f"fetching label-to-taskid.json for cron task {task_id}") try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: if e.response.status_code != 404: raise logger.debug(f"No label-to-taskid.json found for {task_id}: {e}") namespace = "{}.v2.{}.revision.{}.cron".format( graph_config["trust-domain"], parameters["project"], parameters["head_rev"] ) for task_id in list_tasks(namespace): fetches.append(e.submit(fetch_cron, task_id)) # now wait for each fetch to complete, raising an exception if there # were any issues for f in futures.as_completed(fetches): f.result() return (decision_task_id, full_task_graph, label_to_taskid)
def fetch_graph_and_labels(parameters): decision_task_id = find_decision_task(parameters) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # Now fetch any modifications made by action tasks and swap out new tasks # for old ones namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format( parameters['project'], parameters['pushlog_id']) for action in list_tasks(namespace): try: run_label_to_id = get_artifact(action, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.info( 'Skipping {} due to missing artifact! Error: {}'.format( action, e)) continue return (decision_task_id, full_task_graph, label_to_taskid)
def fetch_graph_and_labels(parameters, graph_config): decision_task_id = find_decision_task(parameters, graph_config) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # Now fetch any modifications made by action tasks and swap out new tasks # for old ones namespace = '{}.v2.{}.pushlog-id.{}.actions'.format( graph_config['trust-domain'], parameters['project'], parameters['pushlog_id']) for task_id in list_tasks(namespace): logger.info( 'fetching label-to-taskid.json for action task {}'.format(task_id)) try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.debug('No label-to-taskid.json found for {}: {}'.format( task_id, e)) continue # Similarly for cron tasks.. namespace = '{}.v2.{}.revision.{}.cron'.format( graph_config['trust-domain'], parameters['project'], parameters['head_rev']) for task_id in list_tasks(namespace): logger.info( 'fetching label-to-taskid.json for cron task {}'.format(task_id)) try: run_label_to_id = get_artifact(task_id, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.debug('No label-to-taskid.json found for {}: {}'.format( task_id, e)) continue return (decision_task_id, full_task_graph, label_to_taskid)
def fetch_graph_and_labels(parameters): decision_task_id = find_decision_task(parameters) # First grab the graph and labels generated during the initial decision task full_task_graph = get_artifact(decision_task_id, "public/full-task-graph.json") _, full_task_graph = TaskGraph.from_json(full_task_graph) label_to_taskid = get_artifact(decision_task_id, "public/label-to-taskid.json") # Now fetch any modifications made by action tasks and swap out new tasks # for old ones namespace = 'gecko.v2.{}.pushlog-id.{}.actions'.format( parameters['project'], parameters['pushlog_id']) for action in list_tasks(namespace): try: run_label_to_id = get_artifact(action, "public/label-to-taskid.json") label_to_taskid.update(run_label_to_id) except HTTPError as e: logger.info('Skipping {} due to missing artifact! Error: {}'.format(action, e)) continue return (decision_task_id, full_task_graph, label_to_taskid)