async def do_upload(context, files): """Upload artifacts and return status. Returns the integer status of the upload. args: context (scriptworker.context.Context): the scriptworker context. files (list of str): list of files to be uploaded as artifacts Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: await upload_artifacts(context, files) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except (aiohttp.ClientError, asyncio.TimeoutError) as e: status = worst_level(status, STATUSES["intermittent-task"]) log.error("Hit {}: {}".format(type(e), e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION upload {}".format(e)) raise return status
async def do_upload(context): """Upload artifacts and return status. Returns the integer status of the upload. args: context (scriptworker.context.Context): the scriptworker context. Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION upload {}".format(e)) raise return status
async def do_upload(context, files): """Upload artifacts and return status. Returns the integer status of the upload. args: context (scriptworker.context.Context): the scriptworker context. files (list of str): list of files to be uploaded as artifacts Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: await upload_artifacts(context, files) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION upload {}".format(e)) raise return status
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: # TODO download and verify chain of trust artifacts if # context.config['verify_chain_of_trust'] # write an audit logfile to task_log_dir; copy cot into # artifact_dir/cot ? status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval'])
async def run_tasks(context, creds_key="credentials"): """Run any tasks returned by claimWork. Returns the integer status of the task that was run, or None if no task was run. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status None: if no task run. """ loop = asyncio.get_event_loop() tasks = await claim_work(context) status = None if not tasks or not tasks.get('tasks', []): await asyncio.sleep(context.config['poll_interval']) return status # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. for task_defn in tasks.get('tasks', []): status = 0 prepare_to_run_task(context, task_defn) loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) return status
async def do_run_task(context, run_cancellable, to_cancellable_process): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. run_cancellable (typing.Callable): wraps future such that it'll cancel upon worker shutdown to_cancellable_process (typing.Callable): wraps ``TaskProcess`` such that it will stop if the worker is shutting down Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config["verify_chain_of_trust"]: chain = ChainOfTrust(context, context.config["cot_job_type"]) await run_cancellable(verify_chain_of_trust(chain)) status = await run_task(context, to_cancellable_process) generate_cot(context) except asyncio.CancelledError: log.info("CoT cancelled asynchronously") raise WorkerShutdownDuringTask except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) status = STATUSES["internal-error"] return status
async def do_run_task(context, run_cancellable, to_cancellable_process): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. run_cancellable (typing.Callable): wraps future such that it'll cancel upon worker shutdown to_cancellable_process (typing.Callable): wraps ``TaskProcess`` such that it will stop if the worker is shutting down Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await run_cancellable(verify_chain_of_trust(chain)) status = await run_task(context, to_cancellable_process) generate_cot(context) except asyncio.CancelledError: log.info("CoT cancelled asynchronously") raise WorkerShutdownDuringTask except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) raise return status
async def do_run_task(context): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) raise return status
async def invoke(self, context): """Claims and processes Taskcluster work. Args: context (scriptworker.context.Context): context of worker Returns: status code of build """ try: # Note: claim_work(...) might not be safely interruptible! See # https://bugzilla.mozilla.org/show_bug.cgi?id=1524069 tasks = await self._run_cancellable(claim_work(context)) if not tasks or not tasks.get("tasks", []): await self._run_cancellable( asyncio.sleep(context.config["poll_interval"])) return None # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. status = None for task_defn in tasks.get("tasks", []): prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task( reclaim_task(context, context.task)) try: status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process) artifacts_paths = filepaths_in_dir( context.config["artifact_dir"]) except WorkerShutdownDuringTask: shutdown_artifact_paths = [ os.path.join("public", "logs", log_file) for log_file in ["chain_of_trust.log", "live_backing.log"] ] artifacts_paths = [ path for path in shutdown_artifact_paths if os.path.isfile( os.path.join(context.config["artifact_dir"], path)) ] status = STATUSES["worker-shutdown"] status = worst_level(status, await do_upload(context, artifacts_paths)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status except asyncio.CancelledError: return None
async def invoke(self, context): """Claims and processes Taskcluster work. Args: context (scriptworker.context.Context): context of worker Returns: status code of build """ try: # Note: claim_work(...) might not be safely interruptible! See # https://bugzilla.mozilla.org/show_bug.cgi?id=1524069 tasks = await self._run_cancellable(claim_work(context)) if not tasks or not tasks.get('tasks', []): await self._run_cancellable(asyncio.sleep(context.config['poll_interval'])) return None # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. status = None for task_defn in tasks.get('tasks', []): prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task)) try: status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process) artifacts_paths = filepaths_in_dir(context.config['artifact_dir']) except WorkerShutdownDuringTask: shutdown_artifact_paths = [os.path.join('public', 'logs', log_file) for log_file in ['chain_of_trust.log', 'live_backing.log']] artifacts_paths = [path for path in shutdown_artifact_paths if os.path.isfile(os.path.join(context.config['artifact_dir'], path))] status = STATUSES['worker-shutdown'] status = worst_level(status, await do_upload(context, artifacts_paths)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status except asyncio.CancelledError: return None
async def run_tasks(context, creds_key="credentials"): """Run any tasks returned by claimWork. Returns the integer status of the task that was run, or None if no task was run. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Raises: Exception: on unexpected exception. Returns: int: exit status None: if no task run. """ tasks = await claim_work(context) status = None if not tasks or not tasks.get('tasks', []): await asyncio.sleep(context.config['poll_interval']) return status # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. for task_defn in tasks.get('tasks', []): status = 0 prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task)) status = await do_run_task(context) status = worst_level(status, await do_upload(context)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status
def test_worst_level(one, two, expected): assert task.worst_level(one, two) == expected
def test_worst_level(one, two, expected): assert swtask.worst_level(one, two) == expected