async def do_run_task(context, run_cancellable, to_cancellable_process): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. run_cancellable (typing.Callable): wraps future such that it'll cancel upon worker shutdown to_cancellable_process (typing.Callable): wraps ``TaskProcess`` such that it will stop if the worker is shutting down Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await run_cancellable(verify_chain_of_trust(chain)) status = await run_task(context, to_cancellable_process) generate_cot(context) except asyncio.CancelledError: log.info("CoT cancelled asynchronously") raise WorkerShutdownDuringTask except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) raise return status
async def do_run_task(context, run_cancellable, to_cancellable_process): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. run_cancellable (typing.Callable): wraps future such that it'll cancel upon worker shutdown to_cancellable_process (typing.Callable): wraps ``TaskProcess`` such that it will stop if the worker is shutting down Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config["verify_chain_of_trust"]: chain = ChainOfTrust(context, context.config["cot_job_type"]) await run_cancellable(verify_chain_of_trust(chain)) status = await run_task(context, to_cancellable_process) generate_cot(context) except asyncio.CancelledError: log.info("CoT cancelled asynchronously") raise WorkerShutdownDuringTask except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) status = STATUSES["internal-error"] return status
async def do_run_task(context): """Run the task logic. Returns the integer status of the task. args: context (scriptworker.context.Context): the scriptworker context. Raises: Exception: on unexpected exception. Returns: int: exit status """ status = 0 try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except Exception as e: log.exception("SCRIPTWORKER_UNEXPECTED_EXCEPTION task {}".format(e)) raise return status
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval'])
async def run_tasks(context, creds_key="credentials"): """Run any tasks returned by claimWork. Returns the integer status of the task that was run, or None if no task was run. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status None: if no task run. """ loop = asyncio.get_event_loop() tasks = await claim_work(context) status = None if not tasks or not tasks.get('tasks', []): await asyncio.sleep(context.config['poll_interval']) return status # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. for task_defn in tasks.get('tasks', []): status = 0 prepare_to_run_task(context, task_defn) loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) return status
def test_generate_cot(artifacts, context): path = os.path.join(context.config['work_dir'], "foo") signed_body = cot.generate_cot(context, path=path) with open(path, "r") as fh: assert fh.read() == signed_body body = sgpg.get_body(sgpg.GPG(context), signed_body) log.info(body) assert body.rstrip() == cot.format_json(cot.generate_cot_body(context))
def test_generate_cot_unsigned(artifacts, context): context.config['sign_chain_of_trust'] = False path = os.path.join(context.config['work_dir'], "foo") body = cot.generate_cot(context, path=path) assert body == cot.format_json(cot.generate_cot_body(context))
def test_generate_cot_exception(artifacts, context): context.config['cot_schema_path'] = os.path.join( context.config['work_dir'], "not_a_file") with pytest.raises(ScriptWorkerException): cot.generate_cot(context)
def test_generate_cot_unsigned(artifacts, context): context.config["sign_chain_of_trust"] = False body = cot.generate_cot(context, parent_path=context.config["work_dir"]) assert body == cot.format_json(cot.generate_cot_body(context))
def test_generate_cot(artifacts, context): path = os.path.join(context.config["work_dir"], "chain-of-trust.json") body = cot.generate_cot(context, parent_path=context.config["work_dir"]) with open(path, "r") as fh: assert fh.read() == body
def test_generate_cot_exception(artifacts, context): context.config['cot_schema_path'] = os.path.join(context.config['work_dir'], "not_a_file") with pytest.raises(ScriptWorkerException): cot.generate_cot(context)
def test_generate_cot_unsigned(artifacts, context): context.config['sign_chain_of_trust'] = False body = cot.generate_cot(context, parent_path=context.config['work_dir']) assert body == cot.format_json(cot.generate_cot_body(context))
def test_generate_cot(artifacts, context): path = os.path.join(context.config['work_dir'], "chain-of-trust.json") body = cot.generate_cot(context, parent_path=context.config['work_dir']) with open(path, "r") as fh: assert fh.read() == body