def test_reclaim_task_non_409(context, successful_queue, event_loop):
successful_queue.status = 500
context.temp_queue = successful_queue
with pytest.raises(taskcluster.exceptions.TaskclusterRestFailure):
event_loop.run_until_complete(
task.reclaim_task(context, context.task)
)
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context,
context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow(
).timestamp - context.credentials_timestamp > context.config[
'credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
status = 0
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context, context.task))
try:
# TODO download and verify chain of trust artifacts if
# context.config['verify_chain_of_trust']
# write an audit logfile to task_log_dir; copy cot into
# artifact_dir/cot ?
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
await complete_task(context, status)
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context,
context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
status = 0
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context, context.task))
try:
if context.config['verify_chain_of_trust']:
chain = ChainOfTrust(context,
context.config['cot_job_type'])
await verify_chain_of_trust(chain)
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
except aiohttp.ClientError as e:
status = worst_level(status, STATUSES['intermittent-task'])
log.error("Hit aiohttp error: {}".format(e))
await complete_task(context, status)
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
async def invoke(self, context):
"""Claims and processes Taskcluster work.
Args:
context (scriptworker.context.Context): context of worker
Returns: status code of build
"""
try:
# Note: claim_work(...) might not be safely interruptible! See
# https://bugzilla.mozilla.org/show_bug.cgi?id=1524069
tasks = await self._run_cancellable(claim_work(context))
if not tasks or not tasks.get("tasks", []):
await self._run_cancellable(
asyncio.sleep(context.config["poll_interval"]))
return None
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
status = None
for task_defn in tasks.get("tasks", []):
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(
reclaim_task(context, context.task))
try:
status = await do_run_task(context, self._run_cancellable,
self._to_cancellable_process)
artifacts_paths = filepaths_in_dir(
context.config["artifact_dir"])
except WorkerShutdownDuringTask:
shutdown_artifact_paths = [
os.path.join("public", "logs", log_file) for log_file
in ["chain_of_trust.log", "live_backing.log"]
]
artifacts_paths = [
path for path in shutdown_artifact_paths
if os.path.isfile(
os.path.join(context.config["artifact_dir"], path))
]
status = STATUSES["worker-shutdown"]
status = worst_level(status, await
do_upload(context, artifacts_paths))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
except asyncio.CancelledError:
return None
async def run_tasks(context, creds_key="credentials"):
"""Run any tasks returned by claimWork.
Returns the integer status of the task that was run, or None if no task was
run.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
None: if no task run.
"""
loop = asyncio.get_event_loop()
tasks = await claim_work(context)
status = None
if not tasks or not tasks.get('tasks', []):
await asyncio.sleep(context.config['poll_interval'])
return status
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
for task_defn in tasks.get('tasks', []):
status = 0
prepare_to_run_task(context, task_defn)
loop.create_task(reclaim_task(context, context.task))
try:
if context.config['verify_chain_of_trust']:
chain = ChainOfTrust(context, context.config['cot_job_type'])
await verify_chain_of_trust(chain)
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
except aiohttp.ClientError as e:
status = worst_level(status, STATUSES['intermittent-task'])
log.error("Hit aiohttp error: {}".format(e))
await complete_task(context, status)
cleanup(context)
return status
async def invoke(self, context):
"""Claims and processes Taskcluster work.
Args:
context (scriptworker.context.Context): context of worker
Returns: status code of build
"""
try:
# Note: claim_work(...) might not be safely interruptible! See
# https://bugzilla.mozilla.org/show_bug.cgi?id=1524069
tasks = await self._run_cancellable(claim_work(context))
if not tasks or not tasks.get('tasks', []):
await self._run_cancellable(asyncio.sleep(context.config['poll_interval']))
return None
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
status = None
for task_defn in tasks.get('tasks', []):
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task))
try:
status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process)
artifacts_paths = filepaths_in_dir(context.config['artifact_dir'])
except WorkerShutdownDuringTask:
shutdown_artifact_paths = [os.path.join('public', 'logs', log_file)
for log_file in ['chain_of_trust.log', 'live_backing.log']]
artifacts_paths = [path for path in shutdown_artifact_paths
if os.path.isfile(os.path.join(context.config['artifact_dir'], path))]
status = STATUSES['worker-shutdown']
status = worst_level(status, await do_upload(context, artifacts_paths))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
except asyncio.CancelledError:
return None
async def run_tasks(context, creds_key="credentials"):
"""Run any tasks returned by claimWork.
Returns the integer status of the task that was run, or None if no task was
run.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Raises:
Exception: on unexpected exception.
Returns:
int: exit status
None: if no task run.
"""
tasks = await claim_work(context)
status = None
if not tasks or not tasks.get('tasks', []):
await asyncio.sleep(context.config['poll_interval'])
return status
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
for task_defn in tasks.get('tasks', []):
status = 0
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task))
status = await do_run_task(context)
status = worst_level(status, await do_upload(context))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
def test_skip_reclaim_task(context, successful_queue, event_loop):
context.temp_queue = successful_queue
event_loop.run_until_complete(
task.reclaim_task(context, {"unrelated": "task"}))
def test_reclaim_task(context, successful_queue, event_loop):
context.temp_queue = successful_queue
event_loop.run_until_complete(task.reclaim_task(context, context.task))
