def main(event_loop=None):
"""Scriptworker entry point: get everything set up, then enter the main loop.
Args:
event_loop (asyncio.BaseEventLoop, optional): the event loop to use.
If None, use ``asyncio.get_event_loop()``. Defaults to None.
"""
context, credentials = get_context_from_cmdln(sys.argv[1:])
log.info("Scriptworker starting up at {} UTC".format(
arrow.utcnow().format()))
cleanup(context)
context.event_loop = event_loop or asyncio.get_event_loop()
done = False
async def _handle_sigterm():
log.info("SIGTERM received; shutting down")
nonlocal done
done = True
if context.running_tasks is not None:
await context.running_tasks.cancel()
context.event_loop.add_signal_handler(
signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))
while not done:
try:
context.event_loop.run_until_complete(
async_main(context, credentials))
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
def main(event_loop=None):
"""Scriptworker entry point: get everything set up, then enter the main loop.
Args:
event_loop (asyncio.BaseEventLoop, optional): the event loop to use.
If None, use ``asyncio.get_event_loop()``. Defaults to None.
"""
context, credentials = get_context_from_cmdln(sys.argv[1:])
log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format()))
cleanup(context)
context.event_loop = event_loop or asyncio.get_event_loop()
done = False
async def _handle_sigterm():
log.info("SIGTERM received; shutting down")
nonlocal done
done = True
if context.running_tasks is not None:
await context.running_tasks.cancel()
context.event_loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))
while not done:
try:
context.event_loop.run_until_complete(async_main(context, credentials))
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context,
context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow(
).timestamp - context.credentials_timestamp > context.config[
'credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
def main():
"""Scriptworker entry point: get everything set up, then enter the main loop
"""
context = Context()
kwargs = {}
if len(sys.argv) > 1:
if len(sys.argv) > 2:
print("Usage: {} [configfile]".format(sys.argv[0]),
file=sys.stderr)
sys.exit(1)
kwargs['path'] = sys.argv[1]
context.config, credentials = create_config(**kwargs)
update_logging_config(context)
cleanup(context)
conn = aiohttp.TCPConnector(limit=context.config["max_connections"])
loop = asyncio.get_event_loop()
with aiohttp.ClientSession(connector=conn) as session:
context.session = session
context.credentials = credentials
while True:
try:
loop.create_task(async_main(context))
loop.run_forever()
except RuntimeError:
pass
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
def main(event_loop=None):
"""Scriptworker entry point: get everything set up, then enter the main loop.
Args:
event_loop (asyncio.BaseEventLoop, optional): the event loop to use.
If None, use ``asyncio.get_event_loop()``. Defaults to None.
"""
context, credentials = get_context_from_cmdln(sys.argv[1:])
log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format()))
cleanup(context)
context.event_loop = event_loop or asyncio.get_event_loop()
done = False
def _handle_sigterm(signum, frame):
nonlocal done
log.info("SIGTERM received; shutting down after next task")
done = True
signal.signal(signal.SIGTERM, _handle_sigterm)
while not done:
try:
context.event_loop.run_until_complete(async_main(context, credentials))
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
def get_context(config_override):
context = Context()
context.config, credentials = build_config(config_override)
swlog.update_logging_config(context)
utils.cleanup(context)
with aiohttp.ClientSession() as session:
context.session = session
context.credentials = credentials
yield context
def get_context(config_override):
context = Context()
context.config, credentials = build_config(config_override)
swlog.update_logging_config(context)
utils.cleanup(context)
with aiohttp.ClientSession() as session:
context.session = session
context.credentials = credentials
yield context
async def get_context(config_override=None):
context = Context()
with tempfile.TemporaryDirectory() as tmp:
context.config, credentials = build_config(config_override, basedir=tmp)
swlog.update_logging_config(context)
utils.cleanup(context)
async with aiohttp.ClientSession() as session:
context.session = session
context.credentials = credentials
yield context
def get_context(config_override):
context = Context()
with tempfile.TemporaryDirectory() as tmp:
context.config, credentials = build_config(config_override, basedir=tmp)
swlog.update_logging_config(context)
utils.cleanup(context)
with aiohttp.ClientSession() as session:
context.session = session
context.credentials = credentials
yield context
def test_cleanup(context):
for name in 'work_dir', 'artifact_dir', 'task_log_dir':
path = context.config[name]
os.makedirs(path)
open(os.path.join(path, 'tempfile'), "w").close()
assert os.path.exists(os.path.join(path, "tempfile"))
utils.cleanup(context)
for name in 'work_dir', 'artifact_dir':
path = context.config[name]
assert os.path.exists(path)
assert not os.path.exists(os.path.join(path, "tempfile"))
def test_cleanup(self, context):
for name in 'work_dir', 'artifact_dir':
path = context.config[name]
os.makedirs(path)
open(os.path.join(path, 'tempfile'), "w").close()
assert os.path.exists(os.path.join(path, "tempfile"))
utils.cleanup(context)
for name in 'work_dir', 'artifact_dir':
path = context.config[name]
assert os.path.exists(path)
assert not os.path.exists(os.path.join(path, "tempfile"))
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
status = 0
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context, context.task))
try:
# TODO download and verify chain of trust artifacts if
# context.config['verify_chain_of_trust']
# write an audit logfile to task_log_dir; copy cot into
# artifact_dir/cot ?
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
await complete_task(context, status)
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
async def get_context(config_override=None):
context = Context()
with tempfile.TemporaryDirectory() as tmp:
context.config = await build_config(config_override, basedir=tmp)
credentials = read_integration_creds()
swlog.update_logging_config(context)
utils.cleanup(context)
async with aiohttp.ClientSession() as session:
context.session = session
context.credentials = credentials
yield context
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context,
context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
status = 0
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context, context.task))
try:
if context.config['verify_chain_of_trust']:
chain = ChainOfTrust(context,
context.config['cot_job_type'])
await verify_chain_of_trust(chain)
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
except aiohttp.ClientError as e:
status = worst_level(status, STATUSES['intermittent-task'])
log.error("Hit aiohttp error: {}".format(e))
await complete_task(context, status)
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
async def invoke(self, context):
"""Claims and processes Taskcluster work.
Args:
context (scriptworker.context.Context): context of worker
Returns: status code of build
"""
try:
# Note: claim_work(...) might not be safely interruptible! See
# https://bugzilla.mozilla.org/show_bug.cgi?id=1524069
tasks = await self._run_cancellable(claim_work(context))
if not tasks or not tasks.get("tasks", []):
await self._run_cancellable(
asyncio.sleep(context.config["poll_interval"]))
return None
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
status = None
for task_defn in tasks.get("tasks", []):
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(
reclaim_task(context, context.task))
try:
status = await do_run_task(context, self._run_cancellable,
self._to_cancellable_process)
artifacts_paths = filepaths_in_dir(
context.config["artifact_dir"])
except WorkerShutdownDuringTask:
shutdown_artifact_paths = [
os.path.join("public", "logs", log_file) for log_file
in ["chain_of_trust.log", "live_backing.log"]
]
artifacts_paths = [
path for path in shutdown_artifact_paths
if os.path.isfile(
os.path.join(context.config["artifact_dir"], path))
]
status = STATUSES["worker-shutdown"]
status = worst_level(status, await
do_upload(context, artifacts_paths))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
except asyncio.CancelledError:
return None
def test_cleanup(rw_context):
for name in "work_dir", "artifact_dir", "task_log_dir":
path = rw_context.config[name]
open(os.path.join(path, "tempfile"), "w").close()
assert os.path.exists(os.path.join(path, "tempfile"))
utils.cleanup(rw_context)
for name in "work_dir", "artifact_dir":
path = rw_context.config[name]
assert os.path.exists(path)
assert not os.path.exists(os.path.join(path, "tempfile"))
# 2nd pass
utils.rm(rw_context.config["work_dir"])
utils.cleanup(rw_context)
def test_cleanup(context):
for name in 'work_dir', 'artifact_dir', 'task_log_dir':
path = context.config[name]
open(os.path.join(path, 'tempfile'), "w").close()
assert os.path.exists(os.path.join(path, "tempfile"))
utils.cleanup(context)
for name in 'work_dir', 'artifact_dir':
path = context.config[name]
assert os.path.exists(path)
assert not os.path.exists(os.path.join(path, "tempfile"))
# 2nd pass
utils.rm(context.config['work_dir'])
utils.cleanup(context)
async def run_tasks(context, creds_key="credentials"):
"""Run any tasks returned by claimWork.
Returns the integer status of the task that was run, or None if no task was
run.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
None: if no task run.
"""
loop = asyncio.get_event_loop()
tasks = await claim_work(context)
status = None
if not tasks or not tasks.get('tasks', []):
await asyncio.sleep(context.config['poll_interval'])
return status
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
for task_defn in tasks.get('tasks', []):
status = 0
prepare_to_run_task(context, task_defn)
loop.create_task(reclaim_task(context, context.task))
try:
if context.config['verify_chain_of_trust']:
chain = ChainOfTrust(context, context.config['cot_job_type'])
await verify_chain_of_trust(chain)
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(e))
except aiohttp.ClientError as e:
status = worst_level(status, STATUSES['intermittent-task'])
log.error("Hit aiohttp error: {}".format(e))
await complete_task(context, status)
cleanup(context)
return status
def main(event_loop=None):
"""Scriptworker entry point: get everything set up, then enter the main loop.
Args:
event_loop (asyncio.BaseEventLoop, optional): the event loop to use.
If None, use ``asyncio.get_event_loop()``. Defaults to None.
"""
context, credentials = get_context_from_cmdln(sys.argv[1:])
log.info("Scriptworker starting up at {} UTC".format(
arrow.utcnow().format()))
log.info("Worker FQDN: {}".format(socket.getfqdn()))
log_worker_metric(context, "instanceBoot", timestamp=uptime.boottime())
cleanup(context)
context.event_loop = event_loop or asyncio.get_event_loop()
done = False
async def _handle_sigterm():
log.info("SIGTERM received; shutting down")
nonlocal done
done = True
if context.running_tasks is not None:
await context.running_tasks.cancel()
async def _handle_sigusr1():
"""Stop accepting new tasks."""
log.info("SIGUSR1 received; no more tasks will be taken")
nonlocal done
done = True
context.event_loop.add_signal_handler(
signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm()))
context.event_loop.add_signal_handler(
signal.SIGUSR1, lambda: asyncio.ensure_future(_handle_sigusr1()))
log_worker_metric(context, "workerReady")
while not done:
try:
context.event_loop.run_until_complete(
async_main(context, credentials))
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
else:
log.info("Scriptworker stopped at {} UTC".format(
arrow.utcnow().format()))
log.info("Worker FQDN: {}".format(socket.getfqdn()))
def main():
"""Scriptworker entry point: get everything set up, then enter the main loop."""
context, credentials = get_context_from_cmdln(sys.argv[1:])
log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format()))
cleanup(context)
conn = aiohttp.TCPConnector(limit=context.config['aiohttp_max_connections'])
loop = asyncio.get_event_loop()
with aiohttp.ClientSession(connector=conn) as session:
context.session = session
context.credentials = credentials
while True:
try:
loop.run_until_complete(async_main(context))
except Exception:
log.critical("Fatal exception", exc_info=1)
raise
def main():
"""Scriptworker entry point: get everything set up, then enter the main loop
"""
context, credentials = get_context_from_cmdln(sys.argv[1:])
cleanup(context)
conn = aiohttp.TCPConnector(limit=context.config["max_connections"])
loop = asyncio.get_event_loop()
with aiohttp.ClientSession(connector=conn) as session:
context.session = session
context.credentials = credentials
while True:
try:
loop.create_task(async_main(context))
loop.run_forever()
except RuntimeError:
pass
async def invoke(self, context):
"""Claims and processes Taskcluster work.
Args:
context (scriptworker.context.Context): context of worker
Returns: status code of build
"""
try:
# Note: claim_work(...) might not be safely interruptible! See
# https://bugzilla.mozilla.org/show_bug.cgi?id=1524069
tasks = await self._run_cancellable(claim_work(context))
if not tasks or not tasks.get('tasks', []):
await self._run_cancellable(asyncio.sleep(context.config['poll_interval']))
return None
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
status = None
for task_defn in tasks.get('tasks', []):
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task))
try:
status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process)
artifacts_paths = filepaths_in_dir(context.config['artifact_dir'])
except WorkerShutdownDuringTask:
shutdown_artifact_paths = [os.path.join('public', 'logs', log_file)
for log_file in ['chain_of_trust.log', 'live_backing.log']]
artifacts_paths = [path for path in shutdown_artifact_paths
if os.path.isfile(os.path.join(context.config['artifact_dir'], path))]
status = STATUSES['worker-shutdown']
status = worst_level(status, await do_upload(context, artifacts_paths))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
except asyncio.CancelledError:
return None
async def run_tasks(context, creds_key="credentials"):
"""Run any tasks returned by claimWork.
Returns the integer status of the task that was run, or None if no task was
run.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Raises:
Exception: on unexpected exception.
Returns:
int: exit status
None: if no task run.
"""
tasks = await claim_work(context)
status = None
if not tasks or not tasks.get('tasks', []):
await asyncio.sleep(context.config['poll_interval'])
return status
# Assume only a single task, but should more than one fall through,
# run them sequentially. A side effect is our return status will
# be the status of the final task run.
for task_defn in tasks.get('tasks', []):
status = 0
prepare_to_run_task(context, task_defn)
reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task))
status = await do_run_task(context)
status = worst_level(status, await do_upload(context))
await complete_task(context, status)
reclaim_fut.cancel()
cleanup(context)
return status
def main():
"""Scriptworker entry point: get everything set up, then enter the main loop
"""
context = Context()
kwargs = {}
if len(sys.argv) > 1:
if len(sys.argv) > 2:
print("Usage: {} [configfile]".format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
kwargs['path'] = sys.argv[1]
context.config, credentials = create_config(**kwargs)
update_logging_config(context)
cleanup(context)
conn = aiohttp.TCPConnector(limit=context.config["max_connections"])
loop = asyncio.get_event_loop()
with aiohttp.ClientSession(connector=conn) as session:
context.session = session
context.credentials = credentials
while True:
try:
loop.create_task(async_main(context))
loop.run_forever()
except RuntimeError:
pass
