def main(event_loop=None): """Scriptworker entry point: get everything set up, then enter the main loop. Args: event_loop (asyncio.BaseEventLoop, optional): the event loop to use. If None, use ``asyncio.get_event_loop()``. Defaults to None. """ context, credentials = get_context_from_cmdln(sys.argv[1:]) log.info("Scriptworker starting up at {} UTC".format( arrow.utcnow().format())) cleanup(context) context.event_loop = event_loop or asyncio.get_event_loop() done = False async def _handle_sigterm(): log.info("SIGTERM received; shutting down") nonlocal done done = True if context.running_tasks is not None: await context.running_tasks.cancel() context.event_loop.add_signal_handler( signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm())) while not done: try: context.event_loop.run_until_complete( async_main(context, credentials)) except Exception: log.critical("Fatal exception", exc_info=1) raise
def main(event_loop=None): """Scriptworker entry point: get everything set up, then enter the main loop. Args: event_loop (asyncio.BaseEventLoop, optional): the event loop to use. If None, use ``asyncio.get_event_loop()``. Defaults to None. """ context, credentials = get_context_from_cmdln(sys.argv[1:]) log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format())) cleanup(context) context.event_loop = event_loop or asyncio.get_event_loop() done = False async def _handle_sigterm(): log.info("SIGTERM received; shutting down") nonlocal done done = True if context.running_tasks is not None: await context.running_tasks.cancel() context.event_loop.add_signal_handler(signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm())) while not done: try: context.event_loop.run_until_complete(async_main(context, credentials)) except Exception: log.critical("Fatal exception", exc_info=1) raise
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") context.claim_task = claim_task_defn loop.create_task(reclaim_task(context)) running_task = loop.create_task(run_task(context)) status = await running_task await upload_artifacts(context) await complete_task(context, running_task.result()) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow( ).timestamp - context.credentials_timestamp > context.config[ 'credential_update_interval']: credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
def main(): """Scriptworker entry point: get everything set up, then enter the main loop """ context = Context() kwargs = {} if len(sys.argv) > 1: if len(sys.argv) > 2: print("Usage: {} [configfile]".format(sys.argv[0]), file=sys.stderr) sys.exit(1) kwargs['path'] = sys.argv[1] context.config, credentials = create_config(**kwargs) update_logging_config(context) cleanup(context) conn = aiohttp.TCPConnector(limit=context.config["max_connections"]) loop = asyncio.get_event_loop() with aiohttp.ClientSession(connector=conn) as session: context.session = session context.credentials = credentials while True: try: loop.create_task(async_main(context)) loop.run_forever() except RuntimeError: pass
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") context.claim_task = claim_task_defn loop.create_task(reclaim_task(context)) running_task = loop.create_task(run_task(context)) status = await running_task await upload_artifacts(context) await complete_task(context, running_task.result()) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
def main(event_loop=None): """Scriptworker entry point: get everything set up, then enter the main loop. Args: event_loop (asyncio.BaseEventLoop, optional): the event loop to use. If None, use ``asyncio.get_event_loop()``. Defaults to None. """ context, credentials = get_context_from_cmdln(sys.argv[1:]) log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format())) cleanup(context) context.event_loop = event_loop or asyncio.get_event_loop() done = False def _handle_sigterm(signum, frame): nonlocal done log.info("SIGTERM received; shutting down after next task") done = True signal.signal(signal.SIGTERM, _handle_sigterm) while not done: try: context.event_loop.run_until_complete(async_main(context, credentials)) except Exception: log.critical("Fatal exception", exc_info=1) raise
def get_context(config_override): context = Context() context.config, credentials = build_config(config_override) swlog.update_logging_config(context) utils.cleanup(context) with aiohttp.ClientSession() as session: context.session = session context.credentials = credentials yield context
async def get_context(config_override=None): context = Context() with tempfile.TemporaryDirectory() as tmp: context.config, credentials = build_config(config_override, basedir=tmp) swlog.update_logging_config(context) utils.cleanup(context) async with aiohttp.ClientSession() as session: context.session = session context.credentials = credentials yield context
def get_context(config_override): context = Context() with tempfile.TemporaryDirectory() as tmp: context.config, credentials = build_config(config_override, basedir=tmp) swlog.update_logging_config(context) utils.cleanup(context) with aiohttp.ClientSession() as session: context.session = session context.credentials = credentials yield context
def test_cleanup(context): for name in 'work_dir', 'artifact_dir', 'task_log_dir': path = context.config[name] os.makedirs(path) open(os.path.join(path, 'tempfile'), "w").close() assert os.path.exists(os.path.join(path, "tempfile")) utils.cleanup(context) for name in 'work_dir', 'artifact_dir': path = context.config[name] assert os.path.exists(path) assert not os.path.exists(os.path.join(path, "tempfile"))
def test_cleanup(self, context): for name in 'work_dir', 'artifact_dir': path = context.config[name] os.makedirs(path) open(os.path.join(path, 'tempfile'), "w").close() assert os.path.exists(os.path.join(path, "tempfile")) utils.cleanup(context) for name in 'work_dir', 'artifact_dir': path = context.config[name] assert os.path.exists(path) assert not os.path.exists(os.path.join(path, "tempfile"))
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: # TODO download and verify chain of trust artifacts if # context.config['verify_chain_of_trust'] # write an audit logfile to task_log_dir; copy cot into # artifact_dir/cot ? status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
async def get_context(config_override=None): context = Context() with tempfile.TemporaryDirectory() as tmp: context.config = await build_config(config_override, basedir=tmp) credentials = read_integration_creds() swlog.update_logging_config(context) utils.cleanup(context) async with aiohttp.ClientSession() as session: context.session = session context.credentials = credentials yield context
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval'])
async def invoke(self, context): """Claims and processes Taskcluster work. Args: context (scriptworker.context.Context): context of worker Returns: status code of build """ try: # Note: claim_work(...) might not be safely interruptible! See # https://bugzilla.mozilla.org/show_bug.cgi?id=1524069 tasks = await self._run_cancellable(claim_work(context)) if not tasks or not tasks.get("tasks", []): await self._run_cancellable( asyncio.sleep(context.config["poll_interval"])) return None # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. status = None for task_defn in tasks.get("tasks", []): prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task( reclaim_task(context, context.task)) try: status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process) artifacts_paths = filepaths_in_dir( context.config["artifact_dir"]) except WorkerShutdownDuringTask: shutdown_artifact_paths = [ os.path.join("public", "logs", log_file) for log_file in ["chain_of_trust.log", "live_backing.log"] ] artifacts_paths = [ path for path in shutdown_artifact_paths if os.path.isfile( os.path.join(context.config["artifact_dir"], path)) ] status = STATUSES["worker-shutdown"] status = worst_level(status, await do_upload(context, artifacts_paths)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status except asyncio.CancelledError: return None
def test_cleanup(rw_context): for name in "work_dir", "artifact_dir", "task_log_dir": path = rw_context.config[name] open(os.path.join(path, "tempfile"), "w").close() assert os.path.exists(os.path.join(path, "tempfile")) utils.cleanup(rw_context) for name in "work_dir", "artifact_dir": path = rw_context.config[name] assert os.path.exists(path) assert not os.path.exists(os.path.join(path, "tempfile")) # 2nd pass utils.rm(rw_context.config["work_dir"]) utils.cleanup(rw_context)
def test_cleanup(context): for name in 'work_dir', 'artifact_dir', 'task_log_dir': path = context.config[name] open(os.path.join(path, 'tempfile'), "w").close() assert os.path.exists(os.path.join(path, "tempfile")) utils.cleanup(context) for name in 'work_dir', 'artifact_dir': path = context.config[name] assert os.path.exists(path) assert not os.path.exists(os.path.join(path, "tempfile")) # 2nd pass utils.rm(context.config['work_dir']) utils.cleanup(context)
async def run_tasks(context, creds_key="credentials"): """Run any tasks returned by claimWork. Returns the integer status of the task that was run, or None if no task was run. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status None: if no task run. """ loop = asyncio.get_event_loop() tasks = await claim_work(context) status = None if not tasks or not tasks.get('tasks', []): await asyncio.sleep(context.config['poll_interval']) return status # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. for task_defn in tasks.get('tasks', []): status = 0 prepare_to_run_task(context, task_defn) loop.create_task(reclaim_task(context, context.task)) try: if context.config['verify_chain_of_trust']: chain = ChainOfTrust(context, context.config['cot_job_type']) await verify_chain_of_trust(chain) status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(e)) except aiohttp.ClientError as e: status = worst_level(status, STATUSES['intermittent-task']) log.error("Hit aiohttp error: {}".format(e)) await complete_task(context, status) cleanup(context) return status
def main(event_loop=None): """Scriptworker entry point: get everything set up, then enter the main loop. Args: event_loop (asyncio.BaseEventLoop, optional): the event loop to use. If None, use ``asyncio.get_event_loop()``. Defaults to None. """ context, credentials = get_context_from_cmdln(sys.argv[1:]) log.info("Scriptworker starting up at {} UTC".format( arrow.utcnow().format())) log.info("Worker FQDN: {}".format(socket.getfqdn())) log_worker_metric(context, "instanceBoot", timestamp=uptime.boottime()) cleanup(context) context.event_loop = event_loop or asyncio.get_event_loop() done = False async def _handle_sigterm(): log.info("SIGTERM received; shutting down") nonlocal done done = True if context.running_tasks is not None: await context.running_tasks.cancel() async def _handle_sigusr1(): """Stop accepting new tasks.""" log.info("SIGUSR1 received; no more tasks will be taken") nonlocal done done = True context.event_loop.add_signal_handler( signal.SIGTERM, lambda: asyncio.ensure_future(_handle_sigterm())) context.event_loop.add_signal_handler( signal.SIGUSR1, lambda: asyncio.ensure_future(_handle_sigusr1())) log_worker_metric(context, "workerReady") while not done: try: context.event_loop.run_until_complete( async_main(context, credentials)) except Exception: log.critical("Fatal exception", exc_info=1) raise else: log.info("Scriptworker stopped at {} UTC".format( arrow.utcnow().format())) log.info("Worker FQDN: {}".format(socket.getfqdn()))
def main(): """Scriptworker entry point: get everything set up, then enter the main loop.""" context, credentials = get_context_from_cmdln(sys.argv[1:]) log.info("Scriptworker starting up at {} UTC".format(arrow.utcnow().format())) cleanup(context) conn = aiohttp.TCPConnector(limit=context.config['aiohttp_max_connections']) loop = asyncio.get_event_loop() with aiohttp.ClientSession(connector=conn) as session: context.session = session context.credentials = credentials while True: try: loop.run_until_complete(async_main(context)) except Exception: log.critical("Fatal exception", exc_info=1) raise
def main(): """Scriptworker entry point: get everything set up, then enter the main loop """ context, credentials = get_context_from_cmdln(sys.argv[1:]) cleanup(context) conn = aiohttp.TCPConnector(limit=context.config["max_connections"]) loop = asyncio.get_event_loop() with aiohttp.ClientSession(connector=conn) as session: context.session = session context.credentials = credentials while True: try: loop.create_task(async_main(context)) loop.run_forever() except RuntimeError: pass
async def invoke(self, context): """Claims and processes Taskcluster work. Args: context (scriptworker.context.Context): context of worker Returns: status code of build """ try: # Note: claim_work(...) might not be safely interruptible! See # https://bugzilla.mozilla.org/show_bug.cgi?id=1524069 tasks = await self._run_cancellable(claim_work(context)) if not tasks or not tasks.get('tasks', []): await self._run_cancellable(asyncio.sleep(context.config['poll_interval'])) return None # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. status = None for task_defn in tasks.get('tasks', []): prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task)) try: status = await do_run_task(context, self._run_cancellable, self._to_cancellable_process) artifacts_paths = filepaths_in_dir(context.config['artifact_dir']) except WorkerShutdownDuringTask: shutdown_artifact_paths = [os.path.join('public', 'logs', log_file) for log_file in ['chain_of_trust.log', 'live_backing.log']] artifacts_paths = [path for path in shutdown_artifact_paths if os.path.isfile(os.path.join(context.config['artifact_dir'], path))] status = STATUSES['worker-shutdown'] status = worst_level(status, await do_upload(context, artifacts_paths)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status except asyncio.CancelledError: return None
async def run_tasks(context, creds_key="credentials"): """Run any tasks returned by claimWork. Returns the integer status of the task that was run, or None if no task was run. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Raises: Exception: on unexpected exception. Returns: int: exit status None: if no task run. """ tasks = await claim_work(context) status = None if not tasks or not tasks.get('tasks', []): await asyncio.sleep(context.config['poll_interval']) return status # Assume only a single task, but should more than one fall through, # run them sequentially. A side effect is our return status will # be the status of the final task run. for task_defn in tasks.get('tasks', []): status = 0 prepare_to_run_task(context, task_defn) reclaim_fut = context.event_loop.create_task(reclaim_task(context, context.task)) status = await do_run_task(context) status = worst_level(status, await do_upload(context)) await complete_task(context, status) reclaim_fut.cancel() cleanup(context) return status