async def _get_github_token(): if os.environ.get("SCRIPTWORKER_GITHUB_TOKEN"): return os.environ["SCRIPTWORKER_GITHUB_TOKEN"] token = read_worker_creds(key="scriptworker_github_token") if token: return read_worker_creds(key="scriptworker_github_token") try: root_url = os.environ["TASKCLUSTER_PROXY_URL"] except KeyError as e: raise KeyError("You must provide `TASKCLUSTER_PROXY_URL` to run these tests") from e secrets = Secrets({"rootUrl": root_url}) secret = await secrets.get("repo:github.com/mozilla-releng/scriptworker:github") return secret["secret"]["token"]
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") context.claim_task = claim_task_defn loop.create_task(reclaim_task(context)) running_task = loop.create_task(run_task(context)) status = await running_task await upload_artifacts(context) await complete_task(context, running_task.result()) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
def test_environ_creds(params, t_env): t_env.update(params[0]) with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']): with mock.patch.object(os, 'environ', new=t_env): assert config.read_worker_creds() == params[1]
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") context.claim_task = claim_task_defn loop.create_task(reclaim_task(context)) running_task = loop.create_task(run_task(context)) status = await running_task await upload_artifacts(context) await complete_task(context, running_task.result()) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow( ).timestamp - context.credentials_timestamp > context.config[ 'credential_update_interval']: credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
async def run_loop(context, creds_key="credentials"): """Split this out of the async_main while loop for easier testing. args: context (scriptworker.context.Context): the scriptworker context. creds_key (str, optional): when reading the creds file, this dict key corresponds to the credentials value we want to use. Defaults to "credentials". Returns: int: status """ loop = asyncio.get_event_loop() await update_poll_task_urls( context, context.queue.pollTaskUrls, args=(context.config['provisioner_id'], context.config['worker_type']), ) for poll_url, delete_url in get_azure_urls(context): try: claim_task_defn = await find_task(context, poll_url, delete_url, retry_request) except ScriptWorkerException: await asyncio.sleep(context.config['poll_interval']) break if claim_task_defn: log.info("Going to run task!") status = 0 context.claim_task = claim_task_defn loop.create_task(reclaim_task(context, context.task)) try: # TODO download and verify chain of trust artifacts if # context.config['verify_chain_of_trust'] # write an audit logfile to task_log_dir; copy cot into # artifact_dir/cot ? status = await run_task(context) generate_cot(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) try: await upload_artifacts(context) except ScriptWorkerException as e: status = worst_level(status, e.exit_code) log.error("Hit ScriptWorkerException: {}".format(str(e))) await complete_task(context, status) cleanup(context) await asyncio.sleep(1) return status else: await asyncio.sleep(context.config['poll_interval']) if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch credentials = read_worker_creds(key=creds_key) if credentials and credentials != context.credentials: context.credentials = credentials
def read_integration_creds(): creds = read_worker_creds(key="integration_credentials") if creds: return creds raise Exception( """To run integration tests, put your worker-test clientId creds, in json format, in one of these files: {files} with the format {{"integration_credentials": {{"accessToken": "...", "clientId": "...", "certificate": "..."}}}} (only specify "certificate" if using temporary credentials) This clientId will need the scope assume:project:taskcluster:worker-test-scopes To skip integration tests, set the environment variable NO_TESTS_OVER_WIRE""". format(files=CREDS_FILES))
def read_integration_creds(): creds = read_worker_creds(key="integration_credentials") if creds: return creds raise Exception( """To run integration tests, put your worker-test clientId creds, in json format, in one of these files: {files} with the format {{"integration_credentials": {{"accessToken": "...", "clientId": "...", "certificate": "..."}}}} (only specify "certificate" if using temporary credentials) This clientId will need the scope assume:project:taskcluster:worker-test-scopes To skip integration tests, set the environment variable NO_TESTS_OVER_WIRE""".format(files=CREDS_FILES) )
def test_good_worker_creds(mocker): path = os.path.join(os.path.dirname(__file__), "data", "client_credentials.json") mocker.patch.object(config, 'CREDS_FILES', new=(path, )) assert config.read_worker_creds()
def test_missing_creds(t_env): with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']): with mock.patch.object(os, 'environ', new=t_env): assert config.read_worker_creds() is None
def test_missing_creds(t_env): with mock.patch.object(config, "CREDS_FILES", new=["this_file_does_not_exist"]): with mock.patch.object(os, "environ", new=t_env): assert config.read_worker_creds() is None
def test_bad_worker_creds(): path = os.path.join(os.path.dirname(__file__), "data", "good.json") with mock.patch.object(config, 'CREDS_FILES', new=(path, )): assert config.read_worker_creds(key="nonexistent_key") is None
def verify_cot_cmdln(args=None): """Test the chain of trust from the commandline, for debugging purposes. Args: args (list, optional): the commandline args to parse. If None, use ``sys.argv[1:]`` . Defaults to None. """ args = args or sys.argv[1:] parser = argparse.ArgumentParser( description="""Verify a given task's chain of trust. Given a task's `task_id`, get its task definition, then trace its chain of trust back to the tree. This doesn't verify chain of trust artifact signatures, but does run the other tests in `scriptworker.cot.verify.verify_chain_of_trust`. This is helpful in debugging chain of trust changes or issues. To use, first either set your taskcluster creds in your env http://bit.ly/2eDMa6N or in the CREDS_FILES http://bit.ly/2fVMu0A""") parser.add_argument('task_id', help='the task id to test') parser.add_argument('--task-type', help='the task type to test', choices=['signing', 'balrog', 'beetmover', 'pushapk'], required=True) parser.add_argument('--cleanup', help='clean up the temp dir afterwards', dest='cleanup', action='store_true', default=False) opts = parser.parse_args(args) tmp = tempfile.mkdtemp() log = logging.getLogger('scriptworker') log.setLevel(logging.DEBUG) logging.basicConfig() loop = asyncio.get_event_loop() conn = aiohttp.TCPConnector() try: with aiohttp.ClientSession(connector=conn) as session: context = Context() context.session = session context.credentials = read_worker_creds() context.task = loop.run_until_complete( context.queue.task(opts.task_id)) context.config = dict(deepcopy(DEFAULT_CONFIG)) context.config.update({ 'work_dir': os.path.join(tmp, 'work'), 'artifact_dir': os.path.join(tmp, 'artifacts'), 'task_log_dir': os.path.join(tmp, 'artifacts', 'public', 'logs'), 'base_gpg_home_dir': os.path.join(tmp, 'gpg'), 'verify_cot_signature': False, }) cot = ChainOfTrust(context, opts.task_type, task_id=opts.task_id) loop.run_until_complete(verify_chain_of_trust(cot)) log.info(pprint.pformat(cot.dependent_task_ids())) log.info("Cot task_id: {}".format(cot.task_id)) for link in cot.links: log.info("task_id: {}".format(link.task_id)) context.session.close() context.queue.session.close() loop.close() finally: if opts.cleanup: rm(tmp) else: log.info("Artifacts are in {}".format(tmp))
def read_integration_creds(): return read_worker_creds(key="integration_credentials")