async def _get_github_token():
if os.environ.get("SCRIPTWORKER_GITHUB_TOKEN"):
return os.environ["SCRIPTWORKER_GITHUB_TOKEN"]
token = read_worker_creds(key="scriptworker_github_token")
if token:
return read_worker_creds(key="scriptworker_github_token")
try:
root_url = os.environ["TASKCLUSTER_PROXY_URL"]
except KeyError as e:
raise KeyError("You must provide `TASKCLUSTER_PROXY_URL` to run these tests") from e
secrets = Secrets({"rootUrl": root_url})
secret = await secrets.get("repo:github.com/mozilla-releng/scriptworker:github")
return secret["secret"]["token"]
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
def test_environ_creds(params, t_env):
t_env.update(params[0])
with mock.patch.object(config,
'CREDS_FILES',
new=['this_file_does_not_exist']):
with mock.patch.object(os, 'environ', new=t_env):
assert config.read_worker_creds() == params[1]
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context,
context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context))
running_task = loop.create_task(run_task(context))
status = await running_task
await upload_artifacts(context)
await complete_task(context, running_task.result())
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow(
).timestamp - context.credentials_timestamp > context.config[
'credential_update_interval']:
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
async def run_loop(context, creds_key="credentials"):
"""Split this out of the async_main while loop for easier testing.
args:
context (scriptworker.context.Context): the scriptworker context.
creds_key (str, optional): when reading the creds file, this dict key
corresponds to the credentials value we want to use. Defaults to
"credentials".
Returns:
int: status
"""
loop = asyncio.get_event_loop()
await update_poll_task_urls(
context, context.queue.pollTaskUrls,
args=(context.config['provisioner_id'], context.config['worker_type']),
)
for poll_url, delete_url in get_azure_urls(context):
try:
claim_task_defn = await find_task(context, poll_url, delete_url,
retry_request)
except ScriptWorkerException:
await asyncio.sleep(context.config['poll_interval'])
break
if claim_task_defn:
log.info("Going to run task!")
status = 0
context.claim_task = claim_task_defn
loop.create_task(reclaim_task(context, context.task))
try:
# TODO download and verify chain of trust artifacts if
# context.config['verify_chain_of_trust']
# write an audit logfile to task_log_dir; copy cot into
# artifact_dir/cot ?
status = await run_task(context)
generate_cot(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
try:
await upload_artifacts(context)
except ScriptWorkerException as e:
status = worst_level(status, e.exit_code)
log.error("Hit ScriptWorkerException: {}".format(str(e)))
await complete_task(context, status)
cleanup(context)
await asyncio.sleep(1)
return status
else:
await asyncio.sleep(context.config['poll_interval'])
if arrow.utcnow().timestamp - context.credentials_timestamp > context.config['credential_update_interval']: # pragma: no branch
credentials = read_worker_creds(key=creds_key)
if credentials and credentials != context.credentials:
context.credentials = credentials
def read_integration_creds():
creds = read_worker_creds(key="integration_credentials")
if creds:
return creds
raise Exception(
"""To run integration tests, put your worker-test clientId creds, in json format,
in one of these files:
{files}
with the format
{{"integration_credentials": {{"accessToken": "...", "clientId": "...", "certificate": "..."}}}}
(only specify "certificate" if using temporary credentials)
This clientId will need the scope assume:project:taskcluster:worker-test-scopes
To skip integration tests, set the environment variable NO_TESTS_OVER_WIRE""".
format(files=CREDS_FILES))
def read_integration_creds():
creds = read_worker_creds(key="integration_credentials")
if creds:
return creds
raise Exception(
"""To run integration tests, put your worker-test clientId creds, in json format,
in one of these files:
{files}
with the format
{{"integration_credentials": {{"accessToken": "...", "clientId": "...", "certificate": "..."}}}}
(only specify "certificate" if using temporary credentials)
This clientId will need the scope assume:project:taskcluster:worker-test-scopes
To skip integration tests, set the environment variable NO_TESTS_OVER_WIRE""".format(files=CREDS_FILES)
)
def test_good_worker_creds(mocker):
path = os.path.join(os.path.dirname(__file__), "data",
"client_credentials.json")
mocker.patch.object(config, 'CREDS_FILES', new=(path, ))
assert config.read_worker_creds()
def test_missing_creds(t_env):
with mock.patch.object(config,
'CREDS_FILES',
new=['this_file_does_not_exist']):
with mock.patch.object(os, 'environ', new=t_env):
assert config.read_worker_creds() is None
def test_missing_creds(t_env):
with mock.patch.object(config,
"CREDS_FILES",
new=["this_file_does_not_exist"]):
with mock.patch.object(os, "environ", new=t_env):
assert config.read_worker_creds() is None
def test_bad_worker_creds():
path = os.path.join(os.path.dirname(__file__), "data", "good.json")
with mock.patch.object(config, 'CREDS_FILES', new=(path, )):
assert config.read_worker_creds(key="nonexistent_key") is None
def test_bad_worker_creds():
path = os.path.join(os.path.dirname(__file__), "data", "good.json")
with mock.patch.object(config, 'CREDS_FILES', new=(path, )):
assert config.read_worker_creds(key="nonexistent_key") is None
def verify_cot_cmdln(args=None):
"""Test the chain of trust from the commandline, for debugging purposes.
Args:
args (list, optional): the commandline args to parse. If None, use
``sys.argv[1:]`` . Defaults to None.
"""
args = args or sys.argv[1:]
parser = argparse.ArgumentParser(
description="""Verify a given task's chain of trust.
Given a task's `task_id`, get its task definition, then trace its chain of
trust back to the tree. This doesn't verify chain of trust artifact signatures,
but does run the other tests in `scriptworker.cot.verify.verify_chain_of_trust`.
This is helpful in debugging chain of trust changes or issues.
To use, first either set your taskcluster creds in your env http://bit.ly/2eDMa6N
or in the CREDS_FILES http://bit.ly/2fVMu0A""")
parser.add_argument('task_id', help='the task id to test')
parser.add_argument('--task-type',
help='the task type to test',
choices=['signing', 'balrog', 'beetmover', 'pushapk'],
required=True)
parser.add_argument('--cleanup',
help='clean up the temp dir afterwards',
dest='cleanup',
action='store_true',
default=False)
opts = parser.parse_args(args)
tmp = tempfile.mkdtemp()
log = logging.getLogger('scriptworker')
log.setLevel(logging.DEBUG)
logging.basicConfig()
loop = asyncio.get_event_loop()
conn = aiohttp.TCPConnector()
try:
with aiohttp.ClientSession(connector=conn) as session:
context = Context()
context.session = session
context.credentials = read_worker_creds()
context.task = loop.run_until_complete(
context.queue.task(opts.task_id))
context.config = dict(deepcopy(DEFAULT_CONFIG))
context.config.update({
'work_dir':
os.path.join(tmp, 'work'),
'artifact_dir':
os.path.join(tmp, 'artifacts'),
'task_log_dir':
os.path.join(tmp, 'artifacts', 'public', 'logs'),
'base_gpg_home_dir':
os.path.join(tmp, 'gpg'),
'verify_cot_signature':
False,
})
cot = ChainOfTrust(context, opts.task_type, task_id=opts.task_id)
loop.run_until_complete(verify_chain_of_trust(cot))
log.info(pprint.pformat(cot.dependent_task_ids()))
log.info("Cot task_id: {}".format(cot.task_id))
for link in cot.links:
log.info("task_id: {}".format(link.task_id))
context.session.close()
context.queue.session.close()
loop.close()
finally:
if opts.cleanup:
rm(tmp)
else:
log.info("Artifacts are in {}".format(tmp))
def read_integration_creds():
return read_worker_creds(key="integration_credentials")
def test_good_worker_creds(mocker):
path = os.path.join(os.path.dirname(__file__), "data", "client_credentials.json")
mocker.patch.object(config, 'CREDS_FILES', new=(path, ))
assert config.read_worker_creds()
def test_missing_creds(t_env):
with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
with mock.patch.object(os, 'environ', new=t_env):
assert config.read_worker_creds() is None
def test_environ_creds(params, t_env):
t_env.update(params[0])
with mock.patch.object(config, 'CREDS_FILES', new=['this_file_does_not_exist']):
with mock.patch.object(os, 'environ', new=t_env):
assert config.read_worker_creds() == params[1]
