def build_worker_pool(workerPoolId, cfg, secret_values):
try:
wp = TYPES[cfg["type"]](secret_values=secret_values, **cfg)
except Exception as e:
raise RuntimeError(
"Error generating worker pool configuration for {}".format(
workerPoolId)) from e
if secret_values:
if "secret" in wp:
secret = Secret(name="worker-pool:{}".format(workerPoolId),
secret=wp.pop("secret"))
else:
secret = None
else:
secret = Secret(name="worker-pool:{}".format(workerPoolId))
workerpool = WorkerPool(
workerPoolId=workerPoolId,
description=cfg.get("description", ""),
owner=cfg.get("owner", "*****@*****.**"),
emailOnError=cfg.get("emailOnError", False),
**wp,
)
return workerpool, secret
async def build_worker_pool(workerPoolId, cfg, secret_values):
try:
image_set = await get_image_set(cfg["imageset"])
wp = CLOUD_FUNCS[cfg["cloud"]](
secret_values=secret_values,
image_set=image_set,
**cfg,
)
if wp.supports_worker_config():
wp.merge_worker_config(
# The order is important here: earlier entries take precendence
# over later entries.
cfg.get("workerConfig", {}),
image_set.workerConfig,
WorkerPoolSettings.EXISTING_CONFIG,
)
wp = WORKER_IMPLEMENTATION_FUNCS[
image_set.workerImplementation.replace("-", "_")](
secret_values=secret_values,
wp=wp,
**cfg,
)
except Exception as e:
raise RuntimeError(
"Error generating worker pool configuration for {}".format(
workerPoolId)) from e
if wp.secret_tpl:
if secret_values:
secret = Secret(
name="worker-pool:{}".format(workerPoolId),
secret=secret_values.render(wp.secret_tpl),
)
else:
secret = Secret(name="worker-pool:{}".format(workerPoolId))
else:
secret = None
if wp.scopes:
role = Role(
roleId="worker-pool:{}".format(workerPoolId),
description="Scopes for image set `{}` and cloud `{}`.".format(
image_set.name, cfg["cloud"]),
scopes=wp.scopes,
)
else:
role = None
workerpool = WorkerPool(
workerPoolId=workerPoolId,
description=cfg.get("description", ""),
owner=cfg.get("owner", "*****@*****.**"),
emailOnError=cfg.get("emailOnError", False),
providerId=wp.provider_id,
config=wp.config,
)
return workerpool, secret, role
def test_secret_formatter_different_secrets():
"Secrets are properly formatted with a string"
secret1 = Secret("my-secret1", {"password": "******"})
secret2 = Secret("my-secret2", {"password": "******"})
# even though these have the same value, they should format their last lines differently
fmt1 = str(secret1)
fmt2 = str(secret2)
assert fmt1.split("\n")[-1] != fmt2.split("\n")[-1]
def test_secret_to_api():
"Secrets are properly output to the API, including the actual secret value"
secret = Secret("my-secret", {"password": "******"})
assert secret.to_api() == {
"expires": FOREVER,
"secret": {
"password": "******"
}
}
async def test_fetch_secrets_with_secrets(Secrets):
"When there are secrets and --with-secrets, we get names and values"
with test_options(with_secrets=True):
Secrets.secrets.append({"name": "secret1", "secret": "AA"})
Secrets.secrets.append({"name": "secret2", "secret": "BB"})
resources = Resources([], [".*"])
await fetch_secrets(resources)
assert list(sorted(resources)) == [
Secret(name="secret1", secret="AA"),
Secret(name="secret2", secret="BB"),
]
async def test_fetch_secrets_without_secrets(Secrets):
"When there are secrets but --without-secrets, we just get names"
with test_options(with_secrets=False):
Secrets.secrets.append({"name": "secret1"})
Secrets.secrets.append({"name": "secret2"})
resources = Resources([], [".*"])
await fetch_secrets(resources)
assert list(sorted(resources)) == [
Secret(name="secret1"),
Secret(name="secret2"),
]
async def test_fetch_secrets_managed(Secrets):
"Only managed secrets are returned"
with test_options(with_secrets=True):
Secrets.secrets.append({"name": "secret1", "secret": "AA"})
Secrets.secrets.append({"name": "unmanaged-secret2"})
resources = Resources([], ["Secret=secret"])
await fetch_secrets(resources)
assert list(sorted(resources)) == [Secret(name="secret1", secret="AA")]
def test_secret_formatter_no_secret():
"Secrets with no secret are properly formatted with a string"
secret = Secret("my-secret")
# check that the formatter is stable
assert str(secret) == str(secret)
assert str(secret) == textwrap.dedent("""\
Secret=my-secret:
name: my-secret
secret: <unknown>""")
def test_secret_formatter():
"Secrets are properly formatted with a string"
secret = Secret("my-secret", {"password": "******"})
# check that the formatter is stable
assert str(secret) == str(secret)
assert re.sub(r"secret: [a-z0-9]+", "secret: <hash>",
str(secret)) == textwrap.dedent("""\
Secret=my-secret:
name: my-secret
secret: <hash>""")
def test_secret_from_api():
"Secrets are properly read from a Taskcluster API result"
api_result = {
"expires": datetime.datetime(3000, 1, 1, 0, 0, 0),
"secret": {
"password": "******"
},
}
secret = Secret.from_api("my-secret", api_result)
assert secret.name == "my-secret"
assert secret.secret == {"password": "******"}
async def test_registry_run():
reg = CallbacksRegistry()
# Simple case to trigger on a secret creation
func = AsyncMock()
reg.add("before_apply", func)
await reg.run("before_apply", "create", Secret("xxx"))
func.assert_called_once()
# Should not run on resource mismatch
func = AsyncMock()
reg.add("after_apply", func, resources=[Role, Hook])
await reg.run("before_apply", "delete", Secret("xxx"))
await reg.run("after_apply", "delete", Secret("xxx"))
assert not func.called
# Should not run on action mismatch
func = AsyncMock()
reg.add("after_apply", func, actions=["delete"])
await reg.run("before_apply", "update", Secret("xxx"))
await reg.run("after_apply", "update", Secret("xxx"))
assert not func.called
def build_worker_pool(workerPoolId, cfg, secret_values, image_set):
try:
wp = CLOUD_FUNCS[cfg["cloud"]](
secret_values=secret_values,
image_set=image_set,
**cfg,
)
wp = WORKER_IMPLEMENTATION_FUNCS[
image_set.workerImplementation.replace("-", "_")](
secret_values=secret_values,
image_set=image_set,
wp=wp,
**cfg,
)
except Exception as e:
raise RuntimeError(
"Error generating worker pool configuration for {}".format(
workerPoolId)) from e
if secret_values:
if "secret" in wp:
secret = Secret(name="worker-pool:{}".format(workerPoolId),
secret=wp.pop("secret"))
else:
secret = None
else:
secret = Secret(name="worker-pool:{}".format(workerPoolId))
workerpool = WorkerPool(
workerPoolId=workerPoolId,
description=cfg.get("description", ""),
owner=cfg.get("owner", "*****@*****.**"),
emailOnError=cfg.get("emailOnError", False),
**wp,
)
return workerpool, secret
def test_secret_to_api_no_secret():
"A secret with no value is not output to the API"
secret = Secret("my-secret")
with pytest.raises(ValueError):
secret.to_api()
def test_secret_from_api_no_secret():
"Secrets are properly read from a Taskcluster API result"
secret = Secret.from_api("my-secret")
assert secret.name == "my-secret"
assert secret.secret == NoSecret
def test_secret_json():
"Secrets are properly output as JSON, including the description preamble and sorted scopes"
secret = Secret("my-secret", {"password": "******"})
assert secret.to_json() == {"name": "my-secret", "kind": "Secret"}
async def update_resources(resources, secret_values):
projects = await Projects.load(loader)
for project in projects.values():
for roleId in project.adminRoles:
assert any(roleId.startswith(p) for p in ADMIN_ROLE_PREFIXES)
resources.add(
Role(
roleId=roleId,
description="",
scopes=["assume:project-admin:{}".format(project.name)],
))
if project.repos:
for repo in project.repos:
assert repo.endswith("/*") or repo.endswith(
":*"
), "project.repos should end with `/*` or `:*`, got {}".format(
repo)
resources.add(
Role(
roleId="project-admin:{}".format(project.name),
description="",
scopes=[
"assume:repo-admin:{}".format(repo)
for repo in project.repos
],
))
if project.workerPools:
for name, worker_pool in project.workerPools.items():
worker_pool_id = "proj-{}/{}".format(project.name, name)
worker_pool["description"] = "Workers for " + project.name
worker_pool, secret = build_worker_pool(
worker_pool_id, worker_pool, secret_values)
if project.externallyManaged.manage_individual_resources():
resources.manage("WorkerPool={}".format(worker_pool_id))
if secret:
resources.manage(
"Secret=worker-pool:{}".format(worker_pool_id))
resources.add(worker_pool)
if secret:
resources.add(secret)
if project.clients:
for name, info in project.clients.items():
clientId = "project/{}/{}".format(project.name, name)
if project.externallyManaged.manage_individual_resources():
resources.manage("Client={}".format(clientId))
description = info.get("description", "")
scopes = info["scopes"]
resources.add(
Client(clientId=clientId,
description=description,
scopes=scopes))
if project.secrets:
for nameSuffix, info in project.secrets.items():
if info is True:
continue
name = "project/{}/{}".format(project.name, nameSuffix)
if project.externallyManaged.manage_individual_resources():
resources.manage("Secret={}".format(name))
if secret_values:
resources.add(
Secret(name=name, secret=secret_values.render(info)))
else:
resources.add(Secret(name=name))
if project.hooks:
for hookId, info in project.hooks.items():
hookGroupId = "project-{}".format(project.name)
if project.externallyManaged.manage_individual_resources():
resources.manage("Hook={}/{}".format(hookGroupid, hookId))
assert (
"bindings" not in info
), "Please add support for bindings to use this feature"
resources.add(
Hook(
hookGroupId=hookGroupId,
hookId=hookId,
name=info.get("name", hookId),
description=info.get("description", ""),
owner=info["owner"],
emailOnError=info.get("emailOnError", False),
schedule=info.get("schedule", ()),
bindings=info.get("bindings", ()),
task=info["task"],
triggerSchema=info.get("triggerSchema", {}),
))
for grant in Grants.from_project(project):
if project.externallyManaged.manage_individual_resources():
for role in grant.to:
resources.manage("Role=" + re.escape(role))
grant.update_resources(resources)