def decrypt_secret_environment_variables(
secret_provider_name,
environment,
soa_dir,
service_name,
cluster_name,
secret_provider_kwargs,
):
secret_environment = {}
secret_env_vars = {k: v for k, v in environment.items() if is_secret_ref(v)}
if secret_env_vars:
secret_provider = get_secret_provider(
secret_provider_name=secret_provider_name,
soa_dir=soa_dir,
service_name=service_name,
cluster_names=[cluster_name],
secret_provider_kwargs=secret_provider_kwargs,
)
try:
secret_environment = secret_provider.decrypt_environment(
secret_env_vars,
)
except Exception as e:
paasta_print(f"Failed to retrieve secrets with {e.__class__.__name__}: {e}")
paasta_print("If you don't need the secrets for local-run, you can add --skip-secrets")
sys.exit(1)
return secret_environment
def check_secrets_for_instance(instance_config_dict, soa_dir, service_path,
vault_env):
return_value = True
for env_value in instance_config_dict.get("env", {}).values():
if is_secret_ref(env_value):
secret_name = get_secret_name_from_ref(env_value)
if is_shared_secret(env_value):
secret_file_name = f"{soa_dir}/_shared/secrets/{secret_name}.json"
else:
secret_file_name = f"{service_path}/secrets/{secret_name}.json"
if os.path.isfile(secret_file_name):
secret_json = get_config_file_dict(secret_file_name)
if "ciphertext" not in secret_json["environments"].get(
vault_env, {}):
print(
failure(
f"Secret {secret_name} not defined for ecosystem {vault_env} on secret file {secret_file_name}",
"",
))
return_value = False
else:
print(
failure(f"Secret file {secret_file_name} not defined", ""))
return_value = False
return return_value
def get_secret_env(self) -> Mapping[str, dict]:
base_env = self.config_dict.get("env", {})
secret_env = {}
for k, v in base_env.items():
if is_secret_ref(v):
secret = get_secret_name_from_ref(v)
sanitised_secret = sanitise_kubernetes_name(secret)
service = (
self.service if not is_shared_secret(v) else SHARED_SECRET_SERVICE
)
sanitised_service = sanitise_kubernetes_name(service)
secret_env[k] = {
"secret_name": f"tron-secret-{sanitised_service}-{sanitised_secret}",
"key": secret,
}
return secret_env
def test_is_secret_ref():
assert is_secret_ref("SECRET(aaa-bbb-222_111)")
assert not is_secret_ref("SECRET(#!$)")
# herein is a lesson on how tests are hard:
assert not is_secret_ref("anything_else")
assert not is_secret_ref("")
# this is just incase a non string leaks in somewhere
# if it is not a string it can't be a secret ref
# so this checks that we are catching the TypeError
assert not is_secret_ref(None)
assert not is_secret_ref(3)
def decrypt_secret_environment_variables(
secret_provider_name,
environment,
soa_dir,
service_name,
cluster_name,
secret_provider_kwargs,
):
decrypted_secrets = {}
service_secret_env = {}
shared_secret_env = {}
for k, v in environment.items():
if is_secret_ref(v):
if is_shared_secret(v):
shared_secret_env[k] = v
else:
service_secret_env[k] = v
provider_args = {
"secret_provider_name": secret_provider_name,
"soa_dir": soa_dir,
"cluster_name": cluster_name,
"secret_provider_kwargs": secret_provider_kwargs,
}
secret_provider_kwargs["vault_num_uses"] = len(service_secret_env) + len(
shared_secret_env
)
try:
decrypted_secrets.update(
decrypt_secret_environment_for_service(
service_secret_env, service_name, **provider_args
)
)
decrypted_secrets.update(
decrypt_secret_environment_for_service(
shared_secret_env, SHARED_SECRET_SERVICE, **provider_args
)
)
except Exception as e:
paasta_print(f"Failed to retrieve secrets with {e.__class__.__name__}: {e}")
paasta_print(
"If you don't need the secrets for local-run, you can add --skip-secrets"
)
sys.exit(1)
return decrypted_secrets
def decrypt_secret_environment_variables(
secret_provider_name,
environment,
soa_dir,
service_name,
cluster_name,
secret_provider_kwargs,
):
secret_environment = {}
secret_env_vars = {k: v for k, v in environment.items() if is_secret_ref(v)}
if secret_env_vars:
secret_provider = get_secret_provider(
secret_provider_name=secret_provider_name,
soa_dir=soa_dir,
service_name=service_name,
cluster_names=[cluster_name],
secret_provider_kwargs=secret_provider_kwargs,
)
secret_environment = secret_provider.decrypt_environment(
secret_env_vars,
)
return secret_environment
def test_is_secret_ref():
assert is_secret_ref('SECRET(aaa-bbb-222_111)')
assert not is_secret_ref('SECRET(#!$)')
# herein is a lesson on how tests are hard:
assert not is_secret_ref('anything_else')
def test_is_secret_ref_shared():
assert is_secret_ref("SHARED_SECRET(foo)")
def format_tron_action_dict(action_config: TronActionConfig, use_k8s: bool = False):
"""Generate a dict of tronfig for an action, from the TronActionConfig.
:param job_config: TronActionConfig
"""
executor = action_config.get_executor()
result = {
"command": action_config.get_cmd(),
"executor": executor,
"requires": action_config.get_requires(),
"node": action_config.get_node(),
"retries": action_config.get_retries(),
"retries_delay": action_config.get_retries_delay(),
"expected_runtime": action_config.get_expected_runtime(),
"trigger_downstreams": action_config.get_trigger_downstreams(),
"triggered_by": action_config.get_triggered_by(),
"on_upstream_rerun": action_config.get_on_upstream_rerun(),
"trigger_timeout": action_config.get_trigger_timeout(),
}
# while we're tranisitioning, we want to be able to cleanly fallback to Mesos
# so we'll default to Mesos unless k8s usage is enabled for both the cluster
# and job.
# there are slight differences between k8s and Mesos configs, so we'll translate
# whatever is in soaconfigs to the k8s equivalent here as well.
if executor in KUBERNETES_EXECUTOR_NAMES and use_k8s:
# we'd like Tron to be able to distinguish between spark and normal actions
# even though they both run on k8s
result["executor"] = EXECUTOR_NAME_TO_TRON_EXECUTOR_TYPE.get(
executor, "kubernetes"
)
result["secret_env"] = action_config.get_secret_env()
result["field_selector_env"] = action_config.get_field_selector_env()
all_env = action_config.get_env()
# For k8s, we do not want secret envvars to be duplicated in both `env` and `secret_env`
# or for field selector env vars to be overwritten
result["env"] = {
k: v
for k, v in all_env.items()
if not is_secret_ref(v) and k not in result["field_selector_env"]
}
# for Tron-on-K8s, we want to ship tronjob output through logspout
# such that this output eventually makes it into our per-instance
# log streams automatically
# however, we're missing infrastructure in the superregion where we run spark jobs (and
# some normal tron jobs), so we take a slightly different approach here
if _use_suffixed_log_streams_k8s():
result["env"]["STREAM_SUFFIX_LOGSPOUT"] = (
"spark" if executor == "spark" else "tron"
)
else:
result["env"]["ENABLE_PER_INSTANCE_LOGSPOUT"] = "1"
result["node_selectors"] = action_config.get_node_selectors()
result["node_affinities"] = action_config.get_node_affinities()
# XXX: once we're off mesos we can make get_cap_* return just the cap names as a list
result["cap_add"] = [cap["value"] for cap in action_config.get_cap_add()]
result["cap_drop"] = [cap["value"] for cap in action_config.get_cap_drop()]
result["labels"] = {
"paasta.yelp.com/cluster": action_config.get_cluster(),
"paasta.yelp.com/pool": action_config.get_pool(),
"paasta.yelp.com/service": action_config.get_service(),
"paasta.yelp.com/instance": limit_size_with_hash(
action_config.get_instance(),
limit=63,
suffix=4,
),
}
# we can hardcode this for now as batches really shouldn't
# need routable IPs and we know that Spark probably does.
result["annotations"] = {
"paasta.yelp.com/routable_ip": "true" if executor == "spark" else "false",
}
if action_config.get_team() is not None:
result["labels"]["yelp.com/owner"] = action_config.get_team()
# create_or_find_service_account_name requires k8s credentials, and we don't
# have those available for CI to use (nor do we check these for normal PaaSTA
# services, so we're not doing anything "new" by skipping this)
if (
action_config.get_iam_role_provider() == "aws"
and action_config.get_iam_role()
and not action_config.for_validation
):
# this service account will be used for normal Tron batches as well as for Spark executors
result["service_account_name"] = create_or_find_service_account_name(
iam_role=action_config.get_iam_role(),
namespace=EXECUTOR_TYPE_TO_NAMESPACE[executor],
k8s_role=None,
dry_run=action_config.for_validation,
)
if executor == "spark":
# this service account will only be used by Spark drivers since executors don't
# need Kubernetes access permissions
result["service_account_name"] = create_or_find_service_account_name(
iam_role=action_config.get_iam_role(),
namespace=EXECUTOR_TYPE_TO_NAMESPACE[executor],
k8s_role=_spark_k8s_role(),
dry_run=action_config.for_validation,
)
# spark, unlike normal batches, needs to expose several ports for things like the spark
# ui and for executor->driver communication
result["ports"] = list(
set(
_get_spark_ports(
system_paasta_config=load_system_paasta_config()
).values()
)
)
elif executor in MESOS_EXECUTOR_NAMES:
result["executor"] = "mesos"
constraint_labels = ["attribute", "operator", "value"]
result["constraints"] = [
dict(zip(constraint_labels, constraint))
for constraint in action_config.get_calculated_constraints()
]
result["docker_parameters"] = [
{"key": param["key"], "value": param["value"]}
for param in action_config.format_docker_parameters()
]
result["env"] = action_config.get_env()
# the following config is only valid for k8s/Mesos since we're not running SSH actions
# in a containerized fashion
if executor in (KUBERNETES_EXECUTOR_NAMES + MESOS_EXECUTOR_NAMES):
result["cpus"] = action_config.get_cpus()
result["mem"] = action_config.get_mem()
result["disk"] = action_config.get_disk()
result["extra_volumes"] = format_volumes(action_config.get_extra_volumes())
result["docker_image"] = action_config.get_docker_url()
# Only pass non-None values, so Tron will use defaults for others
return {key: val for key, val in result.items() if val is not None}
