def decrypt_secret_environment_variables( secret_provider_name, environment, soa_dir, service_name, cluster_name, secret_provider_kwargs, ): secret_environment = {} secret_env_vars = {k: v for k, v in environment.items() if is_secret_ref(v)} if secret_env_vars: secret_provider = get_secret_provider( secret_provider_name=secret_provider_name, soa_dir=soa_dir, service_name=service_name, cluster_names=[cluster_name], secret_provider_kwargs=secret_provider_kwargs, ) try: secret_environment = secret_provider.decrypt_environment( secret_env_vars, ) except Exception as e: paasta_print(f"Failed to retrieve secrets with {e.__class__.__name__}: {e}") paasta_print("If you don't need the secrets for local-run, you can add --skip-secrets") sys.exit(1) return secret_environment
def check_secrets_for_instance(instance_config_dict, soa_dir, service_path, vault_env): return_value = True for env_value in instance_config_dict.get("env", {}).values(): if is_secret_ref(env_value): secret_name = get_secret_name_from_ref(env_value) if is_shared_secret(env_value): secret_file_name = f"{soa_dir}/_shared/secrets/{secret_name}.json" else: secret_file_name = f"{service_path}/secrets/{secret_name}.json" if os.path.isfile(secret_file_name): secret_json = get_config_file_dict(secret_file_name) if "ciphertext" not in secret_json["environments"].get( vault_env, {}): print( failure( f"Secret {secret_name} not defined for ecosystem {vault_env} on secret file {secret_file_name}", "", )) return_value = False else: print( failure(f"Secret file {secret_file_name} not defined", "")) return_value = False return return_value
def get_secret_env(self) -> Mapping[str, dict]: base_env = self.config_dict.get("env", {}) secret_env = {} for k, v in base_env.items(): if is_secret_ref(v): secret = get_secret_name_from_ref(v) sanitised_secret = sanitise_kubernetes_name(secret) service = ( self.service if not is_shared_secret(v) else SHARED_SECRET_SERVICE ) sanitised_service = sanitise_kubernetes_name(service) secret_env[k] = { "secret_name": f"tron-secret-{sanitised_service}-{sanitised_secret}", "key": secret, } return secret_env
def test_is_secret_ref(): assert is_secret_ref("SECRET(aaa-bbb-222_111)") assert not is_secret_ref("SECRET(#!$)") # herein is a lesson on how tests are hard: assert not is_secret_ref("anything_else") assert not is_secret_ref("") # this is just incase a non string leaks in somewhere # if it is not a string it can't be a secret ref # so this checks that we are catching the TypeError assert not is_secret_ref(None) assert not is_secret_ref(3)
def decrypt_secret_environment_variables( secret_provider_name, environment, soa_dir, service_name, cluster_name, secret_provider_kwargs, ): decrypted_secrets = {} service_secret_env = {} shared_secret_env = {} for k, v in environment.items(): if is_secret_ref(v): if is_shared_secret(v): shared_secret_env[k] = v else: service_secret_env[k] = v provider_args = { "secret_provider_name": secret_provider_name, "soa_dir": soa_dir, "cluster_name": cluster_name, "secret_provider_kwargs": secret_provider_kwargs, } secret_provider_kwargs["vault_num_uses"] = len(service_secret_env) + len( shared_secret_env ) try: decrypted_secrets.update( decrypt_secret_environment_for_service( service_secret_env, service_name, **provider_args ) ) decrypted_secrets.update( decrypt_secret_environment_for_service( shared_secret_env, SHARED_SECRET_SERVICE, **provider_args ) ) except Exception as e: paasta_print(f"Failed to retrieve secrets with {e.__class__.__name__}: {e}") paasta_print( "If you don't need the secrets for local-run, you can add --skip-secrets" ) sys.exit(1) return decrypted_secrets
def decrypt_secret_environment_variables( secret_provider_name, environment, soa_dir, service_name, cluster_name, secret_provider_kwargs, ): secret_environment = {} secret_env_vars = {k: v for k, v in environment.items() if is_secret_ref(v)} if secret_env_vars: secret_provider = get_secret_provider( secret_provider_name=secret_provider_name, soa_dir=soa_dir, service_name=service_name, cluster_names=[cluster_name], secret_provider_kwargs=secret_provider_kwargs, ) secret_environment = secret_provider.decrypt_environment( secret_env_vars, ) return secret_environment
def test_is_secret_ref(): assert is_secret_ref('SECRET(aaa-bbb-222_111)') assert not is_secret_ref('SECRET(#!$)') # herein is a lesson on how tests are hard: assert not is_secret_ref('anything_else')
def test_is_secret_ref_shared(): assert is_secret_ref("SHARED_SECRET(foo)")
def format_tron_action_dict(action_config: TronActionConfig, use_k8s: bool = False): """Generate a dict of tronfig for an action, from the TronActionConfig. :param job_config: TronActionConfig """ executor = action_config.get_executor() result = { "command": action_config.get_cmd(), "executor": executor, "requires": action_config.get_requires(), "node": action_config.get_node(), "retries": action_config.get_retries(), "retries_delay": action_config.get_retries_delay(), "expected_runtime": action_config.get_expected_runtime(), "trigger_downstreams": action_config.get_trigger_downstreams(), "triggered_by": action_config.get_triggered_by(), "on_upstream_rerun": action_config.get_on_upstream_rerun(), "trigger_timeout": action_config.get_trigger_timeout(), } # while we're tranisitioning, we want to be able to cleanly fallback to Mesos # so we'll default to Mesos unless k8s usage is enabled for both the cluster # and job. # there are slight differences between k8s and Mesos configs, so we'll translate # whatever is in soaconfigs to the k8s equivalent here as well. if executor in KUBERNETES_EXECUTOR_NAMES and use_k8s: # we'd like Tron to be able to distinguish between spark and normal actions # even though they both run on k8s result["executor"] = EXECUTOR_NAME_TO_TRON_EXECUTOR_TYPE.get( executor, "kubernetes" ) result["secret_env"] = action_config.get_secret_env() result["field_selector_env"] = action_config.get_field_selector_env() all_env = action_config.get_env() # For k8s, we do not want secret envvars to be duplicated in both `env` and `secret_env` # or for field selector env vars to be overwritten result["env"] = { k: v for k, v in all_env.items() if not is_secret_ref(v) and k not in result["field_selector_env"] } # for Tron-on-K8s, we want to ship tronjob output through logspout # such that this output eventually makes it into our per-instance # log streams automatically # however, we're missing infrastructure in the superregion where we run spark jobs (and # some normal tron jobs), so we take a slightly different approach here if _use_suffixed_log_streams_k8s(): result["env"]["STREAM_SUFFIX_LOGSPOUT"] = ( "spark" if executor == "spark" else "tron" ) else: result["env"]["ENABLE_PER_INSTANCE_LOGSPOUT"] = "1" result["node_selectors"] = action_config.get_node_selectors() result["node_affinities"] = action_config.get_node_affinities() # XXX: once we're off mesos we can make get_cap_* return just the cap names as a list result["cap_add"] = [cap["value"] for cap in action_config.get_cap_add()] result["cap_drop"] = [cap["value"] for cap in action_config.get_cap_drop()] result["labels"] = { "paasta.yelp.com/cluster": action_config.get_cluster(), "paasta.yelp.com/pool": action_config.get_pool(), "paasta.yelp.com/service": action_config.get_service(), "paasta.yelp.com/instance": limit_size_with_hash( action_config.get_instance(), limit=63, suffix=4, ), } # we can hardcode this for now as batches really shouldn't # need routable IPs and we know that Spark probably does. result["annotations"] = { "paasta.yelp.com/routable_ip": "true" if executor == "spark" else "false", } if action_config.get_team() is not None: result["labels"]["yelp.com/owner"] = action_config.get_team() # create_or_find_service_account_name requires k8s credentials, and we don't # have those available for CI to use (nor do we check these for normal PaaSTA # services, so we're not doing anything "new" by skipping this) if ( action_config.get_iam_role_provider() == "aws" and action_config.get_iam_role() and not action_config.for_validation ): # this service account will be used for normal Tron batches as well as for Spark executors result["service_account_name"] = create_or_find_service_account_name( iam_role=action_config.get_iam_role(), namespace=EXECUTOR_TYPE_TO_NAMESPACE[executor], k8s_role=None, dry_run=action_config.for_validation, ) if executor == "spark": # this service account will only be used by Spark drivers since executors don't # need Kubernetes access permissions result["service_account_name"] = create_or_find_service_account_name( iam_role=action_config.get_iam_role(), namespace=EXECUTOR_TYPE_TO_NAMESPACE[executor], k8s_role=_spark_k8s_role(), dry_run=action_config.for_validation, ) # spark, unlike normal batches, needs to expose several ports for things like the spark # ui and for executor->driver communication result["ports"] = list( set( _get_spark_ports( system_paasta_config=load_system_paasta_config() ).values() ) ) elif executor in MESOS_EXECUTOR_NAMES: result["executor"] = "mesos" constraint_labels = ["attribute", "operator", "value"] result["constraints"] = [ dict(zip(constraint_labels, constraint)) for constraint in action_config.get_calculated_constraints() ] result["docker_parameters"] = [ {"key": param["key"], "value": param["value"]} for param in action_config.format_docker_parameters() ] result["env"] = action_config.get_env() # the following config is only valid for k8s/Mesos since we're not running SSH actions # in a containerized fashion if executor in (KUBERNETES_EXECUTOR_NAMES + MESOS_EXECUTOR_NAMES): result["cpus"] = action_config.get_cpus() result["mem"] = action_config.get_mem() result["disk"] = action_config.get_disk() result["extra_volumes"] = format_volumes(action_config.get_extra_volumes()) result["docker_image"] = action_config.get_docker_url() # Only pass non-None values, so Tron will use defaults for others return {key: val for key, val in result.items() if val is not None}