def get_env(self): env = super().get_env() if self.get_executor() == "spark": env["EXECUTOR_CLUSTER"] = self.get_spark_paasta_cluster() env["EXECUTOR_POOL"] = self.get_spark_paasta_pool() # Run spark (and mesos framework) as root. env["SPARK_USER"] = "******" env["SPARK_OPTS"] = stringify_spark_env( self.get_spark_config_dict()) env.update(get_mesos_spark_auth_env()) env["CLUSTERMAN_RESOURCES"] = json.dumps( dict( get_spark_resource_requirements( spark_config_dict=self.get_spark_config_dict(), webui_url=get_webui_url(self.spark_ui_port), ).values())) if "AWS_ACCESS_KEY_ID" not in env or "AWS_SECRET_ACCESS_KEY" not in env: try: access_key, secret_key = get_aws_credentials( service=self.get_service(), aws_credentials_yaml=self.config_dict.get( "aws_credentials_yaml"), ) env["AWS_ACCESS_KEY_ID"] = access_key env["AWS_SECRET_ACCESS_KEY"] = secret_key except Exception: log.warning( f"Cannot set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment " f"variables for tron action {self.get_instance()} of service " f"{self.get_service()} via credentail file. Traceback:\n" f"{traceback.format_exc()}") if "AWS_DEFAULT_REGION" not in env: env["AWS_DEFAULT_REGION"] = DEFAULT_AWS_REGION return env
def get_env(self): env = super().get_env() spark_env = {} if self.get_executor() == "spark": spark_env = get_mesos_spark_env( spark_app_name= "tron_spark_{self.get_service()}_{self.get_instance()}", spark_ui_port=pick_random_port( f"{self.get_service()}{self.get_instance()}".encode()), mesos_leader=find_mesos_leader(self.get_cluster()), mesos_secret=load_mesos_secret_for_spark(), paasta_cluster=self.get_cluster(), paasta_pool=self.get_pool(), paasta_service=self.get_service(), paasta_instance=self.get_instance(), docker_img=self.get_docker_url(), volumes=format_volumes( self.get_volumes( load_system_paasta_config().get_volumes())), user_spark_opts=self.config_dict.get("spark_args"), event_log_dir=get_default_event_log_dir( service=self.get_service(), aws_credentials_yaml=self.config_dict.get( "aws_credentials"), ), ) env["SPARK_OPTS"] = stringify_spark_env(spark_env) return env
def get_cmd(self): command = self.config_dict.get("command") if self.get_executor() == "spark": # Spark expects to be able to write to MESOS_SANDBOX if it is set # but the default value (/mnt/mesos/sandbox) doesn't get mounted in # our Docker containers, so we unset it here. (Un-setting is fine, # since Spark will just write to /tmp instead). command = "unset MESOS_DIRECTORY MESOS_SANDBOX; " + inject_spark_conf_str( command, stringify_spark_env(self.get_spark_config_dict())) return command
def get_env(self): env = super().get_env() if self.get_executor() == "spark": spark_config_dict = self.get_spark_config_dict() env["EXECUTOR_CLUSTER"] = self.get_spark_paasta_cluster() env["EXECUTOR_POOL"] = self.get_spark_paasta_pool() env["SPARK_OPTS"] = stringify_spark_env(spark_config_dict) # The actual mesos secret will be decrypted and injected on mesos master when assigning # tasks. env["SPARK_MESOS_SECRET"] = "SHARED_SECRET(SPARK_MESOS_SECRET)" if clusterman_metrics: env["CLUSTERMAN_RESOURCES"] = json.dumps( generate_clusterman_metrics_entries( clusterman_metrics, get_resources_requested(spark_config_dict), spark_config_dict["spark.app.name"], get_webui_url(spark_config_dict["spark.ui.port"]), )) else: env["CLUSTERMAN_RESOURCES"] = "{}" if "AWS_ACCESS_KEY_ID" not in env or "AWS_SECRET_ACCESS_KEY" not in env: try: access_key, secret_key, session_token = get_aws_credentials( service=self.get_service(), aws_credentials_yaml=self.config_dict.get( "aws_credentials_yaml"), ) env["AWS_ACCESS_KEY_ID"] = access_key env["AWS_SECRET_ACCESS_KEY"] = secret_key except Exception: log.warning( f"Cannot set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment " f"variables for tron action {self.get_instance()} of service " f"{self.get_service()} via credentail file. Traceback:\n" f"{traceback.format_exc()}") if "AWS_DEFAULT_REGION" not in env: env["AWS_DEFAULT_REGION"] = DEFAULT_AWS_REGION return env
def test_stringify_spark_env(): conf = {'spark.mesos.leader': '1234', 'spark.mesos.principal': 'spark'} assert spark_config.stringify_spark_env(conf) == ( '--conf spark.mesos.leader=1234 --conf spark.mesos.principal=spark')