class MockRedis(): """ Constructor of the mock of a redis object Returns: MockRedis: The simulation of a redis object """ def __init__(self): self.map = {"job": []} self.logger = Log('redis_mock_log', 'redis_mock.log') """ Function the simulates the push of a job in the redis queue Args: metric_queue (string): Representing the metric queue metric (Object): Representing the metric to be pushed in the queue. Returns: None """ def rpush(self, metric_queue, metric): if self.map.get(metric_queue) is None: self.map[metric_queue] = [] self.map[metric_queue].append(metric) """ Function the simulates the pop of a job from the redis queue Args: metric_queue (string): Representing the metric queue Returns: Object: Representing the metric pop from the queue """ def rpop(self, metric_queue): try: return self.map.get(metric_queue).pop(0) except Exception as e: self.logger.log(e) """ Function the simulates the deletion of a redis queue Args: queue_name (string): Representing the name of the queue to be deleted. Returns: None """ def delete(self, queue_name): self.map.pop(queue_name)
class OpenStackSparkApplicationExecutor(GenericApplicationExecutor): def __init__(self, app_id): self.application_state = "None" self.state_lock = threading.RLock() self.application_time = -1 self.start_time = -1 self.app_id = app_id self._verify_existing_log_paths(app_id) self._clean_log_files(app_id) self.running_log = Log("Running_Application_%s" % app_id, "logs/apps/%s/execution" % app_id) self.stdout = Log("stdout_%s" % app_id, "logs/apps/%s/stdout" % app_id) self.stderr = Log("stderr_%s" % app_id, "logs/apps/%s/stderr" % app_id) def get_application_state(self): with self.state_lock: state = self.application_state return state def update_application_state(self, state): with self.state_lock: self.application_state = state def get_application_execution_time(self): return self.application_time def get_application_start_time(self): return self.start_time def start_application(self, data, spark_applications_ids, app_id): try: self.update_application_state("Running") # Broker Parameters cluster_id = None user = api.user password = api.password project_id = api.project_id auth_ip = api.auth_ip domain = api.domain public_key = api.public_key key_path = api.key_path log_path = api.log_path container = api.container hosts = api.hosts remote_hdfs = api.remote_hdfs swift_logdir = api.swift_logdir number_of_attempts = api.number_of_attempts dummy_opportunistic = api.dummy_opportunistic # User Request Parameters net_id = data['net_id'] master_ng = data['master_ng'] slave_ng = data['slave_ng'] op_slave_ng = data['opportunistic_slave_ng'] opportunism = str(data['opportunistic']) plugin = data['openstack_plugin'] percentage = int(data['percentage']) job_type = data['job_type'] version = data['version'] args = data['args'] main_class = data['main_class'] dependencies = data['dependencies'] job_template_name = data['job_template_name'] job_binary_name = data['job_binary_name'] job_binary_url = data['job_binary_url'] image_id = data['image_id'] monitor_plugin = data['monitor_plugin'] expected_time = data['expected_time'] collect_period = data['collect_period'] number_of_jobs = data['number_of_jobs'] image_id = data['image_id'] starting_cap = data['starting_cap'] # Optimizer Parameters app_name = data['app_name'] days = 0 if app_name.lower() == 'bulma': if 'days' in data.keys(): days = data['days'] else: self._log("""%s | 'days' parameter missing""" % (time.strftime("%H:%M:%S"))) raise ex.ConfigurationError() # Openstack Components connector = os_connector.OpenStackConnector(plugin_log) sahara = connector.get_sahara_client(user, password, project_id, auth_ip, domain) swift = connector.get_swift_client(user, password, project_id, auth_ip, domain) nova = connector.get_nova_client(user, password, project_id, auth_ip, domain) # Optimizer gets the vcpu size of flavor cores_per_slave = connector.get_vcpus_by_nodegroup( nova, sahara, slave_ng) cores, vms = optimizer.get_info(api.optimizer_url, expected_time, app_name, days) if cores <= 0: if 'cluster_size' in data.keys(): req_cluster_size = data['cluster_size'] else: self._log("""%s | 'cluster_size' parameter missing""" % (time.strftime("%H:%M:%S"))) raise ex.ConfigurationError() else: req_cluster_size = int( math.ceil(cores / float(cores_per_slave))) # Check Oportunism if opportunism == "True": self._log("""%s | Checking if opportunistic instances are available""" % (time.strftime("%H:%M:%S"))) pred_cluster_size = optimizer.get_cluster_size( api.optimizer_url, hosts, percentage, dummy_opportunistic) else: pred_cluster_size = req_cluster_size if pred_cluster_size > req_cluster_size: cluster_size = pred_cluster_size else: cluster_size = req_cluster_size self._log("%s | Cluster size: %s" % (time.strftime("%H:%M:%S"), str(cluster_size))) self._log("%s | Creating cluster..." % (time.strftime("%H:%M:%S"))) cluster_id = self._create_cluster(sahara, connector, req_cluster_size, pred_cluster_size, public_key, net_id, image_id, plugin, version, master_ng, slave_ng, op_slave_ng) self._log("%s | Cluster id: %s" % (time.strftime("%H:%M:%S"), cluster_id)) swift_path = self._is_swift_path(args) if cluster_id: master = connector.get_master_instance( sahara, cluster_id)['internal_ip'] self._log("%s | Master is %s" % (time.strftime("%H:%M:%S"), master)) workers = connector.get_worker_instances(sahara, cluster_id) workers_id = [] for worker in workers: workers_id.append(worker['instance_id']) self._log("%s | Configuring controller" % (time.strftime("%H:%M:%S"))) controller.setup_environment(api.controller_url, workers_id, starting_cap, data) if swift_path: job_status = self._swift_spark_execution( master, key_path, sahara, connector, job_binary_name, job_binary_url, user, password, job_template_name, job_type, plugin, cluster_size, args, main_class, cluster_id, spark_applications_ids, workers_id, app_id, expected_time, monitor_plugin, collect_period, number_of_jobs, log_path, swift, container, data, number_of_attempts) else: job_status = self._hdfs_spark_execution( master, remote_hdfs, key_path, args, job_binary_url, main_class, dependencies, spark_applications_ids, expected_time, monitor_plugin, collect_period, number_of_jobs, workers_id, data, connector, swift, swift_logdir, container, number_of_attempts) else: # FIXME: exception type self.update_application_state("Error") raise ex.ClusterNotCreatedException() # Delete cluster self._log("%s | Delete cluster: %s" % (time.strftime("%H:%M:%S"), cluster_id)) connector.delete_cluster(sahara, cluster_id) self._log("%s | Finished application execution" % (time.strftime("%H:%M:%S"))) return job_status except KeyError as ke: self._log("%s | Parameter missing in submission: %s, " "please check the config file" % (time.strftime("%H:%M:%S"), str(ke))) self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") except ex.ConfigurationError: self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") except SaharaAPIException: self._log("%s | There is not enough resource to create a cluster" % (time.strftime("%H:%M:%S"))) self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") except Exception: if cluster_id is not None: self._log("%s | Delete cluster: %s" % (time.strftime("%H:%M:%S"), cluster_id)) connector.delete_cluster(sahara, cluster_id) self._log("%s | Unknown error, please report to administrators " "of WP3 infrastructure" % (time.strftime("%H:%M:%S"))) self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") def get_application_time(self): return self.application_time def _get_job_binary_id(self, sahara, connector, job_binary_name, job_binary_url, user, password): extra = dict(user=user, password=password) job_binary_id = connector.get_job_binary(sahara, job_binary_url) if not job_binary_id: job_binary_id = connector.create_job_binary( sahara, job_binary_name, job_binary_url, extra) return job_binary_id def _get_job_template_id(self, sahara, connector, mains, job_template_name, job_type): job_template_id = connector.get_job_template(sahara, mains) if not job_template_id: job_template_id = connector.create_job_template( sahara, job_template_name, job_type, mains) return job_template_id def _wait_on_job_finish(self, sahara, connector, job_exec_id, spark_app_id): completed = failed = False start_time = datetime.datetime.now() self.start_time = time.mktime(start_time.timetuple()) while not (completed or failed): job_status = connector.get_job_status(sahara, job_exec_id) self._log("%s | Sahara current job status: %s" % (time.strftime("%H:%M:%S"), job_status)) if job_status == 'RUNNING': time.sleep(2) current_time = datetime.datetime.now() current_job_time = (current_time - start_time).total_seconds() if current_job_time > 3600: self._log("%s | Job execution killed due to inactivity" % time.strftime("%H:%M:%S")) job_status = 'TIMEOUT' completed = connector.is_job_completed(job_status) failed = connector.is_job_failed(job_status) end_time = datetime.datetime.now() total_time = end_time - start_time application_time_log.log( "%s|%.0f|%.0f" % (spark_app_id, float(time.mktime( start_time.timetuple())), float(total_time.total_seconds()))) self.application_time = total_time.total_seconds() self._log("%s | Sahara job took %s seconds to execute" % (time.strftime("%H:%M:%S"), str(total_time.total_seconds()))) return job_status def _create_cluster(self, sahara, connector, req_cluster_size, pred_cluster_size, public_key, net_id, image_id, plugin, version, master_ng, slave_ng, op_slave_ng): self._log('Creating cluster') try: cluster_id = connector.create_cluster(sahara, req_cluster_size, pred_cluster_size, public_key, net_id, image_id, plugin, version, master_ng, slave_ng, op_slave_ng) except SaharaAPIException: raise SaharaAPIException('Could not create clusters') return cluster_id def _is_swift_path(self, args): for arg in args: if arg.startswith('hdfs://') or arg.startswith('swift://'): if arg.startswith('swift://'): return True else: return False def _swift_spark_execution(self, master, key_path, sahara, connector, job_binary_name, job_binary_url, user, password, job_template_name, job_type, plugin, cluster_size, args, main_class, cluster_id, spark_applications_ids, workers_id, app_id, expected_time, monitor_plugin, collect_period, number_of_jobs, log_path, swift, container, data, number_of_attempts): # Preparing job job_binary_id = self._get_job_binary_id(sahara, connector, job_binary_name, job_binary_url, user, password) mains = [job_binary_id] job_template_id = self._get_job_template_id(sahara, connector, mains, job_template_name, job_type) self._log("%s | Starting job..." % (time.strftime("%H:%M:%S"))) # Running job # What is os_utils? # configs = os_utils.get_job_config(connector, plugin, # cluster_size, user, password, # args, main_class) configs = None job = connector.create_job_execution(sahara, job_template_id, cluster_id, configs=configs) self._log("%s | Created job" % (time.strftime("%H:%M:%S"))) spark_app_id = spark.get_running_app(master, spark_applications_ids, number_of_attempts) spark_applications_ids.append(spark_app_id) self._log("%s | Spark app id" % (time.strftime("%H:%M:%S"))) job_exec_id = job.id for worker_id in workers_id: instances_log.log("%s|%s" % (app_id, worker_id)) job_status = connector.get_job_status(sahara, job_exec_id) self._log("%s | Sahara job status: %s" % (time.strftime("%H:%M:%S"), job_status)) info_plugin = { "spark_submisson_url": "http://" + master, "expected_time": expected_time, "number_of_jobs": number_of_jobs } self._log("%s | Starting monitor" % (time.strftime("%H:%M:%S"))) monitor.start_monitor(api.monitor_url, spark_app_id, monitor_plugin, info_plugin, collect_period) self._log("%s | Starting controller" % (time.strftime("%H:%M:%S"))) controller.start_controller(api.controller_url, spark_app_id, workers_id, data) job_status = self._wait_on_job_finish(sahara, connector, job_exec_id, app_id) self._log("%s | Stopping monitor" % (time.strftime("%H:%M:%S"))) monitor.stop_monitor(api.monitor_url, spark_app_id) self._log("%s | Stopping controller" % (time.strftime("%H:%M:%S"))) controller.stop_controller(api.controller_url, spark_app_id) spark_applications_ids.remove(spark_app_id) self._log("Finished application execution") if connector.is_job_completed(job_status): self.update_application_state("OK") if connector.is_job_failed(job_status): self.update_application_state("Error") return job_status def _hdfs_spark_execution(self, master, remote_hdfs, key_path, args, job_bin_url, main_class, dependencies, spark_applications_ids, expected_time, monitor_plugin, collect_period, number_of_jobs, workers_id, data, connector, swift, swift_logdir, container, number_of_attempts): job_exec_id = str(uuid.uuid4())[0:7] self._log("%s | Job execution ID: %s" % (time.strftime("%H:%M:%S"), job_exec_id)) # Defining params local_path = '/tmp/spark-jobs/' + job_exec_id + '/' # remote_path = 'ubuntu@' + master + ':' + local_path job_input_paths, job_output_path, job_params = (hdfs.get_job_params( key_path, remote_hdfs, args)) job_binary_path = hdfs.get_path(job_bin_url) # Create temporary job directories self._log("%s | Create temporary job directories" % (time.strftime("%H:%M:%S"))) self._mkdir(local_path) # Create cluster directories self._log("%s | Creating cluster directories" % (time.strftime("%H:%M:%S"))) remote.execute_command(master, key_path, 'mkdir -p %s' % local_path) # Get job binary from hdfs self._log("%s | Get job binary from hdfs" % (time.strftime("%H:%M:%S"))) remote.copy_from_hdfs(master, key_path, remote_hdfs, job_binary_path, local_path) # Enabling event log on cluster self._log("%s | Enabling event log on cluster" % (time.strftime("%H:%M:%S"))) self._enable_event_log(master, key_path, local_path) # Submit job self._log("%s | Starting job" % (time.strftime("%H:%M:%S"))) local_binary_file = ( local_path + remote.list_directory(key_path, master, local_path)) spark_job = self._submit_job(master, key_path, main_class, dependencies, local_binary_file, args) spark_app_id = spark.get_running_app(master, spark_applications_ids, number_of_attempts) if spark_app_id is None: self._log("%s | Error on submission of application, " "please check the config file" % (time.strftime("%H:%M:%S"))) (output, err) = spark_job.communicate() self.stdout.log(output) self.stderr.log(err) raise ex.ConfigurationError() spark_applications_ids.append(spark_app_id) info_plugin = { "spark_submisson_url": "http://" + master, "expected_time": expected_time, "number_of_jobs": number_of_jobs } self._log("%s | Starting monitor" % (time.strftime("%H:%M:%S"))) monitor.start_monitor(api.monitor_url, spark_app_id, monitor_plugin, info_plugin, collect_period) self._log("%s | Starting controller" % (time.strftime("%H:%M:%S"))) controller.start_controller(api.controller_url, spark_app_id, workers_id, data) (output, err) = spark_job.communicate() self._log("%s | Stopping monitor" % (time.strftime("%H:%M:%S"))) monitor.stop_monitor(api.monitor_url, spark_app_id) self._log("%s | Stopping controller" % (time.strftime("%H:%M:%S"))) controller.stop_controller(api.controller_url, spark_app_id) self.stdout.log(output) self.stderr.log(err) self._log("%s | Copy log from cluster" % (time.strftime("%H:%M:%S"))) event_log_path = local_path + 'eventlog/' self._mkdir(event_log_path) remote_event_log_path = 'ubuntu@%s:%s%s' % (master, local_path, spark_app_id) remote.copy(key_path, remote_event_log_path, event_log_path) self._log("%s | Upload log to Swift" % (time.strftime("%H:%M:%S"))) connector.upload_directory(swift, event_log_path, swift_logdir, container) spark_applications_ids.remove(spark_app_id) self.update_application_state("OK") return 'OK' def _submit_job(self, remote_instance, key_path, main_class, dependencies, job_binary_file, args): args_line = '' for arg in args: args_line += arg + ' ' spark_submit = ('/opt/spark/bin/spark-submit ' '--packages %(dependencies)s ' '--class %(main_class)s ' '--master spark://%(master)s:7077 ' '%(job_binary_file)s %(args)s ' % { 'dependencies': dependencies, 'main_class': main_class, 'master': remote_instance, 'job_binary_file': 'file://' + job_binary_file, 'args': args_line }) if main_class == '': spark_submit = spark_submit.replace('--class', '') if dependencies == '': spark_submit = spark_submit.replace('--packages', '') self._log("%s | spark-submit: %s" % (time.strftime("%H:%M:%S"), spark_submit)) job = remote.execute_command_popen(remote_instance, key_path, spark_submit) return job def _enable_event_log(self, master, key_path, path): enable_event_log_command = ( "echo -e 'spark.executor.extraClassPath " "/usr/lib/hadoop-mapreduce/hadoop-openstack.jar\n" "spark.eventLog.enabled true\n" "spark.eventLog.dir " "file://%(path)s' > " "/opt/spark/conf/spark-defaults.conf" % { 'path': path }) remote.execute_command(master, key_path, enable_event_log_command) def _log(self, string): plugin_log.log(string) self.running_log.log(string) def _verify_existing_log_paths(self, app_id): if not os.path.exists('logs'): os.mkdir('logs') elif not os.path.exists('logs/apps'): os.mkdir('logs/apps') if not os.path.exists('logs/apps/%s' % app_id): os.mkdir('logs/apps/%s' % app_id) def _clean_log_files(self, app_id): # Commented because isn't used # running_log_file = open("logs/apps/%s/execution" \ # % app_id, "w").close() # stdout_file = open("logs/apps/%s/stdout" % app_id, "w").close() # stderr_file = open("logs/apps/%s/stderr" % app_id, "w").close() pass def _mkdir(self, path): subprocess.call('mkdir -p %s' % path, shell=True)
cluster_username = config.get('spark_mesos', 'cluster_username') cluster_password = config.get('spark_mesos', 'cluster_password') cluster_key_path = config.get('spark_mesos', 'key_path') one_url = config.get('spark_mesos', 'one_url') one_password = config.get('spark_mesos', 'one_password') one_username = config.get('spark_mesos', 'one_username') spark_path = config.get('spark_mesos', 'spark_path') if 'chronos' in plugins: chronos_url = config.get('chronos', 'url') chronos_username = config.get('chronos', 'username') chronos_password = config.get('chronos', 'password') supervisor_url = config.get('chronos', 'supervisor_url') except Exception as e: API_LOG.log("Error: %s" % e.message) quit() def get_node_cluster(k8s_conf_path): """ Gets the IP address of one slave node contained in a Kubernetes cluster. The k8s API aways returns information about the master node followed by the information of the slaves. Therefore, in order to avoid get the IP of the master node, this function always get the last node listed by the API. Raises: Exception -- It was not possible to connect with the Kubernetes cluster. Returns:
# Setting default values for the necessary variables k8s_conf_path = CONFIG_PATH # If explicitly stated in the cfg file, overwrite the variables if(config.has_section('kubejobs')): if(config.has_option('kubejobs', 'k8s_conf_path')): k8s_conf_path = config.get('kubejobs', 'k8s_conf_path') if(config.has_option('kubejobs', 'count_queue')): count_queue = config.get('kubejobs', 'count_queue') if(config.has_option('kubejobs', 'redis_ip')): redis_ip = config.get('kubejobs', 'redis_ip') except Exception as e: API_LOG.log("Error: %s" % e) quit() def get_node_cluster(k8s_conf_path): """ Gets the IP address of one slave node contained in a Kubernetes cluster. The k8s API aways returns information about the master node followed by the information of the slaves. Therefore, in order to avoid get the IP of the master node, this function always get the last node listed by the API. Raises: Exception -- It was not possible to connect with the Kubernetes cluster. Returns: string -- The node IP """
class SparkGenericApplicationExecutor(GenericApplicationExecutor): def __init__(self, app_id, master_ip): self.application_state = "None" self.state_lock = threading.RLock() self.application_time = -1 self.start_time = -1 self.app_id = app_id self.master = master_ip self._verify_existing_log_paths(app_id) self._clean_log_files(app_id) self.running_log = Log("Running_Application_%s" % app_id, "logs/apps/%s/execution" % app_id) self.stdout = Log("stdout_%s" % app_id, "logs/apps/%s/stdout" % app_id) self.stderr = Log("stderr_%s" % app_id, "logs/apps/%s/stderr" % app_id) def get_application_state(self): with self.state_lock: state = self.application_state return state def update_application_state(self, state): with self.state_lock: self.application_state = state def get_application_execution_time(self): return self.application_time def get_application_start_time(self): return self.start_time def start_application(self, data, spark_applications_ids, app_id): try: self.update_application_state("Running") # Broker Parameters key_path = api.key_path remote_hdfs = api.remote_hdfs number_of_attempts = api.number_of_attempts master_ip = self.master # User Request Parameters args = data['args'] main_class = data['main_class'] dependencies = data['dependencies'] job_binary_url = data['job_binary_url'] self._log("%s | Master is %s" % (time.strftime("%H:%M:%S"), master_ip)) job_status = self._hdfs_spark_execution(master_ip, remote_hdfs, key_path, args, job_binary_url, main_class, dependencies, spark_applications_ids, number_of_attempts) self._log("%s | Finished application execution" % time.strftime("%H:%M:%S")) return job_status except KeyError as ke: self._log("%s | Parameter missing in submission: %s, " "please check the config file" % (time.strftime("%H:%M:%S"), str(ke))) self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") except Exception: self._log("%s | Unknown error, please report to administrators " "of WP3 infrastructure" % (time.strftime("%H:%M:%S"))) self._log("%s | Finished application execution with error" % (time.strftime("%H:%M:%S"))) self.update_application_state("Error") def get_application_time(self): return self.application_time def _hdfs_spark_execution(self, master, remote_hdfs, key_path, args, job_bin_url, main_class, dependencies, spark_applications_ids, number_of_attempts): job_exec_id = str(uuid.uuid4())[0:7] self._log("%s | Job execution ID: %s" % (time.strftime("%H:%M:%S"), job_exec_id)) # Defining params local_path = '/tmp/spark-jobs/' + job_exec_id + '/' job_binary_path = hdfs.get_path(job_bin_url) # Create temporary job directories self._log("%s | Create temporary job directories" % time.strftime("%H:%M:%S")) self._mkdir(local_path) # Create cluster directories self._log("%s | Creating cluster directories" % time.strftime("%H:%M:%S")) remote.execute_command(master, key_path, 'mkdir -p %s' % local_path) # Get job binary from hdfs self._log("%s | Get job binary from hdfs" % time.strftime("%H:%M:%S")) remote.copy_from_hdfs(master, key_path, remote_hdfs, job_binary_path, local_path) # Enabling event log on cluster self._log("%s | Enabling event log on cluster" % time.strftime("%H:%M:%S")) self._enable_event_log(master, key_path, local_path) # Submit job self._log("%s | Starting job" % time.strftime("%H:%M:%S")) local_binary_file = ( local_path + remote.list_directory(key_path, master, local_path)) spark_job = self._submit_job(master, key_path, main_class, dependencies, local_binary_file, args) spark_app_id = spark.get_running_app(master, spark_applications_ids, number_of_attempts) if spark_app_id is None: self._log("%s | Error on submission of application, " "please check the config file" % time.strftime("%H:%M:%S")) (output, err) = spark_job.communicate() self.stdout.log(output) self.stderr.log(err) raise ex.ConfigurationError() spark_applications_ids.append(spark_app_id) (output, err) = spark_job.communicate() self.stdout.log(output) self.stderr.log(err) self._log("%s | Copy log from cluster" % (time.strftime("%H:%M:%S"))) event_log_path = local_path + 'eventlog/' self._mkdir(event_log_path) remote_event_log_path = 'ubuntu@%s:%s%s' % (master, local_path, spark_app_id) remote.copy(key_path, remote_event_log_path, event_log_path) spark_applications_ids.remove(spark_app_id) self.update_application_state("OK") return 'OK' def _submit_job(self, remote_instance, key_path, main_class, dependencies, job_binary_file, args): args_line = '' for arg in args: args_line += arg + ' ' spark_submit = ('/opt/spark/bin/spark-submit ' '--packages %(dependencies)s ' '--class %(main_class)s ' '--master spark://%(master)s:7077 ' '%(job_binary_file)s %(args)s ' % { 'dependencies': dependencies, 'main_class': main_class, 'master': remote_instance, 'job_binary_file': 'file://' + job_binary_file, 'args': args_line }) if main_class == '': spark_submit = spark_submit.replace('--class', '') if dependencies == '': spark_submit = spark_submit.replace('--packages', '') job = remote.execute_command_popen(remote_instance, key_path, spark_submit) return job def _enable_event_log(self, master, key_path, path): enable_event_log_command = ( "echo -e 'spark.executor.extraClassPath " "/usr/lib/hadoop-mapreduce/hadoop-openstack.jar\n" "spark.eventLog.enabled true\n" "spark.eventLog.dir " "file://%(path)s' > " "/opt/spark/conf/spark-defaults.conf" % { 'path': path }) remote.execute_command(master, key_path, enable_event_log_command) def _log(self, string): plugin_log.log(string) self.running_log.log(string) def _verify_existing_log_paths(self, app_id): if not os.path.exists('logs'): os.mkdir('logs') elif not os.path.exists('logs/apps'): os.mkdir('logs/apps') if not os.path.exists('logs/apps/%s' % app_id): os.mkdir('logs/apps/%s' % app_id) def _clean_log_files(self, app_id): open("logs/apps/%s/execution" % app_id, "w").close() open("logs/apps/%s/stdout" % app_id, "w").close() open("logs/apps/%s/stderr" % app_id, "w").close() def _mkdir(self, path): subprocess.call('mkdir -p %s' % path, shell=True)