def test_prepare_cluster_for_ds(self, configure): data_source_urls = {'1': '1_runtime', '2': '2_runtime'} data_source = mock.Mock() data_source.type = 'hdfs' data_source.id = '1' cluster = mock.Mock() job_configs = mock.Mock() job_utils.prepare_cluster_for_ds([data_source], cluster, job_configs, data_source_urls) configure.assert_called_once() configure.assert_called_with(cluster, '1_runtime')
def run_job(self, job_execution): ctx = context.ctx() job = conductor.job_get(ctx, job_execution.job_id) # This will be a dictionary of tuples, (native_url, runtime_url) # keyed by data_source id data_source_urls = {} additional_sources, updated_job_configs = ( job_utils.resolve_data_source_references(job_execution.job_configs, job_execution.id, data_source_urls, self.cluster) ) job_execution = conductor.job_execution_update( ctx, job_execution, {"data_source_urls": job_utils.to_url_dict(data_source_urls)}) # Now that we've recorded the native urls, we can switch to the # runtime urls data_source_urls = job_utils.to_url_dict(data_source_urls, runtime=True) job_utils.prepare_cluster_for_ds(additional_sources, self.cluster, updated_job_configs, data_source_urls) # It is needed in case we are working with Spark plugin self.plugin_params['master'] = ( self.plugin_params['master'] % {'host': self.master.hostname()}) # TODO(tmckay): wf_dir should probably be configurable. # The only requirement is that the dir is writable by the image user wf_dir = job_utils.create_workflow_dir(self.master, '/tmp/spark-edp', job, job_execution.id, "700") paths, builtin_paths = self._upload_job_files( self.master, wf_dir, job, updated_job_configs) # We can shorten the paths in this case since we'll run out of wf_dir paths = [os.path.basename(p) if p.startswith(wf_dir) else p for p in paths] builtin_paths = [os.path.basename(p) for p in builtin_paths] cmd = self._build_command(wf_dir, paths, builtin_paths, updated_job_configs) job_execution = conductor.job_execution_get(ctx, job_execution.id) if job_execution.info['status'] == edp.JOB_STATUS_TOBEKILLED: return (None, edp.JOB_STATUS_KILLED, None) # If an exception is raised here, the job_manager will mark # the job failed and log the exception # The redirects of stdout and stderr will preserve output in the wf_dir with remote.get_remote(self.master) as r: # Upload the command launch script launch = os.path.join(wf_dir, "launch_command") r.write_file_to(launch, self._job_script()) r.execute_command("chmod u+rwx,g+rx,o+rx %s" % wf_dir) r.execute_command("chmod +x %s" % launch) ret, stdout = r.execute_command( "cd %s; ./launch_command %s > /dev/null 2>&1 & echo $!" % (wf_dir, cmd)) if ret == 0: # Success, we'll add the wf_dir in job_execution.extra and store # pid@instance_id as the job id # We know the job is running so return "RUNNING" return (stdout.strip() + "@" + self.master.id, edp.JOB_STATUS_RUNNING, {'spark-path': wf_dir}) # Hmm, no execption but something failed. # Since we're using backgrounding with redirect, this is unlikely. raise e.EDPError(_("Spark job execution failed. Exit status = " "%(status)s, stdout = %(stdout)s") % {'status': ret, 'stdout': stdout})
def run_job(self, job_execution): ctx = context.ctx() job = conductor.job_get(ctx, job_execution.job_id) # This will be a dictionary of tuples, (native_url, runtime_url) # keyed by data_source id data_source_urls = {} additional_sources, updated_job_configs = ( job_utils.resolve_data_source_references(job_execution.job_configs, job_execution.id, data_source_urls, self.cluster)) job_execution = conductor.job_execution_update( ctx, job_execution, {"data_source_urls": job_utils.to_url_dict(data_source_urls)}) # Now that we've recorded the native urls, we can switch to the # runtime urls data_source_urls = job_utils.to_url_dict(data_source_urls, runtime=True) job_utils.prepare_cluster_for_ds(additional_sources, self.cluster, updated_job_configs, data_source_urls) # We'll always run the driver program on the master master = plugin_utils.get_instance(self.cluster, "nimbus") # TODO(tmckay): wf_dir should probably be configurable. # The only requirement is that the dir is writable by the image user wf_dir = job_utils.create_workflow_dir(master, '/tmp/storm-edp', job, job_execution.id, "700") paths = self._upload_job_files(master, wf_dir, job, updated_job_configs) topology_name = self._set_topology_name(job_execution, job.name) # Launch the storm job using storm jar host = master.hostname() cmd = self._build_command(paths, updated_job_configs, host, topology_name) job_execution = conductor.job_execution_get(ctx, job_execution.id) if job_execution.info['status'] == edp.JOB_STATUS_TOBEKILLED: return (None, edp.JOB_STATUS_KILLED, None) ret, stdout = self._execute_remote_job(master, wf_dir, cmd) if ret == 0: # Success, we'll add the wf_dir in job_execution.extra and store # topology_name@instance_id as the job id # We know the job is running so return "RUNNING" return (topology_name + "@" + master.id, edp.JOB_STATUS_RUNNING, { 'storm-path': wf_dir }) # Hmm, no execption but something failed. # Since we're using backgrounding with redirect, this is unlikely. raise e.EDPError( _("Storm job execution failed. Exit status = " "%(status)s, stdout = %(stdout)s") % { 'status': ret, 'stdout': stdout })
def run_job(self, job_execution): ctx = context.ctx() job = conductor.job_get(ctx, job_execution.job_id) # This will be a dictionary of tuples, (native_url, runtime_url) # keyed by data_source id data_source_urls = {} additional_sources, updated_job_configs = ( job_utils.resolve_data_source_references(job_execution.job_configs, job_execution.id, data_source_urls, self.cluster) ) job_execution = conductor.job_execution_update( ctx, job_execution, {"data_source_urls": job_utils.to_url_dict(data_source_urls)}) # Now that we've recorded the native urls, we can switch to the # runtime urls data_source_urls = job_utils.to_url_dict(data_source_urls, runtime=True) job_utils.prepare_cluster_for_ds(additional_sources, self.cluster, updated_job_configs, data_source_urls) # We'll always run the driver program on the master master = plugin_utils.get_instance(self.cluster, "nimbus") # TODO(tmckay): wf_dir should probably be configurable. # The only requirement is that the dir is writable by the image user wf_dir = job_utils.create_workflow_dir(master, '/tmp/storm-edp', job, job_execution.id, "700") paths = self._upload_job_files(master, wf_dir, job, updated_job_configs) topology_name = self._set_topology_name(job_execution, job.name) # Launch the storm job using storm jar host = master.hostname() cmd = self._build_command(paths, updated_job_configs, host, topology_name) job_execution = conductor.job_execution_get(ctx, job_execution.id) if job_execution.info['status'] == edp.JOB_STATUS_TOBEKILLED: return (None, edp.JOB_STATUS_KILLED, None) ret, stdout = self._execute_remote_job(master, wf_dir, cmd) if ret == 0: # Success, we'll add the wf_dir in job_execution.extra and store # topology_name@instance_id as the job id # We know the job is running so return "RUNNING" return (topology_name + "@" + master.id, edp.JOB_STATUS_RUNNING, {'storm-path': wf_dir}) # Hmm, no execption but something failed. # Since we're using backgrounding with redirect, this is unlikely. raise e.EDPError(_("Storm job execution failed. Exit status = " "%(status)s, stdout = %(stdout)s") % {'status': ret, 'stdout': stdout})
def run_job(self, job_execution): ctx = context.ctx() job = conductor.job_get(ctx, job_execution.job_id) # This will be a dictionary of tuples, (native_url, runtime_url) # keyed by data_source id data_source_urls = {} additional_sources, updated_job_configs = ( job_utils.resolve_data_source_references(job_execution.job_configs, job_execution.id, data_source_urls, self.cluster) ) job_execution = conductor.job_execution_update( ctx, job_execution, {"data_source_urls": job_utils.to_url_dict(data_source_urls)}) # Now that we've recorded the native urls, we can switch to the # runtime urls data_source_urls = job_utils.to_url_dict(data_source_urls, runtime=True) job_utils.prepare_cluster_for_ds(additional_sources, self.cluster, updated_job_configs, data_source_urls) # It is needed in case we are working with Spark plugin self.plugin_params['master'] = ( self.plugin_params['master'] % {'host': self.master.hostname()}) # TODO(tmckay): wf_dir should probably be configurable. # The only requirement is that the dir is writable by the image user wf_dir = job_utils.create_workflow_dir(self.master, '/tmp/spark-edp', job, job_execution.id, "700") paths, builtin_paths = self._upload_job_files( self.master, wf_dir, job, updated_job_configs) # We can shorten the paths in this case since we'll run out of wf_dir paths = [os.path.basename(p) if p.startswith(wf_dir) else p for p in paths] builtin_paths = [os.path.basename(p) for p in builtin_paths] cmd = self._build_command(wf_dir, paths, builtin_paths, updated_job_configs) job_execution = conductor.job_execution_get(ctx, job_execution.id) if job_execution.info['status'] == edp.JOB_STATUS_TOBEKILLED: return (None, edp.JOB_STATUS_KILLED, None) # If an exception is raised here, the job_manager will mark # the job failed and log the exception # The redirects of stdout and stderr will preserve output in the wf_dir with remote.get_remote(self.master) as r: # Upload the command launch script launch = os.path.join(wf_dir, "launch_command") python_version = r.get_python_version() r.write_file_to(launch, self._job_script(python_version)) r.execute_command("chmod u+rwx,g+rx,o+rx %s" % wf_dir) r.execute_command("chmod +x %s" % launch) ret, stdout = r.execute_command( "cd %s; ./launch_command %s > /dev/null 2>&1 & echo $!" % (wf_dir, cmd)) if ret == 0: # Success, we'll add the wf_dir in job_execution.extra and store # pid@instance_id as the job id # We know the job is running so return "RUNNING" return (stdout.strip() + "@" + self.master.id, edp.JOB_STATUS_RUNNING, {'spark-path': wf_dir}) # Hmm, no execption but something failed. # Since we're using backgrounding with redirect, this is unlikely. raise e.EDPError(_("Spark job execution failed. Exit status = " "%(status)s, stdout = %(stdout)s") % {'status': ret, 'stdout': stdout})
def _prepare_run_job(self, job_execution): ctx = context.ctx() # This will be a dictionary of tuples, (native_url, runtime_url) # keyed by data_source id data_source_urls = {} prepared_job_params = {} job = conductor.job_get(ctx, job_execution.job_id) input_source, output_source = job_utils.get_input_output_data_sources( job_execution, job, data_source_urls, self.cluster) # Updated_job_configs will be a copy of job_execution.job_configs with # any name or uuid references to data_sources resolved to paths # assuming substitution is enabled. # If substitution is not enabled then updated_job_configs will # just be a reference to job_execution.job_configs to avoid a copy. # Additional_sources will be a list of any data_sources found. additional_sources, updated_job_configs = ( job_utils.resolve_data_source_references(job_execution.job_configs, job_execution.id, data_source_urls, self.cluster) ) job_execution = conductor.job_execution_update( ctx, job_execution, {"data_source_urls": job_utils.to_url_dict(data_source_urls)}) # Now that we've recorded the native urls, we can switch to the # runtime urls data_source_urls = job_utils.to_url_dict(data_source_urls, runtime=True) data_sources = additional_sources + [input_source, output_source] job_utils.prepare_cluster_for_ds(data_sources, self.cluster, updated_job_configs, data_source_urls) proxy_configs = updated_job_configs.get('proxy_configs') configs = updated_job_configs.get('configs', {}) use_hbase_lib = configs.get('edp.hbase_common_lib', {}) # Extract all the 'oozie.' configs so that they can be set in the # job properties file. These are config values for Oozie itself, # not the job code oozie_params = {} for k in list(configs): if k.startswith('oozie.'): oozie_params[k] = configs[k] external_hdfs_urls = self._resolve_external_hdfs_urls( job_execution.job_configs) for url in external_hdfs_urls: h.configure_cluster_for_hdfs(self.cluster, url) hdfs_user = self.get_hdfs_user() # TODO(tmckay): this should probably be "get_namenode" # but that call does not exist in the oozie engine api now. oozie_server = self.get_oozie_server(self.cluster) wf_dir = self._create_hdfs_workflow_dir(oozie_server, job) self._upload_job_files_to_hdfs(oozie_server, wf_dir, job, configs, proxy_configs) wf_xml = workflow_factory.get_workflow_xml( job, self.cluster, updated_job_configs, input_source, output_source, hdfs_user, data_source_urls) path_to_workflow = self._upload_workflow_file(oozie_server, wf_dir, wf_xml, hdfs_user) prepared_job_params['context'] = ctx prepared_job_params['hdfs_user'] = hdfs_user prepared_job_params['path_to_workflow'] = path_to_workflow prepared_job_params['use_hbase_lib'] = use_hbase_lib prepared_job_params['job_execution'] = job_execution prepared_job_params['oozie_params'] = oozie_params prepared_job_params['wf_dir'] = wf_dir prepared_job_params['oozie_server'] = oozie_server return prepared_job_params