def go(self): """ This is where the work of the recipe gets done. Subclasses should define their own go() method, but remember to call this one to perform necessary initialisation. """ # Every recipe needs a job identifier if "job_name" not in self.inputs: raise PipelineException("Job undefined") if "start_time" not in self.inputs: import datetime self.inputs["start_time"] = datetime.datetime.utcnow().replace(microsecond=0).isoformat() # Config is passed in from spawning recipe. But if this is the start # of a pipeline, it won't have one. if not hasattr(self, "config"): self.config = self._read_config() # Ensure we have a runtime directory if 'runtime_directory' not in self.inputs: self.inputs["runtime_directory"] = self.config.get( "DEFAULT", "runtime_directory" ) else: self.config.set('DEFAULT', 'runtime_directory', self.inputs['runtime_directory']) if not os.access(self.inputs['runtime_directory'], os.F_OK): raise IOError("Runtime directory doesn't exist") # ...and task files, if applicable if "task_files" not in self.inputs: try: self.inputs["task_files"] = utilities.string_to_list( self.config.get('DEFAULT', "task_files") ) except NoOptionError: self.inputs["task_files"] = [] self.task_definitions = ConfigParser(self.config.defaults()) print("Reading task definition file(s): %s" % \ ",".join(self.inputs["task_files"]), file=sys.stderr) self.task_definitions.read(self.inputs["task_files"]) # Specify the working directory on the compute nodes if 'working_directory' not in self.inputs: self.inputs['working_directory'] = self.config.get( "DEFAULT", "working_directory" ) else: self.config.set("DEFAULT", "working_directory", self.inputs['working_directory']) try: self.recipe_path = [ os.path.join(root, 'master') for root in utilities.string_to_list( self.config.get('DEFAULT', "recipe_directories") ) ] except NoOptionError: self.recipe_path = [] # At this point, the recipe inputs must be complete. If not, exit. if not self.inputs.complete(): raise PipelineException( "Required inputs not available: %s" % " ".join(self.inputs.missing()) ) # Only configure handlers if our parent is the root logger. # Otherwise, our parent should have done it for us. if isinstance(self.logger.parent, logging.RootLogger): self._setup_logging() self.logger.debug("Pipeline start time: %s" % self.inputs['start_time'])
def coerce(self, value): if isinstance(value, str): return string_to_list(value) else: return value
"cwd": os.getcwd(), "start_time": start_time, }) config.read(config_file) config.set('DEFAULT', 'runtime_directory', runtime_directory) config.set('DEFAULT', 'default_working_directory', working_directory) # Extract input file list from parset to_process = input_parset.getStringVector( 'ObsSW.Observation.DataProducts.measurementSets') # Read config file to establish location of parset directory to use parset_directory = config.get("layout", "parset_directory") create_directory(parset_directory) # For each task (currently only ndppp), extract and write parset tasks = ConfigParser(config.defaults()) tasks.read(string_to_list(config.get("DEFAULT", "task_files"))) ndppp_parset_location = tasks.get("ndppp", "parset") input_parset.makeSubset( "ObsSW.Observation.ObservationControl.PythonControl.DPPP.").writeFile( ndppp_parset_location) # Run pipeline & wait for result subprocess.check_call([ 'python', pipeline_definition, '-j', tree_id, '-d', '--config', config_file, '--runtime-directory', runtime_directory, '--default-working-directory', working_directory, '--start-time', start_time ])
# Extract runtime, working, results directories from input parset runtime_directory = input_parset.getString("ObsSW.Observation.ObservationControl.PythonControl.runtimeDirectory") working_directory = input_parset.getString("ObsSW.Observation.ObservationControl.PythonControl.workingDirectory") results_directory = input_parset.getString("ObsSW.Observation.ObservationControl.PythonControl.resultDirectory") # Set up configuration for later processing stages config = ConfigParser({ "job_name": tree_id, "cwd": os.getcwd(), "start_time": start_time, }) config.read(config_file) config.set('DEFAULT', 'runtime_directory', runtime_directory) config.set('DEFAULT', 'default_working_directory', working_directory) # Extract input file list from parset to_process = input_parset.getStringVector('ObsSW.Observation.DataProducts.measurementSets') # Read config file to establish location of parset directory to use parset_directory = config.get("layout", "parset_directory") create_directory(parset_directory) # For each task (currently only ndppp), extract and write parset tasks = ConfigParser(config.defaults()) tasks.read(string_to_list(config.get("DEFAULT", "task_files"))) ndppp_parset_location = tasks.get("ndppp", "parset") input_parset.makeSubset("ObsSW.Observation.ObservationControl.PythonControl.DPPP.").writeFile(ndppp_parset_location) # Run pipeline & wait for result subprocess.check_call(['python', pipeline_definition, '-j', tree_id, '-d', '--config', config_file, '--runtime-directory', runtime_directory, '--default-working-directory', working_directory, '--start-time', start_time])