def __init__(self, task, inputs, outputs, logger, recipe_path): super(PipelineCook, self).__init__(task, inputs, outputs, logger) # Ensures the recipe to be run can be imported from the recipe path self.logger.warn("Ignoring this recipe_path: " + str(recipe_path)) try: try: module = importlib.import_module("lofarpipe.recipes.master." + task) except ImportError: # ...also support lower-cased file names. module = importlib.import_module("lofarpipe.recipes.master." + task.lower()) self.recipe = None try: self.recipe = getattr(module, task)() except AttributeError: # Try with first letter capital (python type nameconvention) self.recipe = getattr(module, task.capitalize())() self.recipe.logger = getSearchingLogger("%s.%s" % (self.logger.name, task)) self.recipe.logger.setLevel(self.logger.level) except Exception as e: self.logger.exception("Exception caught: " + str(e)) self.recipe = None raise CookError(task + ' can not be loaded')
def call_generic_pipeline(op_name, direction_name, parset, config, logbasename, genericpipeline_executable): """ Creates a GenericPipeline object and runs the pipeline Parameters ---------- op_name : str Name of operation direction_name : str Name of direction parset : str Name of pipeline parset file config : str Name of pipeline config file logbasename : str Log file base name genericpipeline_executable : str Path to genericpipeline.py executable """ from lofarpipe.support.pipelinelogging import getSearchingLogger from factor.lib.context import RedirectStdStreams import time genericpipeline_path = os.path.dirname(genericpipeline_executable) loader = imp.load_source('loader', os.path.join(genericpipeline_path, 'loader.py')) gp = imp.load_source('gp', genericpipeline_executable) # Initalize pipeline object pipeline = gp.GenericPipeline() # Add needed attr/methods pipeline.name = '{0}_{1}'.format(op_name, direction_name) pipeline.logger = getSearchingLogger(pipeline.name) pipeline.inputs['args'] = [parset] pipeline.inputs['config'] = config pipeline.inputs['job_name'] = direction_name # Set pipeline logging to DEBUG level logging.root.setLevel(logging.DEBUG) pipeline.logger.setLevel(logging.DEBUG) for handler in pipeline.logger.handlers: handler.setLevel(logging.DEBUG) # Run the pipeline, redirecting screen output to log files time.sleep(2.0) # pause to allow result_callback() to transfer resources log.info('<-- Operation {0} started (direction: {1})'.format(op_name, direction_name)) with open("{0}.out.log".format(logbasename), "wb") as out, \ open("{0}.err.log".format(logbasename), "wb") as err: with RedirectStdStreams(stdout=out, stderr=err): status = pipeline.run(pipeline.name) return (op_name, direction_name, status)
def main_init(self): """Main initialization for stand alone execution, reading input from the command line""" # The root logger has a null handler; we'll override in recipes. logging.getLogger().addHandler(NullLogHandler()) self.logger = getSearchingLogger(self.name) opts = sys.argv[1:] try: myParset = parset.Parset(self.name + ".parset") for p in myParset.keys(): opts[0:0] = "--" + p, myParset.getString(p) except IOError: logging.debug("Unable to open parset") (options, args) = self.optionparser.parse_args(opts) if options.help: return 1 else: for key, value in vars(options).iteritems(): if value is not None: self.inputs[key] = value self.inputs['args'] = args return 0
def __init__(self, task, inputs, outputs, logger, recipe_path): super(PipelineCook, self).__init__(task, inputs, outputs, logger) # Ensures the recipe to be run can be imported from the recipe path try: try: module_details = imp.find_module(task, recipe_path) except ImportError: # ...also support lower-cased file names. module_details = imp.find_module(task.lower(), recipe_path) module = imp.load_module(task, *module_details) self.recipe = None try: self.recipe = getattr(module, task)() except AttributeError: # Try with first letter capital (python type nameconvention) self.recipe = getattr(module, task.capitalize())() self.recipe.logger = getSearchingLogger("%s.%s" % (self.logger.name, task)) self.recipe.logger.setLevel(self.logger.level) except Exception, e: self.logger.exception("Exception caught: " + str(e)) self.recipe = None raise CookError (task + ' can not be loaded')
def go(self): #""" #Read the parset-file that was given as input argument, and set the #jobname before calling the base-class's `go()` method. #""" try: parset_file = os.path.abspath(self.inputs['args'][0]) except IndexError: #return self.usage() self.usage() # Set job-name to basename of parset-file w/o extension, if it's not # set on the command-line with '-j' or '--job-name' if not 'job_name' in self.inputs: self.inputs['job_name'] = ( os.path.splitext(os.path.basename(parset_file))[0]) self.name = self.inputs['job_name'] try: self.logger except: self.logger = getSearchingLogger(self.name) self.logger.setLevel(self.inputs['loglevel']) # Call the base-class's `go()` method. return super(GenericPipeline, self).go()
def go(self): #""" #Read the parset-file that was given as input argument, and set the #jobname before calling the base-class's `go()` method. #""" try: parset_file = os.path.abspath(self.inputs['args'][0]) except IndexError: #return self.usage() self.usage() # Set job-name to basename of parset-file w/o extension, if it's not # set on the command-line with '-j' or '--job-name' if not 'job_name' in self.inputs: self.inputs['job_name'] = (os.path.splitext( os.path.basename(parset_file))[0]) self.name = self.inputs['job_name'] try: self.logger except: self.logger = getSearchingLogger(self.name) self.logger.setLevel(self.inputs['loglevel']) # Call the base-class's `go()` method. return super(GenericPipeline, self).go()