class AsynchronJob: """ is instantiated in child process instantiate the object corresponding to the execution manager defined in Config, and after the completion of the job manage the results """ def __init__(self, commandLine, dirPath, serviceName, resultsMask , userEmail = None, email_notify = 'auto' , jobState = None , xmlEnv = None): """ @param commandLine: the command to be executed @type commandLine: String @param dirPath: the absolute path to directory where the job will be executed (normaly we are already in) @type dirPath: String @param serviceName: the name of the service @type serviceName: string @param resultsMask: the unix mask to retrieve the results of this job @type resultsMask: a dictionary { paramName : [ string prompt , ( string class , string or None superclass ) , string mask ] } @param userEmail: the user email adress @type userEmail: string @param email_notify: if the user must be or not notify of the results at the end of the Job. the 3 authorized values for this argument are: - 'true' to notify the results to the user - 'false' to Not notify the results to the user - 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY @type email_notify: string @param jobState: the jobState link to this job @type jobState: a L{JobState} instance @param xmlEnv: the environement variable need by the program @type xmlEnv: dictionnary @call: by the main of this module which is call by L{AsynchronRunner} """ self._command = commandLine self._dirPath = dirPath self.serviceName = serviceName self.father_pid = os.getppid() self.father_done = False if jobState is None: self.jobState = JobState( self._dirPath ) else: self.jobState = jobState self.userEmail = userEmail self.email_notify = email_notify if self._dirPath[-1] == '/': self._dirPath = self._dirPath[:-1] self.jobKey = os.path.split( self._dirPath )[ 1 ] atexit.register( self.childExit , "------------------- %s : %s -------------------" %( serviceName , self.jobKey ) ) t0 = time.time() ############################################ self._run( serviceName , xmlEnv ) ############################################# t1 = time.time() self.results = {} for paramName in resultsMask.keys(): resultsFiles = [] #type is a tuple ( klass , superKlass ) masks = resultsMask[ paramName ] for mask in masks : for File in glob.glob( mask ): size = os.path.getsize( File ) if size != 0: resultsFiles.append( ( str( File ) , size , None ) ) #we have not information about the output format if resultsFiles: self.results[ paramName ] = resultsFiles #a list of tuple (string file name , int size , string format or None ) self.jobState.setOutputDataFile( paramName , resultsFiles ) self.jobState.commit() try: zipFileName = self.zipResults() except Exception : msg = "an error occured during the zipping results :\n\n" rc_log.critical( "%s/%s : %s" %( self.serviceName , self.jobKey , msg ) , exc_info = True) zipFileName = None if self.userEmail: if self.email_notify == 'auto': # we test email_delay() to see if it is >= to 0, # as it seems that sometimes it is not >0. if ( t1 - t0 ) >= _cfg.email_delay(): emailResults(_cfg, self.userEmail, registry , self.jobState.getID(), self._dirPath , self.serviceName , self.jobKey , FileName = zipFileName ) elif self.email_notify == 'true': emailResults( _cfg, self.userEmail, registry , self.jobState.getID(), self._dirPath , self.serviceName , self.jobKey , FileName = zipFileName ) else: pass def childExit(self , message ): print >> sys.stderr , message #rc_log.log( 12 , "runnerChild %d ending, send a SIGCHLD to %d" %( os.getpid() , self.father_pid ) ) def _run(self , serviceName, xmlEnv ): dispatcher = _cfg.getDispatcher() execution_config = dispatcher.getExecutionConfig( self.jobState ) try: exec_engine = executionLoader( execution_config = execution_config ) except MobyleError ,err : msg = "unknown execution system : %s" %err rc_log.critical("%s : %s" %( serviceName , msg ), exc_info = True ) sm = StatusManager() sm.setStatus( self._dirPath , Status( code = 5 , message = 'Mobyle internal server error' ) ) raise MobyleError, msg except Exception , err: rc_log.error( str(err ), exc_info=True) raise err
class WorkflowJob(object): def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None): """ @param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id) @type id: string @param workflow: the workflow definition used to create a new job @type workflow: a L{Workflow} instance @param email: the user email address @type email: L{EmailAddress} instance or a string @param email_notify: if the user must be or not notify of the results at the end of the Job. the 3 authorized values for this argument are: - 'true' to notify the results to the user - 'false' to Not notify the results to the user - 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY @type email_notify: string @param session: the session owner of this workflow (if session is set workflowID mut be None ) @type session: a L{Session} instance @param workflowID: the ID of a the workflow owner of this workflow @type workflowID: string """ self.cfg = ConfigManager.Config() self.status_manager = StatusManager() if id: log.debug("accessing WorkflowJob %s" %(id)) self.id = id self.jobState = JobState( id ) else: log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name)) self.workflow = workflow if session and workflowID: msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(), workflowID ) log.error( msg ) raise MobyleError( msg ) self.session = session if session : email = session.getEmail() if email: self.email = EmailAddress( email ) else: self.email = None elif email : #there is an email without session if not isinstance( email , EmailAddress ): self.email = EmailAddress( email ) else: self.email = email self.email_notify = email_notify if self.email_notify != 'false' and not self.email: raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify ) self.parameters = {} for parameter in self.workflow.parameters: # setting parameters which have a default value (important for hidden parameters which are not # accessed by JobFacade... if not(parameter.isout) and parameter.vdef is not None: self.set_value(parameter.name, value=str(parameter.vdef)) # job is just an "environment" folder for the job # it contains the instanciation of the job runner which seems to be hardcoded as "command runner"... self._job = Job( service = self.workflow, cfg = self.cfg, userEmail = self.email, session = self.session, workflowID = workflowID , ) self.jobState = self._job.jobState self.id = self._job.getURL() def getDir(self): """ returns the absolute path of the workflow job directory """ return self.jobState.getDir() def set_status(self, status): log.debug("setting job %s status to %s" % (self.id, status)) self.status_manager.setStatus( self.getDir() , status ) def set_value(self, parameter_name, value=None, src=None, srcFileName=None): wf_parameter = [p for p in self.workflow.parameters if p.name==parameter_name][0] if value is not None: log.debug("setting %s parameter value to %s" %(parameter_name, value)) elif src is not None: log.debug("copying %s parameter value from %s/%s" %(parameter_name, src,srcFileName)) else: log.error("no VALUE or SOURCE URL specified for %s parameter." % parameter_name) """ set a parameter value """ self.parameters[parameter_name] = value self.parameters[parameter_name + '.src'] = src self.parameters[parameter_name + '.srcFileName'] = srcFileName if value and value==wf_parameter.vdef: log.debug("setting %s parameter value to default value %s" %(parameter_name, wf_parameter.vdef)) return # save input value in a file # link this file from the JobState xml datatype_class = wf_parameter.type.datatype.class_name datatype_superclass = wf_parameter.type.datatype.superclass_name df = DataTypeFactory() if (datatype_superclass in [None,""] ): dt = df.newDataType(datatype_class) else: dt = df.newDataType(datatype_superclass, datatype_class) mt = MobyleType(dt) p = Parameter(mt, name=parameter_name) p._isout = wf_parameter.isout if dt.isFile(): file_name = parameter_name+'.data' if src: src = DataProvider.get(src) file_name, size = mt.toFile( value , self , file_name, src , srcFileName ) if not(wf_parameter.isout): self.jobState.setInputDataFile(parameter_name, (file_name, size, None)) else: self.jobState.setOutputDataFile(parameter_name, [(file_name, size, None)]) else: if not(wf_parameter.isout): self.jobState.setInputDataValue(parameter_name, value) else: raise NotImplementedError() # so far Mobyle does not manage non-file outputs self.jobState.commit() def setValue(self, parameter_name, value=None, src=None, srcFileName=None): """MobyleJob-style set value method, called from JobFacade""" if type(value)==tuple: return self.set_value(parameter_name, value=value[1], src=value[2],srcFileName=value[3]) else: return self.set_value(parameter_name, value=value, src=src,srcFileName=srcFileName) def getJobid(self): """MobyleJob-style get job id method, called from JobFacade""" return self.id def getDate(self): """MobyleJob-style get date method, called from JobFacade""" return time.strptime(self.get_date(),"%x %X") def getStatus(self): """MobyleJob-style get status method, called from JobFacade""" return self.status_manager.getStatus( self.getDir() ) def get_value(self, parameter_name): """get a parameter value""" return self.parameters.get(parameter_name,None) def get_date(self): """get the job date as a string""" return self.jobState.getDate() def get_id(self): """get the job id""" return self.id def run(self): """submit the job asynchronously""" self.validate() self.set_status(Status( code = 1 )) # status = submitted #raise a UserValueError if nb of job is over the limit accepted if( self.email is not None ): self._job.over_limit( self.email , '' ) self._child_pid = os.fork() if self._child_pid==0: #Child code os.setsid() log_fd = os.open("%s/log" % self.jobState.getDir(), os.O_APPEND | os.O_WRONLY | os.O_CREAT , 0664 ) devnull = os.open( "/dev/null" , os.O_RDWR ) os.dup2( devnull , sys.stdin.fileno() ) os.close( devnull) os.dup2( log_fd , sys.stdout.fileno() ) os.dup2( log_fd , sys.stderr.fileno() ) os.close( log_fd ) atexit.register( self.log , "child exit for workflow id: %s" % self.get_id()) ################################################ service = self._job.getService() serviceName = service.getName() jobKey = self._job.getKey() linkName = ( "%s/%s.%s" %( self.cfg.admindir() , serviceName , jobKey ) ) try: os.symlink( os.path.join( self.getDir() , '.admin') , linkName ) except OSError , err: self.set_status(Status(string="error", message="workflow execution failed")) msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err ) log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info = True ) raise WorkflowJobError , msg ################################################ t0 = time.time() self.srun() t1 = time.time() ################################################ try: os.unlink( linkName ) except OSError , err: self.set_status(Status(string="error", message="workflow execution failed")) msg = "can't remove symbolic link %s in ADMINDIR: %s" %( linkName , err ) log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info= True ) raise WorkflowJobError , msg ################################################ try: zipFileName = self.zip_results() except Exception : msg = "an error occured during the zipping results :\n\n" log.critical( "%s : %s" %( self.id , msg ) , exc_info = True) zipFileName = None if self.email_notify == 'auto': if ( t1 - t0 ) > self.cfg.email_delay() : emailResults( self.cfg , self.email, #userEmail, registry, self.id, self.getDir(), self.workflow.getName(), self._job.getKey(), FileName = zipFileName ) elif self.email_notify == 'true': emailResults( self.cfg , self.email, #userEmail, registry, self.id, self.getDir(), self.workflow.getName(), self._job.getKey(), FileName = zipFileName ) else: pass sys.exit(0) #exit with no error
class CommandRunner: """ """ def __init__( self, job , jobState = None ): """ instanciate a L{CommandBuilder} and use it to build the CommandLine, then run it @param service: @type job: a L{Job} instance @param jobState: @type jobState: @call: l{Job.run} @todo: implementation pour le warper de cgi ou WS a faire """ self.job = job self.job_dir = self.job.getDir() self._service = self.job.getService() if jobState is None: self._jobState = JobState( self.job_dir ) else: self._jobState = jobState commandBuilder = CommandBuilder() method = self._service.getCommand()[1].upper() if method == '' or method == 'LOCAL': try: cmd = commandBuilder.buildLocalCommand( self._service ) self._commandLine = cmd[ 'cmd' ] self._xmlEnv = cmd[ 'env' ] paramfiles = cmd[ 'paramfiles' ] except Exception ,err : msg = "an error occured during the command line building: " self._logError( userMsg = "Mobyle Internal Server Error", logMsg = None #the error log is filled by the rf_log.critical ) #this error is already log in build.log msg = "%s/%s : %s" %( self._service.getName() , self.job.getKey() , msg , ) if self.job.cfg.debug( self._service.getName() ) == 0: rf_log.critical( msg , exc_info = True) # send an email else: rf_log.error( msg , exc_info = True) raise MobyleError , "Mobyle Internal Server Error" js_paramfiles = [] if paramfiles : for paramfile_name , string_handle in paramfiles.items(): paramfile_handle = open( os.path.join( self.job_dir , paramfile_name ) , 'w' ) content = string_handle.getvalue() paramfile_handle.write( content ) paramfile_handle.close() js_paramfiles.append( ( os.path.basename( paramfile_name ) , len( content ) ) ) self._jobState.setParamfiles( js_paramfiles ) self._jobState.commit() elif method == "GET" or method == "POST" or method == "POSTM": raise NotImplementedError ,"cgi wrapping is not yet implemented"