def getOutputParameterRefs(self, name): js = JobState(self.jobId) outputRefs = js.getOutput(name) values = [] if outputRefs: for outputRef in outputRefs: values.append({'src':self.jobId,'srcFileName':outputRef[0],'size':outputRef[1],'fmt':outputRef[2]}) return values
def getOutputParameterValues(self, name): js = JobState(self.jobId) outputRefs = js.getOutput(name) values = [] if outputRefs: for outputRef in outputRefs: fh = js.open(outputRef[0]) value = ''.join( fh.readlines() ) values.append({'value':value,'ref':self.jobId+'/'+outputRef[0],'size':outputRef[1],'fmt':outputRef[2]}) return values
def __init__( self, job , jobState = None ): """ instanciate a L{CommandBuilder} and use it to build the CommandLine, then run it @param service: @type job: a L{Job} instance @param jobState: @type jobState: @call: l{Job.run} @todo: implementation pour le warper de cgi ou WS a faire """ self.job = job self.job_dir = self.job.getDir() self._service = self.job.getService() if jobState is None: self._jobState = JobState( self.job_dir ) else: self._jobState = jobState commandBuilder = CommandBuilder() method = self._service.getCommand()[1].upper() if method == '' or method == 'LOCAL': try: cmd = commandBuilder.buildLocalCommand( self._service ) self._commandLine = cmd[ 'cmd' ] self._xmlEnv = cmd[ 'env' ] paramfiles = cmd[ 'paramfiles' ] except Exception ,err : msg = "an error occured during the command line building: " self._logError( userMsg = "Mobyle Internal Server Error", logMsg = None #the error log is filled by the rf_log.critical ) #this error is already log in build.log msg = "%s/%s : %s" %( self._service.getName() , self.job.getKey() , msg , ) if self.job.cfg.debug( self._service.getName() ) == 0: rf_log.critical( msg , exc_info = True) # send an email else: rf_log.error( msg , exc_info = True) raise MobyleError , "Mobyle Internal Server Error" js_paramfiles = [] if paramfiles : for paramfile_name , string_handle in paramfiles.items(): paramfile_handle = open( os.path.join( self.job_dir , paramfile_name ) , 'w' ) content = string_handle.getvalue() paramfile_handle.write( content ) paramfile_handle.close() js_paramfiles.append( ( os.path.basename( paramfile_name ) , len( content ) ) ) self._jobState.setParamfiles( js_paramfiles ) self._jobState.commit()
def killJob(jobID): """ @param jobID: the url of the job or a sequence of jobID @type jobID: string or sequence of jobID @return: @rtype: @raise MobyleError: if the job has no number or if the job doesn't exist anymore @todo: tester la partie sge """ from types import StringTypes, TupleType, ListType from Mobyle.MobyleError import MobyleError from Mobyle.JobState import JobState, normUri from Mobyle.Job import Job from Mobyle.WorkflowJob import WorkflowJob if isinstance(jobID, StringTypes): jobIDs = [jobID] elif isinstance(jobID, (ListType, TupleType)): jobIDs = jobID else: raise MobyleError, "jobID must be a string or a Sequence of strings :%s" % type(jobID) errors = [] for jobID in jobIDs: try: path = normUri(jobID) except MobyleError, err: errors.append((jobID, str(err))) continue if path[:4] == "http": # the jobID is not on this Mobyle server errors.append((jobID, "can't kill distant job")) continue js = JobState(uri=jobID) if js.isWorkflow(): job = WorkflowJob(id=jobID) else: job = Job(ID=jobID) try: job.kill() except MobyleError, err: errors.append((jobID, str(err))) continue
def getSubJobs(self): """ gets the list of subjobs (along with their status) @return: subjobs list of dictionaries @rtype: list """ js = JobState(self.jobId) if hasattr(js, 'getSubJobs'): subjobs = js.getSubJobs() subsubjobs = [] for s in subjobs: s['jobPID'] = registry.getJobPID(self.jobId) + '::'+ registry.getJobPID(s['jobID']) child_jf = JobFacade.getFromJobId(s['jobID']) s['status'] = child_jf.getStatus() for ss in child_jf.getSubJobs(): ss['jobPID'] = s['jobPID'] + '::' + registry.getJobPID(ss['jobID']) schild_jf = JobFacade.getFromJobId(ss['jobID']) ss['status'] = schild_jf.getStatus() subsubjobs.append(ss) subjobs.extend(subsubjobs) return subjobs else: return []
def getFromJobId(cls, jobId): """ create a JobFacade to access an existing job. @param jobId: the job identifier @type jobId: string @return: the appropriate job facade @rtype: JobFacade """ jobState = JobState(jobId) jfargs = {'jobId': jobState.getID(),'programUrl':None,'workflowUrl':None} if jobState.isWorkflow(): jfargs['workflowUrl'] = jobState.getName() else: jfargs['programUrl']= jobState.getName() # this id is identical to the one in parameter, # except it has been normalized (may have removed # trailing index.xml from the id) if jobState.isLocal(): return(LocalJobFacade(**jfargs)) else: return(RemoteJobFacade(**jfargs))
def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None): """ @param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id) @type id: string @param workflow: the workflow definition used to create a new job @type workflow: a L{Workflow} instance @param email: the user email address @type email: L{EmailAddress} instance or a string @param email_notify: if the user must be or not notify of the results at the end of the Job. the 3 authorized values for this argument are: - 'true' to notify the results to the user - 'false' to Not notify the results to the user - 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY @type email_notify: string @param session: the session owner of this workflow (if session is set workflowID mut be None ) @type session: a L{Session} instance @param workflowID: the ID of a the workflow owner of this workflow @type workflowID: string """ self.cfg = ConfigManager.Config() self.status_manager = StatusManager() if id: log.debug("accessing WorkflowJob %s" %(id)) self.id = id self.jobState = JobState( id ) else: log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name)) self.workflow = workflow if session and workflowID: msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(), workflowID ) log.error( msg ) raise MobyleError( msg ) self.session = session if session : email = session.getEmail() if email: self.email = EmailAddress( email ) else: self.email = None elif email : #there is an email without session if not isinstance( email , EmailAddress ): self.email = EmailAddress( email ) else: self.email = email self.email_notify = email_notify if self.email_notify != 'false' and not self.email: raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify ) self.parameters = {} for parameter in self.workflow.parameters: # setting parameters which have a default value (important for hidden parameters which are not # accessed by JobFacade... if not(parameter.isout) and parameter.vdef is not None: self.set_value(parameter.name, value=str(parameter.vdef)) # job is just an "environment" folder for the job # it contains the instanciation of the job runner which seems to be hardcoded as "command runner"... self._job = Job( service = self.workflow, cfg = self.cfg, userEmail = self.email, session = self.session, workflowID = workflowID , ) self.jobState = self._job.jobState self.id = self._job.getURL()
class WorkflowJob(object): def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None): """ @param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id) @type id: string @param workflow: the workflow definition used to create a new job @type workflow: a L{Workflow} instance @param email: the user email address @type email: L{EmailAddress} instance or a string @param email_notify: if the user must be or not notify of the results at the end of the Job. the 3 authorized values for this argument are: - 'true' to notify the results to the user - 'false' to Not notify the results to the user - 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY @type email_notify: string @param session: the session owner of this workflow (if session is set workflowID mut be None ) @type session: a L{Session} instance @param workflowID: the ID of a the workflow owner of this workflow @type workflowID: string """ self.cfg = ConfigManager.Config() self.status_manager = StatusManager() if id: log.debug("accessing WorkflowJob %s" %(id)) self.id = id self.jobState = JobState( id ) else: log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name)) self.workflow = workflow if session and workflowID: msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(), workflowID ) log.error( msg ) raise MobyleError( msg ) self.session = session if session : email = session.getEmail() if email: self.email = EmailAddress( email ) else: self.email = None elif email : #there is an email without session if not isinstance( email , EmailAddress ): self.email = EmailAddress( email ) else: self.email = email self.email_notify = email_notify if self.email_notify != 'false' and not self.email: raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify ) self.parameters = {} for parameter in self.workflow.parameters: # setting parameters which have a default value (important for hidden parameters which are not # accessed by JobFacade... if not(parameter.isout) and parameter.vdef is not None: self.set_value(parameter.name, value=str(parameter.vdef)) # job is just an "environment" folder for the job # it contains the instanciation of the job runner which seems to be hardcoded as "command runner"... self._job = Job( service = self.workflow, cfg = self.cfg, userEmail = self.email, session = self.session, workflowID = workflowID , ) self.jobState = self._job.jobState self.id = self._job.getURL() def getDir(self): """ returns the absolute path of the workflow job directory """ return self.jobState.getDir() def set_status(self, status): log.debug("setting job %s status to %s" % (self.id, status)) self.status_manager.setStatus( self.getDir() , status ) def set_value(self, parameter_name, value=None, src=None, srcFileName=None): wf_parameter = [p for p in self.workflow.parameters if p.name==parameter_name][0] if value is not None: log.debug("setting %s parameter value to %s" %(parameter_name, value)) elif src is not None: log.debug("copying %s parameter value from %s/%s" %(parameter_name, src,srcFileName)) else: log.error("no VALUE or SOURCE URL specified for %s parameter." % parameter_name) """ set a parameter value """ self.parameters[parameter_name] = value self.parameters[parameter_name + '.src'] = src self.parameters[parameter_name + '.srcFileName'] = srcFileName if value and value==wf_parameter.vdef: log.debug("setting %s parameter value to default value %s" %(parameter_name, wf_parameter.vdef)) return # save input value in a file # link this file from the JobState xml datatype_class = wf_parameter.type.datatype.class_name datatype_superclass = wf_parameter.type.datatype.superclass_name df = DataTypeFactory() if (datatype_superclass in [None,""] ): dt = df.newDataType(datatype_class) else: dt = df.newDataType(datatype_superclass, datatype_class) mt = MobyleType(dt) p = Parameter(mt, name=parameter_name) p._isout = wf_parameter.isout if dt.isFile(): file_name = parameter_name+'.data' if src: src = DataProvider.get(src) file_name, size = mt.toFile( value , self , file_name, src , srcFileName ) if not(wf_parameter.isout): self.jobState.setInputDataFile(parameter_name, (file_name, size, None)) else: self.jobState.setOutputDataFile(parameter_name, [(file_name, size, None)]) else: if not(wf_parameter.isout): self.jobState.setInputDataValue(parameter_name, value) else: raise NotImplementedError() # so far Mobyle does not manage non-file outputs self.jobState.commit() def setValue(self, parameter_name, value=None, src=None, srcFileName=None): """MobyleJob-style set value method, called from JobFacade""" if type(value)==tuple: return self.set_value(parameter_name, value=value[1], src=value[2],srcFileName=value[3]) else: return self.set_value(parameter_name, value=value, src=src,srcFileName=srcFileName) def getJobid(self): """MobyleJob-style get job id method, called from JobFacade""" return self.id def getDate(self): """MobyleJob-style get date method, called from JobFacade""" return time.strptime(self.get_date(),"%x %X") def getStatus(self): """MobyleJob-style get status method, called from JobFacade""" return self.status_manager.getStatus( self.getDir() ) def get_value(self, parameter_name): """get a parameter value""" return self.parameters.get(parameter_name,None) def get_date(self): """get the job date as a string""" return self.jobState.getDate() def get_id(self): """get the job id""" return self.id def run(self): """submit the job asynchronously""" self.validate() self.set_status(Status( code = 1 )) # status = submitted #raise a UserValueError if nb of job is over the limit accepted if( self.email is not None ): self._job.over_limit( self.email , '' ) self._child_pid = os.fork() if self._child_pid==0: #Child code os.setsid() log_fd = os.open("%s/log" % self.jobState.getDir(), os.O_APPEND | os.O_WRONLY | os.O_CREAT , 0664 ) devnull = os.open( "/dev/null" , os.O_RDWR ) os.dup2( devnull , sys.stdin.fileno() ) os.close( devnull) os.dup2( log_fd , sys.stdout.fileno() ) os.dup2( log_fd , sys.stderr.fileno() ) os.close( log_fd ) atexit.register( self.log , "child exit for workflow id: %s" % self.get_id()) ################################################ service = self._job.getService() serviceName = service.getName() jobKey = self._job.getKey() linkName = ( "%s/%s.%s" %( self.cfg.admindir() , serviceName , jobKey ) ) try: os.symlink( os.path.join( self.getDir() , '.admin') , linkName ) except OSError , err: self.set_status(Status(string="error", message="workflow execution failed")) msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err ) log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info = True ) raise WorkflowJobError , msg ################################################ t0 = time.time() self.srun() t1 = time.time() ################################################ try: os.unlink( linkName ) except OSError , err: self.set_status(Status(string="error", message="workflow execution failed")) msg = "can't remove symbolic link %s in ADMINDIR: %s" %( linkName , err ) log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info= True ) raise WorkflowJobError , msg ################################################ try: zipFileName = self.zip_results() except Exception : msg = "an error occured during the zipping results :\n\n" log.critical( "%s : %s" %( self.id , msg ) , exc_info = True) zipFileName = None if self.email_notify == 'auto': if ( t1 - t0 ) > self.cfg.email_delay() : emailResults( self.cfg , self.email, #userEmail, registry, self.id, self.getDir(), self.workflow.getName(), self._job.getKey(), FileName = zipFileName ) elif self.email_notify == 'true': emailResults( self.cfg , self.email, #userEmail, registry, self.id, self.getDir(), self.workflow.getName(), self._job.getKey(), FileName = zipFileName ) else: pass sys.exit(0) #exit with no error
def __init__(self, commandLine, dirPath, serviceName, resultsMask , userEmail = None, email_notify = 'auto' , jobState = None , xmlEnv = None): """ @param commandLine: the command to be executed @type commandLine: String @param dirPath: the absolute path to directory where the job will be executed (normaly we are already in) @type dirPath: String @param serviceName: the name of the service @type serviceName: string @param resultsMask: the unix mask to retrieve the results of this job @type resultsMask: a dictionary { paramName : [ string prompt , ( string class , string or None superclass ) , string mask ] } @param userEmail: the user email adress @type userEmail: string @param email_notify: if the user must be or not notify of the results at the end of the Job. the 3 authorized values for this argument are: - 'true' to notify the results to the user - 'false' to Not notify the results to the user - 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY @type email_notify: string @param jobState: the jobState link to this job @type jobState: a L{JobState} instance @param xmlEnv: the environement variable need by the program @type xmlEnv: dictionnary @call: by the main of this module which is call by L{AsynchronRunner} """ self._command = commandLine self._dirPath = dirPath self.serviceName = serviceName self.father_pid = os.getppid() self.father_done = False if jobState is None: self.jobState = JobState( self._dirPath ) else: self.jobState = jobState self.userEmail = userEmail self.email_notify = email_notify if self._dirPath[-1] == '/': self._dirPath = self._dirPath[:-1] self.jobKey = os.path.split( self._dirPath )[ 1 ] atexit.register( self.childExit , "------------------- %s : %s -------------------" %( serviceName , self.jobKey ) ) t0 = time.time() ############################################ self._run( serviceName , xmlEnv ) ############################################# t1 = time.time() self.results = {} for paramName in resultsMask.keys(): resultsFiles = [] #type is a tuple ( klass , superKlass ) masks = resultsMask[ paramName ] for mask in masks : for File in glob.glob( mask ): size = os.path.getsize( File ) if size != 0: resultsFiles.append( ( str( File ) , size , None ) ) #we have not information about the output format if resultsFiles: self.results[ paramName ] = resultsFiles #a list of tuple (string file name , int size , string format or None ) self.jobState.setOutputDataFile( paramName , resultsFiles ) self.jobState.commit() try: zipFileName = self.zipResults() except Exception : msg = "an error occured during the zipping results :\n\n" rc_log.critical( "%s/%s : %s" %( self.serviceName , self.jobKey , msg ) , exc_info = True) zipFileName = None if self.userEmail: if self.email_notify == 'auto': # we test email_delay() to see if it is >= to 0, # as it seems that sometimes it is not >0. if ( t1 - t0 ) >= _cfg.email_delay(): emailResults(_cfg, self.userEmail, registry , self.jobState.getID(), self._dirPath , self.serviceName , self.jobKey , FileName = zipFileName ) elif self.email_notify == 'true': emailResults( _cfg, self.userEmail, registry , self.jobState.getID(), self._dirPath , self.serviceName , self.jobKey , FileName = zipFileName ) else: pass