Esempio n. 1
0
 def getOutputParameterRefs(self, name):
     js = JobState(self.jobId)
     outputRefs = js.getOutput(name)
     values = []
     if outputRefs:
         for outputRef in outputRefs:
             values.append({'src':self.jobId,'srcFileName':outputRef[0],'size':outputRef[1],'fmt':outputRef[2]})
     return values
Esempio n. 2
0
 def getOutputParameterValues(self, name):
     js = JobState(self.jobId)
     outputRefs = js.getOutput(name)
     values = []
     if outputRefs:
         for outputRef in outputRefs:
             fh = js.open(outputRef[0])
             value = ''.join( fh.readlines() )
             values.append({'value':value,'ref':self.jobId+'/'+outputRef[0],'size':outputRef[1],'fmt':outputRef[2]})
     return values
    def __init__( self, job , jobState = None  ):
        """
        instanciate a L{CommandBuilder} and use it to build the CommandLine, then run it
        @param service:
        @type job: a L{Job} instance
        @param jobState:
        @type jobState:
        @call: l{Job.run}
        @todo: implementation pour le warper de cgi ou WS a faire
        """
        self.job = job
        self.job_dir = self.job.getDir()
        self._service = self.job.getService()
        if jobState is None:
            self._jobState = JobState( self.job_dir )
        else:
            self._jobState = jobState
        
        commandBuilder = CommandBuilder()
        
        method = self._service.getCommand()[1].upper()

        if method == '' or method == 'LOCAL':

            try:
                cmd = commandBuilder.buildLocalCommand( self._service )
                self._commandLine = cmd[ 'cmd' ]
                self._xmlEnv = cmd[ 'env' ]
                paramfiles = cmd[ 'paramfiles' ]
            except Exception ,err :
                msg = "an error occured during the command line building: " 
                self._logError( userMsg = "Mobyle Internal Server Error",
                                logMsg = None #the error log is filled by the rf_log.critical
                                )
                #this error is already log in build.log
                msg = "%s/%s : %s" %( self._service.getName() ,
                                                   self.job.getKey() , 
                                                   msg , 
                                                   )
                
                if self.job.cfg.debug( self._service.getName() ) == 0:
                    rf_log.critical( msg  , exc_info = True) # send an email
                else:
                    rf_log.error( msg , exc_info = True) 
                 
                raise MobyleError , "Mobyle Internal Server Error"
            js_paramfiles = []
            if paramfiles :
                for paramfile_name , string_handle  in paramfiles.items():
                    paramfile_handle = open( os.path.join(  self.job_dir , paramfile_name ) , 'w' )
                    content = string_handle.getvalue()
                    paramfile_handle.write( content )
                    paramfile_handle.close()
                    js_paramfiles.append( ( os.path.basename( paramfile_name ) , len( content ) ) )
                self._jobState.setParamfiles( js_paramfiles )
                self._jobState.commit()
Esempio n. 4
0
def killJob(jobID):
    """
    @param jobID: the url of the job or a sequence of jobID
    @type jobID: string or sequence of jobID
    @return: 
    @rtype: 
    @raise MobyleError: if the job has no number or if the job doesn't exist anymore
    @todo: tester la partie sge
    """
    from types import StringTypes, TupleType, ListType
    from Mobyle.MobyleError import MobyleError
    from Mobyle.JobState import JobState, normUri
    from Mobyle.Job import Job
    from Mobyle.WorkflowJob import WorkflowJob

    if isinstance(jobID, StringTypes):
        jobIDs = [jobID]
    elif isinstance(jobID, (ListType, TupleType)):
        jobIDs = jobID
    else:
        raise MobyleError, "jobID must be a string or a Sequence of strings :%s" % type(jobID)

    errors = []
    for jobID in jobIDs:
        try:
            path = normUri(jobID)
        except MobyleError, err:
            errors.append((jobID, str(err)))
            continue
        if path[:4] == "http":
            # the jobID is not on this Mobyle server
            errors.append((jobID, "can't kill distant job"))
            continue
        js = JobState(uri=jobID)
        if js.isWorkflow():
            job = WorkflowJob(id=jobID)
        else:
            job = Job(ID=jobID)
        try:
            job.kill()
        except MobyleError, err:
            errors.append((jobID, str(err)))
            continue
Esempio n. 5
0
 def getSubJobs(self):
     """
     gets the list of subjobs (along with their status)
     @return: subjobs list of dictionaries
     @rtype: list
     """
     js = JobState(self.jobId)
     if hasattr(js, 'getSubJobs'):
         subjobs = js.getSubJobs()
         subsubjobs = []
         for s in subjobs:
             s['jobPID'] = registry.getJobPID(self.jobId) + '::'+ registry.getJobPID(s['jobID'])
             child_jf = JobFacade.getFromJobId(s['jobID'])
             s['status'] = child_jf.getStatus()
             for ss in child_jf.getSubJobs():
                 ss['jobPID'] = s['jobPID'] + '::' + registry.getJobPID(ss['jobID'])
                 schild_jf = JobFacade.getFromJobId(ss['jobID'])
                 ss['status'] = schild_jf.getStatus() 
                 subsubjobs.append(ss)                   
         subjobs.extend(subsubjobs)
         return subjobs
     else:
         return []
Esempio n. 6
0
 def getFromJobId(cls, jobId):
     """
     create a JobFacade to access an existing job.
     @param jobId: the job identifier
     @type jobId: string
     @return: the appropriate job facade
     @rtype: JobFacade
     """
     jobState = JobState(jobId)
     jfargs = {'jobId': jobState.getID(),'programUrl':None,'workflowUrl':None}
     if jobState.isWorkflow():
         jfargs['workflowUrl'] = jobState.getName()
     else:
         jfargs['programUrl']= jobState.getName()
     # this id is identical to the one in parameter, 
     # except it has been normalized (may have removed
     # trailing index.xml from the id)
     if jobState.isLocal():
         return(LocalJobFacade(**jfargs))
     else:
         return(RemoteJobFacade(**jfargs))
Esempio n. 7
0
 def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None):
     """
     @param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id)
     @type id: string 
     @param workflow: the workflow definition used to create a new job
     @type workflow: a L{Workflow} instance
     @param email: the user email address
     @type email: L{EmailAddress} instance or a string
     @param email_notify: if the user must be or not notify of the results at the end of the Job.
     the 3 authorized values for this argument are: 
       - 'true' to notify the results to the user
       - 'false' to Not notify the results to the user
       - 'auto' to notify the results based on the job elapsed time and the config  EMAIL_DELAY
     @type email_notify: string 
     @param session: the session owner of this workflow (if session is set workflowID mut be None )
     @type session: a L{Session} instance
     @param workflowID: the ID of a the workflow owner of this workflow
     @type workflowID: string
     """
     self.cfg = ConfigManager.Config()
     self.status_manager = StatusManager()
     if id:
         log.debug("accessing WorkflowJob %s" %(id))
         self.id = id
         self.jobState = JobState( id )
     else:
         log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name))
         self.workflow = workflow
         if session and workflowID:
             msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(),
                                                                                                workflowID
                                                                                               )
             log.error( msg )
             raise MobyleError( msg )
         self.session = session
         if session :
             email = session.getEmail()
             if email:
                 self.email = EmailAddress( email )
             else:
                 self.email = None
         elif email : #there is an email without session
             if  not isinstance( email , EmailAddress ):
                 self.email = EmailAddress( email )
             else:
                 self.email = email
         
         self.email_notify =  email_notify  
         if self.email_notify != 'false' and not self.email:
             raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify )
         
         self.parameters = {}
         for parameter in self.workflow.parameters:
             # setting parameters which have a default value (important for hidden parameters which are not 
             # accessed by JobFacade...
             if not(parameter.isout) and parameter.vdef is not None:
                 self.set_value(parameter.name, value=str(parameter.vdef))
         # job is just an "environment" folder for the job
         # it contains the instanciation of the job runner which seems to be hardcoded as "command runner"...
         self._job = Job( service = self.workflow,
                          cfg = self.cfg,
                          userEmail = self.email,
                          session = self.session,
                          workflowID = workflowID ,
                          )
         self.jobState = self._job.jobState
         self.id = self._job.getURL()
Esempio n. 8
0
class WorkflowJob(object):
    
    def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None):
        """
        @param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id)
        @type id: string 
        @param workflow: the workflow definition used to create a new job
        @type workflow: a L{Workflow} instance
        @param email: the user email address
        @type email: L{EmailAddress} instance or a string
        @param email_notify: if the user must be or not notify of the results at the end of the Job.
        the 3 authorized values for this argument are: 
          - 'true' to notify the results to the user
          - 'false' to Not notify the results to the user
          - 'auto' to notify the results based on the job elapsed time and the config  EMAIL_DELAY
        @type email_notify: string 
        @param session: the session owner of this workflow (if session is set workflowID mut be None )
        @type session: a L{Session} instance
        @param workflowID: the ID of a the workflow owner of this workflow
        @type workflowID: string
        """
        self.cfg = ConfigManager.Config()
        self.status_manager = StatusManager()
        if id:
            log.debug("accessing WorkflowJob %s" %(id))
            self.id = id
            self.jobState = JobState( id )
        else:
            log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name))
            self.workflow = workflow
            if session and workflowID:
                msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(),
                                                                                                   workflowID
                                                                                                  )
                log.error( msg )
                raise MobyleError( msg )
            self.session = session
            if session :
                email = session.getEmail()
                if email:
                    self.email = EmailAddress( email )
                else:
                    self.email = None
            elif email : #there is an email without session
                if  not isinstance( email , EmailAddress ):
                    self.email = EmailAddress( email )
                else:
                    self.email = email
            
            self.email_notify =  email_notify  
            if self.email_notify != 'false' and not self.email:
                raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify )
            
            self.parameters = {}
            for parameter in self.workflow.parameters:
                # setting parameters which have a default value (important for hidden parameters which are not 
                # accessed by JobFacade...
                if not(parameter.isout) and parameter.vdef is not None:
                    self.set_value(parameter.name, value=str(parameter.vdef))
            # job is just an "environment" folder for the job
            # it contains the instanciation of the job runner which seems to be hardcoded as "command runner"...
            self._job = Job( service = self.workflow,
                             cfg = self.cfg,
                             userEmail = self.email,
                             session = self.session,
                             workflowID = workflowID ,
                             )
            self.jobState = self._job.jobState
            self.id = self._job.getURL()
            
    def getDir(self):        
        """ returns the absolute path of the workflow job directory """
        return self.jobState.getDir()
    
    def set_status(self, status):
        log.debug("setting job %s status to %s" % (self.id, status))
        self.status_manager.setStatus( self.getDir() , status )
        
    def set_value(self, parameter_name, value=None, src=None, srcFileName=None):
        wf_parameter = [p for p in self.workflow.parameters if p.name==parameter_name][0]
        if value is not None:
            log.debug("setting %s parameter value to %s" %(parameter_name, value))
        elif src is not None:
            log.debug("copying %s parameter value from %s/%s" %(parameter_name, src,srcFileName))
        else:
            log.error("no VALUE or SOURCE URL specified for %s parameter." % parameter_name)            
        """ set a parameter value """
        self.parameters[parameter_name] = value
        self.parameters[parameter_name + '.src'] = src
        self.parameters[parameter_name + '.srcFileName'] = srcFileName
        if value and value==wf_parameter.vdef:
            log.debug("setting %s parameter value to default value %s" %(parameter_name, wf_parameter.vdef))
            return            
        # save input value in a file
        # link this file from the JobState xml
        datatype_class = wf_parameter.type.datatype.class_name
        datatype_superclass = wf_parameter.type.datatype.superclass_name
        df = DataTypeFactory()
        if (datatype_superclass in [None,""] ):
            dt = df.newDataType(datatype_class)
        else:
            dt = df.newDataType(datatype_superclass, datatype_class)
        mt = MobyleType(dt)
        p = Parameter(mt, name=parameter_name)
        p._isout = wf_parameter.isout
        if dt.isFile():
            file_name = parameter_name+'.data'
            if src:
                src = DataProvider.get(src)
            file_name, size = mt.toFile( value , self , file_name, src , srcFileName  )
            if not(wf_parameter.isout):
                self.jobState.setInputDataFile(parameter_name, (file_name, size, None))
            else:
                self.jobState.setOutputDataFile(parameter_name, [(file_name, size, None)])
        else:
            if not(wf_parameter.isout):            
                self.jobState.setInputDataValue(parameter_name, value)
            else:
                raise NotImplementedError() # so far Mobyle does not manage non-file outputs
        self.jobState.commit()
    
    def setValue(self, parameter_name, value=None, src=None, srcFileName=None):
        """MobyleJob-style set value method, called from JobFacade"""
        if type(value)==tuple:
            return self.set_value(parameter_name, value=value[1], src=value[2],srcFileName=value[3])
        else:
            return self.set_value(parameter_name, value=value, src=src,srcFileName=srcFileName)
        
    def getJobid(self):
        """MobyleJob-style get job id method, called from JobFacade"""
        return self.id

    def getDate(self):
        """MobyleJob-style get date method, called from JobFacade"""
        return time.strptime(self.get_date(),"%x  %X")
    
    def getStatus(self):
        """MobyleJob-style get status method, called from JobFacade"""
        return self.status_manager.getStatus( self.getDir() )
            
    def get_value(self, parameter_name):
        """get a parameter value"""
        return self.parameters.get(parameter_name,None)

    def get_date(self):
        """get the job date as a string"""
        return self.jobState.getDate()

    def get_id(self):
        """get the job id"""
        return self.id
    
    def run(self):
        """submit the job asynchronously"""
        self.validate()
        self.set_status(Status( code = 1 )) # status = submitted
        
        #raise a UserValueError if nb of job is over the limit accepted
        if( self.email is not None ):
            self._job.over_limit( self.email , '' )
        
        self._child_pid = os.fork()
        if self._child_pid==0:
            #Child code
            os.setsid()
            log_fd = os.open("%s/log" % self.jobState.getDir(), os.O_APPEND | os.O_WRONLY | os.O_CREAT , 0664 )  
            devnull = os.open( "/dev/null" , os.O_RDWR )
            os.dup2( devnull , sys.stdin.fileno() )
            os.close( devnull)
            os.dup2( log_fd  , sys.stdout.fileno() )
            os.dup2( log_fd  , sys.stderr.fileno() )
            os.close( log_fd )
            atexit.register( self.log , "child exit for workflow id: %s" % self.get_id())
            
            ################################################
            service = self._job.getService()
            serviceName = service.getName()
            jobKey = self._job.getKey()
             
            linkName = ( "%s/%s.%s" %( self.cfg.admindir() ,
                                       serviceName ,
                                       jobKey
                                       )
                                       )
            try:
                
                os.symlink(
                           os.path.join( self.getDir() , '.admin') ,
                           linkName
                           )
            except OSError , err:
                self.set_status(Status(string="error", message="workflow execution failed"))
                msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err )
                log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info = True )
                raise WorkflowJobError , msg
        
            ################################################       
            t0 = time.time()
            self.srun()
            t1 = time.time()
            ################################################
            try:
                os.unlink( linkName )
            except OSError , err:
                self.set_status(Status(string="error", message="workflow execution failed"))
                msg = "can't remove symbolic link %s in ADMINDIR: %s" %( linkName , err )
                log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info= True )
                raise WorkflowJobError , msg
            ################################################
            try:
                zipFileName = self.zip_results()
            except Exception :
                msg = "an error occured during the zipping results :\n\n"
                log.critical( "%s : %s" %( self.id , msg ) , exc_info = True)
                zipFileName = None
                
            if self.email_notify == 'auto':
                if ( t1 - t0 ) > self.cfg.email_delay() :
                    emailResults(  self.cfg ,
                                   self.email, #userEmail, 
                                   registry, 
                                   self.id, 
                                   self.getDir(), 
                                   self.workflow.getName(),
                                   self._job.getKey(),  
                                   FileName = zipFileName )
                elif self.email_notify == 'true':
                    emailResults(  self.cfg ,
                                   self.email, #userEmail, 
                                   registry, 
                                   self.id, 
                                   self.getDir(), 
                                   self.workflow.getName(),
                                   self._job.getKey(),  
                                   FileName = zipFileName )
                else:
                    pass    
            sys.exit(0) #exit with no error
Esempio n. 9
0
    def __init__(self, commandLine, dirPath, serviceName, resultsMask , userEmail = None, email_notify = 'auto' , jobState = None , xmlEnv = None):
        """
        @param commandLine: the command to be executed
        @type commandLine: String
        @param dirPath: the absolute path to directory where the job will be executed (normaly we are already in)
        @type dirPath: String
        @param serviceName: the name of the service
        @type serviceName: string
        @param resultsMask: the unix mask to retrieve the results of this job
        @type resultsMask: a dictionary { paramName : [ string prompt , ( string class , string or None superclass ) , string mask ] }
        @param userEmail: the user email adress
        @type userEmail: string
        @param email_notify: if the user must be or not notify of the results at the end of the Job.
        the 3 authorized values for this argument are: 
          - 'true' to notify the results to the user
          - 'false' to Not notify the results to the user
          - 'auto' to notify the results based on the job elapsed time and the config  EMAIL_DELAY
        @type email_notify: string 
        @param jobState: the jobState link to this job
        @type jobState: a L{JobState} instance
        @param xmlEnv: the environement variable need by the program
        @type xmlEnv: dictionnary
        @call: by the main of this module which is call by L{AsynchronRunner}
        """
        self._command = commandLine
        self._dirPath = dirPath
        self.serviceName = serviceName
        self.father_pid = os.getppid()
        self.father_done = False
        if jobState is None:
            self.jobState = JobState( self._dirPath )
        else:
            self.jobState = jobState
        self.userEmail = userEmail
        self.email_notify =  email_notify 
        
        if self._dirPath[-1] == '/':
            self._dirPath = self._dirPath[:-1]

        self.jobKey = os.path.split( self._dirPath )[ 1 ]
        
        atexit.register( self.childExit , "------------------- %s : %s -------------------" %( serviceName , self.jobKey ) )
        
        t0 = time.time()
        ############################################
        self._run( serviceName , xmlEnv )
        #############################################
        t1 = time.time()
        
        self.results = {}
        for paramName in resultsMask.keys():
            resultsFiles = []
            #type is a tuple ( klass , superKlass )
            masks = resultsMask[ paramName ]
            for mask in masks :
                for File in  glob.glob( mask ):
                    size = os.path.getsize( File )
                    if size != 0:
                        resultsFiles.append(  ( str( File ) , size , None ) ) #we have not information about the output format 
            if resultsFiles: 
                self.results[ paramName ] = resultsFiles  #a list of tuple (string file name , int size ,  string format or None )
                self.jobState.setOutputDataFile( paramName , resultsFiles )

        self.jobState.commit()
        try:
            zipFileName = self.zipResults()
        except Exception :
            msg = "an error occured during the zipping results :\n\n"
            rc_log.critical( "%s/%s : %s" %( self.serviceName , self.jobKey , msg ) , exc_info = True)
            zipFileName = None
        if self.userEmail:
            if self.email_notify == 'auto':
                # we test email_delay() to see if it is >= to 0, 
                # as it seems that sometimes it is not >0.
                if ( t1 - t0 ) >= _cfg.email_delay():
                    emailResults(_cfg,
                                       self.userEmail,
                                       registry ,
                                       self.jobState.getID(),
                                       self._dirPath ,
                                       self.serviceName ,
                                       self.jobKey ,
                                      FileName = zipFileName )
            elif self.email_notify == 'true':
                emailResults( _cfg,
                                   self.userEmail,
                                   registry ,
                                   self.jobState.getID(),
                                   self._dirPath ,
                                   self.serviceName ,
                                   self.jobKey ,  
                                   FileName = zipFileName )
            else:
                pass