def getOutputParameterRefs(self, name):
js = JobState(self.jobId)
outputRefs = js.getOutput(name)
values = []
if outputRefs:
for outputRef in outputRefs:
values.append({'src':self.jobId,'srcFileName':outputRef[0],'size':outputRef[1],'fmt':outputRef[2]})
return values
def getOutputParameterValues(self, name):
js = JobState(self.jobId)
outputRefs = js.getOutput(name)
values = []
if outputRefs:
for outputRef in outputRefs:
fh = js.open(outputRef[0])
value = ''.join( fh.readlines() )
values.append({'value':value,'ref':self.jobId+'/'+outputRef[0],'size':outputRef[1],'fmt':outputRef[2]})
return values
def __init__( self, job , jobState = None ):
"""
instanciate a L{CommandBuilder} and use it to build the CommandLine, then run it
@param service:
@type job: a L{Job} instance
@param jobState:
@type jobState:
@call: l{Job.run}
@todo: implementation pour le warper de cgi ou WS a faire
"""
self.job = job
self.job_dir = self.job.getDir()
self._service = self.job.getService()
if jobState is None:
self._jobState = JobState( self.job_dir )
else:
self._jobState = jobState
commandBuilder = CommandBuilder()
method = self._service.getCommand()[1].upper()
if method == '' or method == 'LOCAL':
try:
cmd = commandBuilder.buildLocalCommand( self._service )
self._commandLine = cmd[ 'cmd' ]
self._xmlEnv = cmd[ 'env' ]
paramfiles = cmd[ 'paramfiles' ]
except Exception ,err :
msg = "an error occured during the command line building: "
self._logError( userMsg = "Mobyle Internal Server Error",
logMsg = None #the error log is filled by the rf_log.critical
)
#this error is already log in build.log
msg = "%s/%s : %s" %( self._service.getName() ,
self.job.getKey() ,
msg ,
)
if self.job.cfg.debug( self._service.getName() ) == 0:
rf_log.critical( msg , exc_info = True) # send an email
else:
rf_log.error( msg , exc_info = True)
raise MobyleError , "Mobyle Internal Server Error"
js_paramfiles = []
if paramfiles :
for paramfile_name , string_handle in paramfiles.items():
paramfile_handle = open( os.path.join( self.job_dir , paramfile_name ) , 'w' )
content = string_handle.getvalue()
paramfile_handle.write( content )
paramfile_handle.close()
js_paramfiles.append( ( os.path.basename( paramfile_name ) , len( content ) ) )
self._jobState.setParamfiles( js_paramfiles )
self._jobState.commit()
def killJob(jobID):
"""
@param jobID: the url of the job or a sequence of jobID
@type jobID: string or sequence of jobID
@return:
@rtype:
@raise MobyleError: if the job has no number or if the job doesn't exist anymore
@todo: tester la partie sge
"""
from types import StringTypes, TupleType, ListType
from Mobyle.MobyleError import MobyleError
from Mobyle.JobState import JobState, normUri
from Mobyle.Job import Job
from Mobyle.WorkflowJob import WorkflowJob
if isinstance(jobID, StringTypes):
jobIDs = [jobID]
elif isinstance(jobID, (ListType, TupleType)):
jobIDs = jobID
else:
raise MobyleError, "jobID must be a string or a Sequence of strings :%s" % type(jobID)
errors = []
for jobID in jobIDs:
try:
path = normUri(jobID)
except MobyleError, err:
errors.append((jobID, str(err)))
continue
if path[:4] == "http":
# the jobID is not on this Mobyle server
errors.append((jobID, "can't kill distant job"))
continue
js = JobState(uri=jobID)
if js.isWorkflow():
job = WorkflowJob(id=jobID)
else:
job = Job(ID=jobID)
try:
job.kill()
except MobyleError, err:
errors.append((jobID, str(err)))
continue
def getSubJobs(self):
"""
gets the list of subjobs (along with their status)
@return: subjobs list of dictionaries
@rtype: list
"""
js = JobState(self.jobId)
if hasattr(js, 'getSubJobs'):
subjobs = js.getSubJobs()
subsubjobs = []
for s in subjobs:
s['jobPID'] = registry.getJobPID(self.jobId) + '::'+ registry.getJobPID(s['jobID'])
child_jf = JobFacade.getFromJobId(s['jobID'])
s['status'] = child_jf.getStatus()
for ss in child_jf.getSubJobs():
ss['jobPID'] = s['jobPID'] + '::' + registry.getJobPID(ss['jobID'])
schild_jf = JobFacade.getFromJobId(ss['jobID'])
ss['status'] = schild_jf.getStatus()
subsubjobs.append(ss)
subjobs.extend(subsubjobs)
return subjobs
else:
return []
def getFromJobId(cls, jobId):
"""
create a JobFacade to access an existing job.
@param jobId: the job identifier
@type jobId: string
@return: the appropriate job facade
@rtype: JobFacade
"""
jobState = JobState(jobId)
jfargs = {'jobId': jobState.getID(),'programUrl':None,'workflowUrl':None}
if jobState.isWorkflow():
jfargs['workflowUrl'] = jobState.getName()
else:
jfargs['programUrl']= jobState.getName()
# this id is identical to the one in parameter,
# except it has been normalized (may have removed
# trailing index.xml from the id)
if jobState.isLocal():
return(LocalJobFacade(**jfargs))
else:
return(RemoteJobFacade(**jfargs))
def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None):
"""
@param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id)
@type id: string
@param workflow: the workflow definition used to create a new job
@type workflow: a L{Workflow} instance
@param email: the user email address
@type email: L{EmailAddress} instance or a string
@param email_notify: if the user must be or not notify of the results at the end of the Job.
the 3 authorized values for this argument are:
- 'true' to notify the results to the user
- 'false' to Not notify the results to the user
- 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY
@type email_notify: string
@param session: the session owner of this workflow (if session is set workflowID mut be None )
@type session: a L{Session} instance
@param workflowID: the ID of a the workflow owner of this workflow
@type workflowID: string
"""
self.cfg = ConfigManager.Config()
self.status_manager = StatusManager()
if id:
log.debug("accessing WorkflowJob %s" %(id))
self.id = id
self.jobState = JobState( id )
else:
log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name))
self.workflow = workflow
if session and workflowID:
msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(),
workflowID
)
log.error( msg )
raise MobyleError( msg )
self.session = session
if session :
email = session.getEmail()
if email:
self.email = EmailAddress( email )
else:
self.email = None
elif email : #there is an email without session
if not isinstance( email , EmailAddress ):
self.email = EmailAddress( email )
else:
self.email = email
self.email_notify = email_notify
if self.email_notify != 'false' and not self.email:
raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify )
self.parameters = {}
for parameter in self.workflow.parameters:
# setting parameters which have a default value (important for hidden parameters which are not
# accessed by JobFacade...
if not(parameter.isout) and parameter.vdef is not None:
self.set_value(parameter.name, value=str(parameter.vdef))
# job is just an "environment" folder for the job
# it contains the instanciation of the job runner which seems to be hardcoded as "command runner"...
self._job = Job( service = self.workflow,
cfg = self.cfg,
userEmail = self.email,
session = self.session,
workflowID = workflowID ,
)
self.jobState = self._job.jobState
self.id = self._job.getURL()
class WorkflowJob(object):
def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None):
"""
@param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id)
@type id: string
@param workflow: the workflow definition used to create a new job
@type workflow: a L{Workflow} instance
@param email: the user email address
@type email: L{EmailAddress} instance or a string
@param email_notify: if the user must be or not notify of the results at the end of the Job.
the 3 authorized values for this argument are:
- 'true' to notify the results to the user
- 'false' to Not notify the results to the user
- 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY
@type email_notify: string
@param session: the session owner of this workflow (if session is set workflowID mut be None )
@type session: a L{Session} instance
@param workflowID: the ID of a the workflow owner of this workflow
@type workflowID: string
"""
self.cfg = ConfigManager.Config()
self.status_manager = StatusManager()
if id:
log.debug("accessing WorkflowJob %s" %(id))
self.id = id
self.jobState = JobState( id )
else:
log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name))
self.workflow = workflow
if session and workflowID:
msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(),
workflowID
)
log.error( msg )
raise MobyleError( msg )
self.session = session
if session :
email = session.getEmail()
if email:
self.email = EmailAddress( email )
else:
self.email = None
elif email : #there is an email without session
if not isinstance( email , EmailAddress ):
self.email = EmailAddress( email )
else:
self.email = email
self.email_notify = email_notify
if self.email_notify != 'false' and not self.email:
raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify )
self.parameters = {}
for parameter in self.workflow.parameters:
# setting parameters which have a default value (important for hidden parameters which are not
# accessed by JobFacade...
if not(parameter.isout) and parameter.vdef is not None:
self.set_value(parameter.name, value=str(parameter.vdef))
# job is just an "environment" folder for the job
# it contains the instanciation of the job runner which seems to be hardcoded as "command runner"...
self._job = Job( service = self.workflow,
cfg = self.cfg,
userEmail = self.email,
session = self.session,
workflowID = workflowID ,
)
self.jobState = self._job.jobState
self.id = self._job.getURL()
def getDir(self):
""" returns the absolute path of the workflow job directory """
return self.jobState.getDir()
def set_status(self, status):
log.debug("setting job %s status to %s" % (self.id, status))
self.status_manager.setStatus( self.getDir() , status )
def set_value(self, parameter_name, value=None, src=None, srcFileName=None):
wf_parameter = [p for p in self.workflow.parameters if p.name==parameter_name][0]
if value is not None:
log.debug("setting %s parameter value to %s" %(parameter_name, value))
elif src is not None:
log.debug("copying %s parameter value from %s/%s" %(parameter_name, src,srcFileName))
else:
log.error("no VALUE or SOURCE URL specified for %s parameter." % parameter_name)
""" set a parameter value """
self.parameters[parameter_name] = value
self.parameters[parameter_name + '.src'] = src
self.parameters[parameter_name + '.srcFileName'] = srcFileName
if value and value==wf_parameter.vdef:
log.debug("setting %s parameter value to default value %s" %(parameter_name, wf_parameter.vdef))
return
# save input value in a file
# link this file from the JobState xml
datatype_class = wf_parameter.type.datatype.class_name
datatype_superclass = wf_parameter.type.datatype.superclass_name
df = DataTypeFactory()
if (datatype_superclass in [None,""] ):
dt = df.newDataType(datatype_class)
else:
dt = df.newDataType(datatype_superclass, datatype_class)
mt = MobyleType(dt)
p = Parameter(mt, name=parameter_name)
p._isout = wf_parameter.isout
if dt.isFile():
file_name = parameter_name+'.data'
if src:
src = DataProvider.get(src)
file_name, size = mt.toFile( value , self , file_name, src , srcFileName )
if not(wf_parameter.isout):
self.jobState.setInputDataFile(parameter_name, (file_name, size, None))
else:
self.jobState.setOutputDataFile(parameter_name, [(file_name, size, None)])
else:
if not(wf_parameter.isout):
self.jobState.setInputDataValue(parameter_name, value)
else:
raise NotImplementedError() # so far Mobyle does not manage non-file outputs
self.jobState.commit()
def setValue(self, parameter_name, value=None, src=None, srcFileName=None):
"""MobyleJob-style set value method, called from JobFacade"""
if type(value)==tuple:
return self.set_value(parameter_name, value=value[1], src=value[2],srcFileName=value[3])
else:
return self.set_value(parameter_name, value=value, src=src,srcFileName=srcFileName)
def getJobid(self):
"""MobyleJob-style get job id method, called from JobFacade"""
return self.id
def getDate(self):
"""MobyleJob-style get date method, called from JobFacade"""
return time.strptime(self.get_date(),"%x %X")
def getStatus(self):
"""MobyleJob-style get status method, called from JobFacade"""
return self.status_manager.getStatus( self.getDir() )
def get_value(self, parameter_name):
"""get a parameter value"""
return self.parameters.get(parameter_name,None)
def get_date(self):
"""get the job date as a string"""
return self.jobState.getDate()
def get_id(self):
"""get the job id"""
return self.id
def run(self):
"""submit the job asynchronously"""
self.validate()
self.set_status(Status( code = 1 )) # status = submitted
#raise a UserValueError if nb of job is over the limit accepted
if( self.email is not None ):
self._job.over_limit( self.email , '' )
self._child_pid = os.fork()
if self._child_pid==0:
#Child code
os.setsid()
log_fd = os.open("%s/log" % self.jobState.getDir(), os.O_APPEND | os.O_WRONLY | os.O_CREAT , 0664 )
devnull = os.open( "/dev/null" , os.O_RDWR )
os.dup2( devnull , sys.stdin.fileno() )
os.close( devnull)
os.dup2( log_fd , sys.stdout.fileno() )
os.dup2( log_fd , sys.stderr.fileno() )
os.close( log_fd )
atexit.register( self.log , "child exit for workflow id: %s" % self.get_id())
################################################
service = self._job.getService()
serviceName = service.getName()
jobKey = self._job.getKey()
linkName = ( "%s/%s.%s" %( self.cfg.admindir() ,
serviceName ,
jobKey
)
)
try:
os.symlink(
os.path.join( self.getDir() , '.admin') ,
linkName
)
except OSError , err:
self.set_status(Status(string="error", message="workflow execution failed"))
msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err )
log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info = True )
raise WorkflowJobError , msg
################################################
t0 = time.time()
self.srun()
t1 = time.time()
################################################
try:
os.unlink( linkName )
except OSError , err:
self.set_status(Status(string="error", message="workflow execution failed"))
msg = "can't remove symbolic link %s in ADMINDIR: %s" %( linkName , err )
log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info= True )
raise WorkflowJobError , msg
################################################
try:
zipFileName = self.zip_results()
except Exception :
msg = "an error occured during the zipping results :\n\n"
log.critical( "%s : %s" %( self.id , msg ) , exc_info = True)
zipFileName = None
if self.email_notify == 'auto':
if ( t1 - t0 ) > self.cfg.email_delay() :
emailResults( self.cfg ,
self.email, #userEmail,
registry,
self.id,
self.getDir(),
self.workflow.getName(),
self._job.getKey(),
FileName = zipFileName )
elif self.email_notify == 'true':
emailResults( self.cfg ,
self.email, #userEmail,
registry,
self.id,
self.getDir(),
self.workflow.getName(),
self._job.getKey(),
FileName = zipFileName )
else:
pass
sys.exit(0) #exit with no error
def __init__(self, commandLine, dirPath, serviceName, resultsMask , userEmail = None, email_notify = 'auto' , jobState = None , xmlEnv = None):
"""
@param commandLine: the command to be executed
@type commandLine: String
@param dirPath: the absolute path to directory where the job will be executed (normaly we are already in)
@type dirPath: String
@param serviceName: the name of the service
@type serviceName: string
@param resultsMask: the unix mask to retrieve the results of this job
@type resultsMask: a dictionary { paramName : [ string prompt , ( string class , string or None superclass ) , string mask ] }
@param userEmail: the user email adress
@type userEmail: string
@param email_notify: if the user must be or not notify of the results at the end of the Job.
the 3 authorized values for this argument are:
- 'true' to notify the results to the user
- 'false' to Not notify the results to the user
- 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY
@type email_notify: string
@param jobState: the jobState link to this job
@type jobState: a L{JobState} instance
@param xmlEnv: the environement variable need by the program
@type xmlEnv: dictionnary
@call: by the main of this module which is call by L{AsynchronRunner}
"""
self._command = commandLine
self._dirPath = dirPath
self.serviceName = serviceName
self.father_pid = os.getppid()
self.father_done = False
if jobState is None:
self.jobState = JobState( self._dirPath )
else:
self.jobState = jobState
self.userEmail = userEmail
self.email_notify = email_notify
if self._dirPath[-1] == '/':
self._dirPath = self._dirPath[:-1]
self.jobKey = os.path.split( self._dirPath )[ 1 ]
atexit.register( self.childExit , "------------------- %s : %s -------------------" %( serviceName , self.jobKey ) )
t0 = time.time()
############################################
self._run( serviceName , xmlEnv )
#############################################
t1 = time.time()
self.results = {}
for paramName in resultsMask.keys():
resultsFiles = []
#type is a tuple ( klass , superKlass )
masks = resultsMask[ paramName ]
for mask in masks :
for File in glob.glob( mask ):
size = os.path.getsize( File )
if size != 0:
resultsFiles.append( ( str( File ) , size , None ) ) #we have not information about the output format
if resultsFiles:
self.results[ paramName ] = resultsFiles #a list of tuple (string file name , int size , string format or None )
self.jobState.setOutputDataFile( paramName , resultsFiles )
self.jobState.commit()
try:
zipFileName = self.zipResults()
except Exception :
msg = "an error occured during the zipping results :\n\n"
rc_log.critical( "%s/%s : %s" %( self.serviceName , self.jobKey , msg ) , exc_info = True)
zipFileName = None
if self.userEmail:
if self.email_notify == 'auto':
# we test email_delay() to see if it is >= to 0,
# as it seems that sometimes it is not >0.
if ( t1 - t0 ) >= _cfg.email_delay():
emailResults(_cfg,
self.userEmail,
registry ,
self.jobState.getID(),
self._dirPath ,
self.serviceName ,
self.jobKey ,
FileName = zipFileName )
elif self.email_notify == 'true':
emailResults( _cfg,
self.userEmail,
registry ,
self.jobState.getID(),
self._dirPath ,
self.serviceName ,
self.jobKey ,
FileName = zipFileName )
else:
pass