class AsynchronJob:
"""
is instantiated in child process instantiate the object corresponding
to the execution manager defined in Config, and after the completion of the job
manage the results
"""
def __init__(self, commandLine, dirPath, serviceName, resultsMask , userEmail = None, email_notify = 'auto' , jobState = None , xmlEnv = None):
"""
@param commandLine: the command to be executed
@type commandLine: String
@param dirPath: the absolute path to directory where the job will be executed (normaly we are already in)
@type dirPath: String
@param serviceName: the name of the service
@type serviceName: string
@param resultsMask: the unix mask to retrieve the results of this job
@type resultsMask: a dictionary { paramName : [ string prompt , ( string class , string or None superclass ) , string mask ] }
@param userEmail: the user email adress
@type userEmail: string
@param email_notify: if the user must be or not notify of the results at the end of the Job.
the 3 authorized values for this argument are:
- 'true' to notify the results to the user
- 'false' to Not notify the results to the user
- 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY
@type email_notify: string
@param jobState: the jobState link to this job
@type jobState: a L{JobState} instance
@param xmlEnv: the environement variable need by the program
@type xmlEnv: dictionnary
@call: by the main of this module which is call by L{AsynchronRunner}
"""
self._command = commandLine
self._dirPath = dirPath
self.serviceName = serviceName
self.father_pid = os.getppid()
self.father_done = False
if jobState is None:
self.jobState = JobState( self._dirPath )
else:
self.jobState = jobState
self.userEmail = userEmail
self.email_notify = email_notify
if self._dirPath[-1] == '/':
self._dirPath = self._dirPath[:-1]
self.jobKey = os.path.split( self._dirPath )[ 1 ]
atexit.register( self.childExit , "------------------- %s : %s -------------------" %( serviceName , self.jobKey ) )
t0 = time.time()
############################################
self._run( serviceName , xmlEnv )
#############################################
t1 = time.time()
self.results = {}
for paramName in resultsMask.keys():
resultsFiles = []
#type is a tuple ( klass , superKlass )
masks = resultsMask[ paramName ]
for mask in masks :
for File in glob.glob( mask ):
size = os.path.getsize( File )
if size != 0:
resultsFiles.append( ( str( File ) , size , None ) ) #we have not information about the output format
if resultsFiles:
self.results[ paramName ] = resultsFiles #a list of tuple (string file name , int size , string format or None )
self.jobState.setOutputDataFile( paramName , resultsFiles )
self.jobState.commit()
try:
zipFileName = self.zipResults()
except Exception :
msg = "an error occured during the zipping results :\n\n"
rc_log.critical( "%s/%s : %s" %( self.serviceName , self.jobKey , msg ) , exc_info = True)
zipFileName = None
if self.userEmail:
if self.email_notify == 'auto':
# we test email_delay() to see if it is >= to 0,
# as it seems that sometimes it is not >0.
if ( t1 - t0 ) >= _cfg.email_delay():
emailResults(_cfg,
self.userEmail,
registry ,
self.jobState.getID(),
self._dirPath ,
self.serviceName ,
self.jobKey ,
FileName = zipFileName )
elif self.email_notify == 'true':
emailResults( _cfg,
self.userEmail,
registry ,
self.jobState.getID(),
self._dirPath ,
self.serviceName ,
self.jobKey ,
FileName = zipFileName )
else:
pass
def childExit(self , message ):
print >> sys.stderr , message
#rc_log.log( 12 , "runnerChild %d ending, send a SIGCHLD to %d" %( os.getpid() , self.father_pid ) )
def _run(self , serviceName, xmlEnv ):
dispatcher = _cfg.getDispatcher()
execution_config = dispatcher.getExecutionConfig( self.jobState )
try:
exec_engine = executionLoader( execution_config = execution_config )
except MobyleError ,err :
msg = "unknown execution system : %s" %err
rc_log.critical("%s : %s" %( serviceName ,
msg
), exc_info = True
)
sm = StatusManager()
sm.setStatus( self._dirPath , Status( code = 5 , message = 'Mobyle internal server error' ) )
raise MobyleError, msg
except Exception , err:
rc_log.error( str(err ), exc_info=True)
raise err
class WorkflowJob(object):
def __init__(self, id=None, workflow=None, email=None, email_notify = 'auto', session=None, workflowID = None):
"""
@param id: the identifier of this workflow (it's used to rebuild WorkflowJob using it's id)
@type id: string
@param workflow: the workflow definition used to create a new job
@type workflow: a L{Workflow} instance
@param email: the user email address
@type email: L{EmailAddress} instance or a string
@param email_notify: if the user must be or not notify of the results at the end of the Job.
the 3 authorized values for this argument are:
- 'true' to notify the results to the user
- 'false' to Not notify the results to the user
- 'auto' to notify the results based on the job elapsed time and the config EMAIL_DELAY
@type email_notify: string
@param session: the session owner of this workflow (if session is set workflowID mut be None )
@type session: a L{Session} instance
@param workflowID: the ID of a the workflow owner of this workflow
@type workflowID: string
"""
self.cfg = ConfigManager.Config()
self.status_manager = StatusManager()
if id:
log.debug("accessing WorkflowJob %s" %(id))
self.id = id
self.jobState = JobState( id )
else:
log.debug("creating WorkflowJob for workflow '%s'" %(workflow.name))
self.workflow = workflow
if session and workflowID:
msg = "try to instanciate a workflow with 2 owners: session %s & workflowID %s" %( session.getKey(),
workflowID
)
log.error( msg )
raise MobyleError( msg )
self.session = session
if session :
email = session.getEmail()
if email:
self.email = EmailAddress( email )
else:
self.email = None
elif email : #there is an email without session
if not isinstance( email , EmailAddress ):
self.email = EmailAddress( email )
else:
self.email = email
self.email_notify = email_notify
if self.email_notify != 'false' and not self.email:
raise MobyleError( "email adress must be specified when email_notify is set to %s" % email_notify )
self.parameters = {}
for parameter in self.workflow.parameters:
# setting parameters which have a default value (important for hidden parameters which are not
# accessed by JobFacade...
if not(parameter.isout) and parameter.vdef is not None:
self.set_value(parameter.name, value=str(parameter.vdef))
# job is just an "environment" folder for the job
# it contains the instanciation of the job runner which seems to be hardcoded as "command runner"...
self._job = Job( service = self.workflow,
cfg = self.cfg,
userEmail = self.email,
session = self.session,
workflowID = workflowID ,
)
self.jobState = self._job.jobState
self.id = self._job.getURL()
def getDir(self):
""" returns the absolute path of the workflow job directory """
return self.jobState.getDir()
def set_status(self, status):
log.debug("setting job %s status to %s" % (self.id, status))
self.status_manager.setStatus( self.getDir() , status )
def set_value(self, parameter_name, value=None, src=None, srcFileName=None):
wf_parameter = [p for p in self.workflow.parameters if p.name==parameter_name][0]
if value is not None:
log.debug("setting %s parameter value to %s" %(parameter_name, value))
elif src is not None:
log.debug("copying %s parameter value from %s/%s" %(parameter_name, src,srcFileName))
else:
log.error("no VALUE or SOURCE URL specified for %s parameter." % parameter_name)
""" set a parameter value """
self.parameters[parameter_name] = value
self.parameters[parameter_name + '.src'] = src
self.parameters[parameter_name + '.srcFileName'] = srcFileName
if value and value==wf_parameter.vdef:
log.debug("setting %s parameter value to default value %s" %(parameter_name, wf_parameter.vdef))
return
# save input value in a file
# link this file from the JobState xml
datatype_class = wf_parameter.type.datatype.class_name
datatype_superclass = wf_parameter.type.datatype.superclass_name
df = DataTypeFactory()
if (datatype_superclass in [None,""] ):
dt = df.newDataType(datatype_class)
else:
dt = df.newDataType(datatype_superclass, datatype_class)
mt = MobyleType(dt)
p = Parameter(mt, name=parameter_name)
p._isout = wf_parameter.isout
if dt.isFile():
file_name = parameter_name+'.data'
if src:
src = DataProvider.get(src)
file_name, size = mt.toFile( value , self , file_name, src , srcFileName )
if not(wf_parameter.isout):
self.jobState.setInputDataFile(parameter_name, (file_name, size, None))
else:
self.jobState.setOutputDataFile(parameter_name, [(file_name, size, None)])
else:
if not(wf_parameter.isout):
self.jobState.setInputDataValue(parameter_name, value)
else:
raise NotImplementedError() # so far Mobyle does not manage non-file outputs
self.jobState.commit()
def setValue(self, parameter_name, value=None, src=None, srcFileName=None):
"""MobyleJob-style set value method, called from JobFacade"""
if type(value)==tuple:
return self.set_value(parameter_name, value=value[1], src=value[2],srcFileName=value[3])
else:
return self.set_value(parameter_name, value=value, src=src,srcFileName=srcFileName)
def getJobid(self):
"""MobyleJob-style get job id method, called from JobFacade"""
return self.id
def getDate(self):
"""MobyleJob-style get date method, called from JobFacade"""
return time.strptime(self.get_date(),"%x %X")
def getStatus(self):
"""MobyleJob-style get status method, called from JobFacade"""
return self.status_manager.getStatus( self.getDir() )
def get_value(self, parameter_name):
"""get a parameter value"""
return self.parameters.get(parameter_name,None)
def get_date(self):
"""get the job date as a string"""
return self.jobState.getDate()
def get_id(self):
"""get the job id"""
return self.id
def run(self):
"""submit the job asynchronously"""
self.validate()
self.set_status(Status( code = 1 )) # status = submitted
#raise a UserValueError if nb of job is over the limit accepted
if( self.email is not None ):
self._job.over_limit( self.email , '' )
self._child_pid = os.fork()
if self._child_pid==0:
#Child code
os.setsid()
log_fd = os.open("%s/log" % self.jobState.getDir(), os.O_APPEND | os.O_WRONLY | os.O_CREAT , 0664 )
devnull = os.open( "/dev/null" , os.O_RDWR )
os.dup2( devnull , sys.stdin.fileno() )
os.close( devnull)
os.dup2( log_fd , sys.stdout.fileno() )
os.dup2( log_fd , sys.stderr.fileno() )
os.close( log_fd )
atexit.register( self.log , "child exit for workflow id: %s" % self.get_id())
################################################
service = self._job.getService()
serviceName = service.getName()
jobKey = self._job.getKey()
linkName = ( "%s/%s.%s" %( self.cfg.admindir() ,
serviceName ,
jobKey
)
)
try:
os.symlink(
os.path.join( self.getDir() , '.admin') ,
linkName
)
except OSError , err:
self.set_status(Status(string="error", message="workflow execution failed"))
msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err )
log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info = True )
raise WorkflowJobError , msg
################################################
t0 = time.time()
self.srun()
t1 = time.time()
################################################
try:
os.unlink( linkName )
except OSError , err:
self.set_status(Status(string="error", message="workflow execution failed"))
msg = "can't remove symbolic link %s in ADMINDIR: %s" %( linkName , err )
log.critical( "%s/%s : %s" %( serviceName, jobKey, msg ), exc_info= True )
raise WorkflowJobError , msg
################################################
try:
zipFileName = self.zip_results()
except Exception :
msg = "an error occured during the zipping results :\n\n"
log.critical( "%s : %s" %( self.id , msg ) , exc_info = True)
zipFileName = None
if self.email_notify == 'auto':
if ( t1 - t0 ) > self.cfg.email_delay() :
emailResults( self.cfg ,
self.email, #userEmail,
registry,
self.id,
self.getDir(),
self.workflow.getName(),
self._job.getKey(),
FileName = zipFileName )
elif self.email_notify == 'true':
emailResults( self.cfg ,
self.email, #userEmail,
registry,
self.id,
self.getDir(),
self.workflow.getName(),
self._job.getKey(),
FileName = zipFileName )
else:
pass
sys.exit(0) #exit with no error
class CommandRunner:
"""
"""
def __init__( self, job , jobState = None ):
"""
instanciate a L{CommandBuilder} and use it to build the CommandLine, then run it
@param service:
@type job: a L{Job} instance
@param jobState:
@type jobState:
@call: l{Job.run}
@todo: implementation pour le warper de cgi ou WS a faire
"""
self.job = job
self.job_dir = self.job.getDir()
self._service = self.job.getService()
if jobState is None:
self._jobState = JobState( self.job_dir )
else:
self._jobState = jobState
commandBuilder = CommandBuilder()
method = self._service.getCommand()[1].upper()
if method == '' or method == 'LOCAL':
try:
cmd = commandBuilder.buildLocalCommand( self._service )
self._commandLine = cmd[ 'cmd' ]
self._xmlEnv = cmd[ 'env' ]
paramfiles = cmd[ 'paramfiles' ]
except Exception ,err :
msg = "an error occured during the command line building: "
self._logError( userMsg = "Mobyle Internal Server Error",
logMsg = None #the error log is filled by the rf_log.critical
)
#this error is already log in build.log
msg = "%s/%s : %s" %( self._service.getName() ,
self.job.getKey() ,
msg ,
)
if self.job.cfg.debug( self._service.getName() ) == 0:
rf_log.critical( msg , exc_info = True) # send an email
else:
rf_log.error( msg , exc_info = True)
raise MobyleError , "Mobyle Internal Server Error"
js_paramfiles = []
if paramfiles :
for paramfile_name , string_handle in paramfiles.items():
paramfile_handle = open( os.path.join( self.job_dir , paramfile_name ) , 'w' )
content = string_handle.getvalue()
paramfile_handle.write( content )
paramfile_handle.close()
js_paramfiles.append( ( os.path.basename( paramfile_name ) , len( content ) ) )
self._jobState.setParamfiles( js_paramfiles )
self._jobState.commit()
elif method == "GET" or method == "POST" or method == "POSTM":
raise NotImplementedError ,"cgi wrapping is not yet implemented"
