def testrefresh(self):
adm_queue = 'mobyle'
self.assertEqual(self.adm.getQueue(), None)
adm = Admin(Admin.FILENAME)
adm.setQueue(adm_queue)
adm.commit()
self.assertEqual(self.adm.getQueue(), None)
self.adm.refresh()
self.assertEqual(self.adm.getQueue(), adm_queue)
def testcreate(self):
## Create a minimal admin file
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
## Should fail if already exists
self.assertTrue(os.access(Admin.FILENAME, os.F_OK))
self.assertRaises(MobyleError, Admin.create, self.cfg.test_dir, None, None, None)
## Check alternate path
altdir = os.path.join(self.cfg.test_dir, "alternate")
os.makedirs(altdir)
Admin.create(altdir, None, None, None)
def testcommit(self):
## Commit should update file
bef = os.stat(Admin.FILENAME)
adm = Admin(Admin.FILENAME)
adm.commit()
aft = os.stat(Admin.FILENAME)
self.assertTrue(bef.st_ino != aft.st_ino and bef.st_mtime <= aft.st_mtime)
## Cannot create/rename temporary file
mod = os.stat(".").st_mode
os.chmod(".", mod & ~stat.S_IWUSR)
self.assertRaises(MobyleError, adm.commit)
os.chmod(".", mod)
## Nothing to save (fails silently)
adm.me.clear()
adm.commit()
def getStatus(jobID):
"""
@param jobID: the url of the job
@type jobID: string
@return: the current status of the job
@rtype: string
@raise MobyleError: if the job has no number or if the job doesn't exist anymore
@raise OSError: if the user is not the owner of the process
"""
from Mobyle.JobState import JobState, normUri
from urlparse import urlparse
from Mobyle.StatusManager import StatusManager
path = normUri(jobID)
protocol, host, path, a, b, c = urlparse(path)
if protocol == "http":
raise NotImplementedError, "trying to querying a distant server"
if path[-9:] == "index.xml":
path = path[:-10]
sm = StatusManager()
oldStatus = sm.getStatus(path)
#'killed' , 'finished' , 'error' the status cannot change anymore
#'building' these jobs have not yet batch number
# ( 'finished' , 'error' , 'killed' , 'building' ):
if not oldStatus.isQueryable():
return oldStatus
else:
adm = Admin(path)
batch = adm.getExecutionAlias()
jobNum = adm.getNumber()
if batch is None or jobNum is None:
return oldStatus
try:
exec_engine = executionLoader(jobID=jobID)
newStatus = exec_engine.getStatus(jobNum)
except MobyleError, err:
u_log.error(str(err), exc_info=True)
raise err
if not newStatus.isKnown():
return oldStatus
if newStatus != oldStatus:
sm.setStatus(path, newStatus)
return newStatus
def testJobID(self):
adm_jobid = "adm_dummy"
## Created without job id info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getJobID(), str(None))
## Created with job id info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, adm_jobid)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getJobID(), adm_jobid)
## Missing key
adm.me.clear()
self.assertEqual(adm.getJobID(), None)
def testSession(self):
adm_session = "adm_dummy"
## Created without session info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getSession(), None)
## Created with session info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None, sessionID=adm_session)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getSession(), adm_session)
## Missing key
adm.me.clear()
self.assertEqual(adm.getSession(), None)
def testWorkflow(self):
workflowID = "adm_dummy"
## Created without session info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getWorkflowID(), None)
## Created with session info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir , None, None, None, workflowID=workflowID)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getWorkflowID(), workflowID)
## Missing key
adm.me.clear()
self.assertEqual(adm.getWorkflowID(), None)
def testRemote(self):
adm_remote = "127.0.0.1/localhost"
## Created without remote info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getRemote(), str(None))
## Created with remote info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, adm_remote, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getRemote(), adm_remote)
## Missing key
adm.me.clear()
self.assertEqual(adm.getRemote(), None)
def testEmail(self):
adm_email = "*****@*****.**"
## Created without email info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getEmail(), None)
## Created with email info
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None, userEmail=adm_email)
adm = Admin(".")
self.assertEqual(adm.getEmail(), adm_email)
## Missing key
adm.me.clear()
self.assertEqual(adm.getEmail(), None)
def run(self, commandLine , dirPath , serviceName , jobKey , jobState , queue , xmlEnv ):
"""
Run the commandLine
redirect the standard error and output on service.name.out and service.name.err, then restore the sys.stderr and sys.stdout
@return: the L{Mobyle.Status.Status} of this job and a message
@rtype: Status object
"""
jobKey = self.getKey()
fout = open( serviceName + ".out" , 'w' )
ferr = open( serviceName + ".err" , 'w' )
try:
## execute the commandline through your favorite execution system
pass
except OSError, err:
msg= "System execution failed: " + str(err)
self._logError( dirPath , serviceName ,jobKey,
userMsg = "Mobyle internal server error" ,
logMsg = None )
_log.critical( "%s/%s : %s" %( self.serviceName ,
jobKey ,
msg
)
)
raise MobyleError , msg
adm = Admin( dirPath )
adm.setExecutionAlias( self.execution_config_alias ) ## store the alias of execution config used for this job
adm.setNumber( jobKey ) ## store the key to query/retrieve this job on this system execution
adm.commit()
## link the .admin file in ADMINDIR which looklike to a "process table"
linkName = ( "%s/%s.%s" %(self. _cfg.admindir() ,
serviceName ,
jobKey
)
)
try:
os.symlink(
os.path.join( self.dirPath , '.admin') ,
linkName
)
except OSError , err:
msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err )
self._logError( dirPath , serviceName ,jobKey,
userMsg = "Mobyle internal server error" ,
logMsg = None )
_log.critical( "%s/%s : %s" %( serviceName ,
jobKey ,
msg
)
)
raise MobyleError , msg
def testExecutionAlias(self):
adm_executionalias = "adm_dummy"
## Default
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getExecutionAlias(), None)
## Set value
adm.setExecutionAlias(adm_executionalias)
self.assertEqual(adm.getExecutionAlias(), adm_executionalias)
## Missing key
adm.me.clear()
self.assertEqual(adm.getExecutionAlias(), None)
def testNumber(self):
adm_number = "adm_dummy"
## Default
os.unlink(Admin.FILENAME)
Admin.create(self.cfg.test_dir, None, None, None)
adm = Admin(Admin.FILENAME)
self.assertEqual(adm.getNumber(), None)
## Set value
adm.setNumber(adm_number)
self.assertEqual(adm.getNumber(), adm_number)
## Missing key
adm.me.clear()
self.assertEqual(adm.getNumber(), None)
def isExecuting(jobID):
"""
@param jobID: the url of the job
@type jobID: string
@return True if the job is currently executing ( submitted , running , pending , hold ).
False otherwise ( building, finished , error , killed )
@rtype: boolean
@raise MobyleError: if the job has no number
@raise OSError: if the user is not the owner of the process
"""
from Mobyle.JobState import normUri
from urlparse import urlparse
from Mobyle.StatusManager import StatusManager
path = normUri(jobID)
protocol, host, path, a, b, c = urlparse(path)
if protocol == "http":
raise NotImplementedError, "trying to querying a distant server"
if path[-9:] == "index.xml":
path = path[:-10]
adm = Admin(path)
batch = adm.getExecutionAlias()
jobNum = adm.getNumber()
if batch is None or jobNum is None:
sm = StatusManager()
status = sm.getStatus(path)
if not status.isQueryable():
return False
else:
raise MobyleError("inconsistency in .admin file %s" % path)
try:
execKlass = executionLoader(jobID=jobID)
newStatus = execKlass.getStatus(jobNum)
except MobyleError, err:
u_log.error(str(err), exc_info=True)
raise err
def executionLoader(jobID=None, alias=None, execution_config=None):
assert (
bool(jobID) + bool(alias) + bool(execution_config) == 1
), "please provide either a jobID, an alias or an execution_config"
from Mobyle.ConfigManager import Config
cfg = Config()
if not execution_config:
if jobID:
from Mobyle.JobState import normUri
from urlparse import urlparse
path = normUri(jobID)
protocol, host, path, a, b, c = urlparse(path)
if protocol == "http":
raise NotImplementedError, "trying to instanciate an Execution system from a remote Job"
if path[-9:] == "index.xml":
path = path[:-10]
adm = Admin(path)
alias = adm.getExecutionAlias()
if not alias:
msg = "cant determine the Execution system for %s " % (jobID)
u_log.error(msg)
raise MobyleError(msg)
try:
execution_config = cfg.getExecutionConfigFromAlias(alias)
except KeyError:
msg = "the ExecutionConfig alias %s doesn't match with any alias in Config" % alias
u_log.critical(msg)
raise MobyleError(msg)
klass_name = execution_config.execution_class_name
try:
module = __import__("Mobyle.Execution.%s" % klass_name)
except ImportError, err:
msg = "The Execution.%s module is missing" % klass_name
u_log.critical(msg)
raise MobyleError, msg
def _fakeadmin(self):
Admin.create(self.cfg.test_dir, None, None, None)
return Admin(".")
def emailResults(cfg, userEmail, registry, ID, job_path, serviceName, jobKey, FileName=None):
"""
@param cfg: the configuration of Mobyle
@type cfg: Config instance
@param userEmail: the user email address
@type userEmail: EmailAddress instance
@param registry: the registry of deployed services
@type registry: Registry.registry object
@param ID: the ID of the job
@type ID: string
@param job_path: the absolute path to the job
@type job_path: string
@param serviceName: the name of the service
@type serviceName: string
@param jobKey: the key of the job
@type jobKey: string
@param FileName: the absolute path of zip file to attach to the email
@type FileName: string or None
"""
from Mobyle.Net import Email
from Mobyle.MobyleError import EmailError, TooBigError
import os
dont_email_result, maxmailsize = cfg.mailResults()
if dont_email_result:
return
else:
if userEmail:
mail = Email(userEmail)
jobInPortalUrl = "%s/portal.py#jobs::%s" % (cfg.cgi_url(), registry.getJobPID(ID))
if FileName is not None:
zipSize = os.path.getsize(FileName)
mailDict = {
"SENDER": cfg.sender(),
"HELP": cfg.mailHelp(),
"SERVER_NAME": cfg.portal_url(),
"JOB_URL": jobInPortalUrl,
"RESULTS_REMAIN": cfg.remainResults(),
"JOB_NAME": serviceName,
"JOB_KEY": jobKey,
}
if zipSize > maxmailsize - 2048:
# 2048 octet is an estimated size of email headers
try:
mail.send("RESULTS_TOOBIG", mailDict)
return
except EmailError, err:
msg = str(err)
adm = Admin(job_path)
adm.setMessage(msg)
adm.commit()
u_log.error("%s/%s : %s" % (serviceName, jobKey, msg))
return
else:
try:
mail.send("RESULTS_FILES", mailDict, files=[FileName])
return
except TooBigError, err:
try:
mail.send("RESULTS_TOOBIG", mailDict)
except EmailError, err:
msg = str(err)
adm = Admin(job_path)
adm.setMessage(msg)
adm.commit()
u_log.error("%s/%s : %s" % (serviceName, jobKey, msg))
return
def run( self , commandLine , dirPath , serviceName , jobState , xmlEnv = None):
"""
@param execution_config: the configuration of the Execution
@type execution_config: ExecutionConfig instance
@param commandLine: the command to be executed
@type commandLine: String
@param dirPath: the absolute path to directory where the job will be executed (normaly we are already in)
@type dirPath: String
@param serviceName: the name of the service
@type serviceName: string
@param jobState:
@type jobState: a L{JobState} instance
"""
self.jobState = jobState
if dirPath[-1] == '/':
dirPath = dirPath[:-1]
jobKey = os.path.split( dirPath )[1]
if os.getcwd() != os.path.abspath( dirPath ):
msg = "the child process execute itself in a wrong directory"
self._logError( dirPath , serviceName ,jobKey,
userMsg = "Mobyle internal server error" ,
logMsg = msg )
raise MobyleError , msg
protectedCommandLine = ''
for c in commandLine:
protectedCommandLine += '\\'+ c
if xmlEnv is None:
xmlEnv = {}
dispatcher = self._cfg.getDispatcher()
queue = dispatcher.getQueue( jobState )
adm = Admin( dirPath )
adm.setQueue( queue )
adm.commit()
new_path = ''
binary_path = self._cfg.binary_path()
if binary_path :
new_path = ":".join( binary_path )
if xmlEnv.has_key( 'PATH' ) :
new_path = "%s:%s" %( xmlEnv[ 'PATH' ] , new_path )
if new_path :
xmlEnv[ 'PATH' ] = "%s:%s" %( new_path , os.environ[ 'PATH' ] )
else:
xmlEnv[ 'PATH' ] = os.environ[ 'PATH' ]
for var in os.environ.keys():
if var != 'PATH':
xmlEnv[ var ] = os.environ[ var ]
self._returncode = None
accounting = self._cfg.accounting()
if accounting:
beg_time = time.time()
###################################
mobyleStatus = self._run( commandLine , dirPath , serviceName , jobKey , jobState , queue , xmlEnv )
###################################
if accounting:
end_time = time.time()
elapsed_time = end_time - beg_time
a_log = getLogger( 'Mobyle.account' )
#%d trunc time to second
#%f for millisecond
a_log.info("%(serviceName)s/%(jobkey)s : %(exec_class)s/%(queue)s : %(beg_time)d-%(end_time)d %(ela_time)d : %(status)s" %{ 'serviceName':serviceName ,
'jobkey':jobKey,
'exec_class':self.execution_config.execution_class_name ,
'queue': queue,
'beg_time':beg_time ,
'end_time':end_time ,
'ela_time':elapsed_time ,
'status': mobyleStatus ,
}
)
self.status_manager.setStatus( dirPath , mobyleStatus )
env = xmlEnv
)
except OSError, err:
msg= "SGE execution failed: "+ str(err)
self._logError( dirPath , serviceName , jobKey ,
admMsg = msg ,
userMsg = "Mobyle internal server error" ,
logMsg = None )
_log.critical( "%s/%s : %s" %( serviceName ,
jobKey ,
msg
) ,
exc_info = True
)
raise MobyleError , msg
adm = Admin( dirPath )
adm.setExecutionAlias( self.execution_config_alias )
adm.setNumber( jobKey )
adm.commit()
linkName = ( "%s/%s.%s" %( self._cfg.admindir() ,
serviceName ,
jobKey
)
)
try:
os.symlink(
os.path.join( dirPath , '.admin') ,
linkName
)
except OSError , err:
msg = "can't create symbolic link %s in ADMINDIR: %s" %( linkName , err )
self._logError( dirPath , serviceName , jobKey ,
userMsg = "Mobyle internal server error" ,
logMsg = None )
_log.critical( "%s/%s : %s" %( serviceName ,
jobKey ,
msg
)
)
raise MobyleError , msg
except Exception , err :
_log.debug( "an error occured in drmaa.run method : %s" %err )
raise MobyleError( "Internal Server Error")
adm = Admin( dirPath )
adm.setExecutionAlias( self.execution_config_alias )
adm.setNumber( drmJobid )
adm.commit()
linkName = ( "%s/%s.%s" %( self._cfg.admindir() ,
serviceName ,
jobKey
)
)
try:
os.symlink(
os.path.join( dirPath , '.admin') ,
linkName
)
def over_limit(job):
"""
check if the user (same email) has a similar job (same command line or same workflow name) running
@param job: the job to check before to submit it to Execution
@type job: L{Job} instance
@raise UserValueError: if the number of similar jobs exceed the Config.SIMULTANEOUS_JOBS
"""
from hashlib import md5
import glob
from Mobyle.Admin import Admin
from Mobyle.MobyleError import UserValueError, MobyleError
max_same_jobs = job.cfg.max_similar_job_per_user()
max_user_jobs = job.cfg.max_job_per_user()
user_email = str(job.getEmail())
newMd5 = md5()
newMd5.update(job.getCommandLine())
newDigest = newMd5.hexdigest()
work_dir = job.getDir()
thisJobAdm = Admin(work_dir)
thisJobAdm.setMd5(newDigest)
thisJobAdm.commit()
if thisJobAdm.getWorkflowID():
# this job is a subtask of a workflow
# we allow a workflow to run several identical job in parallel
return True
mask = os.path.normpath("%s/*.*" % (job.cfg.admindir()))
jobs = glob.glob(mask)
same_jobs_nb = 0
jobs_nb = 0
msg = None
for one_job in jobs:
try:
oldAdm = Admin(one_job)
except MobyleError, err:
if os.path.lexists(one_job):
p_log.critical("%s/%s: invalid job in ADMINDIR : %s" % (job.getServiceName(), job.getKey(), err))
continue
if (oldAdm.getWorkflowID()):
# this job is NOT a workflow but
# this job is the "same" than a workflow subtasks
# we allow a workflow to run several identical job in parallel
continue
old_email = oldAdm.getEmail()
oldDigest = oldAdm.getMd5()
if user_email == old_email:
oldStatus = job.getStatus()
if oldStatus.isEnded():
p_log.debug("oldStatus.isEnded() = True")
continue
jobs_nb += 1
if max_user_jobs and jobs_nb >= max_user_jobs:
msg = "%d jobs (%s) have been already submitted (md5 = %s)" % (jobs_nb,
os.path.basename(one_job),
newDigest
)
userMsg = " %d job(s) have been already submitted, and are(is) not finished yet. Please wait for the end of these jobs before you resubmit." % (
jobs_nb)
p_log.warning(msg + " : run aborted ")
raise UserValueError(parameter=None, msg=userMsg)
if newDigest == oldDigest:
same_jobs_nb += 1
if max_same_jobs and same_jobs_nb >= max_same_jobs:
msg = "%d similar jobs (%s) have been already submitted (md5 = %s)" % (
same_jobs_nb,
os.path.basename(one_job),
newDigest
)
userMsg = " %d similar job(s) have been already submitted, and are(is) not finished yet. Please wait for the end of these jobs before you resubmit." % (
same_jobs_nb)
p_log.warning(msg + " : run aborted ")
raise UserValueError(parameter=None, msg=userMsg)