def test_finalize(mocker, failedFlag, expectedRes, finalStates):
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection",
side_effect=getSystemSectionMock)
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ModuleFactory",
side_effect=MagicMock())
jw = JobWrapper()
jw.jobArgs = {"Executable": "/bin/ls"}
jw.failedFlag = failedFlag
res = jw.finalize()
assert res == expectedRes
assert jw.jobReport.jobStatusInfo[0][0] == finalStates[0]
assert jw.jobReport.jobStatusInfo[0][1] == finalStates[1]
def execute(arguments):
""" The only real function executed here
"""
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int(jobID)
if 'WorkingDirectory' in arguments:
wdir = os.path.expandvars(arguments['WorkingDirectory'])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(wdir) # this will raise an exception if wdir already exists (which is ~OK)
if os.path.isdir(wdir):
os.chdir(wdir)
except OSError as osError:
if osError.errno == errno.EEXIST and os.path.isdir(wdir):
gLogger.exception('JobWrapperTemplate found that the working directory already exists')
rescheduleResult = rescheduleFailedJob(jobID, 'Working Directory already exists')
else:
gLogger.exception('JobWrapperTemplate could not create working directory')
rescheduleResult = rescheduleFailedJob(jobID, 'Could Not Create Working Directory')
return 1
gJobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments) # initialize doesn't return S_OK/S_ERROR
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper failed the initialization phase', lException=exc)
rescheduleResult = rescheduleFailedJob(jobID, 'Job Wrapper Initialization', gJobReport)
try:
job.sendJobAccounting(rescheduleResult, 'Job Wrapper Initialization')
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper failed sending job accounting', lException=exc)
return 1
if 'InputSandbox' in arguments['Job']:
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError:
gLogger.exception('JobWrapper failed to download input sandbox')
rescheduleResult = rescheduleFailedJob(jobID, 'Input Sandbox Download', gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper raised exception while downloading input sandbox', lException=exc)
rescheduleResult = rescheduleFailedJob(jobID, 'Input Sandbox Download', gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
else:
gLogger.verbose('Job has no InputSandbox requirement')
gJobReport.commit()
if 'InputData' in arguments['Job']:
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError:
gLogger.exception('JobWrapper failed to resolve input data')
rescheduleResult = rescheduleFailedJob(jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Data Resolution')
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper raised exception while resolving input data', lException=exc)
rescheduleResult = rescheduleFailedJob(jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Data Resolution')
return 1
else:
gLogger.verbose('Job has a null InputData requirement:')
gLogger.verbose(arguments)
else:
gLogger.verbose('Job has no InputData requirement')
gJobReport.commit()
try:
result = job.execute(arguments)
if not result['OK']:
gLogger.error('Failed to execute job', result['Message'])
raise JobWrapperError((result['Message'], result['Errno']))
except JobWrapperError as exc:
if exc.value[1] == 0 or str(exc.value[0]) == '0':
gLogger.verbose('JobWrapper exited with status=0 after execution')
if exc.value[1] == DErrno.EWMSRESC:
gLogger.warn("Asked to reschedule job")
rescheduleResult = rescheduleFailedJob(jobID, 'JobWrapper execution', gJobReport)
job.sendJobAccounting(rescheduleResult, 'JobWrapper execution')
return 1
gLogger.exception('Job failed in execution phase')
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus(
'Failed', 'Exception During Execution', sendFlag=False)
job.sendFailoverRequest('Failed', 'Exception During Execution')
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('Job raised exception during execution phase', lException=exc)
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus('Failed', 'Exception During Execution', sendFlag=False)
job.sendFailoverRequest('Failed', 'Exception During Execution')
return 1
if 'OutputSandbox' in arguments['Job'] or 'OutputData' in arguments['Job']:
try:
result = job.processJobOutputs()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError as exc:
gLogger.exception('JobWrapper failed to process output files')
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus('Failed', 'Uploading Job Outputs', sendFlag=False)
job.sendFailoverRequest('Failed', 'Uploading Job Outputs')
return 2
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper raised exception while processing output files', lException=exc)
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus('Failed', 'Uploading Job Outputs', sendFlag=False)
job.sendFailoverRequest('Failed', 'Uploading Job Outputs')
return 2
else:
gLogger.verbose('Job has no OutputData or OutputSandbox requirement')
try:
# Failed jobs will return 1 / successful jobs will return 0
return job.finalize()
except Exception as exc: # pylint: disable=broad-except
gLogger.exception('JobWrapper raised exception during the finalization phase', lException=exc)
return 2
def execute(arguments):
""" The only real function executed here
"""
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int(jobID)
if 'WorkingDirectory' in arguments:
wdir = os.path.expandvars(arguments['WorkingDirectory'])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(
wdir
) # this will raise an exception if wdir already exists (which is ~OK)
if os.path.isdir(wdir):
os.chdir(wdir)
except OSError as osError:
if osError.errno == errno.EEXIST and os.path.isdir(wdir):
gLogger.exception(
'JobWrapperTemplate found that the working directory already exists'
)
rescheduleResult = rescheduleFailedJob(
jobID, 'Working Directory already exists')
else:
gLogger.exception(
'JobWrapperTemplate could not create working directory'
)
rescheduleResult = rescheduleFailedJob(
jobID, 'Could Not Create Working Directory')
return 1
gJobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments) # initialize doesn't return S_OK/S_ERROR
except Exception as exc: #pylint: disable=broad-except
gLogger.exception('JobWrapper failed the initialization phase',
lException=exc)
rescheduleResult = rescheduleFailedJob(jobID,
'Job Wrapper Initialization',
gJobReport)
try:
job.sendJobAccounting(rescheduleResult,
'Job Wrapper Initialization')
except Exception as exc: #pylint: disable=broad-except
gLogger.exception('JobWrapper failed sending job accounting',
lException=exc)
return 1
if 'InputSandbox' in arguments['Job']:
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError:
gLogger.exception('JobWrapper failed to download input sandbox')
rescheduleResult = rescheduleFailedJob(jobID,
'Input Sandbox Download',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
except Exception as exc: #pylint: disable=broad-except
gLogger.exception(
'JobWrapper raised exception while downloading input sandbox',
lException=exc)
rescheduleResult = rescheduleFailedJob(jobID,
'Input Sandbox Download',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
else:
gLogger.verbose('Job has no InputSandbox requirement')
gJobReport.commit()
if 'InputData' in arguments['Job']:
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError:
gLogger.exception('JobWrapper failed to resolve input data')
rescheduleResult = rescheduleFailedJob(
jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult,
'Input Data Resolution')
return 1
except Exception as exc: #pylint: disable=broad-except
gLogger.exception(
'JobWrapper raised exception while resolving input data',
lException=exc)
rescheduleResult = rescheduleFailedJob(
jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult,
'Input Data Resolution')
return 1
else:
gLogger.verbose('Job has a null InputData requirement:')
gLogger.verbose(arguments)
else:
gLogger.verbose('Job has no InputData requirement')
gJobReport.commit()
try:
result = job.execute(arguments)
if not result['OK']:
gLogger.error('Failed to execute job', result['Message'])
raise JobWrapperError((result['Message'], result['Errno']))
except JobWrapperError as exc:
if exc.value[1] == 0 or str(exc.value[0]) == '0':
gLogger.verbose('JobWrapper exited with status=0 after execution')
if exc.value[1] == DErrno.EWMSRESC:
gLogger.warn("Asked to reschedule job")
rescheduleResult = rescheduleFailedJob(jobID,
'JobWrapper execution',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'JobWrapper execution')
return 1
gLogger.exception('Job failed in execution phase')
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus('Failed',
'Exception During Execution',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Exception During Execution')
return 1
except Exception as exc: #pylint: disable=broad-except
gLogger.exception('Job raised exception during execution phase',
lException=exc)
gJobReport.setJobParameter('Error Message', str(exc), sendFlag=False)
gJobReport.setJobStatus('Failed',
'Exception During Execution',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Exception During Execution')
return 1
if 'OutputSandbox' in arguments['Job'] or 'OutputData' in arguments['Job']:
try:
result = job.processJobOutputs(arguments)
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except JobWrapperError as exc:
gLogger.exception('JobWrapper failed to process output files')
gJobReport.setJobParameter('Error Message',
str(exc),
sendFlag=False)
gJobReport.setJobStatus('Failed',
'Uploading Job Outputs',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Uploading Job Outputs')
return 2
except Exception as exc: # pylint: disable=broad-except
gLogger.exception(
'JobWrapper raised exception while processing output files',
lException=exc)
gJobReport.setJobParameter('Error Message',
str(exc),
sendFlag=False)
gJobReport.setJobStatus('Failed',
'Uploading Job Outputs',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Uploading Job Outputs')
return 2
else:
gLogger.verbose('Job has no OutputData or OutputSandbox requirement')
try:
# Failed jobs will return 1 / successful jobs will return 0
return job.finalize()
except Exception as exc: #pylint: disable=broad-except
gLogger.exception(
'JobWrapper raised exception during the finalization phase',
lException=exc)
return 2
def execute ( arguments ):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int( jobID )
if arguments.has_key( 'WorkingDirectory' ):
wdir = os.path.expandvars( arguments['WorkingDirectory'] )
if os.path.isdir( wdir ):
os.chdir( wdir )
else:
try:
os.makedirs( wdir )
if os.path.isdir( wdir ):
os.chdir( wdir )
except Exception:
gLogger.exception( 'JobWrapperTemplate could not create working directory' )
rescheduleResult = rescheduleFailedJob( jobID, 'Could Not Create Working Directory' )
return 1
gJobReport = JobReport( jobID, 'JobWrapper' )
try:
job = JobWrapper( jobID, gJobReport )
job.initialize( arguments )
except Exception:
gLogger.exception( 'JobWrapper failed the initialization phase' )
rescheduleResult = rescheduleFailedJob( jobID, 'Job Wrapper Initialization', gJobReport )
job.sendJobAccounting( rescheduleResult, 'Job Wrapper Initialization' )
return 1
if arguments['Job'].has_key( 'InputSandbox' ):
gJobReport.commit()
try:
result = job.transferInputSandbox( arguments['Job']['InputSandbox'] )
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception:
gLogger.exception( 'JobWrapper failed to download input sandbox' )
rescheduleResult = rescheduleFailedJob( jobID, 'Input Sandbox Download', gJobReport )
job.sendJobAccounting( rescheduleResult, 'Input Sandbox Download' )
return 1
else:
gLogger.verbose( 'Job has no InputSandbox requirement' )
gJobReport.commit()
if arguments['Job'].has_key( 'InputData' ):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception, x:
gLogger.exception( 'JobWrapper failed to resolve input data' )
rescheduleResult = rescheduleFailedJob( jobID, 'Input Data Resolution', gJobReport )
job.sendJobAccounting( rescheduleResult, 'Input Data Resolution' )
return 1
else:
gLogger.verbose( 'Job has a null InputData requirement:' )
gLogger.verbose( arguments )
def test_InputData(mocker):
mocker.patch(
'DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection',
side_effect=getSystemSectionMock)
mocker.patch(
'DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ModuleFactory',
side_effect=MagicMock())
jw = JobWrapper()
jw.jobArgs['InputData'] = ''
res = jw.resolveInputData()
assert res['OK'] is False
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
assert res['OK']
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.jobArgs['LocalSE'] = 'mySE'
jw.jobArgs['InputDataModule'] = 'aa.bb'
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
assert res['OK']
def test_InputData(self):
myJW = importlib.import_module(
'DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper')
myJW.getSystemSection = MagicMock()
myJW.ModuleFactory = MagicMock()
jw = JobWrapper()
jw.jobArgs['InputData'] = ''
res = jw.resolveInputData()
self.assertFalse(res['OK'])
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
self.assertTrue(res['OK'])
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.jobArgs['LocalSE'] = 'mySE'
jw.jobArgs['InputDataModule'] = 'aa.bb'
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
self.assertTrue(res['OK'])
def test_execute(self, _patch1, _patch2):
jw = JobWrapper()
jw.jobArgs = {'Executable':'/bin/ls'}
res = jw.execute('')
print 'jw.execute() returns',str(res)
self.assertTrue( res['OK'] )
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script-OK.sh', 'script-OK.sh')
jw = JobWrapper()
jw.jobArgs = {'Executable':'script-OK.sh'}
res = jw.execute('')
self.assertTrue( res['OK'] )
os.remove('script-OK.sh')
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script.sh', 'script.sh')
jw = JobWrapper()
jw.jobArgs = {'Executable':'script.sh', 'Arguments':'111'}
res = jw.execute('')
self.assertTrue( res['OK'] ) # In this case the application finished with errors,
# but the JobWrapper executed successfully
os.remove('script.sh')
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script-RESC.sh', 'script-RESC.sh') #this will reschedule
jw = JobWrapper()
jw.jobArgs = {'Executable':'script-RESC.sh'}
res = jw.execute('')
if res['OK']: # FIXME: This may happen depending on the shell - not the best test admittedly!
print "We should not be here, unless the 'Execution thread status' is equal to 1"
self.assertTrue( res['OK'] )
else:
self.assertFalse( res['OK'] ) # In this case the application finished with an error code
# that the JobWrapper interpreted as "to reschedule"
# so in this case the "execute" is considered an error
os.remove('script-RESC.sh')
def test_InputData( self ):
myJW = importlib.import_module( 'DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper' )
myJW.getSystemSection = MagicMock()
myJW.ModuleFactory = MagicMock()
jw = JobWrapper()
jw.jobArgs['InputData'] = ''
res = jw.resolveInputData()
self.assertFalse( res['OK'] )
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.dm = self.mockDM
jw.fc = self.mockFC
res = jw.resolveInputData()
self.assert_( res['OK'] )
jw = JobWrapper()
jw.jobArgs['InputData'] = 'pippo'
jw.jobArgs['LocalSE'] = 'mySE'
jw.jobArgs['InputDataModule'] = 'aa.bb'
jw.dm = self.mockDM
jw.fc = self.mockFC
res = jw.resolveInputData()
self.assert_( res['OK'] )
def execute(arguments):
"""The only real function executed here"""
global gJobReport
jobID = arguments["Job"].get("JobID", 0)
os.environ["JOBID"] = str(jobID)
jobID = int(jobID)
if "WorkingDirectory" in arguments:
wdir = os.path.expandvars(arguments["WorkingDirectory"])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(
wdir
) # this will raise an exception if wdir already exists (which is ~OK)
if os.path.isdir(wdir):
os.chdir(wdir)
except OSError as osError:
if osError.errno == errno.EEXIST and os.path.isdir(wdir):
gLogger.exception(
"JobWrapperTemplate found that the working directory already exists"
)
rescheduleResult = rescheduleFailedJob(
jobID, "Working Directory already exists")
else:
gLogger.exception(
"JobWrapperTemplate could not create working directory"
)
rescheduleResult = rescheduleFailedJob(
jobID, "Could Not Create Working Directory")
return 1
gJobReport = JobReport(jobID, "JobWrapper")
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments) # initialize doesn't return S_OK/S_ERROR
except Exception as exc: # pylint: disable=broad-except
gLogger.exception("JobWrapper failed the initialization phase",
lException=exc)
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.JOB_WRAPPER_INITIALIZATION,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.JOB_WRAPPER_INITIALIZATION)
return 1
if "InputSandbox" in arguments["Job"]:
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments["Job"]["InputSandbox"])
if not result["OK"]:
gLogger.warn(result["Message"])
raise JobWrapperError(result["Message"])
except JobWrapperError:
gLogger.exception("JobWrapper failed to download input sandbox")
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.DOWNLOADING_INPUT_SANDBOX,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.DOWNLOADING_INPUT_SANDBOX)
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception(
"JobWrapper raised exception while downloading input sandbox",
lException=exc)
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.DOWNLOADING_INPUT_SANDBOX,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.DOWNLOADING_INPUT_SANDBOX)
return 1
else:
gLogger.verbose("Job has no InputSandbox requirement")
gJobReport.commit()
if "InputData" in arguments["Job"]:
if arguments["Job"]["InputData"]:
try:
result = job.resolveInputData()
if not result["OK"]:
gLogger.warn(result["Message"])
raise JobWrapperError(result["Message"])
except JobWrapperError:
gLogger.exception("JobWrapper failed to resolve input data")
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.INPUT_DATA_RESOLUTION,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.INPUT_DATA_RESOLUTION)
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception(
"JobWrapper raised exception while resolving input data",
lException=exc)
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.INPUT_DATA_RESOLUTION,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.INPUT_DATA_RESOLUTION)
return 1
else:
gLogger.verbose("Job has a null InputData requirement:")
gLogger.verbose(arguments)
else:
gLogger.verbose("Job has no InputData requirement")
gJobReport.commit()
try:
result = job.execute()
if not result["OK"]:
gLogger.error("Failed to execute job", result["Message"])
raise JobWrapperError((result["Message"], result["Errno"]))
except JobWrapperError as exc:
if exc.value[1] == 0 or str(exc.value[0]) == "0":
gLogger.verbose("JobWrapper exited with status=0 after execution")
if exc.value[1] == DErrno.EWMSRESC:
gLogger.warn("Asked to reschedule job")
rescheduleResult = rescheduleFailedJob(
jobID=jobID,
minorStatus=JobMinorStatus.JOB_WRAPPER_EXECUTION,
jobReport=gJobReport)
job.sendJobAccounting(
status=rescheduleResult,
minorStatus=JobMinorStatus.JOB_WRAPPER_EXECUTION)
return 1
gLogger.exception("Job failed in execution phase")
gJobReport.setJobParameter("Error Message", repr(exc), sendFlag=False)
gJobReport.setJobStatus(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.EXCEPTION_DURING_EXEC,
sendFlag=False)
job.sendFailoverRequest()
job.sendJobAccounting(status=JobStatus.FAILED,
minorStatus=JobMinorStatus.EXCEPTION_DURING_EXEC)
return 1
except Exception as exc: # pylint: disable=broad-except
gLogger.exception("Job raised exception during execution phase",
lException=exc)
gJobReport.setJobParameter("Error Message", repr(exc), sendFlag=False)
gJobReport.setJobStatus(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.EXCEPTION_DURING_EXEC,
sendFlag=False)
job.sendFailoverRequest()
job.sendJobAccounting(status=JobStatus.FAILED,
minorStatus=JobMinorStatus.EXCEPTION_DURING_EXEC)
return 1
if "OutputSandbox" in arguments["Job"] or "OutputData" in arguments["Job"]:
try:
result = job.processJobOutputs()
if not result["OK"]:
gLogger.warn(result["Message"])
raise JobWrapperError(result["Message"])
except JobWrapperError as exc:
gLogger.exception("JobWrapper failed to process output files")
gJobReport.setJobParameter("Error Message",
repr(exc),
sendFlag=False)
gJobReport.setJobStatus(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.UPLOADING_JOB_OUTPUTS,
sendFlag=False)
job.sendFailoverRequest()
job.sendJobAccounting(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.UPLOADING_JOB_OUTPUTS)
return 2
except Exception as exc: # pylint: disable=broad-except
gLogger.exception(
"JobWrapper raised exception while processing output files",
lException=exc)
gJobReport.setJobParameter("Error Message",
repr(exc),
sendFlag=False)
gJobReport.setJobStatus(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.UPLOADING_JOB_OUTPUTS,
sendFlag=False)
job.sendFailoverRequest()
job.sendJobAccounting(
status=JobStatus.FAILED,
minorStatus=JobMinorStatus.UPLOADING_JOB_OUTPUTS)
return 2
else:
gLogger.verbose("Job has no OutputData or OutputSandbox requirement")
try:
# Failed jobs will return !=0 / successful jobs will return 0
return job.finalize()
except Exception as exc: # pylint: disable=broad-except
gLogger.exception(
"JobWrapper raised exception during the finalization phase",
lException=exc)
return 2
os.chdir(wdir)
else:
try:
os.makedirs(wdir)
if os.path.isdir(wdir):
os.chdir(wdir)
except Exception, x:
gLogger.exception('JobWrapperTemplate could not create working directory')
rescheduleFailedJob(jobID,'Could Not Create Working Directory')
return 1
#root = arguments['CE']['Root']
jobReport = JobReport(jobID,'JobWrapper')
try:
job = JobWrapper( jobID, jobReport )
job.initialize(arguments)
except Exception, x:
gLogger.exception('JobWrapper failed the initialization phase')
rescheduleFailedJob(jobID,'Job Wrapper Initialization')
job.sendWMSAccounting('Failed','Job Wrapper Initialization')
return 1
if arguments['Job'].has_key('InputSandbox'):
jobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception, x:
def execute(arguments):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int(jobID)
if arguments.has_key('WorkingDirectory'):
wdir = os.path.expandvars(arguments['WorkingDirectory'])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(wdir)
if os.path.isdir(wdir):
os.chdir(wdir)
except Exception:
gLogger.exception(
'JobWrapperTemplate could not create working directory')
rescheduleResult = rescheduleFailedJob(
jobID, 'Could Not Create Working Directory')
return 1
gJobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments)
except Exception as e:
gLogger.exception('JobWrapper failed the initialization phase',
lException=e)
rescheduleResult = rescheduleFailedJob(jobID,
'Job Wrapper Initialization',
gJobReport)
try:
job.sendJobAccounting(rescheduleResult,
'Job Wrapper Initialization')
except Exception as e:
gLogger.exception('JobWrapper failed sending job accounting',
lException=e)
return 1
if arguments['Job'].has_key('InputSandbox'):
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception:
gLogger.exception('JobWrapper failed to download input sandbox')
rescheduleResult = rescheduleFailedJob(jobID,
'Input Sandbox Download',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
else:
gLogger.verbose('Job has no InputSandbox requirement')
gJobReport.commit()
if arguments['Job'].has_key('InputData'):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception as x:
gLogger.exception('JobWrapper failed to resolve input data')
rescheduleResult = rescheduleFailedJob(
jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult,
'Input Data Resolution')
return 1
else:
gLogger.verbose('Job has a null InputData requirement:')
gLogger.verbose(arguments)
else:
gLogger.verbose('Job has no InputData requirement')
gJobReport.commit()
try:
result = job.execute(arguments)
if not result['OK']:
gLogger.error('Failed to execute job', result['Message'])
raise JobWrapperError(result['Message'])
except Exception as x:
if str(x) == '0':
gLogger.verbose('JobWrapper exited with status=0 after execution')
else:
gLogger.exception('Job failed in execution phase')
gJobReport.setJobParameter('Error Message', str(x), sendFlag=False)
gJobReport.setJobStatus('Failed',
'Exception During Execution',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Exception During Execution')
return 1
if arguments['Job'].has_key('OutputSandbox') or arguments['Job'].has_key(
'OutputData'):
try:
result = job.processJobOutputs(arguments)
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception as x:
gLogger.exception('JobWrapper failed to process output files')
gJobReport.setJobParameter('Error Message', str(x), sendFlag=False)
gJobReport.setJobStatus('Failed',
'Uploading Job Outputs',
sendFlag=False)
job.sendFailoverRequest('Failed', 'Uploading Job Outputs')
return 2
else:
gLogger.verbose('Job has no OutputData or OutputSandbox requirement')
try:
# Failed jobs will return 1 / successful jobs will return 0
return job.finalize(arguments)
except Exception:
gLogger.exception('JobWrapper failed the finalization phase')
return 2
def execute( arguments ):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int( jobID )
if arguments.has_key( 'WorkingDirectory' ):
wdir = os.path.expandvars( arguments['WorkingDirectory'] )
if os.path.isdir( wdir ):
os.chdir( wdir )
else:
try:
os.makedirs( wdir )
if os.path.isdir( wdir ):
os.chdir( wdir )
except Exception:
gLogger.exception( 'JobWrapperTemplate could not create working directory' )
rescheduleResult = rescheduleFailedJob( jobID, 'Could Not Create Working Directory' )
return 1
gJobReport = JobReport( jobID, 'JobWrapper' )
try:
job = JobWrapper( jobID, gJobReport )
job.initialize( arguments )
except Exception as e:
gLogger.exception( 'JobWrapper failed the initialization phase', lException = e )
rescheduleResult = rescheduleFailedJob( jobID, 'Job Wrapper Initialization', gJobReport )
try:
job.sendJobAccounting( rescheduleResult, 'Job Wrapper Initialization' )
except Exception as e:
gLogger.exception( 'JobWrapper failed sending job accounting', lException = e )
return 1
if arguments['Job'].has_key( 'InputSandbox' ):
gJobReport.commit()
try:
result = job.transferInputSandbox( arguments['Job']['InputSandbox'] )
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception:
gLogger.exception( 'JobWrapper failed to download input sandbox' )
rescheduleResult = rescheduleFailedJob( jobID, 'Input Sandbox Download', gJobReport )
job.sendJobAccounting( rescheduleResult, 'Input Sandbox Download' )
return 1
else:
gLogger.verbose( 'Job has no InputSandbox requirement' )
gJobReport.commit()
if arguments['Job'].has_key( 'InputData' ):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception as x:
gLogger.exception( 'JobWrapper failed to resolve input data' )
rescheduleResult = rescheduleFailedJob( jobID, 'Input Data Resolution', gJobReport )
job.sendJobAccounting( rescheduleResult, 'Input Data Resolution' )
return 1
else:
gLogger.verbose( 'Job has a null InputData requirement:' )
gLogger.verbose( arguments )
else:
gLogger.verbose( 'Job has no InputData requirement' )
gJobReport.commit()
try:
result = job.execute( arguments )
if not result['OK']:
gLogger.error( 'Failed to execute job', result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception as x:
if str( x ) == '0':
gLogger.verbose( 'JobWrapper exited with status=0 after execution' )
else:
gLogger.exception( 'Job failed in execution phase' )
gJobReport.setJobParameter( 'Error Message', str( x ), sendFlag = False )
gJobReport.setJobStatus( 'Failed', 'Exception During Execution', sendFlag = False )
job.sendFailoverRequest( 'Failed', 'Exception During Execution' )
return 1
if arguments['Job'].has_key( 'OutputSandbox' ) or arguments['Job'].has_key( 'OutputData' ):
try:
result = job.processJobOutputs( arguments )
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception as x:
gLogger.exception( 'JobWrapper failed to process output files' )
gJobReport.setJobParameter( 'Error Message', str( x ), sendFlag = False )
gJobReport.setJobStatus( 'Failed', 'Uploading Job Outputs', sendFlag = False )
job.sendFailoverRequest( 'Failed', 'Uploading Job Outputs' )
return 2
else:
gLogger.verbose( 'Job has no OutputData or OutputSandbox requirement' )
try:
# Failed jobs will return 1 / successful jobs will return 0
return job.finalize( arguments )
except Exception:
gLogger.exception( 'JobWrapper failed the finalization phase' )
return 2
def test_finalize(self):
jw = JobWrapper()
jw.jobArgs = {'Executable': '/bin/ls'}
res = jw.finalize()
self.assertTrue(res == 1) # by default failed flag is True
try:
os.makedirs(wdir)
if os.path.isdir(wdir):
os.chdir(wdir)
except Exception, x:
gLogger.exception(
'JobWrapperTemplate could not create working directory')
rescheduleFailedJob(jobID,
'Could Not Create Working Directory')
return 1
#root = arguments['CE']['Root']
jobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, jobReport)
job.initialize(arguments)
except Exception, x:
gLogger.exception('JobWrapper failed the initialization phase')
rescheduleFailedJob(jobID, 'Job Wrapper Initialization')
job.sendWMSAccounting('Failed', 'Job Wrapper Initialization')
return 1
if arguments['Job'].has_key('InputSandbox'):
jobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception, x:
def test_InputData(mocker):
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection",
side_effect=getSystemSectionMock)
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ModuleFactory",
side_effect=MagicMock())
jw = JobWrapper()
jw.jobArgs["InputData"] = ""
res = jw.resolveInputData()
assert res["OK"] is False
jw = JobWrapper()
jw.jobArgs["InputData"] = "pippo"
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
assert res["OK"]
jw = JobWrapper()
jw.jobArgs["InputData"] = "pippo"
jw.jobArgs["LocalSE"] = "mySE"
jw.jobArgs["InputDataModule"] = "aa.bb"
jw.dm = dm_mock
jw.fc = fc_mock
res = jw.resolveInputData()
assert res["OK"]
def execute(arguments):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int(jobID)
# Fix in the environment to get a reasonable performance from dCache,
# until we move to a new version of root
# os.environ['DCACHE_RAHEAD'] = str(1)
# os.environ['DCACHE_RA_BUFFER'] = str(50*1024)
if arguments.has_key('WorkingDirectory'):
wdir = os.path.expandvars(arguments['WorkingDirectory'])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(wdir)
if os.path.isdir(wdir):
os.chdir(wdir)
except Exception:
gLogger.exception(
'JobWrapperTemplate could not create working directory')
rescheduleFailedJob(jobID,
'Could Not Create Working Directory')
return 1
#root = arguments['CE']['Root']
gJobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments)
except Exception:
gLogger.exception('JobWrapper failed the initialization phase')
rescheduleFailedJob(jobID, 'Job Wrapper Initialization', gJobReport)
job.sendWMSAccounting('Failed', 'Job Wrapper Initialization')
return 1
if arguments['Job'].has_key('InputSandbox'):
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception:
gLogger.exception('JobWrapper failed to download input sandbox')
rescheduleFailedJob(jobID, 'Input Sandbox Download')
job.sendWMSAccounting('Failed', 'Input Sandbox Download')
return 1
else:
gLogger.verbose('Job has no InputSandbox requirement')
gJobReport.commit()
if arguments['Job'].has_key('InputData'):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception, x:
gLogger.exception('JobWrapper failed to resolve input data')
rescheduleFailedJob(jobID, 'Input Data Resolution')
job.sendWMSAccounting('Failed', 'Input Data Resolution')
return 1
else:
gLogger.verbose('Job has a null InputData requirement:')
gLogger.verbose(arguments)
def test_execute(mocker):
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection",
side_effect=getSystemSectionMock)
mocker.patch(
"DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog.getSystemInstance",
side_effect=getSystemSectionMock)
jw = JobWrapper()
jw.jobArgs = {"Executable": "/bin/ls"}
res = jw.execute()
print("jw.execute() returns", str(res))
assert res["OK"]
shutil.copy(
"src/DIRAC/WorkloadManagementSystem/JobWrapper/test/script-OK.sh",
"script-OK.sh")
jw = JobWrapper()
jw.jobArgs = {"Executable": "script-OK.sh"}
res = jw.execute()
assert res["OK"]
os.remove("script-OK.sh")
shutil.copy("src/DIRAC/WorkloadManagementSystem/JobWrapper/test/script.sh",
"script.sh")
jw = JobWrapper()
jw.jobArgs = {"Executable": "script.sh", "Arguments": "111"}
res = jw.execute()
assert res["OK"] # In this case the application finished with errors,
# but the JobWrapper executed successfully
os.remove("script.sh")
# this will reschedule
shutil.copy(
"src/DIRAC/WorkloadManagementSystem/JobWrapper/test/script-RESC.sh",
"script-RESC.sh")
jw = JobWrapper()
jw.jobArgs = {"Executable": "script-RESC.sh"}
res = jw.execute()
if res["OK"]: # FIXME: This may happen depending on the shell - not the best test admittedly!
print(
"We should not be here, unless the 'Execution thread status' is equal to 1"
)
assert res["OK"]
else:
assert res[
"OK"] is False # In this case the application finished with an error code
# that the JobWrapper interpreted as "to reschedule"
# so in this case the "execute" is considered an error
os.remove("script-RESC.sh")
os.remove("std.out")
def execute ( arguments ):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int( jobID )
# Fix in the environment to get a reasonable performance from dCache,
# until we move to a new version of root
# os.environ['DCACHE_RAHEAD'] = str(1)
# os.environ['DCACHE_RA_BUFFER'] = str(50*1024)
if arguments.has_key( 'WorkingDirectory' ):
wdir = os.path.expandvars( arguments['WorkingDirectory'] )
if os.path.isdir( wdir ):
os.chdir( wdir )
else:
try:
os.makedirs( wdir )
if os.path.isdir( wdir ):
os.chdir( wdir )
except Exception:
gLogger.exception( 'JobWrapperTemplate could not create working directory' )
rescheduleFailedJob( jobID, 'Could Not Create Working Directory' )
return 1
#root = arguments['CE']['Root']
gJobReport = JobReport( jobID, 'JobWrapper' )
try:
job = JobWrapper( jobID, gJobReport )
job.initialize( arguments )
except Exception:
gLogger.exception( 'JobWrapper failed the initialization phase' )
rescheduleFailedJob( jobID, 'Job Wrapper Initialization', gJobReport )
job.sendWMSAccounting( 'Failed', 'Job Wrapper Initialization' )
return 1
if arguments['Job'].has_key( 'InputSandbox' ):
gJobReport.commit()
try:
result = job.transferInputSandbox( arguments['Job']['InputSandbox'] )
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception:
gLogger.exception( 'JobWrapper failed to download input sandbox' )
rescheduleFailedJob( jobID, 'Input Sandbox Download' )
job.sendWMSAccounting( 'Failed', 'Input Sandbox Download' )
return 1
else:
gLogger.verbose( 'Job has no InputSandbox requirement' )
gJobReport.commit()
if arguments['Job'].has_key( 'InputData' ):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn( result['Message'] )
raise JobWrapperError( result['Message'] )
except Exception, x:
gLogger.exception( 'JobWrapper failed to resolve input data' )
rescheduleFailedJob( jobID, 'Input Data Resolution' )
job.sendWMSAccounting( 'Failed', 'Input Data Resolution' )
return 1
else:
gLogger.verbose( 'Job has a null InputData requirement:' )
gLogger.verbose( arguments )
def test_execute(self, _patch1, _patch2):
jw = JobWrapper()
jw.jobArgs = {'Executable': '/bin/ls'}
res = jw.execute('')
self.assertTrue(res['OK'])
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script-OK.sh',
'script-OK.sh')
jw = JobWrapper()
jw.jobArgs = {'Executable': 'script-OK.sh'}
res = jw.execute('')
self.assertTrue(res['OK'])
os.remove('script-OK.sh')
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script.sh',
'script.sh')
jw = JobWrapper()
jw.jobArgs = {'Executable': 'script.sh', 'Arguments': '111'}
res = jw.execute('')
self.assertTrue(
res['OK']) # In this case the application finished with errors,
# but the JobWrapper executed successfully
os.remove('script.sh')
shutil.copy('WorkloadManagementSystem/JobWrapper/test/script-RESC.sh',
'script-RESC.sh') #this will reschedule
jw = JobWrapper()
jw.jobArgs = {'Executable': 'script-RESC.sh'}
res = jw.execute('')
if res['OK']: # FIXME: This may happen depending on the shell - not the best test admittedly!
print "We should not be here, unless the 'Execution thread status' is equal to 1"
self.assertTrue(res['OK'])
else:
self.assertFalse(
res['OK']
) # In this case the application finished with an error code
# that the JobWrapper interpreted as "to reschedule"
# so in this case the "execute" is considered an error
os.remove('script-RESC.sh')
def execute(arguments):
global gJobReport
jobID = arguments['Job']['JobID']
os.environ['JOBID'] = jobID
jobID = int(jobID)
if arguments.has_key('WorkingDirectory'):
wdir = os.path.expandvars(arguments['WorkingDirectory'])
if os.path.isdir(wdir):
os.chdir(wdir)
else:
try:
os.makedirs(wdir)
if os.path.isdir(wdir):
os.chdir(wdir)
except Exception:
gLogger.exception(
'JobWrapperTemplate could not create working directory')
rescheduleResult = rescheduleFailedJob(
jobID, 'Could Not Create Working Directory')
return 1
gJobReport = JobReport(jobID, 'JobWrapper')
try:
job = JobWrapper(jobID, gJobReport)
job.initialize(arguments)
except Exception:
gLogger.exception('JobWrapper failed the initialization phase')
rescheduleResult = rescheduleFailedJob(jobID,
'Job Wrapper Initialization',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'Job Wrapper Initialization')
return 1
if arguments['Job'].has_key('InputSandbox'):
gJobReport.commit()
try:
result = job.transferInputSandbox(arguments['Job']['InputSandbox'])
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception:
gLogger.exception('JobWrapper failed to download input sandbox')
rescheduleResult = rescheduleFailedJob(jobID,
'Input Sandbox Download',
gJobReport)
job.sendJobAccounting(rescheduleResult, 'Input Sandbox Download')
return 1
else:
gLogger.verbose('Job has no InputSandbox requirement')
gJobReport.commit()
if arguments['Job'].has_key('InputData'):
if arguments['Job']['InputData']:
try:
result = job.resolveInputData()
if not result['OK']:
gLogger.warn(result['Message'])
raise JobWrapperError(result['Message'])
except Exception, x:
gLogger.exception('JobWrapper failed to resolve input data')
rescheduleResult = rescheduleFailedJob(
jobID, 'Input Data Resolution', gJobReport)
job.sendJobAccounting(rescheduleResult,
'Input Data Resolution')
return 1
else:
gLogger.verbose('Job has a null InputData requirement:')
gLogger.verbose(arguments)