def test_result_ok():
assert not result_ok(None), "Didn't return False with None arg"
assert not result_ok(''), "Didn't return False with non-dict arg"
assert not result_ok({}), "Didn't return False as default dict extraction"
assert not result_ok({'OK': False}), "OK not handled properly"
assert result_ok({'OK': True}), "Didn't return True"
def test_result_ok():
assert not result_ok(None), "Didn't return False with None arg"
assert not result_ok(''), "Didn't return False with non-dict arg"
assert not result_ok({}), "Didn't return False as default dict extraction"
assert not result_ok({'OK': False}), "OK not handled properly"
assert result_ok({'OK': True}), "Didn't return True"
def test_result_ok():
from GangaDirac.Lib.Backends.DiracUtils import result_ok
assert not result_ok(None), "Didn't return False with None arg"
assert not result_ok(''), "Didn't return False with non-dict arg"
assert not result_ok({}), "Didn't return False as default dict extraction"
assert not result_ok({'OK': False}), "OK not handled properly"
assert result_ok({'OK': True}), "Didn't return True"
def test_result_ok(self):
self.assertFalse(result_ok(None), "Didn't return False with None arg")
self.assertFalse(
result_ok(''), "Didn't return False with non-dict arg")
self.assertFalse(
result_ok({}), "Didn't return False as default dict extraction")
self.assertFalse(result_ok({'OK': False}), "OK not handled properly")
self.assertTrue(result_ok({'OK': True}), "Didn't return True")
def test_result_ok(self):
self.assertFalse(result_ok(None), "Didn't return False with None arg")
self.assertFalse(result_ok(''),
"Didn't return False with non-dict arg")
self.assertFalse(result_ok({}),
"Didn't return False as default dict extraction")
self.assertFalse(result_ok({'OK': False}), "OK not handled properly")
self.assertTrue(result_ok({'OK': True}), "Didn't return True")
def DiracRunSplitter(inputs, filesPerJob, maxFiles, ignoremissing):
"""
Generator that yields datasets for dirac split jobs by run
"""
metadata = inputs.bkMetadata()
if not result_ok(metadata):
logger.error('Error getting input metadata: %s' % str(metadata))
raise SplittingError('Error splitting files.')
if metadata['Value']['Failed']:
logger.error('Error getting part of metadata')
raise SplittingError('Error splitting files.')
runs = defaultdict(list)
for lfn,v in metadata['Value']['Successful'].items():
f = [f for f in inputs.files if f.lfn == lfn][0]
runs[v['RunNumber']].append(f)
logger.info('Found %d runs in inputdata'%len(runs))
for run,files in sorted(runs.items()):
run_inputs = inputs.__class__()
run_inputs.files = files
if len(files) > filesPerJob:
datasets = list(DiracSplitter(run_inputs, filesPerJob, None, ignoremissing))
else:
datasets = [files]
logger.info('Run %d with %d files was split in %d subjobs'%(run, len(files), len(datasets)))
for ds in datasets:
yield ds
def DiracSizeSplitter(inputs, filesPerJob, maxSize, ignoremissing):
"""
Generator that yields a datasets for LHCbdirac split jobs by size
"""
#logger.debug( "DiracSplitter" )
#logger.debug( "inputs: %s" % str( inputs ) )
split_files = []
i = inputs.__class__()
if inputs.getLFNs() != len(inputs.files):
raise SplittingError(
"Error trying to split dataset using DIRAC backend with non-DiracFile in the inputdata")
all_files = igroup(inputs.files[:maxFiles], getConfig('DIRAC')['splitFilesChunks'],
leftovers=True)
#logger.debug( "Looping over all_files" )
#logger.debug( "%s" % str( all_files ) )
for files in all_files:
i.files = files
LFNsToSplit = i.getLFNs()
if(len(LFNsToSplit)) > 1:
result = execute('splitInputDataBySize(%s,%d)'
% (i.getLFNs(), filesPerJob))
if not result_ok(result):
logger.error('DIRAC:: Error splitting files: %s' % str(result))
raise SplittingError('Error splitting files.')
split_files += result.get('Value', [])
else:
split_files = [LFNsToSplit]
if len(split_files) == 0:
raise SplittingError('An unknown error occured.')
# FIXME
# check that all files were available on the grid
big_list = []
for l in split_files:
big_list.extend(l)
diff = set(inputs.getFileNames()[:maxFiles]).difference(big_list)
if len(diff) > 0:
for f in diff:
logger.warning('Ignored file: %s' % f)
if not ignoremissing:
raise SplittingError('Some files not found!')
###
logger.debug("Split Files: %s" % str(split_files))
for dataset in split_files:
yield dataset
def checkTier1s(self):
cmd = 'checkTier1s()'
result = execute(cmd, cred_req=self.credential_requirements)
if not result_ok(result):
logger.warning('Could not obtain Tier-1 info: %s' % str(result))
return
return result.get('Value', {})
def checkTier1s(self):
cmd = 'checkTier1s()'
result = execute(cmd)
if not result_ok(result):
logger.warning('Could not obtain Tier-1 info: %s' % str(result))
return
return result.get('Value', {})
def DiracRunSplitter(inputs, filesPerJob, maxFiles, ignoremissing):
"""
Generator that yields datasets for dirac split jobs by run
"""
metadata = inputs.bkMetadata()
if not result_ok(metadata):
logger.error('Error getting input metadata: %s' % str(metadata))
raise SplittingError('Error splitting files.')
if metadata['Value']['Failed']:
logger.error('Error getting part of metadata')
raise SplittingError('Error splitting files.')
runs = defaultdict(list)
for lfn, v in metadata['Value']['Successful'].items():
f = [f for f in inputs.files if f.lfn == lfn][0]
runs[v['RunNumber']].append(f)
logger.info('Found %d runs in inputdata' % len(runs))
for run, files in sorted(runs.items()):
run_inputs = inputs.__class__()
run_inputs.files = files
if len(files) > filesPerJob:
datasets = list(
DiracSplitter(run_inputs, filesPerJob, None, ignoremissing))
else:
datasets = [files]
logger.info('Run %d with %d files was split in %d subjobs' %
(run, len(files), len(datasets)))
for ds in datasets:
yield ds
def checkTier1s(self):
cmd = "checkTier1s()"
result = execute(cmd, cred_req=self.credential_requirements)
if not result_ok(result):
logger.warning("Could not obtain Tier-1 info: %s" % str(result))
return
return result.get("Value", {})
def peek(self, filename=None, command=None):
"""Peek at the output of a job (Note: filename/command are ignored)."""
dirac_cmd = 'peek(%d)' % self.id
result = execute(dirac_cmd)
if result_ok(result):
logger.info(result['Value'])
else:
logger.error("No peeking available for Dirac job '%i'.", self.id)
def debug(self):
'''Obtains some (possibly) useful DIRAC debug info. '''
# check services
cmd = 'getServicePorts()'
result = execute(cmd)
if type(result) == str:
import datetime
try:
result = eval(result)
except Exception as err:
logger.debug("Exception, err: %s" % str(err))
pass
if not result_ok(result):
logger.warning('Could not obtain services: %s' % str(result))
return
services = result.get('Value', {})
for category in services:
system, service = category.split('/')
cmd = "ping('%s','%s')" % (system, service)
result = execute(cmd)
if type(result) == str:
import datetime
try:
result = eval(result)
except Exception as err:
logger.debug("Exception: %s" % str(err))
pass
msg = 'OK.'
if not result_ok(result):
msg = '%s' % result['Message']
logger.info('%s: %s' % (category, msg))
# get pilot info for this job
if type(self.id) != int:
return
j = self.getJobObject()
cwd = os.getcwd()
debug_dir = j.getDebugWorkspace().getPath()
cmd = "getJobPilotOutput(%d,'%s')" % \
(self.id, debug_dir)
result = execute(cmd)
if result_ok(result):
logger.info('Pilot Info: %s/pilot_%d/std.out.' %
(debug_dir, self.id))
else:
logger.error(result.get('Message', ''))
def kill(self):
""" Kill a Dirac jobs"""
if not self.id:
return None
dirac_cmd = 'kill(%d)' % self.id
result = execute(dirac_cmd)
if not result_ok(result):
raise BackendError('Dirac', 'Could not kill job: %s' % str(result))
return result['OK']
def peek(self, filename=None, command=None):
"""Peek at the output of a job (Note: filename/command are ignored).
Args:
filename (str): Ignored but is filename of a file in the sandbox
command (str): Ignored but is a command which could be executed"""
dirac_cmd = 'peek(%d)' % self.id
result = execute(dirac_cmd)
if result_ok(result):
logger.info(result['Value'])
else:
logger.error("No peeking available for Dirac job '%i'.", self.id)
def peek(self, filename=None, command=None):
"""Peek at the output of a job (Note: filename/command are ignored).
Args:
filename (str): Ignored but is filename of a file in the sandbox
command (str): Ignored but is a command which could be executed"""
dirac_cmd = 'peek(%d)' % self.id
result = execute(dirac_cmd)
if result_ok(result):
logger.info(result['Value'])
else:
logger.error("No peeking available for Dirac job '%i'.", self.id)
def getOutputSandbox(self, outputDir=None):
j = self.getJobObject()
if outputDir is None:
outputDir = j.getOutputWorkspace().getPath()
dirac_cmd = "getOutputSandbox(%d,'%s')" % (self.id, outputDir)
result = execute(dirac_cmd)
if not result_ok(result):
msg = 'Problem retrieving output: %s' % str(result)
logger.warning(msg)
return False
return True
def getOutputSandbox(self, outputDir=None):
"""Get the outputsandbox for the job object controlling this backend
Args:
outputDir (str): This string represents the output dir where the sandbox is to be placed
"""
j = self.getJobObject()
if outputDir is None:
outputDir = j.getOutputWorkspace().getPath()
dirac_cmd = "getOutputSandbox(%d,'%s')" % (self.id, outputDir)
result = execute(dirac_cmd)
if not result_ok(result):
msg = 'Problem retrieving output: %s' % str(result)
logger.warning(msg)
return False
return True
def getOutputSandbox(self, outputDir=None):
"""Get the outputsandbox for the job object controlling this backend
Args:
outputDir (str): This string represents the output dir where the sandbox is to be placed
"""
j = self.getJobObject()
if outputDir is None:
outputDir = j.getOutputWorkspace().getPath()
dirac_cmd = "getOutputSandbox(%d,'%s')" % (self.id, outputDir)
result = execute(dirac_cmd)
if not result_ok(result):
msg = 'Problem retrieving output: %s' % str(result)
logger.warning(msg)
return False
return True
def _common_submit(self, dirac_script):
'''Submit the job via the Dirac server.
Args:
dirac_script (str): filename of the JDL which is to be submitted to DIRAC
'''
j = self.getJobObject()
self.id = None
self.actualCE = None
self.status = None
self.extraInfo = None
self.statusInfo = ''
j.been_queued = False
dirac_cmd = """execfile(\'%s\')""" % dirac_script
result = execute(dirac_cmd)
# Could use the below code instead to submit on a thread
# If submitting many then user may terminate ganga before
# all jobs submitted
# def submit_checker(result, job, script):
# err_msg = 'Error submitting job to Dirac: %s' % str(result)
# if not result_ok(result) or 'Value' not in result:
# logger.error(err_msg)
# raise BackendError('Dirac',err_msg)
#
# idlist = result['Value']
# if type(idlist) is list:
# return job._setup_bulk_subjobs(idlist, script)
# job.id = idlist
# server.execute_nonblocking(dirac_cmd, callback_func=submit_checker, args=(self, dirac_script))
# return True
err_msg = 'Error submitting job to Dirac: %s' % str(result)
if not result_ok(result) or 'Value' not in result:
logger.error(err_msg)
logger.error("\n\n===\n%s\n===\n" % dirac_script)
logger.error("\n\n====\n")
with open(dirac_script, 'r') as file_in:
logger.error("%s" % file_in.read())
logger.error("\n====\n")
raise BackendError('Dirac', err_msg)
idlist = result['Value']
if type(idlist) is list:
return self._setup_bulk_subjobs(idlist, dirac_script)
self.id = idlist
return type(self.id) == int
def _common_submit(self, dirac_script):
'''Submit the job via the Dirac server.
Args:
dirac_script (str): filename of the JDL which is to be submitted to DIRAC
'''
j = self.getJobObject()
self.id = None
self.actualCE = None
self.status = None
self.extraInfo = None
self.statusInfo = ''
j.been_queued = False
dirac_cmd = """execfile(\'%s\')""" % dirac_script
result = execute(dirac_cmd)
# Could use the below code instead to submit on a thread
# If submitting many then user may terminate ganga before
# all jobs submitted
# def submit_checker(result, job, script):
# err_msg = 'Error submitting job to Dirac: %s' % str(result)
# if not result_ok(result) or 'Value' not in result:
# logger.error(err_msg)
# raise BackendError('Dirac',err_msg)
#
# idlist = result['Value']
# if type(idlist) is list:
# return job._setup_bulk_subjobs(idlist, script)
# job.id = idlist
# server.execute_nonblocking(dirac_cmd, callback_func=submit_checker, args=(self, dirac_script))
# return True
err_msg = 'Error submitting job to Dirac: %s' % str(result)
if not result_ok(result) or 'Value' not in result:
logger.error(err_msg)
logger.error("\n\n===\n%s\n===\n" % dirac_script)
logger.error("\n\n====\n")
with open(dirac_script, 'r') as file_in:
logger.error("%s" % file_in.read())
logger.error("\n====\n")
raise BackendError('Dirac', err_msg)
idlist = result['Value']
if type(idlist) is list:
return self._setup_bulk_subjobs(idlist, dirac_script)
self.id = idlist
return type(self.id) == int
def _internal_job_finalisation(job, updated_dirac_status):
logger = getLogger()
if updated_dirac_status == 'completed':
# firstly update job to completing
DiracBase._getStateTime(job, 'completing')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completing')
if job.master:
job.master.updateMasterJobStatus()
import time
start = time.time()
# contact dirac for information
job.backend.normCPUTime = execute('normCPUTime(%d)' % job.backend.id)
getSandboxResult = execute("getOutputSandbox(%d,'%s')" % (job.backend.id, job.getOutputWorkspace().getPath()))
file_info_dict = execute('getOutputDataInfo(%d)' % job.backend.id)
now = time.time()
logger.debug('Job ' + job.fqid + ' Time for Dirac metadata : ' + str(now - start))
logger.debug('Job ' + job.fqid + ' OutputDataInfo: ' + str(file_info_dict))
logger.debug('Job ' + job.fqid + ' OutputSandbox: ' + str(getSandboxResult))
# Set DiracFile metadata
wildcards = [f.namePattern for f in job.outputfiles.get(DiracFile) if regex.search(f.namePattern) is not None]
with open(os.path.join(job.getOutputWorkspace().getPath(), getConfig('Output')['PostProcessLocationsFileName']), 'ab') as postprocesslocationsfile:
if not hasattr(file_info_dict, 'keys'):
logger.error("Error understanding OutputDataInfo: %s" % str(file_info_dict))
from Ganga.Core.exceptions import GangaException
raise GangaException("Error understanding OutputDataInfo: %s" % str(file_info_dict))
for file_name in file_info_dict.get('Value', []):
file_name = os.path.basename(file_name)
info = file_info_dict.get(file_name)
logger.debug("file_name: %s,\tinfo: %s" % (str(file_name), str(info)))
valid_wildcards = [wc for wc in wildcards if fnmatch.fnmatch(file_name, wc)]
if len(valid_wildcards) == 0: valid_wildcards.append('')
if not hasattr(info, 'get'):
logger.error("Error getting OutputDataInfo for: %s" % str(job.getFQID('.')))
logger.error("Please check the Dirac Job still exists or attempt a job.backend.reset() to try again!")
logger.error("Err: %s" % str(info))
logger.error("file_info_dict: %s" % str(file_info_dict))
from Ganga.Core.exceptions import GangaException
raise GangaException("Error getting OutputDataInfo")
for wc in valid_wildcards:
logger.debug("wildcard: %s" % str(wc))
DiracFileData = 'DiracFile:::%s&&%s->%s:::%s:::%s\n' % (wc,
file_name,
info.get('LFN', 'Error Getting LFN!'),
str(info.get('LOCATIONS', ['NotAvailable'])),
info.get('GUID', 'NotAvailable')
)
logger.debug("DiracFileData: %s" % str(DiracFileData))
postprocesslocationsfile.write(DiracFileData)
# check outputsandbox downloaded correctly
if not result_ok(getSandboxResult):
logger.warning('Problem retrieving outputsandbox: %s' % str(getSandboxResult))
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
raise BackendError('Problem retrieving outputsandbox: %s' % str(getSandboxResult))
# finally update job to completed
DiracBase._getStateTime(job, 'completed')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completed')
if job.master:
job.master.updateMasterJobStatus()
now = time.time()
logger.debug('Job ' + job.fqid + ' Time for complete update : ' + str(now - start))
elif updated_dirac_status == 'failed':
# firstly update status to failed
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
# if requested try downloading outputsandbox anyway
if getConfig('DIRAC')['failed_sandbox_download']:
execute("getOutputSandbox(%d,'%s')" %
(job.backend.id, job.getOutputWorkspace().getPath()))
else:
logger.error("Unexpected dirac status '%s' encountered" % updated_dirac_status)
def DiracSplitter(inputs, filesPerJob, maxFiles, ignoremissing):
"""
Generator that yields a datasets for dirac split jobs
"""
#logger.debug( "DiracSplitter" )
#logger.debug( "inputs: %s" % str( inputs ) )
split_files = []
i = inputs.__class__()
if len(inputs.getLFNs()) != len(inputs.files):
raise SplittingError(
"Error trying to split dataset using DIRAC backend with non-DiracFile in the inputdata"
)
all_files = igroup(inputs.files[:maxFiles],
getConfig('DIRAC')['splitFilesChunks'],
leftovers=True)
#logger.debug( "Looping over all_files" )
#logger.debug( "%s" % str( all_files ) )
for files in all_files:
i.files = files
LFNsToSplit = i.getLFNs()
if (len(LFNsToSplit)) > 1:
result = execute('splitInputData(%s, %d)' %
(i.getLFNs(), filesPerJob))
if not result_ok(result):
logger.error('DIRAC:: Error splitting files: %s' % str(result))
raise SplittingError('Error splitting files.')
split_files += result.get('Value', [])
else:
split_files = [LFNsToSplit]
if len(split_files) == 0:
raise SplittingError('An unknown error occured.')
# FIXME
# check that all files were available on the grid
big_list = []
for l in split_files:
big_list.extend(l)
diff = set(inputs.getFileNames()[:maxFiles]).difference(big_list)
if len(diff) > 0:
for f in diff:
logger.warning('Ignored file: %s' % f)
if not ignoremissing:
raise SplittingError('Some files not found!')
###
logger.debug("Split Files: %s" % str(split_files))
for _dataset in split_files:
dataset = []
for _lfn in _dataset:
dataset.append(DiracFile(lfn=_lfn))
yield dataset
def _internal_job_finalisation(job, updated_dirac_status):
"""
This method performs the main job finalisation
Args:
job (Job): Thi is the job we want to finalise
updated_dirac_status (str): String representing the Ganga finalisation state of the job failed/completed
"""
if updated_dirac_status == 'completed':
start = time.time()
# firstly update job to completing
DiracBase._getStateTime(job, 'completing')
if job.status in ['removed', 'killed']:
return
elif (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completing')
if job.master:
job.master.updateMasterJobStatus()
output_path = job.getOutputWorkspace().getPath()
logger.info('Contacting DIRAC for job: %s' % job.fqid)
# Contact dirac which knows about the job
job.backend.normCPUTime, getSandboxResult, file_info_dict, completeTimeResult = execute("finished_job(%d, '%s')" % (job.backend.id, output_path))
now = time.time()
logger.info('%0.2fs taken to download output from DIRAC for Job %s' % ((now - start), job.fqid))
#logger.info('Job ' + job.fqid + ' OutputDataInfo: ' + str(file_info_dict))
#logger.info('Job ' + job.fqid + ' OutputSandbox: ' + str(getSandboxResult))
#logger.info('Job ' + job.fqid + ' normCPUTime: ' + str(job.backend.normCPUTime))
# Set DiracFile metadata
wildcards = [f.namePattern for f in job.outputfiles.get(DiracFile) if regex.search(f.namePattern) is not None]
lfn_store = os.path.join(output_path, getConfig('Output')['PostProcessLocationsFileName'])
# Make the file on disk with a nullop...
if not os.path.isfile(lfn_store):
with open(lfn_store, 'w'):
pass
if job.outputfiles.get(DiracFile):
# Now we can iterate over the contents of the file without touching it
with open(lfn_store, 'ab') as postprocesslocationsfile:
if not hasattr(file_info_dict, 'keys'):
logger.error("Error understanding OutputDataInfo: %s" % str(file_info_dict))
from Ganga.Core.exceptions import GangaException
raise GangaException("Error understanding OutputDataInfo: %s" % str(file_info_dict))
## Caution is not clear atm whether this 'Value' is an LHCbism or bug
list_of_files = file_info_dict.get('Value', file_info_dict.keys())
for file_name in list_of_files:
file_name = os.path.basename(file_name)
info = file_info_dict.get(file_name)
#logger.debug("file_name: %s,\tinfo: %s" % (str(file_name), str(info)))
if not hasattr(info, 'get'):
logger.error("Error getting OutputDataInfo for: %s" % str(job.getFQID('.')))
logger.error("Please check the Dirac Job still exists or attempt a job.backend.reset() to try again!")
logger.error("Err: %s" % str(info))
logger.error("file_info_dict: %s" % str(file_info_dict))
from Ganga.Core.exceptions import GangaException
raise GangaException("Error getting OutputDataInfo")
valid_wildcards = [wc for wc in wildcards if fnmatch.fnmatch(file_name, wc)]
if not valid_wildcards:
valid_wildcards.append('')
for wc in valid_wildcards:
#logger.debug("wildcard: %s" % str(wc))
DiracFileData = 'DiracFile:::%s&&%s->%s:::%s:::%s\n' % (wc,
file_name,
info.get('LFN', 'Error Getting LFN!'),
str(info.get('LOCATIONS', ['NotAvailable'])),
info.get('GUID', 'NotAvailable')
)
#logger.debug("DiracFileData: %s" % str(DiracFileData))
postprocesslocationsfile.write(DiracFileData)
postprocesslocationsfile.flush()
logger.debug("Written: %s" % open(lfn_store, 'r').readlines())
# check outputsandbox downloaded correctly
if not result_ok(getSandboxResult):
logger.warning('Problem retrieving outputsandbox: %s' % str(getSandboxResult))
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
elif (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
raise BackendError('Problem retrieving outputsandbox: %s' % str(getSandboxResult))
# finally update job to completed
DiracBase._getStateTime(job, 'completed', completeTimeResult)
if job.status in ['removed', 'killed']:
return
elif (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completed')
if job.master:
job.master.updateMasterJobStatus()
now = time.time()
logger.debug('Job ' + job.fqid + ' Time for complete update : ' + str(now - start))
elif updated_dirac_status == 'failed':
# firstly update status to failed
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
# if requested try downloading outputsandbox anyway
if configDirac['failed_sandbox_download']:
execute("getOutputSandbox(%d,'%s')" % (job.backend.id, job.getOutputWorkspace().getPath()))
else:
logger.error("Unexpected dirac status '%s' encountered" % updated_dirac_status)
def _internal_job_finalisation(job, updated_dirac_status):
"""
This method performs the main job finalisation
Args:
job (Job): Thi is the job we want to finalise
updated_dirac_status (str): String representing the Ganga finalisation state of the job failed/completed
"""
if updated_dirac_status == 'completed':
start = time.time()
# firstly update job to completing
DiracBase._getStateTime(job, 'completing')
if job.status in ['removed', 'killed']:
return
elif (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completing')
if job.master:
job.master.updateMasterJobStatus()
output_path = job.getOutputWorkspace().getPath()
logger.info('Contacting DIRAC for job: %s' % job.fqid)
# Contact dirac which knows about the job
job.backend.normCPUTime, getSandboxResult, file_info_dict, completeTimeResult = execute(
"finished_job(%d, '%s')" % (job.backend.id, output_path),
cred_req=job.backend.credential_requirements)
now = time.time()
logger.info(
'%0.2fs taken to download output from DIRAC for Job %s' %
((now - start), job.fqid))
#logger.info('Job ' + job.fqid + ' OutputDataInfo: ' + str(file_info_dict))
#logger.info('Job ' + job.fqid + ' OutputSandbox: ' + str(getSandboxResult))
#logger.info('Job ' + job.fqid + ' normCPUTime: ' + str(job.backend.normCPUTime))
# Set DiracFile metadata
wildcards = [
f.namePattern for f in job.outputfiles.get(DiracFile)
if regex.search(f.namePattern) is not None
]
lfn_store = os.path.join(
output_path,
getConfig('Output')['PostProcessLocationsFileName'])
# Make the file on disk with a nullop...
if not os.path.isfile(lfn_store):
with open(lfn_store, 'w'):
pass
if job.outputfiles.get(DiracFile):
# Now we can iterate over the contents of the file without touching it
with open(lfn_store, 'ab') as postprocesslocationsfile:
if not hasattr(file_info_dict, 'keys'):
logger.error("Error understanding OutputDataInfo: %s" %
str(file_info_dict))
raise GangaDiracError(
"Error understanding OutputDataInfo: %s" %
str(file_info_dict))
## Caution is not clear atm whether this 'Value' is an LHCbism or bug
list_of_files = file_info_dict.get('Value',
file_info_dict.keys())
for file_name in list_of_files:
file_name = os.path.basename(file_name)
info = file_info_dict.get(file_name)
#logger.debug("file_name: %s,\tinfo: %s" % (str(file_name), str(info)))
if not hasattr(info, 'get'):
logger.error(
"Error getting OutputDataInfo for: %s" %
str(job.getFQID('.')))
logger.error(
"Please check the Dirac Job still exists or attempt a job.backend.reset() to try again!"
)
logger.error("Err: %s" % str(info))
logger.error("file_info_dict: %s" %
str(file_info_dict))
raise GangaDiracError(
"Error getting OutputDataInfo")
valid_wildcards = [
wc for wc in wildcards
if fnmatch.fnmatch(file_name, wc)
]
if not valid_wildcards:
valid_wildcards.append('')
for wc in valid_wildcards:
#logger.debug("wildcard: %s" % str(wc))
DiracFileData = 'DiracFile:::%s&&%s->%s:::%s:::%s\n' % (
wc, file_name,
info.get('LFN', 'Error Getting LFN!'),
str(info.get('LOCATIONS', ['NotAvailable'])),
info.get('GUID', 'NotAvailable'))
#logger.debug("DiracFileData: %s" % str(DiracFileData))
postprocesslocationsfile.write(DiracFileData)
postprocesslocationsfile.flush()
logger.debug("Written: %s" % open(lfn_store, 'r').readlines())
# check outputsandbox downloaded correctly
if not result_ok(getSandboxResult):
logger.warning('Problem retrieving outputsandbox: %s' %
str(getSandboxResult))
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
elif (job.master
and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
raise BackendError(
'Dirac', 'Problem retrieving outputsandbox: %s' %
str(getSandboxResult))
# finally update job to completed
DiracBase._getStateTime(job, 'completed', completeTimeResult)
if job.status in ['removed', 'killed']:
return
elif (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('completed')
if job.master:
job.master.updateMasterJobStatus()
now = time.time()
logger.debug('Job ' + job.fqid + ' Time for complete update : ' +
str(now - start))
elif updated_dirac_status == 'failed':
# firstly update status to failed
DiracBase._getStateTime(job, 'failed')
if job.status in ['removed', 'killed']:
return
if (job.master and job.master.status in ['removed', 'killed']):
return # user changed it under us
job.updateStatus('failed')
if job.master:
job.master.updateMasterJobStatus()
# if requested try downloading outputsandbox anyway
if configDirac['failed_sandbox_download']:
execute("getOutputSandbox(%d,'%s')" %
(job.backend.id, job.getOutputWorkspace().getPath()),
cred_req=job.backend.credential_requirements)
else:
logger.error("Job #%s Unexpected dirac status '%s' encountered" %
(job.getFQID('.'), updated_dirac_status))