def test_h_MultiUpload(self):
"""Test that multiple 'uploads' work"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
for i in range(0, TestMassStorageWN.sj_len):
# Check that we correctly have expanded the wildcard still
assert len(
stripProxy(stripProxy(
j.subjobs[i]).outputfiles[0]).subfiles) == 2
assert len(j.subjobs[i].outputfiles) == 2
file_prep = os.path.join(TestMassStorageWN.outputFilePath,
str(j.id) + '_' + str(i) + '_')
# Check that the files were placed in the correct place on storage
for file_ in j.inputfiles:
assert os.path.isfile(file_prep + file_.namePattern)
TestMassStorageWN.cleanUp()
def createNewJob(self):
"""Create any jobs required for this unit"""
import copy
j = GPI.Job()
stripProxy(j).backend = self._getParent().backend.clone()
stripProxy(j).application = self._getParent().application.clone()
if self.inputdata:
j.inputdata = self.inputdata.clone()
stripProxy(j).inputfiles = copy.deepcopy(self._getParent().inputfiles)
trf = self._getParent()
task = trf._getParent()
j.inputsandbox = self._getParent().inputsandbox
j.outputfiles = copy.deepcopy(self._getParent().outputfiles)
if len(self._getParent().postprocessors.process_objects) > 0:
j.postprocessors = copy.deepcopy(
addProxy(self._getParent()).postprocessors)
if trf.splitter:
j.splitter = trf.splitter.clone()
# change the first event for GaussSplitter
from Ganga.GPI import GaussSplitter
if isType(trf.splitter, GaussSplitter):
events_per_unit = j.splitter.eventsPerJob * \
j.splitter.numberOfJobs
j.splitter.firstEventNumber = self.getID() * events_per_unit
else:
j.splitter = SplitByFiles()
return j
def test_f_MultiUpload(self):
"""Test that multiple 'uploads' work"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
for i in range(0, TestMassStorageWN.sj_len):
# Check that the subfiles were expended correctly
assert len(
stripProxy(stripProxy(
j.subjobs[i]).outputfiles[0]).subfiles) == 2
assert len(
stripProxy(stripProxy(
j.subjobs[i]).outputfiles[1]).subfiles) == 1
# Check we have the correct total number of files
assert len(j.subjobs[i].outputfiles) == 3
output_dir = os.path.join(TestMassStorageWN.outputFilePath,
str(j.id), str(i))
assert os.path.isdir(output_dir)
# Checl all of the files were put into storage
for file_ in j.inputfiles:
assert os.path.isfile(
os.path.join(output_dir, file_.namePattern))
TestMassStorageWN.cleanUp()
def test_b_testClientSideComplete(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import jobs
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
j = jobs[-1]
assert sleep_until_completed(j)
for sj in j.subjobs:
output_dir = stripProxy(sj).getOutputWorkspace(create=False).getPath()
assert os.path.isdir(output_dir) == True
# Check that the files have been removed from the output worker dir
for input_f in j.inputfiles:
assert not os.path.isfile(os.path.join(output_dir, input_f.namePattern))
# Check that the files were placed in the correct place on storage
output_dir = os.path.join(self.outputFilePath, str(j.id), str(sj.id))
for file_ in j.inputfiles:
assert os.path.isfile(os.path.join(output_dir, file_.namePattern))
# Check that wildcard expansion happened correctly
assert len(stripProxy(stripProxy(sj).outputfiles[0]).subfiles) == 2
assert len(sj.outputfiles) == 2
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
import inspect
script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
script_location = os.path.join( script_path, 'uploadScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
upload_script = FileUtils.loadScript(script_location, '')
WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py' )
script = FileUtils.loadScript(WNscript_location, '###INDENT###')
selfConstructedLFNs = False
if self.remoteDir == '' and self.lfn == '':
import datetime
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
self.lfn = os.path.join(configDirac['DiracLFNBase'], 'GangaFiles_%s' % this_date)
selfConstructedLFNs = True
if self.remoteDir == '' and self.lfn != '':
self.remoteDir = configDirac['DiracLFNBase']
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
for this_file in outputFiles:
isCompressed = this_file.namePattern in patternsToZip
if not regex.search(this_file.namePattern) is None:
script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
else:
script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))
if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
script_env = self._getDiracEnvStr()
else:
script_env = str(None)
script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])
replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
'###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
'###INDENT###' : indent,
'###LOCATIONSFILE###' : postProcessLocationsFP,
'###DIRAC_ENV###' : script_env}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def test_b_Completed(self):
"""Test the job completed and the output files exit `in storage`"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
# Just has to have reached completed state for checks to make sense
assert j.status == 'completed'
# Check that we've still got 1 file everywhere we expect 1
assert len(j.inputfiles) == 1
assert len(j.outputfiles) == 1
# 1 file after wildcard expansion
assert len(stripProxy(stripProxy(j).outputfiles[0]).subfiles) == 1
assert len(j.outputfiles) == 1
# Test that these strings are sensible
assert j.outputfiles[0].namePattern != '' and j.outputfiles[0].namePattern[0] != '*'
assert j.outputfiles[0].locations != [''] and isinstance(j.outputfiles[0].locations[0], str) is True
assert j.outputfiles[0].accessURL() != [''] and isinstance(j.outputfiles[0].accessURL()[0], str) is True
# Check that the output file exists on 'storage'
output_dir = os.path.join(TestMassStorageWN.outputFilePath, str(j.id))
assert os.path.isdir(output_dir)
assert os.path.isfile(os.path.join(output_dir, j.inputfiles[0].namePattern))
TestMassStorageWN.cleanUp()
def test_b_testClientSideComplete(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import jobs
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
j = jobs[-1]
assert sleep_until_completed(j)
for sj in j.subjobs:
output_dir = stripProxy(sj).getOutputWorkspace(create=False).getPath()
assert os.path.isdir(output_dir) == True
# Check that the files have been removed from the output worker dir
for input_f in j.inputfiles:
assert not os.path.isfile(os.path.join(output_dir, input_f.namePattern))
# Check that the files were placed in the correct place on storage
output_dir = os.path.join(self.outputFilePath, str(j.id), str(sj.id))
for file_ in j.inputfiles:
assert os.path.isfile(os.path.join(output_dir, file_.namePattern))
# Check that wildcard expansion happened correctly
assert len(stripProxy(stripProxy(sj).outputfiles[0]).subfiles) == 2
assert len(sj.outputfiles) == 2
def test_f_MultiUpload(self):
"""Test that multiple 'uploads' work"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
for i in range(0, TestMassStorageWN.sj_len):
# Check that the subfiles were expended correctly
assert len(stripProxy(stripProxy(j.subjobs[i]).outputfiles[0]).subfiles) == 2
assert len(stripProxy(stripProxy(j.subjobs[i]).outputfiles[1]).subfiles) == 1
# Check we have the correct total number of files
assert len(j.subjobs[i].outputfiles) == 3
output_dir = os.path.join(TestMassStorageWN.outputFilePath, str(j.id), str(i))
assert os.path.isdir(output_dir)
# Checl all of the files were put into storage
for file_ in j.inputfiles:
assert os.path.isfile(os.path.join(output_dir, file_.namePattern))
TestMassStorageWN.cleanUp()
def test_h_MultiUpload(self):
"""Test that multiple 'uploads' work"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
for i in range(0, TestMassStorageWN.sj_len):
# Check that we correctly have expanded the wildcard still
assert len(stripProxy(stripProxy(j.subjobs[i]).outputfiles[0]).subfiles) == 2
assert len(j.subjobs[i].outputfiles) == 2
file_prep = os.path.join(TestMassStorageWN.outputFilePath, str(j.id) + '_' + str(i) + '_')
# Check that the files were placed in the correct place on storage
for file_ in j.inputfiles:
assert os.path.isfile(file_prep + file_.namePattern)
TestMassStorageWN.cleanUp()
def createChainUnit(self, parent_units, use_copy_output=True):
"""Create an output unit given this output data"""
# check parent units/jobs are complete
if not self.checkUnitsAreCompleted(parent_units):
return None
# get the include/exclude masks
incl_pat_list, excl_pat_list = self.getChainInclExclMasks(parent_units)
# go over the output files and transfer to input data
flist = []
for sj in self.getParentUnitJobs(parent_units):
for f in sj.outputfiles:
temp_flist = stripProxy(f).getSubFiles() if len(
stripProxy(f).getSubFiles()) > 0 else [stripProxy(f)]
for f2 in temp_flist:
if len(incl_pat_list) > 0:
for pat in incl_pat_list:
if re.search(pat, f2.namePattern):
flist.append(f2)
else:
flist.append(f2)
for pat in excl_pat_list:
if re.search(pat, f2.namePattern):
flist.remove(f2)
# now create the unit with a GangaDataset
unit = CoreUnit()
unit.name = "Unit %d" % len(self.units)
unit.inputdata = GangaDataset(files=flist)
unit.inputdata.treat_as_inputfiles = self.chaindata_as_inputfiles
return unit
def test_d_CompletedSJ(self):
"""Test that the subjobs ave completed"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
assert len(stripProxy(stripProxy(j.subjobs[0]).outputfiles[0]).subfiles) == 1
assert len(j.subjobs[0].outputfiles) == 1
for i in range(0, TestMassStorageWN.sj_len):
output_dir = os.path.join(TestMassStorageWN.outputFilePath, str(j.id), str(i))
assert os.path.isdir(output_dir)
# Check each inputfile has been placed in storage like we asked
for _input_file in j.inputfiles:
assert os.path.isfile(os.path.join(output_dir, _input_file.namePattern))
TestMassStorageWN.cleanUp()
def test_b_testClientSideComplete(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
assert sleep_until_completed(j)
for sj in j.subjobs:
output_dir = stripProxy(sj).getOutputWorkspace(
create=False).getPath()
assert os.path.isdir(output_dir)
# Check that the files were placed in the correct place on storage
for file_ in j.inputfiles:
for this_file in glob.glob(
os.path.join(output_dir, file_.namePattern)):
assert os.path.isfile(this_file)
# Check that wildcard expansion happened correctly
assert len(stripProxy(sj).outputfiles[0].subfiles) == 2
assert len(sj.outputfiles) == 2
def test_e_TestCorruptLoad(self):
# Test loading of backup when corrupt
from Ganga.GPI import jobs
assert len(jobs) == 1
assert len(jobs(0).subjobs) == 2
backend2 = jobs(0).subjobs(0).backend
assert backend2 is not None
XMLFileName = getSJXMLFile(jobs(0).subjobs(0))
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(delete=False) as myTempfile:
myTempfile.write(badStr)
myTempfile.flush()
myTempName=myTempfile.name
from Ganga.GPIDev.Base.Proxy import stripProxy
assert stripProxy(jobs(0).subjobs(0))._dirty is True
assert stripProxy(jobs(0))._dirty is True
stripProxy(jobs(0).subjobs(0))._getRegistry().flush_all()
assert open(XMLFileName).read() != open(myTempName).read()
unlink(myTempName)
def test_d_TestCorruptXML(self):
# Corrupt the XML file
from Ganga.GPI import jobs, Job
assert isinstance(jobs(0).subjobs(0), Job)
XMLFileName = getSJXMLFile(jobs(0).subjobs(0))
unlink(XMLFileName)
assert not path.isfile(XMLFileName)
with open(XMLFileName, 'w') as handler:
handler.write(badStr)
handler.flush()
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(delete=False) as myTempfile:
myTempfile.write(badStr)
myTempfile.flush()
myTempName = myTempfile.name
from Ganga.GPIDev.Base.Proxy import stripProxy
assert stripProxy(jobs(0).subjobs(0))._dirty is False
assert stripProxy(jobs(0))._dirty is False
assert open(XMLFileName, 'r').read() == open(myTempName, 'r').read()
unlink(myTempName)
global global_AutoStartReg
global_AutoStartReg = True
def updatePartitionStatus(self, partition):
""" Calculate the correct status of the given partition."""
running_status = set(["submitting", "submitted", "running", "completing"])
nonProcessing_states = set(["bad", "ignored", "unknown"])
# If the partition has status, and is not in a fixed state...
if partition in self._partition_status and (not self._partition_status[partition] in nonProcessing_states):
# if we have no applications, we are in "ready" state
if not partition in self.getPartitionApps():
if self._partition_status[partition] != "hold":
self._partition_status[partition] = "ready"
elif self._partition_status[partition] != "hold":
status = set([pj.status for pj in self.getPartitionJobs(partition)])
if status.intersection(running_status):
self._partition_status[partition] = "running"
elif "new" in status:
self._partition_status[partition] = "ready"
elif "failed" in status:
mj = self._getPartitionMasterJob(partition)
if mj.status not in running_status:
failures = mj.info.submit_counter
if failures >= self.run_limit:
self._partition_status[partition] = "failed"
elif failures > 0:
if len(mj.subjobs):
# Catches the fact that master job submit
# counter doesnt increment when subjobs
# resubmitted.
stripProxy(mj).info.submit_counter += 1
self._partition_status[partition] = "attempted"
else:
self._partition_status[partition] = "completed"
def test_b_testClientSideComplete(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
assert sleep_until_completed(j)
for sj in j.subjobs:
output_dir = stripProxy(sj).getOutputWorkspace(create=False).getPath()
assert os.path.isdir(output_dir)
# Check that the files were placed in the correct place on storage
for file_ in j.inputfiles:
for this_file in glob.glob(os.path.join(output_dir, file_.namePattern)):
assert os.path.isfile(this_file)
# Check that wildcard expansion happened correctly
assert len(stripProxy(sj).outputfiles[0].subfiles) == 2
assert len(sj.outputfiles) == 2
def test_i_testSJXMLIndex(self):
# Check index of all sj
from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file
from Ganga.GPI import jobs
assert len(jobs) == 2
j = jobs(0)
with open(getSJXMLIndex(j)) as handler:
obj, errs = from_file(handler)
assert isinstance(obj, dict)
from Ganga.GPIDev.Base.Proxy import stripProxy, getName
raw_j = stripProxy(j)
new_dict = {}
for sj in j.subjobs:
raw_sj = stripProxy(sj)
temp_index = raw_sj._getRegistry().getIndexCache(raw_sj)
new_dict[sj.id] = temp_index
assert raw_sj._category == raw_j._category
for k, v in new_dict.iteritems():
for k1, v1 in v.iteritems():
if k1 != 'modified':
assert obj[k][k1] == new_dict[k][k1]
def isA(self, _what):
from Ganga.GPIDev.Base.Proxy import stripProxy
what = stripProxy(_what)
this_type = type(what)
try:
# for backwards compatibility with Ganga3 CLIP: if a string --
# first convert to the class name
if isinstance(what, str):
# get access to all Item classes defined in this module
# (schema)
if hasattr(Schema, what):
what = getattr(Schema, what)
else:
return False
elif isinstance(stripProxy(what), types.InstanceType):
if hasattr(what, '__class__'):
what = what.__class__
else:
return False
except AttributeError:
# class not found
return False
return issubclass(self.__class__, what)
def test_c_JobLoaded(self):
""" Third do something to trigger a loading of a Job and then test if it's loaded"""
from Ganga.GPI import jobs
self.assertEqual(len(jobs), 1)
j = jobs(0)
from Ganga.GPIDev.Base.Proxy import stripProxy
raw_j = stripProxy(j)
## ANY COMMAND TO LOAD A JOB CAN BE USED HERE
raw_j.printSummaryTree()
has_loaded_job = raw_j._getRegistry().has_loaded(raw_j)
for i in range(len(j.subjobs)):
self.assertFalse(raw_j.subjobs.isLoaded(i))
self.assertTrue(has_loaded_job)
stripProxy(j.subjobs(0)).printSummaryTree()
self.assertTrue(raw_j.subjobs.isLoaded(0))
for i in range(1, len(j.subjobs)):
self.assertFalse(raw_j.subjobs.isLoaded(i))
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
self._hasStarted = True
from Ganga.Utility.logging import getLogger
self.shouldRun = True
## Aparently this shuts down the metadata repo before we want to shut it down...
#super(PrepRegistry, self).shutdown()
logger = getLogger()
#logger.info("Geting id: %s" % self.metadata.ids()[-1])
self.shareref = self.metadata._objects[self.metadata.ids()[-1]]
#logger.info("ShareRef: %s" % getName(self.shareref))
self._lock.acquire()
## THIS IS DISABLED AS IT REQUIRES ACCESS TO REPO OBJECTS THROUGH GETREADACCES...
## THIS NEEDS TO BE FIXED OR IMPLEMENTED AS A SHUTDOWN SERVICE!!!
try:
stripProxy(self.shareref).closedown() ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
except Exception as err:
logger.error("Shutdown Error in ShareRef")
logger.error("Err: %s" % str(err))
try:
self._safe_shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % str(err))
finally:
self._hasStarted = False
self._lock.release()
def find(self, id, path=None):
"""For a job with given id tries to find all references in the job tree.
The return value is a list of found paths.
"""
if isType(id, Job):
id = stripProxy(id).getFQID('.')
if isType(id, GPIProxyObject):
id = stripProxy(id)
pp = self.__get_path(path)
tp = os.path.join(*pp)
top_level = self.__folder_cd(pp)
search_dirs = []
found_items = []
for _item in top_level.keys():
if isinstance(top_level[_item], dict):
search_dirs.append(_item)
else:
if top_level[_item] == id:
found_items.append(tp)
result = []
for item in found_items:
result.append(item)
for dir in search_dirs:
new_path = os.path.join(tp, dir)
for found in self.find(id, new_path):
result.append(found)
return result
def test_d_TestCorruptXML(self):
# Corrupt the XML file
from Ganga.GPI import jobs, Job
assert isinstance(jobs(0).subjobs(0), Job)
XMLFileName = getSJXMLFile(jobs(0).subjobs(0))
unlink(XMLFileName)
assert not path.isfile(XMLFileName)
with open(XMLFileName, 'w') as handler:
handler.write(badStr)
handler.flush()
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(delete=False) as myTempfile:
myTempfile.write(badStr)
myTempfile.flush()
myTempName = myTempfile.name
from Ganga.GPIDev.Base.Proxy import stripProxy
assert stripProxy(jobs(0).subjobs(0))._dirty is False
assert stripProxy(jobs(0))._dirty is False
assert open(XMLFileName, 'r').read() == open(myTempName, 'r').read()
unlink(myTempName)
global global_AutoStartReg
global_AutoStartReg = True
def updateMonitoringInformation(jobs):
for j in jobs:
stripProxy(j)._getWriteAccess()
raw_backend = stripProxy(j.backend)
if not j.backend.id:
id = raw_backend._getIntFromOutfile("PID:", "__id__")
if id > 0:
raw_backend.id = id
if ("submitted" == j.backend.status):
raw_backend.status = "running"
# Check that the process is still alive
if j.backend.id:
try:
os.kill(j.backend.id, 0)
except Exception as err:
logger.debug("Err: %s" % str(err))
raw_backend.status = "completed"
if j.backend.status in ["completed", "failed", "killed"]:
raw_backend.exitcode = raw_backend._getIntFromOutfile("EXITCODE:", "__jobstatus__")
# Set job status to failed for non-zero exit code
if j.backend.exitcode:
if j.backend.exitcode in [2, 9, 256]:
raw_backend.status = "killed"
else:
raw_backend.status = "failed"
if (j.backend.status != j.status):
j.updateStatus(j.backend.status)
return None
def _getColour(self, obj):
if stripProxy(obj).getNodeIndexCache():
status_attr = stripProxy(obj).getNodeIndexCache()['display:status']
else:
status_attr = obj.status
returnable = self.status_colours.get(status_attr, self.fx.normal)
return returnable
def transition_update(self, new_status):
# print "Transition Update of app ", self.id, " to ",new_status
try:
transform = self.getTransform()
if self.tasks_id.startswith("00"): # Master job
if new_status == "new": # something went wrong with submission
for sj in self._getParent().subjobs:
sj.application.transition_update(new_status)
if transform:
stripProxy(transform).setMasterJobStatus(
self._getParent(), new_status)
else:
if transform:
stripProxy(transform).setAppStatus(self, new_status)
except Exception as x:
import traceback
import sys
logger.error(
"Exception in call to transform[%s].setAppStatus(%i, %s)", self.tasks_id, self.id, new_status)
logger.error( getName(x) + " : " + x)
tb = sys.exc_info()[2]
if tb:
traceback.print_tb(tb)
else:
logger.error("No Traceback available")
logger.error("%s", x)
def createChainUnit(self, parent_units, use_copy_output=True):
"""Create an output unit given this output data"""
# check parent units/jobs are complete
if not self.checkUnitsAreCompleted(parent_units):
return None
# get the include/exclude masks
incl_pat_list, excl_pat_list = self.getChainInclExclMasks(parent_units)
# go over the output files and transfer to input data
flist = []
for sj in self.getParentUnitJobs(parent_units):
for f in sj.outputfiles:
temp_flist = stripProxy(f).getSubFiles() if len(stripProxy(f).getSubFiles()) > 0 else [stripProxy(f)]
for f2 in temp_flist:
if len(incl_pat_list) > 0:
for pat in incl_pat_list:
if re.search(pat, f2.namePattern):
flist.append(f2)
else:
flist.append(f2)
for pat in excl_pat_list:
if re.search(pat, f2.namePattern):
flist.remove(f2)
# now create the unit with a GangaDataset
unit = CoreUnit()
unit.name = "Unit %d" % len(self.units)
unit.inputdata = GangaDataset(files=flist)
unit.inputdata.treat_as_inputfiles = self.chaindata_as_inputfiles
return unit
def append(self, obj, my_filter=True):
if isType(obj, GangaList):
stripped_o = stripProxy(obj)
stripped_o._setParent(self._getParent())
self._list.append(stripped_o)
return
elem = self.strip_proxy(obj, my_filter)
list_objs = (list, tuple)
if isType(elem, GangaObject):
stripped_e = stripProxy(elem)
stripped_e._setParent(self._getParent())
self._list.append(stripped_e)
elif isType(elem, list_objs):
new_list = []
def my_append(_obj):
if isType(_obj, GangaObject):
stripped_o = stripProxy(_obj)
stripped_o._setParent(self._getParent())
return stripped_o
else:
return _obj
self._list.append([my_append(l) for l in elem])
else:
self._list.append(elem)
def print_config_file():
from cStringIO import StringIO
sio = StringIO()
sections = sorted(stripProxy(config).keys())
def print_doc_text(text):
for line in text.splitlines():
sio.write('# ' + line + '\n')
for p in sections:
sect = stripProxy(config)[p]
if not sect.cfile:
continue
sio.write('\n')
sio.write(
"#=======================================================================\n"
)
sio.write("[%s]" % p + '\n')
print_doc_text(sect.docstring)
opts = sorted(sect.options.keys())
for o in opts:
if sect.options[o].cfile:
sio.write('\n')
print_doc_text(sect.options[o].docstring)
sio.write('#%s = %s' % (o, sect.options[o].default_value) +
'\n')
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType, stripProxy
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(random.choice(string.ascii_uppercase) for _ in range(5))
stripProxy(mj)._getRegistry()._flush([stripProxy(mj)])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert stripProxy(sj)._getRoot() is stripProxy(j)
assert stripProxy(sj.application)._getRoot() is stripProxy(j)
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType, stripProxy
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(
random.choice(string.ascii_uppercase) for _ in range(5))
stripProxy(mj)._getRegistry()._flush([stripProxy(mj)])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert stripProxy(sj)._getRoot() is stripProxy(j)
assert stripProxy(sj.application)._getRoot() is stripProxy(j)
def test_c_JobLoaded(self):
""" Third do something to trigger a loading of a Job and then test if it's loaded"""
from Ganga.GPI import jobs
self.assertEqual(len(jobs), 1)
j = jobs(0)
from Ganga.GPIDev.Base.Proxy import stripProxy
raw_j = stripProxy(j)
## ANY COMMAND TO LOAD A JOB CAN BE USED HERE
raw_j.printSummaryTree()
has_loaded_job = raw_j._getRegistry().has_loaded(raw_j)
for i in range(len(j.subjobs)):
self.assertFalse(raw_j.subjobs.isLoaded(i))
self.assertTrue(has_loaded_job)
stripProxy(j.subjobs(0)).printSummaryTree()
self.assertTrue(raw_j.subjobs.isLoaded(0))
for i in range(1, len(j.subjobs)):
self.assertFalse(raw_j.subjobs.isLoaded(i))
def _display(self, colour):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Effects
if colour:
markup = ANSIMarkup()
else:
markup = NoMarkup()
fg = Foreground()
from cStringIO import StringIO
sio = StringIO()
sio.write("Ganga Configuration" + '\n')
sections = sorted(stripProxy(self).keys())
maxcol = 0
for p in sections:
if len(p) > maxcol:
maxcol = len(p)
if maxcol > 50:
maxcol = 50
for p in sections:
sio.write('%-*s : %s' % (maxcol, p,
markup(
stripProxy(self)[p].docstring.split(
'\n')[0], fg.boldgrey)) + '\n')
return sio.getvalue()
def transition_update(self, new_status):
# print "Transition Update of app ", self.id, " to ",new_status
try:
transform = self.getTransform()
if self.tasks_id.startswith("00"): # Master job
if new_status == "new": # something went wrong with submission
for sj in self._getParent().subjobs:
sj.application.transition_update(new_status)
if transform:
stripProxy(transform).setMasterJobStatus(
self._getParent(), new_status)
else:
if transform:
stripProxy(transform).setAppStatus(self, new_status)
except Exception as x:
import traceback
import sys
logger.error(
"Exception in call to transform[%s].setAppStatus(%i, %s)",
self.tasks_id, self.id, new_status)
logger.error(getName(x) + " : " + x)
tb = sys.exc_info()[2]
if tb:
traceback.print_tb(tb)
else:
logger.error("No Traceback available")
logger.error("%s", x)
def test__init__(self):
self.assertEqual(self.df.namePattern, 'np',
'namePattern not initialised as np')
self.assertEqual(self.df.lfn, 'lfn', 'lfn not initialised as lfn')
self.assertEqual(self.df.localDir, 'ld',
'localDir not initialised as ld')
d1 = stripProxy(DiracFile())
self.assertEqual(d1.namePattern, '',
'namePattern not default initialised as empty')
self.assertEqual(d1.lfn, '', 'lfn not default initialised as empty')
self.assertEqual(d1.localDir, None,
'localDir not default initialised as None')
self.assertEqual(d1.locations, [],
'locations not initialised as empty list')
d2 = stripProxy(DiracFile(namePattern='np', lfn='lfn', localDir='ld'))
self.assertEqual(
d2.namePattern, 'np',
'namePattern not keyword initialised as np, initialized as: %s\n%s'
% (d2.namePattern, str(d2)))
self.assertEqual(
d2.lfn, 'lfn',
'lfn not keyword initialised as lfn, initialized as: %s\n%s' %
(d2.lfn, str(d2)))
self.assertEqual(
d2.localDir, 'ld',
'localDir not keyword initialised as ld, initializes as %s\n%s' %
(d2.localDir, str(d2.localDir)))
def test_Savannah14799(self):
from Ganga.GPI import Job, jobtree, jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
j = Job()
jobtree.add(j)
self.assertNotEqual(
stripProxy(j)._getRegistry(),
stripProxy(jobtree)._getRegistry())
self.assertTrue(str(j.id) in jobtree.listjobs())
jt2 = jobtree.copy()
self.assertTrue(str(j.id) in jt2.listjobs())
jobs(j.id).remove()
jt2.cleanlinks()
self.assertFalse(str(j.id) in jobtree.listjobs())
print jt2.listjobs()
print jt2
self.assertFalse(str(j.id) in jt2.listjobs())
jt3 = jobtree.copy()
l1 = jobtree.listjobs()
l3 = jt3.listjobs()
l1.sort()
l3.sort()
self.assertEqual(l1, l3)
def load(self):
ids = self.reg.ids()
if len(ids) == 0:
return
_id = self.rng.sample(ids, 1)[0]
self.logger.info(str(self.id) + ' load(%s)' % _id)
try:
self.logger.info('Getting ReadAccess: %s from %s' %
(_id, self.reg.ids()))
from Ganga.GPIDev.Base.Proxy import stripProxy
stripProxy(self.reg[_id])._loadObject()
# self.logger.info('Looking at: %s' % self.owned_objs[_id])
# self.logger.info('stripped: %s' % stripProxy(self.owned_objs[_id]))
self.logger.info('name: %s' % self.reg[_id].name)
self.logger.info('Wanting: %s' % _id)
assert self.reg[_id].name.startswith('HT')
if _id in self.owned_ids:
assert self.reg[
_id].name == 'HT%i' % self.id, '{0} == {1}'.format(
self.reg[_id].name, 'HT%i' % self.id)
except KeyError: # If the object has been deleted in the meantime, it must be gone from the registry
assert _id not in self.reg.ids()
self.logger.info(
str(self.id) +
' %s deleted after KeyError (as per specification)' % _id)
self.logger.info(str(self.id) + ' load(%s) done!' % _id)
def print_config_file():
sections = sorted(stripProxy(config).keys())
def print_doc_text(text):
for line in text.splitlines():
sio.write('# ' + line + '\n')
for p in sections:
sect = stripProxy(config)[p]
if not sect.cfile:
continue
sio.write('\n')
sio.write(
"#=======================================================================\n")
sio.write("[%s]" % p + '\n')
print_doc_text(sect.docstring)
opts = sorted(sect.options.keys())
for o in opts:
if sect.options[o].cfile:
sio.write('\n')
print_doc_text(sect.options[o].docstring)
sio.write('#%s = %s' %
(o, sect.options[o].default_value) + '\n')
def _checkActiveBackends(self, activeBackendsFunc):
log.debug("calling function _checkActiveBackends")
activeBackends = activeBackendsFunc()
summary = '{'
for this_backend, these_jobs in activeBackends.iteritems():
summary += '"' + this_backend + '" : ['
for this_job in these_jobs:
summary += str(stripProxy(this_job).getFQID('.')) + ', '
summary += '], '
summary += '}'
log.debug("Active Backends: %s" % summary)
for jList in activeBackends.values():
#log.debug("backend: %s" % str(jList))
backendObj = jList[0].backend
b_name = getName(backendObj)
if b_name in config:
pRate = config[b_name]
else:
pRate = config['default_backend_poll_rate']
# TODO: To include an if statement before adding entry to
# updateDict. Entry is added only if credential requirements
# of the particular backend is satisfied.
# This requires backends to hold relevant information on its
# credential requirements.
#log.debug("addEntry: %s, %s, %s, %s" % (str(backendObj), str(self._checkBackend), str(jList), str(pRate)))
self.updateDict_ts.addEntry(backendObj, self._checkBackend, jList, pRate)
summary = str([stripProxy(x).getFQID('.') for x in jList])
log.debug("jList: %s" % str(summary))
def test_d_CompletedSJ(self):
"""Test that the subjobs ave completed"""
from Ganga.GPI import jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
from GangaTest.Framework.utils import sleep_until_completed
j = jobs[-1]
sleep_until_completed(j)
assert j.status == 'completed'
assert len(j.subjobs) == TestMassStorageWN.sj_len
assert len(
stripProxy(stripProxy(j.subjobs[0]).outputfiles[0]).subfiles) == 1
assert len(j.subjobs[0].outputfiles) == 1
for i in range(0, TestMassStorageWN.sj_len):
output_dir = os.path.join(TestMassStorageWN.outputFilePath,
str(j.id), str(i))
assert os.path.isdir(output_dir)
# Check each inputfile has been placed in storage like we asked
for _input_file in j.inputfiles:
assert os.path.isfile(
os.path.join(output_dir, _input_file.namePattern))
TestMassStorageWN.cleanUp()
def test_e_TestCorruptLoad(self):
# Test loading of backup when corrupt
from Ganga.GPI import jobs
assert len(jobs) == 1
assert len(jobs(0).subjobs) == 2
backend2 = jobs(0).subjobs(0).backend
assert backend2 is not None
XMLFileName = getSJXMLFile(jobs(0).subjobs(0))
from tempfile import NamedTemporaryFile
with NamedTemporaryFile(delete=False) as myTempfile:
myTempfile.write(badStr)
myTempfile.flush()
myTempName = myTempfile.name
from Ganga.GPIDev.Base.Proxy import stripProxy
assert stripProxy(jobs(0).subjobs(0))._dirty is True
assert stripProxy(jobs(0))._dirty is True
stripProxy(jobs(0).subjobs(0))._getRegistry().flush_all()
assert open(XMLFileName).read() != open(myTempName).read()
unlink(myTempName)
def test_b_Completed(self):
"""Test the job completed and the output files exit `in storage`"""
from Ganga.GPI import jobs
j = jobs[-1]
assert sleep_until_completed(j)
# Check that we've still got 1 file everywhere we expect 1
assert len(j.inputfiles) == 1
assert len(j.outputfiles) == 1
# 1 file after wildcard expansion
assert len(stripProxy(stripProxy(j).outputfiles[0]).subfiles) == 1
assert len(j.outputfiles) == 1
# Test that these strings are sensible
assert j.outputfiles[0].namePattern != ''
assert j.outputfiles[0].namePattern[0] != '*'
assert j.outputfiles[0].locations != ['']
assert isinstance(j.outputfiles[0].locations[0], str) is True
assert j.outputfiles[0].accessURL() != ['']
assert isinstance(j.outputfiles[0].accessURL()[0], str) is True
# Check that the output file exists on 'storage'
output_dir = os.path.join(self.outputFilePath, str(j.id))
assert os.path.isdir(output_dir)
assert os.path.isfile(os.path.join(output_dir, j.inputfiles[0].namePattern))
self.cleanUp()
def _checkActiveBackends(self, activeBackendsFunc):
log.debug("calling function _checkActiveBackends")
activeBackends = activeBackendsFunc()
summary = '{'
for this_backend, these_jobs in activeBackends.iteritems():
summary += '"' + this_backend + '" : ['
for this_job in these_jobs:
summary += str(stripProxy(this_job).getFQID('.')) + ', '
summary += '], '
summary += '}'
log.debug("Active Backends: %s" % summary)
for jList in activeBackends.values():
#log.debug("backend: %s" % str(jList))
backendObj = jList[0].backend
b_name = getName(backendObj)
if b_name in config:
pRate = config[b_name]
else:
pRate = config['default_backend_poll_rate']
# TODO: To include an if statement before adding entry to
# updateDict. Entry is added only if credential requirements
# of the particular backend is satisfied.
# This requires backends to hold relevant information on its
# credential requirements.
#log.debug("addEntry: %s, %s, %s, %s" % (str(backendObj), str(self._checkBackend), str(jList), str(pRate)))
updateDict_ts.addEntry(backendObj, self._checkBackend, jList,
pRate)
summary = str([stripProxy(x).getFQID('.') for x in jList])
log.debug("jList: %s" % str(summary))
def updateMonitoringInformation(jobs):
for j in jobs:
stripProxy(j)._getWriteAccess()
raw_backend = stripProxy(j.backend)
if not j.backend.id:
id = raw_backend._getIntFromOutfile("PID:", "__id__")
if id > 0:
raw_backend.id = id
if ("submitted" == j.backend.status):
raw_backend.status = "running"
# Check that the process is still alive
if j.backend.id:
try:
os.kill(j.backend.id, 0)
except Exception as err:
logger.debug("Err: %s" % str(err))
raw_backend.status = "completed"
if j.backend.status in ["completed", "failed", "killed"]:
raw_backend.exitcode = raw_backend._getIntFromOutfile(
"EXITCODE:", "__jobstatus__")
# Set job status to failed for non-zero exit code
if j.backend.exitcode:
if j.backend.exitcode in [2, 9, 256]:
raw_backend.status = "killed"
else:
raw_backend.status = "failed"
if (j.backend.status != j.status):
j.updateStatus(j.backend.status)
return None
def shutdown(self):
"""Flush and disconnect the repository. Called from Repository_runtime.py """
self._hasStarted = True
from Ganga.Utility.logging import getLogger
self.shouldRun = True
## Aparently this shuts down the metadata repo before we want to shut it down...
#super(PrepRegistry, self).shutdown()
logger = getLogger()
#logger.info("Geting id: %s" % self.metadata.ids()[-1])
self.shareref = self.metadata._objects[self.metadata.ids()[-1]]
#logger.info("ShareRef: %s" % getName(self.shareref))
with self._flush_lock:
with self._read_lock:
## THIS IS DISABLED AS IT REQUIRES ACCESS TO REPO OBJECTS THROUGH GETREADACCES...
## THIS NEEDS TO BE FIXED OR IMPLEMENTED AS A SHUTDOWN SERVICE!!!
try:
stripProxy(self.shareref).closedown(
) ## Commenting out a potentially EXTREMELY heavy operation from shutdown after ganga dev meeting - rcurrie
except Exception as err:
logger.error("Shutdown Error in ShareRef")
logger.error("Err: %s" % err)
try:
self._safe_shutdown()
except Exception as err:
logger.debug("Shutdown Error: %s" % err)
self._hasStarted = False
self.metadata = None
def find(self, id, path=None):
"""For a job with given id tries to find all references in the job tree.
The return value is a list of found paths.
"""
if isType(id, Job):
id = stripProxy(id).getFQID('.')
if isType(id, GPIProxyObject):
id = stripProxy(id)
pp = self.__get_path(path)
tp = os.path.join(*pp)
top_level = self.__folder_cd(pp)
search_dirs = []
found_items = []
for _item in top_level.keys():
if isinstance(top_level[_item], dict):
search_dirs.append(_item)
else:
if top_level[_item] == id:
found_items.append(tp)
result = []
for item in found_items:
result.append(item)
for dir in search_dirs:
new_path = os.path.join(tp, dir)
for found in self.find(id, new_path):
result.append(found)
return result
def addQuery(self, transform, bkQuery, associate=True):
"""Allows the user to add multiple transforms corresponding to the list of
BKQuery type objects given in the second argument. The first argument
is a transform object to use as the basis for the creation of further
transforms."""
if not isType(transform, LHCbAnalysisTransform):
raise GangaException(
None, 'First argument must be an LHCbAnalysisTransform objects to use as the basis for establishing the new transforms')
# Check if the template transform is associated with the Task
try:
self.transforms.index(transform)
except:
if associate:
logger.info(
'The transform is not associated with this Task, doing so now.')
self.appendTransform(transform)
# Check if the BKQuery input is correct and append/update
if type(bkQuery) is not list:
bkQuery = [bkQuery]
for bk in bkQuery:
if not isType(bk, BKQuery):
raise GangaAttributeError(
None, 'LHCbTransform expects a BKQuery object or list of BKQuery objects passed to the addQuery method')
if transform.query is None: # If template has no query itself
logger.info('Attaching query to transform')
transform.query = stripProxy(bk)
transform.update()
else: # Duplicate from template
logger.info('Duplicating transform to add new query.')
tr = deepcopy(transform)
tr.query = stripProxy(bk)
self.appendTransform(tr)
def postprocessor_filter(value, item):
from Ganga.GPIDev.Lib.Job.Job import Job
from Ganga.GPIDev.Lib.Tasks.ITransform import ITransform
from Ganga.GPIDev.Base.Objects import ObjectMetaclass
#from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPIDev.Base.Proxy import getProxyInterface
valid_jobtypes = [stripProxy(i)._schema.datadict['postprocessors'] for i in getProxyInterface().__dict__.values()
if isinstance(stripProxy(i), ObjectMetaclass)
and (issubclass(stripProxy(i), Job) or issubclass(stripProxy(i), ITransform))
and 'postprocessors' in stripProxy(i)._schema.datadict]
# Alex modified this line to that from above to allow for arbitrary dynamic LHCbJobTemplate etc types
# if item is Job._schema['postprocessors']:
if item in valid_jobtypes:
ds = MultiPostProcessor()
if isinstance(value, list) or isType(value, GangaList):
for item_ in value:
ds.append(item_)
else:
ds.append(value)
return ds
else:
raise PostProcessException("j.postprocessors only takes objects of category 'postprocessor'")
def test_i_testSJXMLIndex(self):
# Check index of all sj
from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file
from Ganga.GPI import jobs
assert len(jobs) == 2
j=jobs(0)
with open(getSJXMLIndex(j)) as handler:
obj, errs = from_file(handler)
assert isinstance(obj, dict)
from Ganga.GPIDev.Base.Proxy import stripProxy, getName
raw_j = stripProxy(j)
new_dict = {}
for sj in j.subjobs:
raw_sj = stripProxy(sj)
temp_index = raw_sj._getRegistry().getIndexCache(raw_sj)
new_dict[sj.id] = temp_index
assert raw_sj._category == raw_j._category
for k, v in new_dict.iteritems():
for k1, v1 in v.iteritems():
if k1 != 'modified':
assert obj[k][k1] == new_dict[k][k1]
def lock(self):
ids = self.reg.ids()
if len(ids) == 0:
return
_id = self.rng.sample(ids, 1)[0]
self.logger.info(str(self.id) + ' lock(%s)' % _id)
from Ganga.Core.GangaRepository import RegistryLockError
try:
self.logger.info('Getting Read, Write access: %s' % _id)
from Ganga.GPIDev.Base.Proxy import stripProxy
stripProxy(self.reg[_id])._loadObject()
stripProxy(self.reg[_id])._getSessionLock()
self.logger.info('Got Access: %s' % _id)
self.logger.info('Name: %s' % self.reg[_id].name)
assert self.reg[_id].name.startswith('HT')
self.reg[_id].name = 'HT%i' % self.id
if _id not in self.owned_ids:
self.owned_ids.append(_id)
except KeyError: # If the object has been deleted in the meantime, it must be gone from the registry
self.logger.info(
str(self.id) +
' %s deleted after KeyError (as per specification)' % _id)
assert _id not in self.reg
except RegistryLockError: # ok, this is already locked
self.logger.info(str(self.id) + ' %s was locked...' % _id)
self.logger.info(str(self.id) + ' lock(%s) done!' % _id)
def _splitter(self, job, inputdata):
logger.debug("_splitter")
indata = inputdata
if indata is not None:
self.depth = indata.depth
self.persistency = indata.persistency
self.XMLCatalogueSlice = indata.XMLCatalogueSlice
else:
self.depth = 0
self.persistency = None
self.XMLCatalogueSlice = None
if stripProxy(job.backend).__module__.find('Dirac') > 0:
logger.debug("found Dirac backend")
if self.filesPerJob > 100:
self.filesPerJob = 100 # see above warning
logger.debug("indata: %s " % str(indata))
if self.splitterBackend == "GangaDiracSplitter":
from GangaDirac.Lib.Splitters.GangaSplitterUtils import GangaDiracSplitter
outdata = GangaDiracSplitter(indata,
self.filesPerJob,
self.maxFiles,
self.ignoremissing)
elif self.splitterBackend == "OfflineGangaDiracSplitter":
from GangaDirac.Lib.Splitters.OfflineGangaDiracSplitter import OfflineGangaDiracSplitter
outdata = OfflineGangaDiracSplitter(indata,
self.filesPerJob,
self.maxFiles,
self.ignoremissing)
elif self.splitterBackend == "splitInputDataBySize":
from GangaLHCb.Lib.Splitters.LHCbSplitterUtils import DiracSizeSplitter
outdata = DiracSizeSplitter(indata,
self.filesPerJob,
self.maxFiles,
self.ignoremissing)
elif self.splitterBackend == "splitInputData":
indata = stripProxy(copy.deepcopy(inputdata))
from GangaDirac.Lib.Splitters.SplitterUtils import DiracSplitter
outdata = DiracSplitter(indata,
self.filesPerJob,
self.maxFiles,
self.ignoremissing)
else:
raise SplitterError("Backend algorithm not selected!")
logger.debug("outdata: %s " % str(outdata))
return outdata
else:
logger.debug("Calling Parent Splitter as not on Dirac")
return super(SplitByFiles, self)._splitter(job, indata)
def getWNInjectedScript(self, outputFiles, indent, patternsToZip, postProcessLocationsFP):
"""
Returns script that have to be injected in the jobscript for postprocessing on the WN
"""
script_path = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
script_location = os.path.join( script_path, 'uploadScript.py')
from Ganga.GPIDev.Lib.File import FileUtils
upload_script = FileUtils.loadScript(script_location, '')
WNscript_location = os.path.join( script_path, 'WNInjectTemplate.py' )
script = FileUtils.loadScript(WNscript_location, '')
if not self.remoteDir:
try:
job = self.getJobObject()
lfn_folder = os.path.join("GangaUploadedFiles", "GangaJob_%s" % job.getFQID('.'))
except AssertionError:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
lfn_folder = os.path.join("GangaUploadedFiles", 'GangaFiles_%s' % this_date)
self.lfn = os.path.join(DiracFile.diracLFNBase(), lfn_folder, self.namePattern)
if self.remoteDir == '':
self.remoteDir = DiracFile.diracLFNBase()
if self.remoteDir[:4] == 'LFN:':
lfn_base = self.remoteDir[4:]
else:
lfn_base = self.remoteDir
for this_file in outputFiles:
isCompressed = this_file.namePattern in patternsToZip
if not regex.search(this_file.namePattern) is None:
script += self._WN_wildcard_script(this_file.namePattern, lfn_base, str(isCompressed))
else:
script += '###INDENT###print("Uploading: %s as: %s")\n' % (this_file.namePattern, str(os.path.join(lfn_base, this_file.namePattern)))
script += '###INDENT###processes.append(uploadFile("%s", "%s", %s))\n' % (this_file.namePattern, lfn_base, str(isCompressed))
if stripProxy(self)._parent is not None and stripProxy(self).getJobObject() and getName(stripProxy(self).getJobObject().backend) != 'Dirac':
script_env = self._getDiracEnvStr()
else:
script_env = str(None)
script = '\n'.join([str('###INDENT###' + str(line)) for line in script.split('\n')])
replace_dict = {'###UPLOAD_SCRIPT###' : upload_script,
'###STORAGE_ELEMENTS###' : str(configDirac['allDiracSE']),
'###INDENT###' : indent,
'###LOCATIONSFILE###' : postProcessLocationsFP,
'###DIRAC_ENV###' : script_env}
for k, v in replace_dict.iteritems():
script = script.replace(str(k), str(v))
return script
def acceptOptional(self, s):
if s is None:
print(None, end='', file=self.out)
else:
if isType(stripProxy(s), list):
print(s, end='', file=self.out)
else:
stripProxy(s).accept(self)
def acceptOptional(self, s):
if s is None:
print(None, end='', file=self.out)
else:
if isType(stripProxy(s), list):
print(s, end='', file=self.out)
else:
stripProxy(s).accept(self)
def getWNCodeForOutputPostprocessing(job, indent):
# dict containing the list of outputfiles that need to be processed on the
# WN for every file type
outputFilesProcessedOnWN = {}
patternsToZip = []
if len(job.outputfiles) == 0:
return ""
else:
for outputFile in job.outputfiles:
outputfileClassName = stripProxy(outputFile).__class__.__name__
backendClassName = stripProxy(job.backend).__class__.__name__
if outputFile.compressed:
if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
if outputfileClassName not in outputFilesProcessedOnWN.keys():
outputFilesProcessedOnWN[outputfileClassName] = []
if outputFilePostProcessingOnWN(job, outputfileClassName):
outputFilesProcessedOnWN[
outputfileClassName].append(outputFile)
shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
os.system("gzip %s" % currentFile)
postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
insertScript = shortScript
insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])
for outputFileName in outputFilesProcessedOnWN.keys():
if len(outputFilesProcessedOnWN[outputFileName]) > 0:
insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')
insertScript += """\n
###INDENT###postprocesslocations.close()
"""
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def config_file_as_text():
text = ''
sections = sorted(stripProxy(config).keys())
INDENT = "# "
for p in sections:
sect = stripProxy(config)[p]
if not sect.cfile:
continue
text += "\n"
text += "#=======================================================================\n"
text += textwrap.fill(sect.docstring.strip(),
width=80,
initial_indent=INDENT,
subsequent_indent=INDENT) + "\n"
text += "[%s]\n\n" % p
opts = sorted(sect.options.keys())
for o in opts:
if sect.options[o].cfile:
text += ""
text += textwrap.fill(sect.options[o].docstring.strip(),
width=80,
initial_indent=INDENT,
subsequent_indent=INDENT) + "\n"
examples = sect.options[o].examples
if examples:
text += INDENT + "Examples:\n"
for e in examples.splitlines():
text += INDENT + " " + e.strip() + "\n"
if sect.getEffectiveLevel(o) == 0:
value = sect[o]
def_value = sect.options[o].default_value
if isinstance(value, str):
try:
lines = value.splitlines()
def_lines = def_value.splitlines()
if len(lines) > 1:
value = "\n# ".join(lines)
def_value = "\n# ".join(def_lines)
except AttributeError as err:
pass
text += '#%s = %s\n' % (o, def_value)
text += '%s = %s\n\n' % (o, value)
else:
value = sect.getEffectiveOption(o)
if isinstance(value, str):
lines = value.splitlines()
if len(lines) > 1:
value = "\n# ".join(lines)
text += '#%s = %s\n\n' % (o, value)
return text
def test_job_submit_and_monitor(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert j.status != 'new'
stripProxy(LCG).master_updateMonitoringInformation([stripProxy(j)])
def test_job_equality(gpi):
"""Check that copies of Jobs are equal to each other"""
j = gpi.Job()
j2 = j.copy()
j3 = gpi.Job(j)
assert j == j2
assert j2 == j3
assert stripProxy(j) == stripProxy(j2)
assert stripProxy(j2) == stripProxy(j3)
def test_job_equality(gpi):
"""Check that copies of Jobs are equal to each other"""
j = gpi.Job()
j2 = j.copy()
j3 = gpi.Job(j)
assert j == j2
assert j2 == j3
assert stripProxy(j) == stripProxy(j2)
assert stripProxy(j2) == stripProxy(j3)