def makeGangaList(_list, mapfunction = None, parent = None, preparable = False):
"""Should be used for makeing full gangalists"""
#work with a simple list always
if isType(_list,list):
_list = _list
elif isType(_list,GangaList):
_list = getProxyAttr(_list,'_list')
else:
_list = [_list]
if mapfunction is not None:
_list = map(mapfunction,_list)
result = GangaList()
result.extend(_list)
result._is_preparable = preparable
#set the parent if possible
if parent is not None:
result._setParent(parent)
for r in result:
if isinstance(r,GangaObject) and r._getParent() is None:
r._setParent(parent)
return result
def append(self, obj, my_filter=True):
if isType(obj, GangaList):
stripped_o = stripProxy(obj)
stripped_o._setParent(self._getParent())
self._list.append(stripped_o)
return
elem = self.strip_proxy(obj, my_filter)
list_objs = (list, tuple)
if isType(elem, GangaObject):
stripped_e = stripProxy(elem)
stripped_e._setParent(self._getParent())
self._list.append(stripped_e)
elif isType(elem, list_objs):
new_list = []
def my_append(_obj):
if isType(_obj, GangaObject):
stripped_o = stripProxy(_obj)
stripped_o._setParent(self._getParent())
return stripped_o
else:
return _obj
self._list.append([my_append(l) for l in elem])
else:
self._list.append(elem)
def find(self, id, path=None):
"""For a job with given id tries to find all references in the job tree.
The return value is a list of found paths.
"""
if isType(id, Job):
id = stripProxy(id).getFQID('.')
if isType(id, GPIProxyObject):
id = stripProxy(id)
pp = self.__get_path(path)
tp = os.path.join(*pp)
top_level = self.__folder_cd(pp)
search_dirs = []
found_items = []
for _item in top_level.keys():
if isinstance(top_level[_item], dict):
search_dirs.append(_item)
else:
if top_level[_item] == id:
found_items.append(tp)
result = []
for item in found_items:
result.append(item)
for dir in search_dirs:
new_path = os.path.join(tp, dir)
for found in self.find(id, new_path):
result.append(found)
return result
def copyPreparables(self):
"""
This method iterates over all attributes in an application and decides\
whether they should be persisted (i.e. copied) in the Shared Directory\
when the application is prepared.
If an IOError is raised when attempting the copy, this method returns 0\
otherwise it returns 1.
"""
send_to_sharedir = []
for name, item in self._schema.allItems():
if item['preparable']:
logger.debug('Found preparable %s' % (name))
logger.debug('adding to sharedir %s' % (self.__getattribute__(name)))
send_to_sharedir.append(self.__getattribute__(name))
for prepitem in send_to_sharedir:
logger.debug('working on %s' % (prepitem))
# we may have a list of files/strings
if isType(prepitem, (list, GangaList)):
logger.debug('found a list')
for subitem in prepitem:
if isType(subitem, str):
# we have a file. if it's an absolute path, copy it to
# the shared dir
if os.path.abspath(subitem) == subitem:
logger.debug('Sending file %s to shared directory.' % (subitem))
try:
self.copyIntoPrepDir(subitem)
except IOError as e:
logger.error(e)
return 0
elif isType(subitem, File) and subitem.name is not '':
logger.debug('Sending file object %s to shared directory' % subitem.name)
try:
self.copyIntoPrepDir(subitem.name)
except IOError as e:
logger.error(e)
return 0
elif isinstance(prepitem, str):
logger.debug('found a string')
# we have a file. if it's an absolute path, copy it to the
# shared dir
if prepitem and os.path.abspath(prepitem) == prepitem:
logger.debug('Sending string file %s to shared directory.' % (prepitem))
try:
self.copyIntoPrepDir(prepitem)
except IOError as e:
logger.error(e)
return 0
elif isType(prepitem, File) and prepitem.name is not '':
logger.debug('found a file')
logger.debug('Sending "File" object %s to shared directory' % prepitem.name)
try:
self.copyIntoPrepDir(prepitem.name)
except IOError as e:
logger.error(e)
return 0
else:
logger.debug('Nothing worth copying found in %s' % (prepitem))
return 1
def __init__(self, files=[], persistency=None, depth=0):
new_files = GangaList()
if isType(files, LHCbDataset):
for this_file in files:
new_files.append(copy.deepcopy(this_file))
elif isType(files, IGangaFile):
new_files.append(copy.deepcopy(this_file))
elif type(files) == type([]):
for this_file in files:
if type(this_file) == type(''):
new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
elif isType(this_file, IGangaFile):
new_files.append(this_file, False)
else:
new_files.append(strToDataFile(this_file))
elif type(files) == type(''):
new_files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
from Ganga.Core.exceptions import GangaException
raise GangaException("Unknown object passed to LHCbDataset constructor!")
new_files._setParent(self)
super(LHCbDataset, self).__init__()
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % files )
self.files = new_files
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType, stripProxy
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(
random.choice(string.ascii_uppercase) for _ in range(5))
stripProxy(mj)._getRegistry()._flush([stripProxy(mj)])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert stripProxy(sj)._getRoot() is stripProxy(j)
assert stripProxy(sj.application)._getRoot() is stripProxy(j)
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(random.choice(string.ascii_uppercase) for _ in range(5))
mj._impl._getRegistry()._flush([j])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert sj._impl._getRoot() is j._impl
def getValidDiracFiles(job, names=None):
"""
This is a generator for all DiracFiles in a jobs outputfiles
TODO: Is this still called anywhere?
Args:
job (Job): The job which is having it's DiracFiles tested
names (list): list of strings of names to be matched to namePatterns in outputfiles
"""
from GangaDirac.Lib.Files.DiracFile import DiracFile
from Ganga.GPIDev.Base.Proxy import isType
if job.subjobs:
for sj in job.subjobs:
for df in (f for f in sj.outputfiles if isType(f, DiracFile)):
if df.subfiles:
for valid_sf in (sf for sf in df.subfiles if sf.lfn != '' and (names is None or sf.namePattern in names)):
yield valid_sf
else:
if df.lfn != '' and (names is None or df.namePattern in names):
yield df
else:
for df in (f for f in job.outputfiles if isType(f, DiracFile)):
if df.subfiles:
for valid_sf in (sf for sf in df.subfiles if sf.lfn != '' and (names is None or sf.namePattern in names)):
yield valid_sf
else:
if df.lfn != '' and (names is None or df.namePattern in names):
yield df
def makeGangaList(_list, mapfunction=None, parent=None, preparable=False):
"""Should be used for makeing full gangalists"""
# work with a simple list always
if isType(_list, list):
_list = _list
elif isType(_list, GangaList):
_list = getProxyAttr(_list, '_list')
else:
_list = [_list]
#logger.debug("_list: %s" % str(_list))
if mapfunction is not None:
_list = map(mapfunction, _list)
result = makeGangaListByRef(_list)
result._is_preparable = preparable
# set the parent if possible
if parent is not None:
result._setParent(parent)
for r in result:
bare_r = stripProxy(r)
if isType(bare_r, GangaObject) and bare_r._getParent() is None:
bare_r._setParent(parent)
return result
def makeGangaList(_list, mapfunction=None, parent=None, preparable=False):
"""Should be used for makeing full gangalists"""
# work with a simple list always
if isType(_list, list):
_list = _list
elif isType(_list, GangaList):
_list = getProxyAttr(_list, '_list')
else:
_list = [_list]
#logger.debug("_list: %s" % str(_list))
if mapfunction is not None:
_list = map(mapfunction, _list)
#logger.debug("Making a GangaList of size: %s" % str(len(_list)))
result = makeGangaListByRef(_list, preparable)
result._is_a_ref = False
# set the parent if possible
if parent is not None:
result._setParent(parent)
return result
def copyFrom(self, srcobj, _ignore_atts=None):
if _ignore_atts is None:
_ignore_atts = []
_srcobj = stripProxy(srcobj)
# Check if this object is derived from the source object, then the copy
# will not throw away information
if not hasattr(_srcobj, '__class__') and not inspect.isclass(_srcobj.__class__):
raise GangaValueError("Can't copyFrom a non-class object: %s isclass: %s" % (str(_srcobj), str(inspect.isclass(_srcobj))))
if not isType(self, _srcobj.__class__) and not isType(_srcobj, self.__class__):
raise GangaValueError("copyFrom: Cannot copy from %s to %s!" % (getName(_srcobj), getName(self)))
if not hasattr(self, '_schema'):
logger.debug("No Schema found for myself")
return
if self._schema is None and _srcobj._schema is None:
logger.debug("Schema object for one of these classes is None!")
return
if _srcobj._schema is None:
self._schema = None
return
self._actually_copyFrom(_srcobj, _ignore_atts)
def copyFrom(self, srcobj, _ignore_atts=[]):
_srcobj = stripProxy(srcobj)
# Check if this object is derived from the source object, then the copy
# will not throw away information
if not isType(self, _srcobj.__class__) and not isType(_srcobj, self.__class__):
raise GangaValueError("copyFrom: Cannot copy from %s to %s!" % (_srcobj.__class__, self.__class__))
if not hasattr(self, '_schema'):
return
if self._schema is None and _srcobj._schema is None:
return
if _srcobj._schema is None:
self._schema = None
return
for name, item in self._schema.allItems():
if name in _ignore_atts:
continue
#logger.debug("Copying: %s : %s" % (str(name), str(item)))
if name is 'application' and hasattr(_srcobj.application, 'is_prepared'):
if _srcobj.application.is_prepared is not None and not _srcobj.application.is_prepared is True:
_srcobj.application.incrementShareCounter(_srcobj.application.is_prepared.name)
if not self._schema.hasAttribute(name):
#raise ValueError('copyFrom: incompatible schema: source=%s destination=%s'%(_srcobj._name,self._name))
setattr(self, name, self._schema.getDefaultValue(name))
elif not item['copyable']:
setattr(self, name, self._schema.getDefaultValue(name))
else:
c = deepcopy(getattr(_srcobj, name))
setattr(self, name, c)
def find(self, id, path=None):
"""For a job with given id tries to find all references in the job tree.
The return value is a list of found paths.
"""
if isType(id, Job):
id = stripProxy(id).getFQID('.')
if isType(id, GPIProxyObject):
id = stripProxy(id)
pp = self.__get_path(path)
tp = os.path.join(*pp)
top_level = self.__folder_cd(pp)
search_dirs = []
found_items = []
for _item in top_level.keys():
if isinstance(top_level[_item], dict):
search_dirs.append(_item)
else:
if top_level[_item] == id:
found_items.append(tp)
result = []
for item in found_items:
result.append(item)
for dir in search_dirs:
new_path = os.path.join(tp, dir)
for found in self.find(id, new_path):
result.append(found)
return result
def add(self, job, path=None):
"""Adds job to the job tree into the current folder.
If path to a folder is provided as a parameter than adds job to that folder.
"""
self._getWriteAccess()
try:
job = stripProxy(job)
mydir = self.__select_dir(path)
if isType(job, Job):
mydir[job.getFQID('.')] = job.getFQID('.') # job.id
elif isType(job, JobRegistrySlice):
for sliceKey in job.objects.iterkeys():
mydir[sliceKey] = sliceKey
elif isType(job, list):
for element in job:
mydir[element.id] = element.id
else:
raise TreeError(4, "Not a job/slice/list object")
self._setDirty()
except Exception, err:
logger.error("Error: %s" % err)
raise
def submit(self, subjobconfig, master_input_sandbox):
"""Submit a DIRAC job"""
j = self.getJobObject()
sboxname = j.createPackedInputSandbox(subjobconfig.getSandboxFiles())
input_sandbox = master_input_sandbox[:]
input_sandbox += sboxname
input_sandbox += self._addition_sandbox_content(subjobconfig)
## Add LFN to the inputfiles section of the file
input_sandbox_userFiles = []
for this_file in j.inputfiles:
if isType(this_file, DiracFile):
input_sandbox_userFiles.append('LFN:'+str(this_file.lfn))
if j.master:
for this_file in j.master.inputfiles:
if isType(this_file, DiracFile):
input_sandbox_userFiles.append('LFN:'+str(this_file.lfn))
for this_file in input_sandbox_userFiles:
input_sandbox.append(this_file)
dirac_script = subjobconfig.getExeString().replace('##INPUT_SANDBOX##', str(input_sandbox))
dirac_script_filename = os.path.join(j.getInputWorkspace().getPath(), 'dirac-script.py')
f = open(dirac_script_filename, 'w')
f.write(dirac_script)
f.close()
return self._common_submit(dirac_script_filename)
def add(self, job, path=None):
"""Adds job to the job tree into the current folder.
If path to a folder is provided as a parameter than adds job to that folder.
"""
self._getWriteAccess()
try:
job = stripProxy(job)
mydir = self.__select_dir(path)
if isType(job, Job):
mydir[job.getFQID('.')] = job.getFQID('.') # job.id
elif isType(job, JobRegistrySlice):
for sliceKey in job.objects.iterkeys():
mydir[sliceKey] = sliceKey
elif isType(job, list):
for element in job:
mydir[element.id] = element.id
else:
raise TreeError(4, "Not a job/slice/list object")
self._setDirty()
except Exception, err:
logger.error("Error: %s" % err)
raise
def addQuery(self, transform, bkQuery, associate=True):
"""Allows the user to add multiple transforms corresponding to the list of
BKQuery type objects given in the second argument. The first argument
is a transform object to use as the basis for the creation of further
transforms."""
if not isType(transform, LHCbAnalysisTransform):
raise GangaException(
None, 'First argument must be an LHCbAnalysisTransform objects to use as the basis for establishing the new transforms')
# Check if the template transform is associated with the Task
try:
self.transforms.index(transform)
except:
if associate:
logger.info(
'The transform is not associated with this Task, doing so now.')
self.appendTransform(transform)
# Check if the BKQuery input is correct and append/update
if type(bkQuery) is not list:
bkQuery = [bkQuery]
for bk in bkQuery:
if not isType(bk, BKQuery):
raise GangaAttributeError(
None, 'LHCbTransform expects a BKQuery object or list of BKQuery objects passed to the addQuery method')
if transform.query is None: # If template has no query itself
logger.info('Attaching query to transform')
transform.query = stripProxy(bk)
transform.update()
else: # Duplicate from template
logger.info('Duplicating transform to add new query.')
tr = deepcopy(transform)
tr.query = stripProxy(bk)
self.appendTransform(tr)
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' %
(item, self.name))
shutil.copy2(expandfilename(item),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(
expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' %
(item.name, self.name))
shutil.copy2(expandfilename(item.name),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def _handle_load_exception(self, err, fn, this_id, load_backup):
if isType(err, XMLFileError):
logger.error("XML File failed to load for Job id: %s" % str(this_id))
logger.error("Actual Error was:\n%s" % str(err))
if load_backup:
logger.debug("Could not load backup object #%i: %s", this_id, str(err))
raise InaccessibleObjectError(self, this_id, err)
logger.debug("Could not load object #%i: %s", this_id, str(err))
# try loading backup
try:
self.load([this_id], load_backup=True)
logger.warning("Object '%s' #%i loaded from backup file - the last changes may be lost.", self.registry.name, this_id)
return True
except Exception as err2:
logger.debug("Exception when loading backup: %s" % str(err2) )
if isType(err2, XMLFileError):
logger.error("XML File failed to load for Job id: %s" % str(this_id))
logger.error("Actual Error was:\n%s" % str(err2))
# add object to incomplete_objects
if not this_id in self.incomplete_objects:
self.incomplete_objects.append(this_id)
# remove index so we do not continue working with wrong
# information
rmrf(os.path.dirname(fn) + ".index")
raise InaccessibleObjectError(self, this_id, err)
return False
def select(self, minid=None, maxid=None, **attrs):
import repr
from Ganga.GPIDev.Lib.Job.Job import Job
if isType(minid, Job):
if minid.master:
minid = minid.master.id
else:
minid = minid.id
if maxid is None:
maxid = minid
if isType(maxid, Job):
if maxid.master:
maxid = maxid.master.id
else:
maxid = maxid.id
logger = getLogger()
this_repr = repr.Repr()
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
attrs_str = ""
## Loop through all possible input combinations to constructa string representation of the attrs from possible inputs
## Reuired to flatten the additional arguments into a flat string in attrs_str
for a in attrs:
from inspect import isclass
if isclass(attrs[a]):
this_attr = GPIProxyObjectFactory(attrs[a]())
else:
from Ganga.GPIDev.Base.Objects import GangaObject
if isType(attrs[a], GangaObject):
this_attr = GPIProxyObjectFactory(attrs[a])
else:
if type(attrs[a]) is str:
from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject
this_attr = getRuntimeGPIObject(attrs[a], True)
else:
this_attr = attrs[a]
full_str = str(this_attr)
split_str = full_str.split('\n')
for line in split_str:
line = line.strip()
flat_str = ''.join(split_str)
attrs_str += ", %s=\"%s\"" % (str(a), flat_str)
logger.debug("Attrs_Str: %s" % str(attrs_str))
logger.debug("Constructing slice: %s" %
str("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str)))
this_slice = self.__class__("%s.select(minid='%s', maxid='%s'%s)" %
(self.name, this_repr.repr(minid),
this_repr.repr(maxid), attrs_str))
def append(id, obj):
this_slice.objects[id] = obj
self.do_select(append, minid, maxid, **attrs)
return this_slice
def select(self, minid=None, maxid=None, **attrs):
import repr
from Ganga.GPIDev.Lib.Job.Job import Job
if isType(minid, Job):
if minid.master:
minid = minid.master.id
else:
minid = minid.id
if maxid is None:
maxid = minid
if isType(maxid, Job):
if maxid.master:
maxid = maxid.master.id
else:
maxid = maxid.id
this_repr = repr.Repr()
attrs_str = "".join([',%s=%s' % (a, this_repr.repr(attrs[a])) for a in attrs])
logger.debug("Constructing slice: %s" % str("%s.select(minid='%s', maxid='%s%s')" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str)))
this_slice = self.__class__("%s.select(minid='%s', maxid='%s%s')" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str))
def append(id, obj):
this_slice.objects[id] = obj
self.do_select(append, minid, maxid, **attrs)
return this_slice
def getValidDiracFiles(job, names=None):
"""
This is a generator for all DiracFiles in a jobs outputfiles
TODO: Is this still called anywhere?
Args:
job (Job): The job which is having it's DiracFiles tested
names (list): list of strings of names to be matched to namePatterns in outputfiles
"""
from GangaDirac.Lib.Files.DiracFile import DiracFile
if job.subjobs:
for sj in job.subjobs:
for df in (f for f in sj.outputfiles if isType(f, DiracFile)):
if df.subfiles:
for valid_sf in (sf for sf in df.subfiles if sf.lfn != '' and (names is None or sf.namePattern in names)):
yield valid_sf
else:
if df.lfn != '' and (names is None or df.namePattern in names):
yield df
else:
for df in (f for f in job.outputfiles if isType(f, DiracFile)):
if df.subfiles:
for valid_sf in (sf for sf in df.subfiles if sf.lfn != '' and (names is None or sf.namePattern in names)):
yield valid_sf
else:
if df.lfn != '' and (names is None or df.namePattern in names):
yield df
def makeGangaList(_list, mapfunction=None, parent=None, preparable=False):
"""Should be used for makeing full gangalists"""
# work with a simple list always
if isType(_list, list):
_list = _list
elif isType(_list, GangaList):
_list = getProxyAttr(_list, '_list')
else:
_list = [_list]
#logger.debug("_list: %s" % str(_list))
if mapfunction is not None:
_list = map(mapfunction, _list)
#logger.debug("Making a GangaList of size: %s" % str(len(_list)))
result = makeGangaListByRef(_list, preparable)
result._is_a_ref = False
# set the parent if possible
if parent is not None:
result._setParent(parent)
return result
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset
if (
job.inputfiles is None
or len(job.inputfiles) == 0
and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles))
):
return ""
insertScript = """\n
"""
# first, go over any LocalFiles in GangaDatasets to be transferred
# The LocalFiles in inputfiles have already been dealt with
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
for inputFile in job.inputdata.files:
inputfileClassName = getName(inputFile)
if inputfileClassName == "LocalFile":
# special case for LocalFile
if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]:
# create symlink
shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
os.symlink(f, os.path.basename(f))
"""
shortScript = FileUtils.indentScript(shortScript, "###INDENT####")
insertScript += shortScript
insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList())
# if GangaDataset is used, check if they want the inputfiles transferred
inputfiles_list = job.inputfiles
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = getName(inputFile)
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace("###INDENT###", indent)
return insertScript
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(files))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(
this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(files, None),
False)
else:
raise GangaException(
"Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def _create_subjob(self, job, dataset):
logger.debug("_create_subjob")
datatmp = []
logger.debug("dataset size: %s" % str(len(dataset)))
#logger.debug( "dataset: %s" % str(dataset) )
if isinstance(dataset, LHCbDataset):
for i in dataset:
if isType(i, DiracFile):
datatmp.append(i)
else:
logger.error(
"Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
from Ganga.Core.exceptions import GangaException
raise GangaException(
"Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
elif type(dataset) == type([]) or isType(dataset, GangaList()):
for file in dataset:
if type(file) == type(''):
datatmp.append(
allComponentFilters['gangafiles'](file, None))
elif isType(file, IGangaFile):
datatmp.append(file)
else:
logger.error("Unexpected type: %s" % str(type(file)))
logger.error(
"Wanted object to inherit from type: %s: %s" % (str(type(IGangaFile()))))
from Ganga.Core.exceptions import GangaException
x = GangaException(
"Unknown(unexpected) file object: %s" % file)
raise x
elif type(dataset) == type(''):
datatmp.append(DiracFile(lfn=dataset))
else:
logger.error("Unkown dataset type, cannot perform split here")
from Ganga.Core.exceptions import GangaException
logger.error("Dataset found: " + str(dataset))
raise GangaException(
"Unkown dataset type, cannot perform split here")
logger.debug("Creating new Job in Splitter")
j = Job()
logger.debug("Copying From Job")
j.copyFrom(
stripProxy(job), ['inputdata', 'inputsandbox', 'inputfiles'])
logger.debug("Unsetting Splitter")
j.splitter = None
logger.debug("Unsetting Merger")
j.merger = None
# j.inputsandbox = [] ## master added automatically
#j.inputfiles = []
logger.debug("Setting InputData")
j.inputdata = LHCbDataset(files=datatmp[:],
persistency=self.persistency,
depth=self.depth)
#j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
logger.debug("Returning new subjob")
return j
def _check_type(self, val, name, enableGangaList=True):
if enableGangaList:
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
else:
GangaList = list
# type checking does not make too much sense for Component Items because component items are
# always checked at the object (_impl) level for category compatibility.
if self.isA(ComponentItem):
return
validTypes = self._meta['typelist']
# setting typelist explicitly to None results in disabling the type
# checking completely
if validTypes is None:
return
if self._meta['sequence']:
from Ganga.GPIDev.Base.Proxy import isType
if not isType(self._meta['defvalue'], (list, tuple, GangaList)):
raise SchemaError('Attribute "%s" defined as a sequence but defvalue is not a list.' % name)
if not isType(val, (GangaList, tuple, list)):
raise TypeMismatchError('Attribute "%s" expects a list.' % name)
if validTypes:
if self._meta['sequence']:
for valItem in val:
if not valueTypeAllowed(valItem, validTypes):
raise TypeMismatchError('List entry %s for %s is invalid. Valid types: %s' % (valItem, name, validTypes))
return
else: # Non-sequence
if isinstance(self._meta['defvalue'], dict):
if isinstance(val, dict):
for dKey, dVal in val.iteritems():
if not valueTypeAllowed(dKey, validTypes) or not valueTypeAllowed(dVal, validTypes):
raise TypeMismatchError('Dictionary entry %s:%s for attribute %s is invalid. Valid types for key/value pairs: %s' % (dKey, dVal, name, validTypes))
else: # new value is not a dict
raise TypeMismatchError('Attribute "%s" expects a dictionary.' % name)
return
else: # a 'simple' (i.e. non-dictionary) non-sequence value
self.__check(valueTypeAllowed(val, validTypes), name, validTypes, val)
return
# typelist is not defined, use the type of the default value
if self._meta['defvalue'] is None:
logger.warning('type-checking disabled: type information not provided for %s, contact plugin developer', name)
else:
if self._meta['sequence']:
logger.warning('type-checking is incomplete: type information not provided for a sequence %s, contact plugin developer', name)
else:
logger.debug("valType: %s defValueType: %s name: %s" % (type(val), type(self._meta['defvalue']), name))
self.__check(self.__actualCheck(val, self._meta['defvalue']), name, type(self._meta['defvalue']), val)
def getDataFile(file):
if isType(file, DiracFile):
return file
if isType(file, LocalFile):
return file
if type(file) == type(''):
return strToDataFile(file)
return None
def _wrap(obj):
if isType(obj, GangaObject):
return addProxy(obj)
if isType(obj, RegistrySlice):
return obj._proxyClass(obj)
if isType(obj, list):
return map(addProxy, obj)
return obj
def _wrap(obj):
if isType(obj, GangaObject):
return GPIProxyObjectFactory(obj)
if isType(obj, RegistrySlice):
return obj._proxyClass(obj)
if isType(obj, list):
return map(GPIProxyObjectFactory, obj)
return obj
def getDataFile(file_):
if isType(file_, DiracFile):
return file_
if isType(file_, LocalFile):
return file_
if isinstance(file_, str):
return strToDataFile(file_)
return None
def getDataFile(file_):
if isType(file_, DiracFile):
return file_
if isType(file_, LocalFile):
return file_
if isinstance(file_, str):
return strToDataFile(file_)
return None
def getDataFile(file):
if isType(file, DiracFile):
return file
if isType(file, LocalFile):
return file
if type(file) == type(''):
return strToDataFile(file)
return None
def _checkOtherFiles(self, other ):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def _checkOtherFiles(self, other ):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def testGet(self):
assert isProxy(self.test_job.application.seq)
assert isProxy(self.test_job.application.gList)
assert isProxy(self.test_job.application.gListComp)
assert isType(self.test_job.application.seq, gangaList)
assert isType(self.test_job.application.gList, gangaList)
assert isType(self.test_job.application.gListComp, gangaList)
def dirac_inputdata(app):
job = stripProxy(app).getJobObject()
input_data = None
parametricinput_data = None
inputLFNs = []
if hasattr(job.inputdata, "getLFNs"):
inputLFNs = job.inputdata.getLFNs()
if job.master:
logger.debug("job.master.inputdata: %s " % str(job.master.inputdata))
logger.debug("job.inputdata: %s" % str(job.inputdata))
if hasattr(job.inputdata, "getLFNs"):
logger.debug("getLFNs(): %s" % job.inputdata.getLFNs())
has_input_DiracFile = False
for this_file in job.inputfiles:
if isType(this_file, DiracFile):
has_input_DiracFile = True
break
if job.master and not has_input_DiracFile:
for this_file in job.master.inputfiles:
if isType(this_file, DiracFile):
has_input_DiracFile = True
break
if len(inputLFNs) > 0:
# master job with a splitter reaching prepare, hence bulk submit
if not job.master and job.splitter:
parametricinput_data = dirac_parametric_split(app)
if parametricinput_data is not None and len(parametricinput_data) > getConfig("DIRAC")["MaxDiracBulkJobs"]:
raise BackendError(
"Dirac",
"Number of bulk submission jobs '%s' exceeds the maximum allowed '%s' if more are needed please modify your config. Note there is a hard limit in Dirac of currently 1000."
% (len(parametricinput_data), getConfig("DIRAC")["MaxDiracBulkJobs"]),
)
# master job with no splitter or subjob already split proceed as normal
else:
input_data = job.inputdata.getLFNs()
elif "Destination" not in job.backend.settings and not has_input_DiracFile:
##THIS IS NOT VERY DIRAC CENTRIC
##PLEASE WHEN TIME MOVE TO LHCBDIRAC where T1 is more applicable rcurrie
##Also editing the settings on the fly is asking for potential problems, should avoid
t1_sites = getConfig("DIRAC")["noInputDataBannedSites"]
logger.info("Job has no inputdata (T1 sites will be banned to help avoid overloading them).")
if "BannedSites" in job.backend.settings:
job.backend.settings["BannedSites"].extend(t1_sites)
job.backend.settings["BannedSites"] = unique(job.backend.settings["BannedSites"])
else:
job.backend.settings["BannedSites"] = t1_sites[:]
# import traceback
# traceback.print_stack()
return input_data, parametricinput_data
def testGet(self):
assert isProxy(self.test_job.application.seq)
assert isProxy(self.test_job.application.gList)
assert isProxy(self.test_job.application.gListComp)
assert isType(self.test_job.application.seq, gangaList)
assert isType(self.test_job.application.gList, gangaList)
assert isType(self.test_job.application.gListComp, gangaList)
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset
if job.inputfiles is None or len(job.inputfiles) == 0 and\
(not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or\
not job.inputdata.treat_as_inputfiles )):
return ""
insertScript = """\n
"""
# first, go over any LocalFiles in GangaDatasets to be transferred
# The LocalFiles in inputfiles have already been dealt with
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
for inputFile in job.inputdata.files:
inputfileClassName = getName(inputFile)
if inputfileClassName == "LocalFile":
# special case for LocalFile
if getName(job.backend) in ['Localhost', 'Batch', 'LSF', 'Condor', 'PBS']:
# create symlink
shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
os.symlink(f, os.path.basename(f))
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT####')
insertScript += shortScript
insertScript = insertScript.replace('###FILELIST###', "%s" % inputFile.getFilenameList())
# if GangaDataset is used, check if they want the inputfiles transferred
inputfiles_list = job.inputfiles
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = getName(inputFile)
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def is_gaudi_child(app):
if isType(app, Gaudi):
return True
if isType(app, TaskApplication):
from Ganga.GPI import GaudiPythonTask, BenderTask
if not isType(app, GaudiPythonTask) and not isType(app, BenderTask):
return True
return False
def is_gaudi_child(app):
if isType(app, Gaudi):
return True
if isType(app, TaskApplication):
from Ganga.GPI import GaudiPythonTask, BenderTask
if not isType(app, GaudiPythonTask) and not isType(app, BenderTask):
return True
return False
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset
def doIHaveInputFiles(job):
""" helper function for determining if a job has inputfiles it needs to make available
Args: job(Job) This is the job which we're testing for inputfiles """
if job.inputfiles is not None and len(job.inputfiles) != 0:
return True
if job.inputdata is not None and isinstance(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
return True
return False
if job.master is not None:
if not doIHaveInputFiles(job.master) and not doIHaveInputFiles(job):
return ""
else:
if not doIHaveInputFiles(job):
return ""
insertScript = """\n
"""
inputfiles_list = []
if not job.inputfiles:
# if GangaDataset is used, check if they want the inputfiles transferred
if job.master is not None:
inputfiles_list = job.master.inputfiles
else:
inputfiles_list = job.inputfiles
if job.inputdata:
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
elif job.master is not None:
if job.master.inputdata and isType(job.master.inputdata, GangaDataset) and job.master.inputdata.treat_as_inputfiles:
inputfiles_list += job.master.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = getName(inputFile)
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def __atomic_set__(self, _obj, _val):
obj = stripProxy(_obj)
val = stripProxy(_val)
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList
checkSet = self._bind_method(obj, self._checkset_name)
if checkSet is not None:
checkSet(val)
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(val)
# LOCKING
obj._getWriteAccess()
# self._check_getter()
item = stripProxy(obj._schema[getName(self)])
def cloneVal(v):
if isType(v, list()):
new_v = GangaList()
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
return new_v
else:
return self.__cloneVal(v, obj)
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
val = GangaList()
else:
if isType(item, Schema.ComponentItem):
val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
if isType(item, Schema.ComponentItem):
newListObj = []
if isinstance(val, list):
for elem in val:
newListObj.append(cloneVal(elem))
val = newListObj
else:
val = cloneVal(val)
if hasattr(val, '_setParent'):
val._setParent(obj)
obj.setNodeAttribute(getName(self), val)
obj._setDirty()
def __str__(self, short=True, id=None):
"""Prints an overview over the currently running tasks"""
if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]:
self.help(short=True)
Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue("tasks_show_help", False)
print("To show this help message again, type 'tasks.help()'.")
print('')
print(" The following is the output of " + markup("tasks.table()", fgcol("blue")))
lenfstring = 0
flist = []
for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns"]:
width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns_width"][thing]
lenfstring += width
flist.append("%" + str(width) + "s ")
fstring = "|".join(flist)
fstring += '\n'
lenfstring += 27
ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float")
ds += "-" * lenfstring + "\n"
from Ganga.GPIDev.Lib.Tasks import ITask
if id is not None and type(id) is int:
iterable_list = [self[id]]
else:
iterable_list = stripProxy(self).objects.values()
for p in iterable_list:
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])
if short is True:
continue
for ti in range(0, len(p.transforms)):
t = p.transforms[ti]
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
ds += "\n" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
ds += "zn" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])
ds += "-" * lenfstring + "\n"
return ds + "\n"
def _create_subjob(self, job, dataset):
logger.debug("_create_subjob")
datatmp = []
logger.debug("dataset size: %s" % str(len(dataset)))
#logger.debug( "dataset: %s" % str(dataset) )
from GangaLHCb.Lib.LHCbDataset.LHCbDataset import LHCbDataset
if isinstance(dataset, LHCbDataset):
for i in dataset:
if isType(i, DiracFile):
datatmp.append(i)
else:
logger.error("Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
from Ganga.Core.exceptions import GangaException
raise GangaException("Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
elif isinstance(dataset, (list, GangaList)):
for this_file in dataset:
if type(this_file) is str:
datatmp.append(allComponentFilters['gangafiles'](this_file, None))
elif isType(this_file, IGangaFile):
datatmp.append(this_file)
else:
logger.error("Unexpected type: %s" % str(type(this_file)))
logger.error("Wanted object to inherit from type: %s: %s" % (str(type(IGangaFile()))))
from Ganga.Core.exceptions import GangaException
x = GangaException("Unknown(unexpected) file object: %s" % this_file)
raise x
elif type(dataset) is str:
datatmp.append(DiracFile(lfn=dataset))
else:
logger.error("Unkown dataset type, cannot perform split here")
from Ganga.Core.exceptions import GangaException
logger.error("Dataset found: " + str(dataset))
raise GangaException("Unkown dataset type, cannot perform split here")
logger.debug("Creating new Job in Splitter")
j = Job()
logger.debug("Copying From Job")
j.copyFrom(stripProxy(job), ['splitter', 'subjobs', 'inputdata', 'inputsandbox', 'inputfiles'])
logger.debug("Unsetting Splitter")
j.splitter = None
#logger.debug("Unsetting Merger")
#j.merger = None
#j.inputsandbox = [] ## master added automatically
#j.inputfiles = []
logger.debug("Setting InputData")
j.inputdata = LHCbDataset(files=datatmp[:],
persistency=self.persistency,
depth=self.depth)
#j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
logger.debug("Returning new subjob")
return j
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(this_file))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(this_file, None), False)
else:
raise GangaException("Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def createChainUnit(self, parent_units, use_copy_output=True):
"""Create an output unit given this output data"""
# we need a parent job that has completed to get the output files
incl_pat_list = []
excl_pat_list = []
for parent in parent_units:
if len(parent.active_job_ids) == 0 or parent.status != "completed":
return None
for inds in self.inputdata:
from Ganga.GPIDev.Lib.Tasks.TaskChainInput import TaskChainInput
if isType(
inds, TaskChainInput
) and inds.input_trf_id == parent._getParent().getID():
incl_pat_list += inds.include_file_mask
excl_pat_list += inds.exclude_file_mask
# go over the output files and copy the appropriates over as input
# files
flist = []
import re
for parent in parent_units:
job = getJobByID(parent.active_job_ids[0])
if job.subjobs:
job_list = job.subjobs
else:
job_list = [job]
for sj in job_list:
for f in sj.outputfiles:
# match any dirac files that are allowed in the file mask
if isType(f, DiracFile):
if len(incl_pat_list) > 0:
for pat in incl_pat_list:
if re.search(pat, f.lfn):
flist.append("LFN:" + f.lfn)
else:
flist.append("LFN:" + f.lfn)
if len(excl_pat_list) > 0:
for pat in excl_pat_list:
if re.search(
pat,
f.lfn) and "LFN:" + f.lfn in flist:
flist.remove("LFN:" + f.lfn)
# just do one unit that uses all data
unit = LHCbUnit()
unit.name = "Unit %d" % len(self.units)
unit.inputdata = LHCbDataset(files=[DiracFile(lfn=f) for f in flist])
return unit
def select(self, minid=None, maxid=None, **attrs):
import repr
from Ganga.GPIDev.Lib.Job.Job import Job
if isType(minid, Job):
if minid.master:
minid = minid.master.id
else:
minid = minid.id
if maxid is None:
maxid = minid
if isType(maxid, Job):
if maxid.master:
maxid = maxid.master.id
else:
maxid = maxid.id
logger = getLogger()
this_repr = repr.Repr()
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
attrs_str = ""
## Loop through all possible input combinations to constructa string representation of the attrs from possible inputs
## Reuired to flatten the additional arguments into a flat string in attrs_str
for a in attrs:
from inspect import isclass
if isclass(attrs[a]):
this_attr = GPIProxyObjectFactory(attrs[a]())
else:
from Ganga.GPIDev.Base.Objects import GangaObject
if isType(attrs[a], GangaObject):
this_attr = GPIProxyObjectFactory(attrs[a])
else:
if type(attrs[a]) is str:
from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject
this_attr = getRuntimeGPIObject(attrs[a], True)
else:
this_attr = attrs[a]
full_str = str(this_attr)
split_str = full_str.split('\n')
for line in split_str:
line = line.strip()
flat_str = ''.join(split_str)
attrs_str += ", %s=\"%s\"" % (str(a), flat_str)
logger.debug("Attrs_Str: %s" % str(attrs_str))
logger.debug("Constructing slice: %s" % str("%s.select(minid='%s', maxid='%s'%s)" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str)))
this_slice = self.__class__("%s.select(minid='%s', maxid='%s'%s)" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str))
def append(id, obj):
this_slice.objects[id] = obj
self.do_select(append, minid, maxid, **attrs)
return this_slice
def testGet(self):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isProxy, isType
assert isProxy(self.test_job.application.seq)
assert isProxy(self.test_job.application.gList)
assert isProxy(self.test_job.application.gListComp)
assert isType(self.test_job.application.seq, gangaList)
assert isType(self.test_job.application.gList, gangaList)
assert isType(self.test_job.application.gListComp, gangaList)
def testGetDefaults(self):
test_job = Job(application=GListApp(), backend=TestSubmitter())
assert isProxy(test_job.application.seq)
assert isProxy(test_job.application.gList)
assert isProxy(test_job.application.gListComp)
assert isType(test_job.application.seq, gangaList)
assert isType(test_job.application.gList, gangaList)
assert isType(test_job.application.gListComp, gangaList)
def master_prepare(self, masterjobconfig):
""" Prepare the master job (shared sandbox files). This method is/should be called by master_submit() exactly once.
The input sandbox is created according to self._packed_input_sandbox flag (a class attribute)
"""
from Ganga.GPIDev.Lib.File.OutputFileManager import getInputFilesPatterns
from Ganga.GPIDev.Lib.File.File import File, ShareDir
job = self.getJobObject()
create_sandbox = job.createInputSandbox
if self._packed_input_sandbox:
create_sandbox = job.createPackedInputSandbox
if masterjobconfig:
if hasattr(job.application, 'is_prepared') and isType(
job.application.is_prepared, ShareDir):
sharedir_pred = lambda f: f.name.find(job.application.
is_prepared.name) > -1
sharedir_files = itertools.ifilter(
sharedir_pred, masterjobconfig.getSandboxFiles())
nonsharedir_files = itertools.ifilterfalse(
sharedir_pred, masterjobconfig.getSandboxFiles())
# ATLAS use bool to bypass the prepare mechanism and some ATLAS
# apps have no is_prepared
else:
sharedir_files = []
nonsharedir_files = masterjobconfig.getSandboxFiles()
inputsandbox = create_sandbox(nonsharedir_files, master=True)
inputsandbox.extend(
itertools.imap(lambda f: f.name, sharedir_files))
return inputsandbox
tmpDir = None
files = []
if len(job.inputfiles) > 0 or (len(job.subjobs) == 0 and job.inputdata and\
isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles):
(fileNames, tmpDir) = getInputFilesPatterns(job)
files = itertools.imap(lambda f: File(f), fileNames)
else:
# RTHandler is not required to produce masterjobconfig, in that
# case just use the inputsandbox
files = job.inputsandbox
result = create_sandbox(files, master=True)
if tmpDir is not None:
import shutil
# remove temp dir
if os.path.exists(tmpDir):
shutil.rmtree(tmpDir)
return result
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError(
'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now'
% (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(
files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(
string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def __actualCheck(val, defVal):
try:
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = (list, tuple, GangaList)
except Exception as err:
knownLists = (list, tuple)
from Ganga.GPIDev.Base.Proxy import isType
if isType(defVal, knownLists) and isType(val, knownLists):
return True
else:
if type(defVal) == type:
return isType(val, type)
else:
return isType(val, type(defVal))
def testGetDefaults(self):
from Ganga.GPI import Job, TestSubmitter, GListApp
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isProxy, isType
test_job = Job(application=GListApp(), backend=TestSubmitter())
assert isProxy(test_job.application.seq)
assert isProxy(test_job.application.gList)
assert isProxy(test_job.application.gListComp)
assert isType(test_job.application.seq, gangaList)
assert isType(test_job.application.gList, gangaList)
assert isType(test_job.application.gListComp, gangaList)
def master_auto_resubmit(self, rjobs):
'''Duplicate of the IBackend.master_resubmit but hooked into auto resubmission
such that the monitoring server is used rather than the user server'''
from Ganga.Core import IncompleteJobSubmissionError, GangaException
from Ganga.Utility.logging import log_user_exception
incomplete = 0
def handleError(x):
if incomplete:
raise x
else:
return 0
try:
for sj in rjobs:
fqid = sj.getFQID('.')
logger.info("resubmitting job %s to %s backend", fqid, getName(sj.backend))
try:
b = sj.backend
sj.updateStatus('submitting')
result = b._resubmit()
if result:
sj.updateStatus('submitted')
# sj._commit() # PENDING: TEMPORARY DISABLED
incomplete = 1
else:
return handleError(IncompleteJobSubmissionError(fqid, 'resubmission failed'))
except Exception as x:
log_user_exception(logger, debug=isType(x, GangaException))
return handleError(IncompleteJobSubmissionError(fqid, str(x)))
finally:
master = self.getJobObject().master
if master:
master.updateMasterJobStatus()
return 1
def __setitem__(self, category, _filter):
if category not in self._dict:
self._dict[category] = {}
# the filter can be registered as a tuple: ('filtername',filterfunction)
# or just as a function in which case the function name is used as an
# alias
if isType(_filter, tuple) and len(_filter) >= 2:
filtername = _filter[0]
filterfunc = _filter[1]
else:
try:
filtername = getName(_filter)
filterfunc = _filter
except AttributeError as e:
raise ValueError(
'FilterManager: Invalid component filter %s.' % _filter)
if filtername in self._dict[category]:
raise ValueError('FilterManager: %s component filter already exists for %s category ' % (filtername, category))
if category not in config.options:
config.addOption(category, "", "")
config.overrideDefaultValue(category, filtername)
self._dict[category][filtername] = filterfunc
def dirac_outputfile_jdl(output_files):
_output_files = [
this_file for this_file in output_files
if isType(this_file, DiracFile)
]
file_SE_dict = {}
for this_file in _output_files:
if not this_file.defaultSE in file_SE_dict:
file_SE_dict[this_file.defaultSE] = []
file_SE_dict[this_file.defaultSE].append(this_file.namePattern)
per_SE_JDL = '''
j.setOutputData(###OUTPUTDATA###, outputPath='###OUTPUT_PATH###', outputSE=###OUTPUT_SE###)
'''
total_JDL = ''
for outputSE, namePatterns in file_SE_dict.iteritems():
myLine = str(per_SE_JDL)
myLine = myLine.replace('###OUTPUTDATA###', str(namePatterns))
if outputSE != '':
myLine = myLine.replace('###OUTPUT_SE###', str([outputSE]))
else:
myLine = myLine.replace('###OUTPUT_SE###', str([]))
total_JDL += myLine + "\n"
return total_JDL
def sharedir_handler(app, root_dir_names, output):
share_path = get_share_path(app)
if '' in root_dir_names:
# get the '' entry and only that entry so dont waste time walking
# others before root.
root_dir_names = ['']
for share_dir in root_dir_names:
if share_dir == '':
share_dir = share_path
else:
share_dir = os.path.join(share_path, share_dir)
for root, dirs, files in os.walk(share_dir):
# [1:] removes the preceeding /
subdir = root.replace(share_dir, '')[1:]
if isType(output, (list, tuple, GangaList)):
output += [File(name=os.path.join(root, f), subdir=subdir) for f in files]
# for f in files:
## output += [File(name=os.path.join(root,f),subdir=subdir)]
elif type(output) is type(''):
for d in dirs:
if not os.path.isdir(d):
os.makedirs(d)
for f in files:
shutil.copy(os.path.join(root, f), os.path.join(output, subdir, f))
else:
raise GangaException('output must be either a list to append to or a path string to copy to')
