class TFile(GangaObject):
"""Test File object with well known equality properties -i.e. Does not reply on proxy!"""
_category = 'files'
_exportmethods = ['__eq__', '__cmp__', '__hash__', '__iadd__', '__imul__']
_name = 'TFile'
_schema = File._schema.inherit_copy()
_schema.datadict['added'] = SimpleItem(defvalue=False)
_schema.datadict['multiplied'] = SimpleItem(defvalue=False)
def __eq__(self, other):
if not isType(other, TFile):
return False
return self.name == other.name and self.subdir == other.subdir
def __cmp__(self, other):
"""A hacky but correct cmp function."""
self_comb = self.name + self.subdir
other_comb = other.name + other.subdir
return cmp(self_comb, other_comb)
def __hash__(self):
return self.name.__hash__() + self.subdir.__hash__()
def __iadd__(self, other):
self.added = True
return self
def __imul__(self, other):
self.multiplied = True
return self
class GListApp(IApplication):
"""Test File object with well known equality properties -i.e. Does not reply on proxy!"""
# summary_print
_category = 'applications'
_exportedmethods = ['configure']
_name = 'GListApp'
_schema = Schema(
Version(1, 0), {
'bound_print_comp':
ComponentItem(
'files',
defvalue=[],
sequence=1,
summary_print='_print_summary_bound_comp',
typelist=['str', 'Ganga.test.GPI.GangaList.TFile.TFile']),
'bound_print_simple':
SimpleItem(defvalue=[],
sequence=1,
summary_print='_print_summary_bound_simple'),
'no_summary':
SimpleItem(defvalue=[],
sequence=1,
summary_sequence_maxlen=-1,
typelist=['str']),
'seq':
SimpleItem(defvalue=[], sequence=1, typelist=['int']),
'gList':
SimpleItem(defvalue=[], sequence=1, typelist=['str']),
'gListComp':
ComponentItem('files', defvalue=[], sequence=1),
'simple_print':
SimpleItem(defvalue='',
summary_print='_print_summary_simple_print'),
'comp_print':
ComponentItem('backends',
defvalue=None,
summary_print='_print_summary_comp_print')
})
def configure(self, master_appconfig):
return (None, None)
def _print_summary_bound_comp(self, value, verbosity_level):
return '_print_summary_bound_comp'
def _print_summary_bound_simple(self, value, verbosity_level):
return '_print_summary_bound_simple'
def _print_summary_simple_print(self, value, verbosity_level):
return '_print_summary_simple_print'
def _print_summary_comp_print(self, value, verbosity_level):
return '_print_summary_comp_print'
class Bender(GaudiBase):
"""The Bender application handler
The user specifies a module file (via Bender.module) which contains a
Bender python module and the number of events they want to run on
(via Bender.events). The user's module is then run on the data by
calling:
USERMODULE.configure(EventSelectorInput,FileCatalogCatalogs)
USERMODULE.run(NUMEVENTS)
"""
_name = 'Bender'
_category = 'applications'
_exportmethods = GaudiBase._exportmethods[:]
_exportmethods += ['prepare', 'unprepare']
_schema = GaudiBase._schema.inherit_copy()
docstr = 'The package the application belongs to (e.g. "Sim", "Phys")'
_schema.datadict['package'] = SimpleItem(defvalue=None,
typelist=['str', 'type(None)'],
doc=docstr)
docstr = 'The package where your top level requirements file is read ' \
'from. Can be written either as a path ' \
'\"Tutorial/Analysis/v6r0\" or in traditional notation ' \
'\"Analysis v6r0 Tutorial\"'
_schema.datadict['masterpackage'] = SimpleItem(
defvalue=None, typelist=['str', 'type(None)'], doc=docstr)
docstr = 'Extra options to be passed onto the SetupProject command '\
'used for configuring the environment. As an example '\
'setting it to \'--dev\' will give access to the DEV area. '\
'For full documentation of the available options see '\
'https://twiki.cern.ch/twiki/bin/view/LHCb/SetupProject'
_schema.datadict['setupProjectOptions'] = SimpleItem(
defvalue='', typelist=['str', 'type(None)'], doc=docstr)
docstr = 'The name of the module to import. A copy will be made ' \
'at submission time'
_schema.datadict['module'] = FileItem(preparable=1,
defvalue=File(),
doc=docstr)
docstr = 'The name of the Gaudi application (Bender)'
_schema.datadict['project'] = SimpleItem(preparable=1,
defvalue='Bender',
hidden=1,
protected=1,
typelist=['str'],
doc=docstr)
docstr = 'The number of events '
_schema.datadict['events'] = SimpleItem(defvalue=-1,
typelist=['int'],
doc=docstr)
docstr = 'Parameres for module '
_schema.datadict['params'] = SimpleItem(
defvalue={},
typelist=['dict', 'str', 'int', 'bool', 'float'],
doc=docstr)
_schema.version.major += 2
_schema.version.minor += 0
#def __init__(self):
# super(Bender, self).__init__()
def _get_default_version(self, gaudi_app):
return guess_version(self, gaudi_app)
def _auto__init__(self):
if (not self.appname) and (not self.project):
self.project = 'Bender' # default
if (not self.appname):
self.appname = self.project
self._init()
def _getshell(self):
import EnvironFunctions
return EnvironFunctions._getshell(self)
def prepare(self, force=False):
super(Bender, self).prepare(force)
self._check_inputs()
share_dir = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'], self.is_prepared.name)
fillPackedSandbox([self.module],
os.path.join(
share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name))
gzipFile(
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name),
True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def master_configure(self):
return (None, StandardJobConfig())
def configure(self, master_appconfig):
# self._configure()
modulename = split(self.module.name)[-1].split('.')[0]
script = """
from copy import deepcopy
from Gaudi.Configuration import *
importOptions('data.py')
import %s as USERMODULE
EventSelectorInput = deepcopy(EventSelector().Input)
FileCatalogCatalogs = deepcopy(FileCatalog().Catalogs)
EventSelector().Input=[]
FileCatalog().Catalogs=[]\n""" % modulename
script_configure = "USERMODULE.configure(EventSelectorInput,FileCatalogCatalogs%s)\n"
if self.params:
param_string = ",params=%s" % self.params
else:
param_string = ""
script_configure = script_configure % param_string
script += script_configure
script += "USERMODULE.run(%d)\n" % self.events
script += getXMLSummaryScript()
# add summary.xml
outputsandbox_temp = XMLPostProcessor._XMLJobFiles()
outputsandbox_temp += unique(self.getJobObject().outputsandbox)
outputsandbox = unique(outputsandbox_temp)
input_files = []
input_files += [FileBuffer('gaudipython-wrapper.py', script)]
logger.debug("Returning StandardJobConfig")
return (None,
StandardJobConfig(inputbox=input_files,
outputbox=outputsandbox))
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
class BenderScript(GaudiBase):
"""The application handler for BenderScript
The user specifies:
- script file(s) which contain BenderScript scripts,
- configuration/Configurables files, to be used for 'importOptions'
- set of command line arguments
At least one 'script' or 'import' file is required.
The application executes the following line:
bender {scripts} {arguments} --import {imports} --no-color --no-castor --import=data.py --batch
e.g.
bender script1.py --import import1.py -w -p5 --no-color --no-castor --import=data.py --batch
where data.py a file with input data and xml catalog, automatically generated by Ganga/DIRAC
"""
_name = 'BenderScript'
_category = 'applications'
_exportmethods = GaudiBase._exportmethods[:]
_exportmethods += ['prepare', 'unprepare']
_schema = GaudiBase._schema.inherit_copy()
_schema.datadict['package'] = SimpleItem(
defvalue = None,
typelist = ['str', 'type(None)'],
doc = """The package the application belongs to (e.g. 'Sim', 'Phys')
"""
)
_schema.datadict['masterpackage'] = SimpleItem (
defvalue = None,
typelist = [ 'str', 'type(None)' ],
doc = """The package where your top level requirements file is read from.
Can be written either as a path 'Tutorial/Analysis/v6r0' or in traditional notation
'Analysis v6r0 Tutorial'
"""
)
_schema.datadict['setupProjectOptions'] = SimpleItem(
defvalue = '' ,
typelist = [ 'str', 'type(None)'],
doc = """Extra options to be passed onto the SetupProject command
used for configuring the environment. As an example
setting it to '--dev' will give access to the DEV area.
For full documentation of the available options see
https://twiki.cern.ch/twiki/bin/view/LHCb/SetupProject
"""
)
_schema.datadict['scripts'] = FileItem(
preparable = 1 ,
sequence = 1 ,
strict_sequence = 0 ,
defvalue = [] ,
doc = """The names of the script files to execute.
A copy will be made at submission time
"""
)
_schema.datadict['imports'] = FileItem (
preparable = 1 ,
sequence = 1 ,
strict_sequence = 0 ,
defvalue = [] ,
doc = """The names of the files to be used for 'importOptions'.
A copy will be made at submission time
"""
)
_schema.datadict['commands'] = SimpleItem(
defvalue = [] ,
typelist = ['str'] ,
sequence = 1 ,
doc = """The commands to be executed,
e.g. [ 'run(10)' , 'print ls()' , 'print dir()' ]
"""
)
_schema.datadict['arguments'] = SimpleItem(
defvalue = [] ,
typelist = ['str'] ,
sequence = 1 ,
doc = """List of command-line arguments for bender script,
e.g. ['-w','-p5'], etc.
For python scripts and configuration/Configurable files for 'importOptions'
it is much better to use the separate options 'scripts' and 'imports'
Following arguments will be appended automatically: --no-color, --no-castor and --batch
"""
)
_schema.version.major += 2
_schema.version.minor += 0
#def __init__(self):
# super(BenderScrip, self).__init__()
def _get_default_version(self, gaudi_app):
return guess_version(self, gaudi_app)
def _auto__init__(self):
if not self.appname : self.appname = 'Bender'
self._init()
def _getshell(self):
import EnvironFunctions
return EnvironFunctions._getshell(self)
def prepare(self, force=False):
super(BenderScript, self).prepare(force)
self._check_inputs()
share_dir = os.path.join (
expandfilename ( getConfig('Configuration')['gangadir'] ) ,
'shared' ,
getConfig('Configuration')['user'] ,
self.is_prepared.name )
input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name )
input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name )
fillPackedSandbox ( self.scripts + self.imports , input_sandbox_tar )
gzipFile ( input_sandbox_tar , input_sandbox_tgz , True )
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def master_configure(self):
return (None, StandardJobConfig())
def configure(self, master_appconfig):
## strip leading and trailing blanks from arguments
self.arguments = [ a.strip() for a in self.arguments ]
## strip leading and trailing blanks from the command
self.commands = [ a.strip() for a in self.commands ]
## the script layout
the_script = layout.format (
scripts = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.scripts ] ,
imports = [ os.path.join ( f.subdir , os.path.basename ( f.name ) ) for f in self.imports ] ,
arguments = self.arguments ,
command = self.commands
)
print 'SCRIPT:\n', the_script
# add summary.xml
outputsandbox_temp = XMLPostProcessor._XMLJobFiles()
outputsandbox_temp += unique(self.getJobObject().outputsandbox)
outputsandbox = unique(outputsandbox_temp)
input_files = []
input_files += [ FileBuffer('gaudipython-wrapper.py', the_script ) ]
logger.debug("Returning StandardJobConfig")
return (None, StandardJobConfig(inputbox=input_files,
outputbox=outputsandbox))
def _check_inputs(self):
"""Checks the validity of user's entries for BenderScript schema"""
if not self.scripts and not self.imports :
raise ApplicationConfigurationError("Application scripts/imports are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
if isinstance ( self.imports , str ) : self.imports = [ File ( self.imports ) ]
for f in self.scripts : f.name = fullpath ( f.name )
for f in self.imports : f.name = fullpath ( f.name )
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
class GangaList(GangaObject):
_category = 'internal'
_exportmethods = [
'__add__', '__contains__', '__delitem__', '__delslice__', '__eq__',
'__ge__', '__getitem__', '__getslice__', '__gt__', '__iadd__',
'__imul__', '__iter__', '__le__', '__len__', '__lt__', '__mul__',
'__ne__', '__reversed__', '__radd__', '__rmul__', '__setitem__',
'__setslice__', 'append', 'count', 'extend', 'index', 'insert', 'pop',
'remove', 'reverse', 'sort', '__hash__', 'get'
]
_hidden = 1
_enable_plugin = 1
_name = 'GangaList'
_schema = Schema(
Version(1, 0), {
'_list':
SimpleItem(
defvalue=[], doc='The raw list', hidden=1,
category='internal'),
'_is_preparable':
SimpleItem(defvalue=False,
doc='defines if prepare lock is checked',
hidden=1),
})
_enable_config = 1
def __init__(self):
self._is_a_ref = False
super(GangaList, self).__init__()
# convenience methods
@staticmethod
def is_list(obj):
"""
This returns a boolean as to if this object is a list or not
Args:
obj (object): object to be tested against known list types
"""
result = (obj is not None) and isType(obj, (GangaList, list, tuple))
return result
@staticmethod
def has_proxy_element(_list):
"""
Returns if a proxy object has crept into the list
Args:
_list (list): Any iterable object
"""
return all([isProxy(l) for l in _list])
## Attempt to prevent raw assignment of _list causing Proxied objects to get inside the GangaList
def _attribute_filter__set__(self, name, value):
logger.debug("GangaList filter")
if name == "_list":
if self.is_list(value):
if self.has_proxy_element(value):
returnable_list = [stripProxy(l) for l in value]
my_parent = self._getParent()
for elem in returnable_list:
if isinstance(elem, GangaObject):
elem._setParent(my_parent)
return
return returnable_list
else:
return value
elif self._list is None:
return None
else:
raise GangaException(
"Attempting to assign a non list item: %s to a GangaList._list!"
% str(value))
else:
return super(GangaList, self)._attribute_filter__set__(name, value)
def _on_attribute__set__(self, obj_type, attrib_name):
if self._is_a_ref is True:
new_list = []
for i in self._list:
if hasattr(i, '_on_attribute__set__'):
new_list.append(
i._on_attribute__set__(obj_type, attrib_name))
else:
new_list.append(i)
self._list = new_list
self._is_a_ref = False
return self
def _getParent(self):
return super(GangaList, self)._getParent()
def _setParent(self, parent):
"""
Set the parent of this object and it's children to the given parent
Args:
parent (GangaObject): Sets this object as the parent of the list and it's children
"""
super(GangaList, self)._setParent(parent)
for elem in self._list:
if isinstance(elem, GangaObject):
elem._setParent(parent)
def get(self, to_match):
def matching_filter(item):
if '_list_get__match__' in dir(item):
return item._list_get__match__(to_match)
return to_match == item
return makeGangaListByRef(filter(matching_filter, self._list),
preparable=self._is_preparable)
def _export_get(self, to_match):
return addProxy(self.get(stripProxy(to_match)))
def strip_proxy(self, obj, filter=False):
"""Removes proxies and calls shortcut if needed"""
def applyFilter(obj, item):
category = item['category']
this_filter = allComponentFilters[category]
filter_obj = this_filter(obj, item)
if filter_obj is None:
raise TypeMismatchError('%s is not of type %s.' %
(str(obj), category))
return filter_obj
raw_obj = stripProxy(obj)
# apply a filter if possible
if filter is True:
parent = self._getParent()
item = self.findSchemaParentSchemaEntry(parent)
if item and item.isA(ComponentItem): # only filter ComponentItems
category = item['category']
if isinstance(raw_obj, GangaObject):
if raw_obj._category != category:
raw_obj = applyFilter(raw_obj, item)
raw_obj._setParent(parent)
else:
raw_obj = applyFilter(raw_obj, item)
return raw_obj
def strip_proxy_list(self, obj_list, filter=False):
if isType(obj_list, GangaList):
return getProxyAttr(obj_list, '_list')
result = [self.strip_proxy(l, filter=filter) for l in obj_list]
return result
def getCategory(self):
"""Returns a list of categories for the objects in the list. Returns [] for an empty list."""
def return_cat(elem):
if hasattr(elem, 'category'):
return elem._category
else:
return type(elem)
return unique([return_cat(l) for l in self._list])
def _readonly(self):
"""
Return if this object is read-only based upon the Schema
"""
if self._is_preparable and hasattr(self, '_getParent'):
if self._getParent()._category == 'applications' and hasattr(
self._getParent(), 'is_prepared'):
from Ganga.GPIDev.Lib.File.File import ShareDir
return (isinstance(self._getParent().is_prepared, ShareDir)
or super(GangaList, self)._readonly())
return super(GangaList, self)._readonly()
def checkReadOnly(self):
"""Puts a hook in to stop mutable access to readonly jobs."""
if self._readonly():
raise ReadOnlyObjectError(
'object %s is readonly and attribute "%s" cannot be modified now'
% (repr(self), getName(self)))
else:
self._getSessionLock()
# TODO: BUG: This should only be set _after_ the change has been
# done! This can lead to data loss!
self._setDirty()
# list methods
# All list methods should be overridden in a way that makes
# sure that no proxy objects end up in the list, and no
# unproxied objects make it out.
def __add__(self, obj_list):
# Savanah 32342
if not self.is_list(obj_list):
raise TypeError('Type %s can not be concatinated to a GangaList' %
type(obj_list))
return makeGangaList(self._list.__add__(
self.strip_proxy_list(obj_list, True)),
preparable=self._is_preparable)
def _export___add__(self, obj_list):
self.checkReadOnly()
return addProxy(self.__add__(obj_list))
def __contains__(self, obj):
return self._list.__contains__(self.strip_proxy(obj))
def __clone__(self):
""" clone this object in a similar way to copy """
# TODO deterine if silently calling __copy__ is more correct
return makeGangaListByRef(_list=copy.copy(self._list),
preparable=self._is_preparable)
def __copy__(self):
"""Bypass any checking when making the copy"""
return makeGangaListByRef(_list=copy.copy(self._list),
preparable=self._is_preparable)
def __delitem__(self, obj):
self._list.__delitem__(self.strip_proxy(obj))
def _export___delitem__(self, obj):
self.checkReadOnly()
self.__delitem__(obj)
def __delslice__(self, start, end):
self._list.__delslice__(start, end)
def _export___delslice__(self, start, end):
self.checkReadOnly()
self.__delslice__(start, end)
def __deepcopy__(self, memo=None):
"""Bypass any checking when making the copy"""
#logger.info("memo: %s" % str(memo))
#logger.info("self.len: %s" % str(len(self._list)))
if self._list != []:
return makeGangaListByRef(_list=copy.deepcopy(self._list, memo),
preparable=self._is_preparable)
else:
new_list = GangaList()
new_list._is_preparable = self._is_preparable
return new_list
def __getListToCompare(self, input_list):
# if the arg isn't a list, just give it back
if not self.is_list(self.strip_proxy(input_list)):
return input_list
# setup up the list correctly
tmp_list = input_list
if isType(input_list, GangaList):
# GangaLists should never contain proxied objects so just return the list
return stripProxy(input_list)._list
elif isinstance(input_list, tuple):
tmp_list = list(input_list)
# Now return the list after stripping any objects of proxies
return self.strip_proxy_list(tmp_list)
def __eq__(self, obj_list):
if obj_list is self: # identity check
return True
return self._list == self.__getListToCompare(obj_list)
def __ge__(self, obj_list):
return self._list.__ge__(self.__getListToCompare(obj_list))
def __getitem__(self, index):
return self._list.__getitem__(index)
def _export___getitem__(self, index):
return addProxy(self.__getitem__(index))
def __getslice__(self, start, end):
return makeGangaList(_list=self._list.__getslice__(start, end),
preparable=self._is_preparable)
def _export___getslice__(self, start, end):
return addProxy(self.__getslice__(start, end))
def __gt__(self, obj_list):
return self._list.__gt__(self.strip_proxy_list(obj_list))
def __hash__(self):
logger.info("hash")
result = 0
for element in result:
result ^= hash(element)
return result
def __iadd__(self, obj_list):
self._list.__iadd__(self.strip_proxy_list(obj_list, True))
return self
def _export___iadd__(self, obj_list):
self.checkReadOnly()
return addProxy(self.__iadd__(obj_list))
def __imul__(self, number):
self._list.__imul__(number)
return self
def _export___imul__(self, number):
self.checkReadOnly()
return addProxy(self.__imul__(number))
def __iter__(self):
return self._list.__iter__()
def _export___iter__(self):
return GangaListIter(iter(self._list))
def __le__(self, obj_list):
return self._list.__le__(self.strip_proxy_list(obj_list))
def __len__(self):
return len(self._list)
def __lt__(self, obj_list):
return self._list.__lt__(self.strip_proxy_list(obj_list))
def __mul__(self, number):
return makeGangaList(self._list.__mul__(number),
preparable=self._is_preparable)
def _export___mul__(self, number):
return addProxy(self.__mul__(number))
def __ne__(self, obj_list):
if obj_list is self: # identity check
return False
result = True
if self.is_list(obj_list):
result = self._list.__ne__(self.strip_proxy_list(obj_list))
return result
def __reversed__(self):
"""Implements the __reversed__ list method introduced in 2.4"""
return reversed(self._list)
def _export___reversed__(self):
return GangaListIter(self.__reversed__())
def __radd__(self, obj):
return obj + self._list
def _export___radd__(self, obj):
# return the proxied objects
cp = [i for i in self._export___iter__()]
return obj + cp
def __rmul__(self, number):
return makeGangaList(self._list.__rmul__(number),
preparable=self._is_preparable)
def _export___rmul__(self, number):
return addProxy(self.__rmul__(number))
def __setitem__(self, index, obj):
self._list.__setitem__(index, self.strip_proxy(obj, True))
def _export___setitem__(self, index, obj):
self.checkReadOnly()
self.__setitem__(index, obj)
def __setslice__(self, start, end, obj_list):
self._list.__setslice__(start, end,
self.strip_proxy_list(obj_list, True))
def _export___setslice__(self, start, end, obj_list):
self.checkReadOnly()
self.__setslice__(start, end, obj_list)
def __repr__(self):
#logger.info("__repr__")
#return self.toString()
#import traceback
#traceback.print_stack()
containsObj = False
for elem in self._list:
if isinstance(elem, GangaObject):
containsObj = True
break
if containsObj is False:
return self.toString()
else:
return str("<GangaList at: %s>" % str(hex(abs(id(self)))))
#return str("<GangaList at: %s>" % str(hex(abs(id(self)))))
def __str__(self):
#logger.info("__str__")
return self.toString()
def append(self, obj, my_filter=True):
if isType(obj, GangaList):
stripped_o = stripProxy(obj)
stripped_o._setParent(self._getParent())
self._list.append(stripped_o)
return
elem = self.strip_proxy(obj, my_filter)
list_objs = (list, tuple)
if isType(elem, GangaObject):
stripped_e = stripProxy(elem)
stripped_e._setParent(self._getParent())
self._list.append(stripped_e)
elif isType(elem, list_objs):
new_list = []
def my_append(_obj):
if isType(_obj, GangaObject):
stripped_o = stripProxy(_obj)
stripped_o._setParent(self._getParent())
return stripped_o
else:
return _obj
self._list.append([my_append(l) for l in elem])
else:
self._list.append(elem)
def _export_append(self, obj):
self.checkReadOnly()
self.append(obj)
def count(self, obj):
return self._list.count(self.strip_proxy(obj))
def extend(self, ittr):
for i in ittr:
self.append(i)
def _export_extend(self, ittr):
self.checkReadOnly()
self.extend(ittr)
def index(self, obj):
return self._list.index(self.strip_proxy(obj))
def insert(self, index, obj):
if isType(obj, GangaObject):
stripProxy(obj)._setParent(stripProxy(self)._getParent())
self._list.insert(index, self.strip_proxy(obj, True))
def _export_insert(self, index, obj):
self.checkReadOnly()
self.insert(index, obj)
def pop(self, index=-1):
return self._list.pop(index)
def _export_pop(self, index=-1):
self.checkReadOnly()
return addProxy(self.pop(index))
def remove(self, obj):
"""
Remove a given object from a list
Args:
obj (unknown): Remove this object from the list if it exists
"""
self._list.remove(self.strip_proxy(obj))
def _export_remove(self, obj):
"""
Remove an object from a list after checking if it's read-only
Args:
obj (unknown): Remove this object from the list if it exists
"""
self.checkReadOnly()
self.remove(obj)
def reverse(self):
self._list.reverse()
def _export_reverse(self):
self.checkReadOnly()
self.reverse()
def sort(self, cmpfunc=None):
# TODO: Should comparitor have access to unproxied objects?
self._list.sort(cmpfunc)
def _export_sort(self, cmpfunc=None):
"""
Args:
compfunc (function): Function to be used in sorting list elements
"""
self.checkReadOnly()
self.sort(cmpfunc)
# now some more ganga specific methods
def findSchemaParentSchemaEntry(self, parent):
"""Finds the schema entry for this GangaList
Args:
parent (GangaObject): This is the parent object which controls this object
"""
result = None
if parent and parent._schema:
for k, v in parent._schema.allItems():
if getattr(parent, k) is self:
result = v
break
return result
def printSummaryTree(self,
level=0,
verbosity_level=0,
whitespace_marker='',
out=sys.stdout,
selection='',
interactive=False):
"""
This funtion displays a summary of the contents of this file.
(Docs from Gaga.GPIDev.Base.Objects # TODO determine how mch of this may be duplicated from there)
Args:
level (int): the hierachy level we are currently at in the object tree.
verbosity_level (int): How verbose the print should be. Currently this is always 0.
whitespace_marker (str): If printing on multiple lines, this allows the default indentation to be replicated.
The first line should never use this, as the substitution is 'name = %s' % printSummaryTree()
out (stream): An output stream to print to. The last line of output should be printed without a newline.'
selection: See VPrinter for an explaintion of this.
interactive (bool): Is this printing code being called from the interactive IPython prompt?
"""
parent = self._getParent()
schema_entry = self.findSchemaParentSchemaEntry(parent)
if parent is None:
full_print(self, out)
return
if schema_entry:
self_len = len(self)
print_summary = schema_entry['summary_print']
maxLen = schema_entry['summary_sequence_maxlen']
if print_summary:
fp = getattr(parent, print_summary)
str_val = fp(self._list, verbosity_level)
out.write(str_val)
return
if (maxLen != -1) and (self_len > maxLen):
out.write(decorateListEntries(self_len,
getName(type(self[0]))))
return
else:
summary_print(self, out)
return
out.write(str(self._list))
return
def toString(self):
"""Returns a simple str of the _list."""
returnable_str = "["
for element in self._list:
if isType(element, GangaObject):
returnable_str += repr(stripProxy(element))
else:
returnable_str += "'"
returnable_str += str(stripProxy(element))
returnable_str += "'"
returnable_str += ", "
returnable_str += "]"
return returnable_str
@synchronised
def accept(self, visitor):
"""
accept a visitor pattern - overrides GangaObject because we need to process _list as a ComponentItem in this
case to allow save/load of nested lists to work.
Could just get away with SimpleItem checks here but have included Shared and Component just in case these
are added in the future
# TODO investigate if it's possible to call GangaObject.accept for 90% of this functionality rather than duplicating it here
This accepts a VPrinter or VStreamer class instance visitor which is used to display the contents of the object according the Schema
Args:
visitor (VPrinter, VStreamer): An instance which will produce a display of this contents of this object
"""
visitor.nodeBegin(self)
for (name, item) in self._schema.simpleItems():
if name == "_list":
visitor.componentAttribute(self, name, getattr(self, name), 1)
elif item['visitable']:
visitor.simpleAttribute(self, name, getattr(self, name),
item['sequence'])
for (name, item) in self._schema.sharedItems():
if item['visitable']:
visitor.sharedAttribute(self, name, getattr(self, name),
item['sequence'])
for (name, item) in self._schema.componentItems():
if item['visitable']:
visitor.componentAttribute(self, name, getattr(self, name),
item['sequence'])
visitor.nodeEnd(self)
def _setFlushed(self):
"""Set flushed like the Node but do the _list by hand to avoid problems"""
self._dirty = False
for elem in self._list:
if isinstance(elem, Node):
elem._setFlushed()
super(GangaList, self)._setFlushed()
class GListApp(IPrepareApp):
"""Test File object with well known equality properties -i.e. Does not reply on proxy!"""
# summary_print
_category = 'applications'
_exportedmethods = ['configure']
_name = 'GListApp'
_schema = Schema(
Version(1, 0), {
'bound_print_comp':
ComponentItem(
'files',
defvalue=[],
sequence=1,
summary_print='_print_summary_bound_comp',
typelist=['str', 'Ganga.test.GPI.GangaList.TFile.TFile']),
'bound_print_simple':
SimpleItem(defvalue=[],
sequence=1,
summary_print='_print_summary_bound_simple'),
'no_summary':
SimpleItem(defvalue=[],
sequence=1,
summary_sequence_maxlen=-1,
typelist=['str']),
'seq':
SimpleItem(defvalue=[], sequence=1, typelist=['int']),
'gList':
SimpleItem(defvalue=[], sequence=1, typelist=['str']),
'gListComp':
ComponentItem('files', defvalue=[], sequence=1),
'simple_print':
SimpleItem(defvalue='',
summary_print='_print_summary_simple_print'),
'comp_print':
ComponentItem('backends',
defvalue=None,
summary_print='_print_summary_comp_print'),
'is_prepared':
SimpleItem(
defvalue=None,
strict_sequence=0,
visitable=1,
copyable=1,
hidden=0,
typelist=['type(None)', 'bool', ShareDir],
protected=0,
comparable=1,
doc=
'Location of shared resources. Presence of this attribute implies the application has been prepared.'
),
})
def configure(self, master_appconfig):
return (None, None)
def _print_summary_bound_comp(self, value, verbosity_level):
return '_print_summary_bound_comp'
def _print_summary_bound_simple(self, value, verbosity_level):
return '_print_summary_bound_simple'
def _print_summary_simple_print(self, value, verbosity_level):
return '_print_summary_simple_print'
def _print_summary_comp_print(self, value, verbosity_level):
return '_print_summary_comp_print'
class GangaList(GangaObject):
_category = 'internal'
_exportmethods = [
'__add__', '__contains__', '__delitem__', '__delslice__', '__eq__',
'__ge__', '__getitem__', '__getslice__', '__gt__', '__iadd__',
'__imul__', '__iter__', '__le__', '__len__', '__lt__', '__mul__',
'__ne__', '__reversed__', '__radd__', '__rmul__', '__setitem__',
'__setslice__', 'append', 'count', 'extend', 'index', 'insert', 'pop',
'remove', 'reverse', 'sort', '__hash__', 'get'
]
_hidden = 1
_enable_plugin = 1
_name = 'GangaList'
_schema = Schema(
Version(1, 0), {
'_list':
ComponentItem(
defvalue=[], doc='The raw list', hidden=1,
category='internal'),
'_is_preparable':
SimpleItem(defvalue=False,
doc='defines if prepare lock is checked',
hidden=1),
})
_enable_config = 1
_data = {}
def __init__(self):
self._is_a_ref = False
super(GangaList, self).__init__()
def __construct__(self, args):
#super(GangaList, self).__construct__(args)
if len(args) == 1:
if isType(args[0], (len, GangaList, tuple)):
for element_i in args[0]:
self._list.expand(self.strip_proxy(element_i))
elif args[0] is None:
self._list = None
else:
raise GangaException(
"Construct: Attempting to assign a non list item: %s to a GangaList._list!"
% str(args[0]))
else:
super(GangaList, self).__construct__(args)
return
# convenience methods
@staticmethod
def is_list(obj):
result = (obj is not None) and isType(obj, (GangaList, list, tuple))
return result
@staticmethod
def has_proxy_element(_list):
return all([isProxy(element) for element in _list])
## Attempt to prevent raw assignment of _list causing Proxied objects to get inside the GangaList
def _attribute_filter__set__(self, name, value):
logger.debug("GangaList filter")
if name == "_list":
if self.is_list(value):
if self.has_proxy_element(value):
returnable_list = [
stripProxy(element) for element in value
]
for elem in returnable_list:
if isType(elem, GangaObject):
elem._setParent(self._getParent())
return returnable_list
else:
return value
elif self._list is None:
return None
else:
raise GangaException(
"Attempting to assign a non list item: %s to a GangaList._list!"
% str(value))
else:
return super(GangaList, self)._attribute_filter__set__(name, value)
def _on_attribute__set__(self, obj_type, attrib_name):
if self._is_a_ref is True:
new_list = []
for i in self._list:
if hasattr(i, '_on_attribute__set__'):
new_list.append(
i._on_attribute__set__(obj_type, attrib_name))
continue
new_list.append(i)
self._list = new_list
self._is_a_ref = False
return self
def _getParent(self):
return super(GangaList, self)._getParent()
def _setParent(self, parent):
super(GangaList, self)._setParent(parent)
for element in self._list:
if isType(element, GangaObject):
stripProxy(element)._setParent(parent)
def get(self, to_match):
def matching_filter(item):
if '_list_get__match__' in dir(item):
return item._list_get__match__(to_match)
return to_match == item
return makeGangaListByRef(filter(matching_filter, self._list),
preparable=self._is_preparable)
def _export_get(self, to_match):
return addProxy(self.get(stripProxy(to_match)))
def strip_proxy(self, obj, filter=False):
"""Removes proxies and calls shortcut if needed"""
def applyFilter(obj, item):
category = item['category']
this_filter = allComponentFilters[category]
filter_obj = this_filter(obj, item)
if filter_obj is None:
raise TypeMismatchError('%s is not of type %s.' %
(str(obj), category))
return filter_obj
raw_obj = stripProxy(obj)
# apply a filter if possible
if filter is True:
parent = self._getParent()
item = self.findSchemaParentSchemaEntry(parent)
if item and item.isA(ComponentItem): # only filter ComponentItems
category = item['category']
if isType(raw_obj, GangaObject):
if raw_obj._category != category:
raw_obj = applyFilter(raw_obj, item)
raw_obj._setParent(parent)
else:
raw_obj = applyFilter(raw_obj, item)
return raw_obj
def strip_proxy_list(self, obj_list, filter=False):
if isType(obj_list, GangaList):
return getProxyAttr(obj_list, '_list')
result = []
for o in obj_list:
result.append(self.strip_proxy(o, filter))
return result
def getCategory(self):
"""Returns a list of categories for the objects in the list. Returns [] for an empty list."""
result = []
for o in self._list:
if hasattr(stripProxy(o), 'category'):
category = o._category
if not category in result:
result.append(category)
else:
result.append(type(o))
return result
def _readonly(self):
if self._is_preparable and hasattr(self, '_getParent'):
if self._getParent()._category == 'applications' and hasattr(
self._getParent(), 'is_prepared'):
from Ganga.GPIDev.Lib.File.File import ShareDir
return (isType(self._getParent().is_prepared, ShareDir)
or super(GangaList, self)._readonly())
return super(GangaList, self)._readonly()
def checkReadOnly(self):
"""Puts a hook in to stop mutable access to readonly jobs."""
if self._readonly():
raise ReadOnlyObjectError(
'object %s is readonly and attribute "%s" cannot be modified now'
% (repr(self), getName(self)))
else:
self._getWriteAccess()
# TODO: BUG: This should only be set _after_ the change has been
# done! This can lead to data loss!
self._setDirty()
# list methods
# All list methods should be overridden in a way that makes
# sure that no proxy objects end up in the list, and no
# unproxied objects make it out.
def __add__(self, obj_list):
# Savanah 32342
if not self.is_list(obj_list):
raise TypeError('Type %s can not be concatinated to a GangaList' %
type(obj_list))
return makeGangaList(self._list.__add__(
self.strip_proxy_list(obj_list, True)),
preparable=self._is_preparable)
def _export___add__(self, obj_list):
self.checkReadOnly()
return addProxy(self.__add__(obj_list))
def __contains__(self, obj):
return self._list.__contains__(self.strip_proxy(obj))
# def __clone__(self):
# return makeGangaListByRef(_list=copy.copy(self._list), preparable=self._is_preparable)
def __copy__(self):
"""Bypass any checking when making the copy"""
return makeGangaListByRef(_list=copy.copy(self._list),
preparable=self._is_preparable)
def __delitem__(self, obj):
self._list.__delitem__(self.strip_proxy(obj))
def _export___delitem__(self, obj):
self.checkReadOnly()
self.__delitem__(obj)
def __delslice__(self, start, end):
self._list.__delslice__(start, end)
def _export___delslice__(self, start, end):
self.checkReadOnly()
self.__delslice__(start, end)
def __deepcopy__(self, memo):
"""Bypass any checking when making the copy"""
#logger.info("memo: %s" % str(memo))
#logger.info("self.len: %s" % str(len(self._list)))
if self._list != []:
return makeGangaListByRef(_list=copy.deepcopy(self._list),
preparable=self._is_preparable)
else:
new_list = GangaList()
new_list._is_preparable = self._is_preparable
return new_list
#super(GangaList, self).__deepcopy__(memo)
@staticmethod
def __getListToCompare(input_list):
if isType(input_list, GangaList):
return stripProxy(input_list)._list
elif isinstance(input_list, tuple):
return list(input_list)
else:
return input_list
def __eq__(self, obj_list):
if obj_list is self: # identity check
return True
return self._list == self.__getListToCompare(obj_list)
def __ge__(self, obj_list):
return self._list.__ge__(self.__getListToCompare(obj_list))
def __getitem__(self, index):
return self._list.__getitem__(index)
def _export___getitem__(self, index):
return addProxy(self.__getitem__(index))
def __getslice__(self, start, end):
return makeGangaList(_list=self._list.__getslice__(start, end),
preparable=self._is_preparable)
def _export___getslice__(self, start, end):
return addProxy(self.__getslice__(start, end))
def __gt__(self, obj_list):
return self._list.__gt__(self.strip_proxy_list(obj_list))
def __hash__(self):
logger.info("hash")
result = 0
for element in self:
result ^= hash(element)
return result
# return self._list.__hash__()
def __iadd__(self, obj_list):
self._list.__iadd__(self.strip_proxy_list(obj_list, True))
return self
def _export___iadd__(self, obj_list):
self.checkReadOnly()
return addProxy(self.__iadd__(obj_list))
def __imul__(self, number):
self._list.__imul__(number)
return self
def _export___imul__(self, number):
self.checkReadOnly()
return addProxy(self.__imul__(number))
def __iter__(self):
return self._list.__iter__()
def _export___iter__(self):
return GangaListIter(iter(self._list))
def __le__(self, obj_list):
return self._list.__le__(self.strip_proxy_list(obj_list))
def __len__(self):
return len(self._list)
def __lt__(self, obj_list):
return self._list.__lt__(self.strip_proxy_list(obj_list))
def __mul__(self, number):
return makeGangaList(self._list.__mul__(number),
preparable=self._is_preparable)
def _export___mul__(self, number):
return addProxy(self.__mul__(number))
def __ne__(self, obj_list):
if obj_list is self: # identity check
return True
result = True
if self.is_list(obj_list):
result = self._list.__ne__(self.strip_proxy_list(obj_list))
return result
def __reversed__(self):
"""Implements the __reversed__ list method introduced in 2.4"""
return reversed(self._list)
def _export___reversed__(self):
return GangaListIter(self.__reversed__())
def __radd__(self, obj):
return obj + self._list
def _export___radd__(self, obj):
# return the proxied objects
cp = []
for i in self._export___iter__():
cp.append(i)
return obj + cp
def __rmul__(self, number):
return makeGangaList(self._list.__rmul__(number),
preparable=self._is_preparable)
def _export___rmul__(self, number):
return addProxy(self.__rmul__(number))
def __setitem__(self, index, obj):
self._list.__setitem__(index, self.strip_proxy(obj, True))
def _export___setitem__(self, index, obj):
self.checkReadOnly()
self.__setitem__(index, obj)
def __setslice__(self, start, end, obj_list):
self._list.__setslice__(start, end,
self.strip_proxy_list(obj_list, True))
def _export___setslice__(self, start, end, obj_list):
self.checkReadOnly()
self.__setslice__(start, end, obj_list)
def __repr__(self):
#logger.info("__repr__")
#return self.toString()
#import traceback
#traceback.print_stack()
containsObj = False
for elem in self._list:
if isType(elem, GangaObject):
containsObj = True
break
if containsObj is False:
return self.toString()
else:
return str("<GangaList at: %s>" % str(hex(abs(id(self)))))
#return str("<GangaList at: %s>" % str(hex(abs(id(self)))))
def __str__(self):
#logger.info("__str__")
return self.toString()
def append(self, obj, my_filter=True):
if isType(obj, GangaList):
self._list.append(obj._list)
return
elem = self.strip_proxy(obj, my_filter)
list_objs = (list, tuple)
if isType(elem, GangaObject):
stripProxy(elem)._setParent(self._getParent())
self._list.append(elem)
elif isinstance(elem, list_objs):
new_list = []
for _obj in elem:
if isType(_obj, GangaObject):
new_list.append(stripProxy(_obj))
stripProxy(_obj)._setParent(self._getParent())
else:
new_list.append(_obj)
self._list.append(new_list)
else:
self._list.append(elem)
def _export_append(self, obj):
self.checkReadOnly()
self.append(obj)
def count(self, obj):
return self._list.count(self.strip_proxy(obj))
def extend(self, ittr):
for i in ittr:
self.append(i)
def _export_extend(self, ittr):
self.checkReadOnly()
self.extend(ittr)
def index(self, obj):
return self._list.index(self.strip_proxy(obj))
def insert(self, index, obj):
if isType(obj, GangaObject):
stripProxy(obj)._setParent(stripProxy(self)._getParent())
self._list.insert(index, self.strip_proxy(obj, True))
def _export_insert(self, index, obj):
self.checkReadOnly()
self.insert(index, obj)
def pop(self, index=-1):
return self._list.pop(index)
def _export_pop(self, index=-1):
self.checkReadOnly()
return addProxy(self.pop(index))
def remove(self, obj):
self._list.remove(self.strip_proxy(obj))
def _export_remove(self, obj):
self.checkReadOnly()
self.remove(obj)
def reverse(self):
self._list.reverse()
def _export_reverse(self):
self.checkReadOnly()
self.reverse()
def sort(self, cmpfunc=None):
# TODO: Should comparitor have access to unproxied objects?
self._list.sort(cmpfunc)
def _export_sort(self, cmpfunc=None):
self.checkReadOnly()
self.sort(cmpfunc)
# now some more ganga specific methods
def findSchemaParentSchemaEntry(self, parent):
"""Finds the schema entry for this GangaList"""
result = None
if parent and parent._schema:
for k, v in parent._schema.allItems():
if getattr(parent, k) is self:
result = v
break
return result
def printSummaryTree(self,
level=0,
verbosity_level=0,
whitespace_marker='',
out=sys.stdout,
selection='',
interactive=False):
parent = self._getParent()
schema_entry = self.findSchemaParentSchemaEntry(parent)
if parent is None:
full_print(self, out)
return
if schema_entry:
self_len = len(self)
print_summary = schema_entry['summary_print']
maxLen = schema_entry['summary_sequence_maxlen']
if print_summary:
fp = getattr(parent, print_summary)
str_val = fp(self._list, verbosity_level)
out.write(str_val)
return
if (maxLen != -1) and (self_len > maxLen):
out.write(decorateListEntries(self_len,
getName(type(self[0]))))
return
else:
summary_print(self, out)
return
out.write(str(self._list))
return
def toString(self):
"""Returns a simple str of the _list."""
returnable_str = "["
for element in self._list:
if isType(element, GangaObject):
returnable_str += repr(stripProxy(element))
else:
returnable_str += "'"
returnable_str += str(stripProxy(element))
returnable_str += "'"
returnable_str += ", "
returnable_str += "]"
return returnable_str
class AMIDataset(DQ2Dataset):
'''ATLAS DDM Dataset With AMI Connection'''
_category = 'datasets'
_name = 'AMIDataset'
_schema = DQ2Dataset._schema.inherit_copy()
_schema.datadict['logicalDatasetName'] = SimpleItem(defvalue='', doc='')
_schema.datadict['project'] = SimpleItem(defvalue='Atlas_Production',
doc='')
_schema.datadict['processingStep'] = SimpleItem(
defvalue='Atlas_Production', doc='')
_schema.datadict['amiStatus'] = SimpleItem(defvalue='VALID', doc='')
_schema.datadict['entity'] = SimpleItem(defvalue='dataset', doc='')
_schema.datadict['metadata'] = SimpleItem(defvalue={}, doc="Metadata")
_schema.datadict['provenance'] = SimpleItem(defvalue=[],
typelist=['str'],
sequence=1,
doc='Dataset provenance chain')
_schema.datadict['goodRunListXML'] = FileItem(
doc='GoodRunList XML file to search on')
_exportmethods = [
'search', 'fill_provenance', 'get_datasets_metadata',
'get_files_metadata', 'get_contents'
]
def __init__(self):
super(AMIDataset, self).__init__()
def fill_provenance(self, extraargs=[]):
dataType = ""
if (len(extraargs) > 1):
dataType = extraargs[1]
self.provenance = []
for d in self.dataset:
logger.info("Filling provenance info for dataset %s...", d)
prov = []
self.provenance.append(prov)
ds = d[:-1]
argument = []
argument.append("ListDatasetProvenance")
argument.append("logicalDatasetName=" + ds)
argument.append('-output=xml')
result = amiclient.execute(argument)
dom = result.getAMIdom()
graph = dom.getElementsByTagName('graph')
nFound = 0
dictOfLists = {}
for line in graph:
nodes = line.getElementsByTagName('node')
for node in nodes:
level = int(node.attributes['level'].value)
dataset = node.attributes['name'].value
if (len(dataType) > 0) and (dataset.find(dataType) >= 0):
# print only selected dataType
levelList = dictOfLists.get(level, [])
levelList.append(dataset)
dictOfLists[level] = levelList
nFound = nFound + 1
elif (len(dataType) == 0):
#print everything
levelList = dictOfLists.get(level, [])
levelList.append(dataset)
dictOfLists[level] = levelList
#print level,dictOfLists[level]
nFound = nFound + 1
if (nFound == 0) and (len(dataType) > 0):
logger.warning("No datasets found of type", dataType)
else:
keys = dictOfLists.keys()
keys.sort()
for key in keys:
datasetList = dictOfLists.get(key)
datasetList.sort()
#print "generation =",key
#for dataset in datasetList:
# print " ",dataset
for dataset in datasetList:
prov.append("%s/" % dataset.strip())
def search(self,
pattern='',
maxresults=config['MaxNumOfDatasets'],
extraargs=[]):
argument = []
dsetList = []
if self.goodRunListXML.name != '':
# open the GRL
if os.path.exists(self.goodRunListXML.name):
logger.info("Good Run List '%s' file selected" %
self.goodRunListXML.name)
grl_text = open(self.goodRunListXML.name).read()
else:
logger.error('Could not read Good Run List XML file')
return []
argument = []
argument.append("GetGoodDatasetList")
argument.append("prodStep=merge")
#argument.append("dataType=%s" % self.dataType)
argument.append("goodRunList=%s" % grl_text)
argument.append("logicalDatasetName=%s" % self.logicalDatasetName)
elif self.logicalDatasetName:
pattern = self.logicalDatasetName
pattern = pattern.replace("/", "")
pattern = pattern.replace('*', '%')
limit = "0,%d" % config['MaxNumOfDatasets']
argument.append("SearchQuery")
argument.append("entity=" + self.entity)
argument.append(
"glite=SELECT logicalDatasetName WHERE amiStatus='" +
self.amiStatus + "' AND logicalDatasetName like '" + pattern +
"' LIMIT " + limit)
argument.append("project=" + self.project)
argument.append("processingStep=" + self.processingStep)
argument.append("mode=defaultField")
argument.extend(extraargs)
else:
logger.error(
"AMI search not set up correctly. No datasetname or good runs list supplied."
)
return []
try:
result = amiclient.execute(argument)
if argument[0] == "GetGoodDatasetList":
# GRL has different output
res_text = result.output()
dsetList = []
for ln in res_text.split('\n'):
if ln.find("logicalDatasetName") != -1:
# add to the dataset list - check container...
if resolve_dataset_name(ln.split('=')[1].strip()):
dsetList.append(ln.split('=')[1].strip() + "/")
else:
dsetList.append(ln.split('=')[1].strip())
else:
resultDict = result.getDict()
resultByRow = resultDict['Element_Info']
for row, vals in resultByRow.iteritems():
dsName = str(vals['logicalDatasetName'])
# check with DQ2 since AMI doesn't store /
if resolve_dataset_name(dsName):
dsName += '/'
dsetList.append(dsName)
except Exception as msg:
logger.error(msg)
return dsetList
def get_datasets_metadata(self):
datasets = self.search()
metadata = []
for dataset in datasets:
dataset = dataset.replace("/", "")
tmp = get_metadata(dataset=dataset)
metadata.append(tmp)
return metadata
def get_files_metadata(self, all=False):
datasets = self.search()
metadata = {}
for dataset in datasets:
dataset = dataset.replace("/", "")
job = self._getParent()
file_info = get_file_metadata(dataset=dataset,
all=all,
numevtsperfile=0)
metadata.update(file_info)
return metadata