def __sequence_set__(stripper, obj, val, name):
item = stripProxy(obj)._schema[name]
# we need to explicitly check for the list type, because simple
# values (such as strings) may be iterable
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if isType(val, getKnownLists()):
# create GangaList
if stripper is not None:
val = makeGangaList(val, stripper)
else:
val = makeGangaList(
ProxyDataDescriptor._stripAttribute(obj, val, name))
else:
# val is not iterable
if item['strict_sequence']:
raise GangaAttributeError(
'cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)'
%
(repr(val), getattr(obj, proxyObject)._schema.name, name))
if stripper is not None:
val = makeGangaList(stripper(val))
else:
val = makeGangaList(
ProxyDataDescriptor._stripAttribute(obj, val, name))
return val
def __sequence_set__(stripper, obj, val, name):
item = stripProxy(obj)._schema[name]
# we need to explicitly check for the list type, because simple
# values (such as strings) may be iterable
new_v = None
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if isType(val, getKnownLists()):
# create GangaList
if stripper is not None:
new_v = makeGangaList(val, stripper)
else:
temp_v = ProxyDataDescriptor._stripAttribute(obj, val, name)
new_v = makeGangaList(temp_v)
else:
# val is not iterable
if item['strict_sequence']:
raise GangaAttributeError('cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)' % (repr(val), getName(obj), name))
if stripper is not None:
new_v = makeGangaList(stripper(val))
else:
temp_v = ProxyDataDescriptor._stripAttribute(obj, val, name)
new_v = makeGangaList(temp_v)
if new_v is None:
new_v = val
return new_v
def __sequence_set__(stripper, obj, val, name):
item = stripProxy(getattr(obj, proxyClass))._schema[name]
# we need to explicitly check for the list type, because simple
# values (such as strings) may be iterable
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if isType(val, getKnownLists()):
# create GangaList
if stripper is not None:
val = makeGangaList(val, stripper)
else:
val = makeGangaList(ProxyDataDescriptor._stripAttribute(obj, val, name))
else:
# val is not iterable
if item["strict_sequence"]:
raise GangaAttributeError(
"cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)"
% (repr(val), getattr(obj, proxyClass)._schema.name, name)
)
if stripper is not None:
val = makeGangaList(stripper(val))
else:
val = makeGangaList(ProxyDataDescriptor._stripAttribute(obj, val, name))
return val
def __atomic_set__(self, _obj, _val):
obj = stripProxy(_obj)
val = stripProxy(_val)
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList
checkSet = self._bind_method(obj, self._checkset_name)
if checkSet is not None:
checkSet(val)
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(val)
# LOCKING
obj._getWriteAccess()
# self._check_getter()
item = stripProxy(obj._schema[getName(self)])
def cloneVal(v):
if isType(v, list()):
new_v = GangaList()
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
return new_v
else:
return self.__cloneVal(v, obj)
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
val = GangaList()
else:
if isType(item, Schema.ComponentItem):
val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
if isType(item, Schema.ComponentItem):
newListObj = []
if isinstance(val, list):
for elem in val:
newListObj.append(cloneVal(elem))
val = newListObj
else:
val = cloneVal(val)
if hasattr(val, '_setParent'):
val._setParent(obj)
obj.setNodeAttribute(getName(self), val)
obj._setDirty()
def __get__(self, obj, cls):
global proxyRef
# at class level return a helper object (for textual description)
if obj is None:
# return Schema.make_helper(getattr(getattr(cls, proxyRef), getName(self)))
return getattr(getattr(cls, proxyRef), getNeme(self))
try:
val = getattr(getattr(obj, proxyRef), getName(self))
except Exception as err:
if getName(self) in getattr(obj, proxyRef).__dict__.keys():
val = getattr(obj, proxyRef).__dict__[getName(self)]
else:
val = getattr(getattr(obj, proxyRef), getName(self))
# wrap proxy
item = getattr(obj, proxyClass)._schema[getName(self)]
if item["proxy_get"]:
return getattr(getattr(obj, proxyRef), item["proxy_get"])()
if isType(item, Schema.ComponentItem):
disguiser = self.disguiseComponentObject
else:
disguiser = self.disguiseAttribute
## FIXME Add GangaList?
if item["sequence"] and isType(val, list):
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
val = makeGangaList(val, disguiser)
return disguiser(val)
def __get__(self, obj, cls):
global proxyRef
# at class level return a helper object (for textual description)
if obj is None:
# return Schema.make_helper(getattr(getattr(cls, proxyRef),self._name))
return getattr(stripProxy(cls), self._name)
val = getattr(stripProxy(obj), self._name)
# wrap proxy
item = stripProxy(obj)._schema[self._name]
if item["proxy_get"]:
return getattr(stripProxy(obj), item["proxy_get"])()
if isType(item, Schema.ComponentItem):
disguiser = self.disguiseComponentObject
else:
disguiser = self.disguiseAttribute
if item["sequence"] and isType(val, list):
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
val = makeGangaList(val, disguiser)
return disguiser(val)
def setUp(self):
# make a list of lists containing GangaObjects
self.filelist = []
for _ in range(10):
self.filelist.append([self._makeRandomTFile() for _ in range(3)])
# make an empty GangaList
self.gangalist = addProxy(makeGangaList([]))
def setUp(self):
# make a list of lists containing GangaObjects
self.filelist = []
for _ in range(10):
self.filelist.append([self._makeRandomTFile() for _ in range(3)])
# make an empty GangaList
self.gangalist = addProxy(makeGangaList([]))
def __get__(self, obj, cls):
# at class level return a helper object (for textual description)
if obj is None:
# return Schema.make_helper(getattr(getattr(cls, proxyRef),self._name))
return getattr( getattr(cls, proxyRef), self._name)
val = getattr( getattr(obj, proxyRef), self._name)
# apply object conversion or if it failes, make the wrapper proxy
def disguiseComponentObject(v):
# get the proxy for implementation object
def getProxy(v):
from Ganga.GPIDev.Base import GangaObject
if not isType(v, GangaObject):
raise GangaAttributeError("invalid type: cannot assign '%s' to attribute '%s'" % (repr(v), self._name))
return GPIProxyObjectFactory(v)
# convert implementation object to GPI value according to the
# static method defined in the implementation object
def object2value(v):
return None
vv = object2value(v)
if vv is None:
# allow None to be a legal value for optional component items
if v is None:
return None
else:
return getProxy(v)
else:
return vv
# apply attribute conversion
def disguiseAttribute(v):
# FIXME: this is obsoleted method
from Ganga.GPIDev.Base import GangaObject
if isType(v, GangaObject):
return GPIProxyObjectFactory(v)
return v
# wrap proxy
item = getattr(obj, proxyRef)._schema[self._name]
if item['proxy_get']:
return getattr(getattr(obj, proxyRef), item['proxy_get'])()
if item.isA(Schema.ComponentItem):
disguiser = disguiseComponentObject
else:
disguiser = disguiseAttribute
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if item['sequence'] and isType(val, list):
val = makeGangaList(val, disguiser)
return disguiser(val)
def setUp(self):
super(TestNestedLists, self).setUp()
# make a list of lists containing GangaObjects
self.filelist = []
self.gangalist = None
for _ in range(10):
self.filelist.append([self._makeRandomTFile() for _ in range(3)])
# make an empty GangaList
self.gangalist = addProxy(makeGangaList([]))
def end_element(name):
#logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
obj.setSchemaAttribute(aname, value)
#logger.info("Setting: %s = %s" % (aname, value))
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
# unescape the special characters
s = unescape(self.value_construct)
#logger.debug('string value: %s',s)
val = eval(s, config_scope)
#logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
alist = makeGangaList(self.stack[pos:])
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
obj = self.stack[-1]
for attr, item in obj._schema.allItems():
if not attr in obj._data:
#logger.info("Opening: %s" % attr)
if item._meta["sequence"] == 1:
obj.setSchemaAttribute(attr, makeGangaListByRef(obj._schema.getDefaultValue(attr)))
#setattr(obj, attr, makeGangaListByRef(obj._schema.getDefaultValue(attr)))
else:
obj.setSchemaAttribute(attr, obj._schema.getDefaultValue(attr))
def end_element(name):
#logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
obj.setSchemaAttribute(aname, value)
#logger.info("Setting: %s = %s" % (aname, value))
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
# unescape the special characters
s = unescape(self.value_construct)
#logger.debug('string value: %s',s)
if s not in _cached_eval_strings:
# This is ugly and classes which use this are bad, but this needs to be fixed in another PR
# TODO Make the scope of objects a lot better than whatever is in the config
# This is a dictionary constructed from eval-ing things in the Config. Why does should it do this?
# Anyway, lets save the result for speed
_cached_eval_strings[s] = eval(s, config_scope)
val = copy.deepcopy(_cached_eval_strings[s])
#logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
alist = makeGangaList(self.stack[pos:])
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
pass
def end_element(name):
#logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
obj.setSchemaAttribute(aname, value)
#logger.info("Setting: %s = %s" % (aname, value))
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
# unescape the special characters
s = unescape(self.value_construct)
#logger.debug('string value: %s',s)
if s not in _cached_eval_strings:
# This is ugly and classes which use this are bad, but this needs to be fixed in another PR
# TODO Make the scope of objects a lot better than whatever is in the config
# This is a dictionary constructed from eval-ing things in the Config. Why does should it do this?
# Anyway, lets save the result for speed
_cached_eval_strings[s] = eval(s, config_scope)
val = copy.deepcopy(_cached_eval_strings[s])
#logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
alist = makeGangaList(self.stack[pos:])
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
pass
def proxy_add(self, obj, name):
"""
Add an object to the box
The object must also be given a descriptive text name, for example:
box.add(Job(),'A job')
or
a=Executable()
box.add(a, 'An executable application')
"""
obj = _unwrap(obj)
if isinstance(obj, list):
obj = makeGangaList(obj)
if not isinstance(obj, GangaObject):
raise BoxTypeError(
"The Box can only contain Ganga Objects (i.e. Applications, Datasets or Backends). Check that the object is first in box.add(obj,'name')"
)
if obj._category == 'jobs':
if hasattr(obj.application, 'is_prepared'):
if obj.application.is_prepared is not None and obj.application.is_prepared is not True:
logger.debug(
'Adding a prepared job to the box and increasing the shareref counter'
)
obj.application.incrementShareCounter(
obj.application.is_prepared.name)
if obj._category == 'applications':
if hasattr(obj, 'is_prepared'):
if obj.is_prepared is not None and obj.is_prepared is not True:
logger.debug(
'Adding a prepared application to the box and increasing the shareref counter'
)
obj.incrementShareCounter(obj.is_prepared.name)
obj = obj.clone()
nobj = BoxMetadataObject()
nobj.name = name
self._add(obj)
self.metadata._add(nobj, self.find(obj))
nobj._setDirty()
obj._setDirty()
def proxy_add(self, obj, name):
"""
Add an object to the box
The object must also be given a descriptive text name, for example:
box.add(Job(),'A job')
or
a=Executable()
box.add(a, 'An executable application')
"""
obj = _unwrap(obj)
if isinstance(obj, list):
obj = makeGangaList(obj)
if not isinstance(obj, GangaObject):
raise BoxTypeError(
"The Box can only contain Ganga Objects (i.e. Applications, Datasets or Backends). Check that the object is first in box.add(obj,'name')")
if obj._category == 'jobs':
if hasattr(obj.application, 'is_prepared'):
if obj.application.is_prepared is not None and obj.application.is_prepared is not True:
logger.debug(
'Adding a prepared job to the box and increasing the shareref counter')
obj.application.incrementShareCounter(
obj.application.is_prepared.name)
if obj._category == 'applications':
if hasattr(obj, 'is_prepared'):
if obj.is_prepared is not None and obj.is_prepared is not True:
logger.debug(
'Adding a prepared application to the box and increasing the shareref counter')
obj.incrementShareCounter(obj.is_prepared.name)
obj = obj.clone()
nobj = BoxMetadataObject()
nobj.name = name
self._add(obj)
self.metadata._add(nobj, self.find(obj))
nobj._setDirty()
obj._setDirty()
def __get__(self, obj, cls):
# at class level return a helper object (for textual description)
if obj is None:
# return Schema.make_helper(getattr(getattr(cls, implRef), getName(self)))
return getattr(stripProxy(cls), getName(self))
try:
val = getattr(stripProxy(obj), getName(self))
except Exception as err:
if getName(self) in stripProxy(obj).__dict__.keys():
val = stripProxy(obj).__dict__[getName(self)]
else:
val = getattr(stripProxy(obj), getName(self))
# wrap proxy
item = stripProxy(obj)._schema[getName(self)]
if item['proxy_get']:
return getattr(stripProxy(obj), item['proxy_get'])()
if isType(item, ComponentItem):
disguiser = self.disguiseComponentObject
else:
disguiser = self.disguiseAttribute
## FIXME Add GangaList?
if item['sequence'] and isType(val, list):
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
val = makeGangaList(val, disguiser)
returnable = disguiser(val)
GangaObject = _getGangaObject()
if isType(returnable, GangaObject):
return addProxy(returnable)
else:
return returnable
def __get__(self, obj, cls):
# at class level return a helper object (for textual description)
if obj is None:
# return Schema.make_helper(getattr(getattr(cls, implRef), getName(self)))
return getattr(stripProxy(cls), getName(self))
try:
val = getattr(stripProxy(obj), getName(self))
except Exception as err:
if getName(self) in stripProxy(obj).__dict__.keys():
val = stripProxy(obj).__dict__[getName(self)]
else:
val = getattr(stripProxy(obj), getName(self))
# wrap proxy
item = stripProxy(obj)._schema[getName(self)]
if item['proxy_get']:
return getattr(stripProxy(obj), item['proxy_get'])()
if isType(item, ComponentItem):
disguiser = self.disguiseComponentObject
else:
disguiser = self.disguiseAttribute
## FIXME Add GangaList?
if item['sequence'] and isType(val, list):
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
val = makeGangaList(val, disguiser)
returnable = disguiser(val)
GangaObject = _getGangaObject()
if isType(returnable, GangaObject):
return addProxy(returnable)
else:
return returnable
def gangaObjectFactory(attrDict, migration_class=None):
# gangaObjectFactory(...) --> (object, migrated, [<list of errors>])
migrated = [False]
errors = []
if attrDict['simple']:
return (None, migrated[0], errors)
if migration_class:
cls = migration_class
else:
try:
cls = allPlugins.find(attrDict['category'], attrDict['name'])
except PluginManagerError as e:
msg = "Plugin Manager Error: %s" % str(e)
errors.append(GangaObjectFactoryError(e, msg=msg))
return (EmptyGangaObject(), migrated[0], errors)
schema = cls._schema
major, minor = attrDict['version']
version = Version(major, minor)
if not schema.version.isCompatible(version):
v1 = '.'.join(map(str, [major, minor]))
v2 = '.'.join(map(str, [schema.version.major, schema.version.minor]))
msg = "Incompatible schema versions of plugin %s in the category %s. Current version %s. Repository version %s." % (
attrDict['name'], attrDict['category'], v2, v1)
if schema.version.major > version.major: # no forward migration
from .MigrationControl import migration
if migration.isAllowed(attrDict['category'], attrDict['name'], attrDict['version'], msg=msg):
# try if migration provided by the plugin class
try:
old_cls = cls.getMigrationClass(version)
except:
old_cls = None
if old_cls:
old_obj, old_migrated, old_errors = gangaObjectFactory(
attrDict, migration_class=old_cls)
if old_migrated:
migrated[0] = old_migrated
if not old_errors:
try:
obj = cls.getMigrationObject(old_obj)
#assert(isinstance(obj, cls))
except Exception as e:
msg += ' Error in object migration: ' + str(e)
else:
obj.__setstate__(obj.__dict__)
migrated[0] = True
return (obj, migrated[0], errors)
else:
msg += ' Errors in object migration ' + \
str(map(str, old_errors))
else:
msg += ' No migration class is provided.'
else:
msg += ' Migration was denied in the migration control object.'
errors.append(GangaObjectFactoryError(msg=msg))
return (EmptyGangaObject(), migrated[0], errors)
obj = super(cls, cls).__new__(cls)
obj._data = {}
def mapper(attrDict):
if attrDict['simple']:
val = attrDict['data']
else:
val, attr_migrated, attr_errors = gangaObjectFactory(attrDict)
if attr_migrated:
migrated[0] = attr_migrated
for err in attr_errors:
if str(err) not in map(str, errors):
# don't duplicate the same errors
errors.append(err)
return val
data = attrDict['data']
for attr, item in schema.allItems():
if attr in data:
val = data[attr]
if item['sequence']:
val = makeGangaList(val, mapper, parent=obj)
else:
val = mapper(val)
else:
val = schema.getDefaultValue(attr)
obj._data[attr] = val
obj.__setstate__(obj.__dict__)
return (obj, migrated[0], errors)
def __set__(self, obj, val):
#self is the attribute we're about to change
#obj is the object we're about to make the change in
#val is the value we're setting the attribute to.
#item is the schema entry of the attribute we're about to change
item = obj._impl._schema[self._name]
if item['protected']:
raise ProtectedAttributeError('"%s" attribute is protected and cannot be modified'%(self._name,))
if obj._impl._readonly() and not (self._name == 'comment' and obj._impl._name == 'Job'):
raise ReadOnlyObjectError('object %s is read-only and attribute "%s" cannot be modified now'%(repr(obj),self._name))
#mechanism for locking of preparable attributes
if item['preparable'] and obj.is_prepared is not None and obj.is_prepared is not True:
raise ProtectedAttributeError('AttributeError: "%s" attribute belongs to a prepared application and so cannot be modified. unprepare() the application or copy the job/application (using j.copy(unprepare=True)) and modify that new instance.'%(self._name,))
#if we set is_prepared to None in the GPI, that should effectively unprepare the application
if self._name == 'is_prepared' and val is None and obj.is_prepared is not None:
logger.info('Unpreparing application.')
obj.unprepare()
#Replace is_prepared on an application for another ShareDir object
from Ganga.GPIDev.Lib.File import ShareDir
if self._name == 'is_prepared' and isType(val, ShareDir) and hasattr(obj._impl,'_getRegistry'):
if obj._impl._getRegistry() is not None:
logger.debug('Overwriting is_prepared attribute with a ShareDir object')
obj.unprepare() #it's safe to unprepare 'not-prepared' applications.
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(val.name)
#catch assignment of 'something' to a preparable application
if self._name == 'application' and hasattr(obj.application,'is_prepared'):
#a=Job(); a.prepare(); a.application=Executable()
if obj.application.is_prepared is not None and obj.application.is_prepared is not True and val.is_prepared is None:
logger.debug('Overwriting a prepared application with one that is unprepared')
obj.application.unprepare()
#a=Job(); b=Executable(); b.prepare(); a.application=b
elif obj.application.is_prepared is not True and hasattr(val,'is_prepared') and val.is_prepared is not None and val.is_prepared is not True:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug('Overwriting application with a prepared one')
obj.application.unprepare()
shareref.increase(val.is_prepared.name)
#check that the shared directory actually exists before assigning the (prepared) application to a job
if hasattr(val, 'is_prepared'):
if val.is_prepared is not None and val.is_prepared is not True:
if not os.path.isdir(os.path.join(shared_path,val.is_prepared.name)):
logger.error('ShareDir directory not found: %s' % val.is_prepared.name)
# apply attribute conversion
def stripAttribute(v):
# just warn
#print '**** checking',v,v.__class__, isinstance(val,GPIProxyObject)
if isinstance(v,GPIProxyObject):
v = v._impl
logger.debug('%s property: assigned a component object (_impl used)',self._name)
return obj._impl._attribute_filter__set__(self._name,v)
# unwrap proxy
if item.isA(Schema.ComponentItem):
from Filters import allComponentFilters
item = obj._impl._schema.getItem(self._name)
cfilter = allComponentFilters[item['category']]
stripper = lambda v: stripComponentObject(v,cfilter,item)
else:
stripper = stripAttribute
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList, makeGangaList
if item['sequence']:
# we need to explicitly check for the list type, because simple values (such as strings) may be iterable
if isType(val, (GangaList,list)):
#create GangaList
val = makeGangaList(val,stripper)
else:
# val is not iterable
if item['strict_sequence']:
raise GangaAttributeError('cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)'%(repr(val),obj._impl._schema.name,self._name))
val = makeGangaList(stripper(val))
else:
val = stripper(val)
# apply attribute filter to component items
if item.isA(Schema.ComponentItem):
val = stripAttribute(val)
self._check_type(obj,val)
setattr(obj._impl, self._name, val)
obj = super(cls, cls).__new__(cls)
obj._data = {}
def mapper(attrDict):
if attrDict["simple"]:
val = attrDict["data"]
else:
val, attr_migrated, attr_errors = gangaObjectFactory(attrDict)
if attr_migrated:
migrated[0] = attr_migrated
for err in attr_errors:
if str(err) not in map(str, errors):
# don't duplicate the same errors
errors.append(err)
return val
data = attrDict["data"]
for attr, item in schema.allItems():
if attr in data:
val = data[attr]
if item["sequence"]:
val = makeGangaList(val, mapper, parent=obj)
else:
val = mapper(val)
else:
val = schema.getDefaultValue(attr)
obj._data[attr] = val
obj.__setstate__(obj.__dict__)
return (obj, migrated[0], errors)
def _checkoutJobs(self, dir, meta={}):
""" Checkout the jobs and return a list of job objects.
"""
###logger.debug('checkoutJobs meta=%s', meta)
jobs = []
# summary of errors (exception reprs are keys, occurence count is
# value)
error_summary = {}
incomplete_ids = [] # ids of incomplete jobs (schema mismatch etc.)
bad_ids = [] # ids of ignored jobs (I/O errors)
entries_cnt = 0
master_ids = self._getJobIds(dir, meta)
# add a new error entry to error_summary
def add_error(e):
#re = repr(e)
re = str(e)
error_summary.setdefault(re, 0)
error_summary[re] += 1
# read a job id from dir and append it to jobs list
# masterid (if set) is used for reporting purposes only
def read_job(dir, id, jobs, masterid=None):
def fqid():
if masterid is None:
return str(id)
else:
return '.'.join([str(masterid), str(id)])
try:
# read job data and reconstruct the object
with open(os.path.join(dir, str(id), 'data')) as data_file:
j, errors = from_file(data_file)
if errors: # data errors
j.status = 'incomplete'
for e in errors:
add_error(e)
incomplete_ids.append(fqid())
jobs.append(j)
return j
except KeyboardInterrupt:
# FIXME: any special cleanup needed?
raise
except Exception as x: # I/O and parsing errors
msg = 'Cannot read job %s: %s' % (fqid(), repr(x))
add_error(msg)
bad_ids.append(fqid())
logger.debug(msg)
Ganga.Utility.logging.log_user_exception(logger, debug=True)
# main loop
import time
progress = 0
t0 = time.time()
for id in master_ids:
# progress log
if progress % 100 == 0 and progress != 0:
logger.info(
'Loaded %d/%d jobs in %d seconds', progress, len(master_ids), time.time() - t0)
progress += 1
# read top-level job
j = read_job(dir, id, jobs)
entries_cnt += 1
if j: # FIXME: hardcoded subjobs handling
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
subjobs_dir = os.path.join(dir, str(id))
if os.path.isdir(subjobs_dir):
# get all subjob ids
subjob_ids = self._getJobIds(subjobs_dir)
entries_cnt += len(subjob_ids)
subjobs = []
for sid in subjob_ids:
# read-in subjob objects
s = read_job(subjobs_dir, sid, subjobs, masterid=id)
s._setParent(j)
# initialize correctly the subjobs attribute
j._data['subjobs'] = makeGangaList(subjobs)
j.__setstate__(j.__dict__)
# print out reports
logger.info(
'Loaded total of %d job entries (including all subjobs)', entries_cnt)
logger.info('Loaded total of %d master job entries:', len(master_ids))
logger.info('Load time %d seconds', time.time() - t0)
if bad_ids:
logger.error(
'Missing job entries due to I/O errors: %d/%d', len(bad_ids), entries_cnt)
if incomplete_ids:
logger.error(
'Job entries loaded in incomplete state due to data errors: %d/%d', len(incomplete_ids), entries_cnt)
if len(incomplete_ids) < 100:
logger.error('Incomplete job ids: %s', incomplete_ids)
if error_summary:
logger.error('Summary of problems:')
for re, cnt in error_summary.iteritems():
logger.error(' - %d job entries: %s', cnt, re)
return jobs
def __set__(self, obj, val):
# self is the attribute we're about to change
# obj is the object we're about to make the change in
# val is the value we're setting the attribute to.
# item is the schema entry of the attribute we're about to change
item = stripProxy(obj)._schema[self._name]
if item["protected"]:
raise ProtectedAttributeError('"%s" attribute is protected and cannot be modified' % (self._name,))
if stripProxy(obj)._readonly():
if not (self._name == "comment" and stripProxy(obj)._name == "Job"):
raise ReadOnlyObjectError(
'object %s is read-only and attribute "%s" cannot be modified now' % (repr(obj), self._name)
)
# mechanism for locking of preparable attributes
if item["preparable"]:
if stripProxy(obj).is_prepared is not None:
if stripProxy(obj).is_prepared is not True:
raise ProtectedAttributeError(
'AttributeError: "%s" attribute belongs to a prepared application and so cannot be modified. unprepare() the application or copy the job/application (using j.copy(unprepare=True)) and modify that new instance.'
% (self._name,)
)
# if we set is_prepared to None in the GPI, that should effectively
# unprepare the application
if self._name == "is_prepared":
if val is None:
if stripProxy(obj).is_prepared is not None:
logger.info("Unpreparing application.")
stripProxy(obj).unprepare()
# Replace is_prepared on an application for another ShareDir object
if self._name == "is_prepared":
if hasattr(stripProxy(obj), "_getRegistry"):
from Ganga.GPIDev.Lib.File import ShareDir
if stripProxy(obj)._getRegistry() is not None and isType(val, ShareDir):
logger.debug("Overwriting is_prepared attribute with a ShareDir object")
# it's safe to unprepare 'not-prepared' applications.
stripProxy(obj).unprepare()
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
shareref.increase(val.name)
# catch assignment of 'something' to a preparable application
if self._name == "application":
if hasattr(stripProxy(obj.application), "is_prepared"):
# a=Job(); a.prepare(); a.application=Executable()
if (
stripProxy(obj.application).is_prepared not in [None, True]
and hasattr(stripProxy(val), "is_prepared")
and stripProxy(val).is_prepared is None
):
logger.debug("Overwriting a prepared application with one that is unprepared")
obj.application.unprepare()
# a=Job(); b=Executable(); b.prepare(); a.application=b
elif stripProxy(obj.application).is_prepared is not True:
if hasattr(val, "is_prepared"):
if stripProxy(val).is_prepared not in [None, True]:
from Ganga.Core.GangaRepository import getRegistry
shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef())
logger.debug("Overwriting application with a prepared one")
if stripProxy(obj.application) != val:
stripProxy(obj.application).unprepare()
shareref.increase(val.is_prepared.name)
# check that the shared directory actually exists before
# assigning the (prepared) application to a job
if hasattr(stripProxy(val), "is_prepared"):
if stripProxy(val).is_prepared not in [None, True]:
if hasattr(stripProxy(val).is_prepared, "name"):
from Ganga.Utility.files import expandfilename
Config_conf = getConfig("Configuration")
shared_path = os.path.join(
expandfilename(Config_conf["gangadir"]), "shared", Config_conf["user"]
)
if not os.path.isdir(os.path.join(shared_path, stripProxy(val).is_prepared.name)):
logger.error("ShareDir directory not found: %s" % stripProxy(val).is_prepared.name)
# unwrap proxy
if item.isA(Schema.ComponentItem):
from .Filters import allComponentFilters
item = getattr(obj, proxyRef)._schema.getItem(self._name)
cfilter = allComponentFilters[item["category"]]
stripper = lambda v: stripComponentObject(v, cfilter, item)
else:
stripper = None
# stripper = self._stripAttribute
if item["sequence"]:
# we need to explicitly check for the list type, because simple
# values (such as strings) may be iterable
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if isType(val, [getGangaList(), list, type([])]):
# create GangaList
if stripper is not None:
val = makeGangaList(val, stripper)
else:
val = makeGangaList(self._stripAttribute(obj, val))
else:
# val is not iterable
if item["strict_sequence"]:
raise GangaAttributeError(
"cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)"
% (repr(val), getattr(obj, proxyRef)._schema.name, self._name)
)
if stripper is not None:
val = makeGangaList(stripper(val))
else:
val = makeGangaList(self._stripAttribute(obj, val))
else:
if stripper is not None:
val = stripper(val)
else:
val = self._stripAttribute(obj, val)
# apply attribute filter to component items
if item.isA(Schema.ComponentItem):
val = self._stripAttribute(obj, val)
self._check_type(obj, val)
setattr(stripProxy(obj), self._name, val)
def __set__(self, obj, val):
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList, makeGangaList
cs = self._bind_method(obj,self._checkset_name)
if cs:
cs(val)
filter = self._bind_method(obj, self._filter_name)
if filter:
val = filter(val)
#LOCKING
obj._getWriteAccess()
#self._check_getter()
def cloneVal(v):
#print 'cloneVal:',self._name,v,item['optional'],item['load_default'], item['defvalue']
if v is None:
assert(item['optional'])
return None
else:
assert(isinstance(v, Node))
if isinstance(v, GangaList):
catagories = v.getCategory()
len_cat = len(catagories)
#we pass on empty lists, as the catagory is yet to be defined
if (len_cat > 1) or ((len_cat == 1) and (catagories[0] != item['category'])):
raise GangaAttributeError('%s: attempt to assign a list containing incompatible objects %s to the property in category "%s"' %(self._name, v,item['category']))
else:
if v._category != item['category']:
raise GangaAttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s"' %(self._name, v,item['category']))
v = v.clone()
v._setParent(obj)
return v
item = obj._schema[self._name]
if item.isA(Schema.ComponentItem):
if item['sequence']:
## checklist=filter(lambda x: not implies(x is None,item['optional']) or x._category != item['category'],val)
## if len(checklist) > 0:
## raise AttributeError('%s: attempt to assign incompatible objects %s to the property in category "%s"'%(self._name, str(checklist),item['category']))
if item['preparable']:
val = makeGangaList(val,cloneVal, parent = obj, preparable = True)
else:
val = makeGangaList(val,cloneVal, parent = obj)
else:
val = cloneVal(val)
## if val is None:
## assert(item['optional'])
## else:
## assert(isinstance(val, Node))
## if val._category != item['category']:
## raise AttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s"' %(self._name, val,item['category']))
## val = cloneVal(val)
else:
if item['sequence']:
if item['preparable']:
val = makeGangaList(val, parent = obj, preparable = True)
else:
val = makeGangaList(val, parent = obj)
obj._data[self._name] = val
obj._setDirty()
def __atomic_set__(self, _obj, _val):
## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed
## _obj: parent class which 'owns' the attribute
## _val: value of the attribute which we're about to set
#if hasattr(_obj, getName(self)):
# if not isinstance(getattr(_obj, getName(self)), GangaObject):
# if type( getattr(_obj, getName(self)) ) == type(_val):
# object.__setattr__(_obj, getName(self), deepcopy(_val))
# return
#
# if not isinstance(_obj, GangaObject) and type(_obj) == type(_val):
# _obj = deepcopy(_val)
# return
obj = _obj
temp_val = _val
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList
if hasattr(obj, '_checkset_name'):
checkSet = self._bind_method(obj, self._checkset_name)
if checkSet is not None:
checkSet(temp_val)
if hasattr(obj, '_filter_name'):
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(temp_val)
else:
val = temp_val
else:
val = temp_val
# LOCKING
obj._getWriteAccess()
#self._check_getter()
item = obj._schema[getName(self)]
def cloneVal(v):
GangaList = _getGangaList()
if isinstance(v, (list, tuple, GangaList)):
new_v = GangaList()
for elem in v:
new_v.append(self.__cloneVal(elem, obj))
return new_v
else:
return self.__cloneVal(v, obj)
## If the item has been defined as a sequence great, let's continue!
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
GangaList = _getGangaList()
new_val = GangaList()
else:
if isinstance(item, Schema.ComponentItem):
new_val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
new_val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
## Else we need to work out what we've got.
if isinstance(item, Schema.ComponentItem):
GangaList = _getGangaList()
if isinstance(val, (list, tuple, GangaList)):
## Can't have a GangaList inside a GangaList easily so lets not
if isinstance(_obj, GangaList):
newListObj = []
else:
newListObj = GangaList()
self.__createNewList(newListObj, val, cloneVal)
#for elem in val:
# newListObj.append(cloneVal(elem))
new_val = newListObj
else:
new_val = cloneVal(val)
else:
new_val = val
pass
#val = deepcopy(val)
if isinstance(new_val, Node):
new_val._setParent(obj)
obj.setNodeAttribute(getName(self), new_val)
obj._setDirty()
def end_element(name):
#logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
try:
obj.setSchemaAttribute(aname, value)
except:
raise GangaException(
"ERROR in loading XML, failed to set attribute %s for class %s"
% (aname, _getName(obj)))
#logger.info("Setting: %s = %s" % (aname, value))
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
try:
# unescape the special characters
s = unescape(self.value_construct)
#logger.debug('string value: %s',s)
if s not in _cached_eval_strings:
# This is ugly and classes which use this are bad, but this needs to be fixed in another PR
# TODO Make the scope of objects a lot better than whatever is in the config
# This is a dictionary constructed from eval-ing things in the Config. Why does should it do this?
# Anyway, lets save the result for speed
_cached_eval_strings[s] = eval(s, config_scope)
eval_str = _cached_eval_strings[s]
if not isinstance(eval_str, str):
val = copy.deepcopy(eval_str)
else:
val = eval_str
#logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
except:
raise GangaException(
"ERROR in loading XML, failed to correctly parse attribute value: \'%s\'"
% str(self.value_construct))
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
try:
alist = makeGangaList(self.stack[pos:])
except:
raise GangaException(
"ERROR in loading XML, failed to construct a sequence(list) properly"
)
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
obj = self.stack[-1]
cls = obj.__class__
if isinstance(cls, GangaObject):
for attr, item in cls._schema.allItems():
if attr not in obj._data:
if item.getProperties()['getter'] is None:
try:
setattr(obj, attr,
self._schema.getDefaultValue(attr))
except:
raise GangaException(
"ERROR in loading XML, failed to set default attribute %s for class %s"
% (attr, _getName(obj)))
pass
def __set__(self, _obj, _val):
obj = stripProxy(_obj)
val = stripProxy(_val)
from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList
cs = self._bind_method(obj, self._checkset_name)
if cs:
cs(val)
this_filter = self._bind_method(obj, self._filter_name)
if this_filter:
val = this_filter(val)
# LOCKING
obj._getWriteAccess()
# self._check_getter()
item = stripProxy(obj._schema[self._name])
def cloneVal(v):
return self.__cloneVal(v, obj)
obj_reg = obj._getRegistry()
full_reg = obj_reg is not None and hasattr(obj_reg, 'dirty_flush_counter')
### THIS CODE HERE CHANGES THE DIRTY FLUSH COUNTER SUCH THAT GANGALIST OBJECTS ARE WRITTEN TO DISK ATOMICALLY
### THIS COMES WITH A MAJOR PERFORMANCE IMPROVEMENT IN SOME CODE THAT REALLY SHOULDN'T BE SLOW
### Maybe delete this if/when we aren't using XML repo in future as allowing a really dirty repo is probably bad m'kay
## NB Should really tie in the default here to defaults from Core
old_count = 10
if hasattr(val, '__len__') and full_reg:
old_count = obj_reg.dirty_flush_counter
val_len = 2*len(val) + 10
obj_reg.dirty_flush_counter = val_len
obj_len = 1
if len(val) > 0:
bare_val = stripProxy(val)
if hasattr(bare_val, '_schema'):
obj_len = len(stripProxy(bare_val._schema).datadict.keys())
obj_len = obj_len*2
val_len = val_len * obj_len
if item['sequence']:
_preparable = True if item['preparable'] else False
if len(val) == 0:
val = GangaList()
else:
if isType(item, Schema.ComponentItem):
val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable)
else:
val = makeGangaList(val, parent=obj, preparable=_preparable)
else:
if isType(item, Schema.ComponentItem):
val = cloneVal(val)
if hasattr(val, '_setParent'):
val._setParent(obj)
### RESET DIRTY COUNT
if full_reg:
obj_reg.dirty_flush_counter = old_count
obj._data[self._name] = val
obj._setDirty()