def end_element(name):
#logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
obj.setNodeAttribute(aname, value)
#logger.info("Setting: %s = %s" % (str(aname), str(value)))
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
# unescape the special characters
s = unescape(self.value_construct)
#logger.debug('string value: %s',s)
val = eval(s, config_scope)
#logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
alist = makeGangaListByRef(self.stack[pos:])
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
obj = self.stack[-1]
for attr, item in obj._schema.allItems():
if not attr in obj.getNodeData():
#logger.info("Opening: %s" % attr)
if item._meta["sequence"] == 1:
obj.setNodeAttribute(
attr,
makeGangaListByRef(
obj._schema.getDefaultValue(attr)))
#setattr(obj, attr, makeGangaListByRef(obj._schema.getDefaultValue(attr)))
else:
obj.setNodeAttribute(
attr, obj._schema.getDefaultValue(attr))
def end_element(name):
###logger.debug('End element: name=%s', name)
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go up
if self.ignore_count:
self.ignore_count -= 1
return
# when </attribute> is seen the current object, attribute name and
# value should be on top of the stack
if name == 'attribute':
value = self.stack.pop()
aname = self.stack.pop()
obj = self.stack[-1]
# update the object's attribute
obj._data[aname] = value
# when </value> is seen the value_construct buffer (CDATA) should
# be a python expression (e.g. quoted string)
if name == 'value':
# unescape the special characters
s = unescape(self.value_construct)
###logger.debug('string value: %s',s)
val = eval(s, config_scope)
###logger.debug('evaled value: %s type=%s',repr(val),type(val))
self.stack.append(val)
self.value_construct = None
# when </sequence> is seen we remove last items from stack (as indicated by sequence_start)
# we make a GangaList from these items and put it on stack
if name == 'sequence':
pos = self.sequence_start.pop()
alist = makeGangaListByRef(self.stack[pos:])
del self.stack[pos:]
self.stack.append(alist)
# when </class> is seen we finish initializing the new object
# by setting remaining attributes to their default values
# the object stays on the stack (will be removed by </attribute> or
# is a root object)
if name == 'class':
obj = self.stack[-1]
for attr, item in obj._schema.allItems():
if not attr in obj._data:
if item._meta["sequence"] == 1:
obj._data[attr] = makeGangaListByRef(
obj._schema.getDefaultValue(attr))
else:
obj._data[attr] = obj._schema.getDefaultValue(attr)
obj.__setstate__(obj.__dict__) # this sets the parent
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError(
'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now'
% (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(
files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(
string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def extend(self, files, unique=False):
'''Extend the dataset. If unique, then only add files which are not
already in the dataset.'''
from Ganga.GPIDev.Base import ReadOnlyObjectError
if self._parent is not None and self._parent._readonly():
raise ReadOnlyObjectError('object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self)))
_external_files = []
if type(files) is str or isType(files, IGangaFile):
_external_files = [files]
elif type(files) in [list, tuple]:
_external_files = files
elif isType(files, LHCbDataset):
_external_files = files.files
else:
if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'):
_external_files = [files]
# just in case they extend w/ self
_to_remove = []
for this_file in _external_files:
if hasattr(this_file, 'subfiles'):
if len(this_file.subfiles) > 0:
_external_files = makeGangaListByRef(this_file.subfiles)
_to_remove.append(this_file)
if type(this_file) is str:
_external_files.append(string_datafile_shortcut_lhcb(this_file, None))
_to_remove.append(this_file)
for _this_file in _to_remove:
_external_files.pop(_external_files.index(_this_file))
for this_f in _external_files:
_file = getDataFile(this_f)
if _file is None:
_file = this_f
if not isinstance(_file, IGangaFile):
raise GangaException('Cannot extend LHCbDataset based on this object type: %s' % type(_file) )
myName = _file.namePattern
from GangaDirac.Lib.Files.DiracFile import DiracFile
if isType(_file, DiracFile):
myName = _file.lfn
if unique and myName in self.getFileNames():
continue
self.files.append(stripProxy(_file))
def load(self, ids, load_backup=False):
for id in ids:
fn = self.get_fn(id)
if load_backup:
fn = fn+"~"
try:
fobj = file(fn,"r")
except IOError, x:
if x.errno == errno.ENOENT:
# remove index so we do not continue working with wrong information
try:
self._internal_del__(id) # remove internal representation
os.unlink(os.path.dirname(fn)+".index")
except OSError:
pass
raise KeyError(id)
else:
raise RepositoryError(self,"IOError: " + str(x))
try:
must_load = (not id in self.objects) or (self.objects[id]._data is None)
tmpobj = None
if must_load or (self._load_timestamp.get(id,0) != os.fstat(fobj.fileno()).st_ctime):
tmpobj, errs = self.from_file(fobj)
do_sub_split = (not self.sub_split is None) and (self.sub_split in tmpobj._data) and len(tmpobj._data[self.sub_split]) == 0
if do_sub_split:
i = 0
ld = os.listdir(os.path.dirname(fn))
l = []
while str(i) in ld:
sfn = os.path.join(os.path.dirname(fn),str(i),"data")
if load_backup:
sfn = sfn+"~"
try:
sfobj = file(sfn,"r")
except IOError, x:
if x.errno == errno.ENOENT:
raise IOError("Subobject %i.%i not found: %s" % (id,i,x))
else:
raise RepositoryError(self,"IOError on loading subobject %i.%i: %s" % (id,i,x))
ff = self.from_file(sfobj)
l.append(ff[0])
errs.extend(ff[1])
i += 1
tmpobj._data[self.sub_split] = makeGangaListByRef(l)
if len(errs) > 0:
raise errs[0]
#if len(errs) > 0 and "status" in tmpobj._data: # MAGIC "status" if incomplete
# tmpobj._data["status"] = "incomplete"
# logger.error("Registry '%s': Could not load parts of object #%i: %s" % (self.registry.name,id,map(str,errs)))
if id in self.objects:
obj = self.objects[id]
obj._data = tmpobj._data
# Fix parent for objects in _data (necessary!)
for n, v in obj._data.items():
if isinstance(v,Node):
v._setParent(obj)
if (isinstance(v,list) or v.__class__.__name__ == "GangaList"):
# set the parent of the list or dictionary (or other iterable) items
for i in v:
if isinstance(i,Node):
i._setParent(obj)
# Check if index cache; if loaded; was valid:
if obj._index_cache:
new_idx_cache = self.registry.getIndexCache(obj)
if new_idx_cache != obj._index_cache:
# index is wrong! Try to get read access - then we can fix this
if len(self.lock([id])) != 0:
self.index_write(id)
self.unlock([id])
logger.warning("Incorrect index cache of '%s' object #%s was corrected!" % (self.registry.name, id))
# if we cannot lock this, the inconsistency is most likely the result of another ganga process modifying the repo
obj._index_cache = None
else:
self._internal_setitem__(id, tmpobj)
if do_sub_split:
try:
for sobj in self.objects[id]._data[self.sub_split]:
sobj._setParent(self.objects[id])
except AttributeError:
pass # not actually Ganga objects in the sub-split field
self.objects[id]._data[self.sub_split]._setParent(self.objects[id])
self._load_timestamp[id] = os.fstat(fobj.fileno()).st_ctime
except RepositoryError:
raise
except Exception, x:
if load_backup:
logger.debug("Could not load backup object #%i: %s %s", id, x.__class__.__name__, x)
raise InaccessibleObjectError(self,id,x)
logger.debug("Could not load object #%i: %s %s", id, x.__class__.__name__, x)
# try loading backup
try:
self.load([id],load_backup=True)
logger.warning("Object '%s' #%i loaded from backup file - the last changes may be lost.", self.registry.name, id)
continue
except Exception:
pass
# add object to incomplete_objects
if not id in self.incomplete_objects:
self.incomplete_objects.append(id)
# remove index so we do not continue working with wrong information
try:
os.unlink(os.path.dirname(fn)+".index")
except OSError:
pass
#self._internal_setitem__(id, EmptyGangaObject()) // NO NO NO! BREAKS EVERYTHING HORRIBLY!
raise InaccessibleObjectError(self,id,x)
