def end_element(name): #logger.debug('End element: name=%s', name) # if higher level element had error, ignore the corresponding part # of the XML tree as we go up if self.ignore_count: self.ignore_count -= 1 return # when </attribute> is seen the current object, attribute name and # value should be on top of the stack if name == 'attribute': value = self.stack.pop() aname = self.stack.pop() obj = self.stack[-1] # update the object's attribute obj.setNodeAttribute(aname, value) #logger.info("Setting: %s = %s" % (str(aname), str(value))) # when </value> is seen the value_construct buffer (CDATA) should # be a python expression (e.g. quoted string) if name == 'value': # unescape the special characters s = unescape(self.value_construct) #logger.debug('string value: %s',s) val = eval(s, config_scope) #logger.debug('evaled value: %s type=%s',repr(val),type(val)) self.stack.append(val) self.value_construct = None # when </sequence> is seen we remove last items from stack (as indicated by sequence_start) # we make a GangaList from these items and put it on stack if name == 'sequence': pos = self.sequence_start.pop() alist = makeGangaListByRef(self.stack[pos:]) del self.stack[pos:] self.stack.append(alist) # when </class> is seen we finish initializing the new object # by setting remaining attributes to their default values # the object stays on the stack (will be removed by </attribute> or # is a root object) if name == 'class': obj = self.stack[-1] for attr, item in obj._schema.allItems(): if not attr in obj.getNodeData(): #logger.info("Opening: %s" % attr) if item._meta["sequence"] == 1: obj.setNodeAttribute( attr, makeGangaListByRef( obj._schema.getDefaultValue(attr))) #setattr(obj, attr, makeGangaListByRef(obj._schema.getDefaultValue(attr))) else: obj.setNodeAttribute( attr, obj._schema.getDefaultValue(attr))
def end_element(name): ###logger.debug('End element: name=%s', name) # if higher level element had error, ignore the corresponding part # of the XML tree as we go up if self.ignore_count: self.ignore_count -= 1 return # when </attribute> is seen the current object, attribute name and # value should be on top of the stack if name == 'attribute': value = self.stack.pop() aname = self.stack.pop() obj = self.stack[-1] # update the object's attribute obj._data[aname] = value # when </value> is seen the value_construct buffer (CDATA) should # be a python expression (e.g. quoted string) if name == 'value': # unescape the special characters s = unescape(self.value_construct) ###logger.debug('string value: %s',s) val = eval(s, config_scope) ###logger.debug('evaled value: %s type=%s',repr(val),type(val)) self.stack.append(val) self.value_construct = None # when </sequence> is seen we remove last items from stack (as indicated by sequence_start) # we make a GangaList from these items and put it on stack if name == 'sequence': pos = self.sequence_start.pop() alist = makeGangaListByRef(self.stack[pos:]) del self.stack[pos:] self.stack.append(alist) # when </class> is seen we finish initializing the new object # by setting remaining attributes to their default values # the object stays on the stack (will be removed by </attribute> or # is a root object) if name == 'class': obj = self.stack[-1] for attr, item in obj._schema.allItems(): if not attr in obj._data: if item._meta["sequence"] == 1: obj._data[attr] = makeGangaListByRef( obj._schema.getDefaultValue(attr)) else: obj._data[attr] = obj._schema.getDefaultValue(attr) obj.__setstate__(obj.__dict__) # this sets the parent
def extend(self, files, unique=False): '''Extend the dataset. If unique, then only add files which are not already in the dataset.''' from Ganga.GPIDev.Base import ReadOnlyObjectError if self._parent is not None and self._parent._readonly(): raise ReadOnlyObjectError( 'object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self))) _external_files = [] if type(files) is str or isType(files, IGangaFile): _external_files = [files] elif type(files) in [list, tuple]: _external_files = files elif isType(files, LHCbDataset): _external_files = files.files else: if not hasattr(files, "__getitem__") or not hasattr( files, '__iter__'): _external_files = [files] # just in case they extend w/ self _to_remove = [] for this_file in _external_files: if hasattr(this_file, 'subfiles'): if len(this_file.subfiles) > 0: _external_files = makeGangaListByRef(this_file.subfiles) _to_remove.append(this_file) if type(this_file) is str: _external_files.append( string_datafile_shortcut_lhcb(this_file, None)) _to_remove.append(this_file) for _this_file in _to_remove: _external_files.pop(_external_files.index(_this_file)) for this_f in _external_files: _file = getDataFile(this_f) if _file is None: _file = this_f myName = _file.namePattern from GangaDirac.Lib.Files.DiracFile import DiracFile if isType(_file, DiracFile): myName = _file.lfn if unique and myName in self.getFileNames(): continue self.files.append(stripProxy(_file))
def extend(self, files, unique=False): '''Extend the dataset. If unique, then only add files which are not already in the dataset.''' from Ganga.GPIDev.Base import ReadOnlyObjectError if self._parent is not None and self._parent._readonly(): raise ReadOnlyObjectError('object Job#%s is read-only and attribute "%s/inputdata" cannot be modified now' % (self._parent.id, getName(self))) _external_files = [] if type(files) is str or isType(files, IGangaFile): _external_files = [files] elif type(files) in [list, tuple]: _external_files = files elif isType(files, LHCbDataset): _external_files = files.files else: if not hasattr(files, "__getitem__") or not hasattr(files, '__iter__'): _external_files = [files] # just in case they extend w/ self _to_remove = [] for this_file in _external_files: if hasattr(this_file, 'subfiles'): if len(this_file.subfiles) > 0: _external_files = makeGangaListByRef(this_file.subfiles) _to_remove.append(this_file) if type(this_file) is str: _external_files.append(string_datafile_shortcut_lhcb(this_file, None)) _to_remove.append(this_file) for _this_file in _to_remove: _external_files.pop(_external_files.index(_this_file)) for this_f in _external_files: _file = getDataFile(this_f) if _file is None: _file = this_f if not isinstance(_file, IGangaFile): raise GangaException('Cannot extend LHCbDataset based on this object type: %s' % type(_file) ) myName = _file.namePattern from GangaDirac.Lib.Files.DiracFile import DiracFile if isType(_file, DiracFile): myName = _file.lfn if unique and myName in self.getFileNames(): continue self.files.append(stripProxy(_file))
def load(self, ids, load_backup=False): for id in ids: fn = self.get_fn(id) if load_backup: fn = fn+"~" try: fobj = file(fn,"r") except IOError, x: if x.errno == errno.ENOENT: # remove index so we do not continue working with wrong information try: self._internal_del__(id) # remove internal representation os.unlink(os.path.dirname(fn)+".index") except OSError: pass raise KeyError(id) else: raise RepositoryError(self,"IOError: " + str(x)) try: must_load = (not id in self.objects) or (self.objects[id]._data is None) tmpobj = None if must_load or (self._load_timestamp.get(id,0) != os.fstat(fobj.fileno()).st_ctime): tmpobj, errs = self.from_file(fobj) do_sub_split = (not self.sub_split is None) and (self.sub_split in tmpobj._data) and len(tmpobj._data[self.sub_split]) == 0 if do_sub_split: i = 0 ld = os.listdir(os.path.dirname(fn)) l = [] while str(i) in ld: sfn = os.path.join(os.path.dirname(fn),str(i),"data") if load_backup: sfn = sfn+"~" try: sfobj = file(sfn,"r") except IOError, x: if x.errno == errno.ENOENT: raise IOError("Subobject %i.%i not found: %s" % (id,i,x)) else: raise RepositoryError(self,"IOError on loading subobject %i.%i: %s" % (id,i,x)) ff = self.from_file(sfobj) l.append(ff[0]) errs.extend(ff[1]) i += 1 tmpobj._data[self.sub_split] = makeGangaListByRef(l) if len(errs) > 0: raise errs[0] #if len(errs) > 0 and "status" in tmpobj._data: # MAGIC "status" if incomplete # tmpobj._data["status"] = "incomplete" # logger.error("Registry '%s': Could not load parts of object #%i: %s" % (self.registry.name,id,map(str,errs))) if id in self.objects: obj = self.objects[id] obj._data = tmpobj._data # Fix parent for objects in _data (necessary!) for n, v in obj._data.items(): if isinstance(v,Node): v._setParent(obj) if (isinstance(v,list) or v.__class__.__name__ == "GangaList"): # set the parent of the list or dictionary (or other iterable) items for i in v: if isinstance(i,Node): i._setParent(obj) # Check if index cache; if loaded; was valid: if obj._index_cache: new_idx_cache = self.registry.getIndexCache(obj) if new_idx_cache != obj._index_cache: # index is wrong! Try to get read access - then we can fix this if len(self.lock([id])) != 0: self.index_write(id) self.unlock([id]) logger.warning("Incorrect index cache of '%s' object #%s was corrected!" % (self.registry.name, id)) # if we cannot lock this, the inconsistency is most likely the result of another ganga process modifying the repo obj._index_cache = None else: self._internal_setitem__(id, tmpobj) if do_sub_split: try: for sobj in self.objects[id]._data[self.sub_split]: sobj._setParent(self.objects[id]) except AttributeError: pass # not actually Ganga objects in the sub-split field self.objects[id]._data[self.sub_split]._setParent(self.objects[id]) self._load_timestamp[id] = os.fstat(fobj.fileno()).st_ctime except RepositoryError: raise except Exception, x: if load_backup: logger.debug("Could not load backup object #%i: %s %s", id, x.__class__.__name__, x) raise InaccessibleObjectError(self,id,x) logger.debug("Could not load object #%i: %s %s", id, x.__class__.__name__, x) # try loading backup try: self.load([id],load_backup=True) logger.warning("Object '%s' #%i loaded from backup file - the last changes may be lost.", self.registry.name, id) continue except Exception: pass # add object to incomplete_objects if not id in self.incomplete_objects: self.incomplete_objects.append(id) # remove index so we do not continue working with wrong information try: os.unlink(os.path.dirname(fn)+".index") except OSError: pass #self._internal_setitem__(id, EmptyGangaObject()) // NO NO NO! BREAKS EVERYTHING HORRIBLY! raise InaccessibleObjectError(self,id,x)