def __sequence_set__(stripper, obj, val, name): item = stripProxy(obj)._schema[name] # we need to explicitly check for the list type, because simple # values (such as strings) may be iterable from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if isType(val, getKnownLists()): # create GangaList if stripper is not None: val = makeGangaList(val, stripper) else: val = makeGangaList( ProxyDataDescriptor._stripAttribute(obj, val, name)) else: # val is not iterable if item['strict_sequence']: raise GangaAttributeError( 'cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)' % (repr(val), getattr(obj, proxyObject)._schema.name, name)) if stripper is not None: val = makeGangaList(stripper(val)) else: val = makeGangaList( ProxyDataDescriptor._stripAttribute(obj, val, name)) return val
def __sequence_set__(stripper, obj, val, name): item = stripProxy(obj)._schema[name] # we need to explicitly check for the list type, because simple # values (such as strings) may be iterable new_v = None from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if isType(val, getKnownLists()): # create GangaList if stripper is not None: new_v = makeGangaList(val, stripper) else: temp_v = ProxyDataDescriptor._stripAttribute(obj, val, name) new_v = makeGangaList(temp_v) else: # val is not iterable if item['strict_sequence']: raise GangaAttributeError('cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)' % (repr(val), getName(obj), name)) if stripper is not None: new_v = makeGangaList(stripper(val)) else: temp_v = ProxyDataDescriptor._stripAttribute(obj, val, name) new_v = makeGangaList(temp_v) if new_v is None: new_v = val return new_v
def __sequence_set__(stripper, obj, val, name): item = stripProxy(getattr(obj, proxyClass))._schema[name] # we need to explicitly check for the list type, because simple # values (such as strings) may be iterable from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if isType(val, getKnownLists()): # create GangaList if stripper is not None: val = makeGangaList(val, stripper) else: val = makeGangaList(ProxyDataDescriptor._stripAttribute(obj, val, name)) else: # val is not iterable if item["strict_sequence"]: raise GangaAttributeError( "cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)" % (repr(val), getattr(obj, proxyClass)._schema.name, name) ) if stripper is not None: val = makeGangaList(stripper(val)) else: val = makeGangaList(ProxyDataDescriptor._stripAttribute(obj, val, name)) return val
def __atomic_set__(self, _obj, _val): obj = stripProxy(_obj) val = stripProxy(_val) from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList checkSet = self._bind_method(obj, self._checkset_name) if checkSet is not None: checkSet(val) this_filter = self._bind_method(obj, self._filter_name) if this_filter: val = this_filter(val) # LOCKING obj._getWriteAccess() # self._check_getter() item = stripProxy(obj._schema[getName(self)]) def cloneVal(v): if isType(v, list()): new_v = GangaList() for elem in v: new_v.append(self.__cloneVal(elem, obj)) return new_v else: return self.__cloneVal(v, obj) if item['sequence']: _preparable = True if item['preparable'] else False if len(val) == 0: val = GangaList() else: if isType(item, Schema.ComponentItem): val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable) else: val = makeGangaList(val, parent=obj, preparable=_preparable) else: if isType(item, Schema.ComponentItem): newListObj = [] if isinstance(val, list): for elem in val: newListObj.append(cloneVal(elem)) val = newListObj else: val = cloneVal(val) if hasattr(val, '_setParent'): val._setParent(obj) obj.setNodeAttribute(getName(self), val) obj._setDirty()
def __get__(self, obj, cls): global proxyRef # at class level return a helper object (for textual description) if obj is None: # return Schema.make_helper(getattr(getattr(cls, proxyRef), getName(self))) return getattr(getattr(cls, proxyRef), getNeme(self)) try: val = getattr(getattr(obj, proxyRef), getName(self)) except Exception as err: if getName(self) in getattr(obj, proxyRef).__dict__.keys(): val = getattr(obj, proxyRef).__dict__[getName(self)] else: val = getattr(getattr(obj, proxyRef), getName(self)) # wrap proxy item = getattr(obj, proxyClass)._schema[getName(self)] if item["proxy_get"]: return getattr(getattr(obj, proxyRef), item["proxy_get"])() if isType(item, Schema.ComponentItem): disguiser = self.disguiseComponentObject else: disguiser = self.disguiseAttribute ## FIXME Add GangaList? if item["sequence"] and isType(val, list): from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList val = makeGangaList(val, disguiser) return disguiser(val)
def __get__(self, obj, cls): global proxyRef # at class level return a helper object (for textual description) if obj is None: # return Schema.make_helper(getattr(getattr(cls, proxyRef),self._name)) return getattr(stripProxy(cls), self._name) val = getattr(stripProxy(obj), self._name) # wrap proxy item = stripProxy(obj)._schema[self._name] if item["proxy_get"]: return getattr(stripProxy(obj), item["proxy_get"])() if isType(item, Schema.ComponentItem): disguiser = self.disguiseComponentObject else: disguiser = self.disguiseAttribute if item["sequence"] and isType(val, list): from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList val = makeGangaList(val, disguiser) return disguiser(val)
def setUp(self): # make a list of lists containing GangaObjects self.filelist = [] for _ in range(10): self.filelist.append([self._makeRandomTFile() for _ in range(3)]) # make an empty GangaList self.gangalist = addProxy(makeGangaList([]))
def __get__(self, obj, cls): # at class level return a helper object (for textual description) if obj is None: # return Schema.make_helper(getattr(getattr(cls, proxyRef),self._name)) return getattr( getattr(cls, proxyRef), self._name) val = getattr( getattr(obj, proxyRef), self._name) # apply object conversion or if it failes, make the wrapper proxy def disguiseComponentObject(v): # get the proxy for implementation object def getProxy(v): from Ganga.GPIDev.Base import GangaObject if not isType(v, GangaObject): raise GangaAttributeError("invalid type: cannot assign '%s' to attribute '%s'" % (repr(v), self._name)) return GPIProxyObjectFactory(v) # convert implementation object to GPI value according to the # static method defined in the implementation object def object2value(v): return None vv = object2value(v) if vv is None: # allow None to be a legal value for optional component items if v is None: return None else: return getProxy(v) else: return vv # apply attribute conversion def disguiseAttribute(v): # FIXME: this is obsoleted method from Ganga.GPIDev.Base import GangaObject if isType(v, GangaObject): return GPIProxyObjectFactory(v) return v # wrap proxy item = getattr(obj, proxyRef)._schema[self._name] if item['proxy_get']: return getattr(getattr(obj, proxyRef), item['proxy_get'])() if item.isA(Schema.ComponentItem): disguiser = disguiseComponentObject else: disguiser = disguiseAttribute from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if item['sequence'] and isType(val, list): val = makeGangaList(val, disguiser) return disguiser(val)
def setUp(self): super(TestNestedLists, self).setUp() # make a list of lists containing GangaObjects self.filelist = [] self.gangalist = None for _ in range(10): self.filelist.append([self._makeRandomTFile() for _ in range(3)]) # make an empty GangaList self.gangalist = addProxy(makeGangaList([]))
def end_element(name): #logger.debug('End element: name=%s', name) # if higher level element had error, ignore the corresponding part # of the XML tree as we go up if self.ignore_count: self.ignore_count -= 1 return # when </attribute> is seen the current object, attribute name and # value should be on top of the stack if name == 'attribute': value = self.stack.pop() aname = self.stack.pop() obj = self.stack[-1] # update the object's attribute obj.setSchemaAttribute(aname, value) #logger.info("Setting: %s = %s" % (aname, value)) # when </value> is seen the value_construct buffer (CDATA) should # be a python expression (e.g. quoted string) if name == 'value': # unescape the special characters s = unescape(self.value_construct) #logger.debug('string value: %s',s) val = eval(s, config_scope) #logger.debug('evaled value: %s type=%s',repr(val),type(val)) self.stack.append(val) self.value_construct = None # when </sequence> is seen we remove last items from stack (as indicated by sequence_start) # we make a GangaList from these items and put it on stack if name == 'sequence': pos = self.sequence_start.pop() alist = makeGangaList(self.stack[pos:]) del self.stack[pos:] self.stack.append(alist) # when </class> is seen we finish initializing the new object # by setting remaining attributes to their default values # the object stays on the stack (will be removed by </attribute> or # is a root object) if name == 'class': obj = self.stack[-1] for attr, item in obj._schema.allItems(): if not attr in obj._data: #logger.info("Opening: %s" % attr) if item._meta["sequence"] == 1: obj.setSchemaAttribute(attr, makeGangaListByRef(obj._schema.getDefaultValue(attr))) #setattr(obj, attr, makeGangaListByRef(obj._schema.getDefaultValue(attr))) else: obj.setSchemaAttribute(attr, obj._schema.getDefaultValue(attr))
def end_element(name): #logger.debug('End element: name=%s', name) # if higher level element had error, ignore the corresponding part # of the XML tree as we go up if self.ignore_count: self.ignore_count -= 1 return # when </attribute> is seen the current object, attribute name and # value should be on top of the stack if name == 'attribute': value = self.stack.pop() aname = self.stack.pop() obj = self.stack[-1] # update the object's attribute obj.setSchemaAttribute(aname, value) #logger.info("Setting: %s = %s" % (aname, value)) # when </value> is seen the value_construct buffer (CDATA) should # be a python expression (e.g. quoted string) if name == 'value': # unescape the special characters s = unescape(self.value_construct) #logger.debug('string value: %s',s) if s not in _cached_eval_strings: # This is ugly and classes which use this are bad, but this needs to be fixed in another PR # TODO Make the scope of objects a lot better than whatever is in the config # This is a dictionary constructed from eval-ing things in the Config. Why does should it do this? # Anyway, lets save the result for speed _cached_eval_strings[s] = eval(s, config_scope) val = copy.deepcopy(_cached_eval_strings[s]) #logger.debug('evaled value: %s type=%s',repr(val),type(val)) self.stack.append(val) self.value_construct = None # when </sequence> is seen we remove last items from stack (as indicated by sequence_start) # we make a GangaList from these items and put it on stack if name == 'sequence': pos = self.sequence_start.pop() alist = makeGangaList(self.stack[pos:]) del self.stack[pos:] self.stack.append(alist) # when </class> is seen we finish initializing the new object # by setting remaining attributes to their default values # the object stays on the stack (will be removed by </attribute> or # is a root object) if name == 'class': pass
def proxy_add(self, obj, name): """ Add an object to the box The object must also be given a descriptive text name, for example: box.add(Job(),'A job') or a=Executable() box.add(a, 'An executable application') """ obj = _unwrap(obj) if isinstance(obj, list): obj = makeGangaList(obj) if not isinstance(obj, GangaObject): raise BoxTypeError( "The Box can only contain Ganga Objects (i.e. Applications, Datasets or Backends). Check that the object is first in box.add(obj,'name')" ) if obj._category == 'jobs': if hasattr(obj.application, 'is_prepared'): if obj.application.is_prepared is not None and obj.application.is_prepared is not True: logger.debug( 'Adding a prepared job to the box and increasing the shareref counter' ) obj.application.incrementShareCounter( obj.application.is_prepared.name) if obj._category == 'applications': if hasattr(obj, 'is_prepared'): if obj.is_prepared is not None and obj.is_prepared is not True: logger.debug( 'Adding a prepared application to the box and increasing the shareref counter' ) obj.incrementShareCounter(obj.is_prepared.name) obj = obj.clone() nobj = BoxMetadataObject() nobj.name = name self._add(obj) self.metadata._add(nobj, self.find(obj)) nobj._setDirty() obj._setDirty()
def proxy_add(self, obj, name): """ Add an object to the box The object must also be given a descriptive text name, for example: box.add(Job(),'A job') or a=Executable() box.add(a, 'An executable application') """ obj = _unwrap(obj) if isinstance(obj, list): obj = makeGangaList(obj) if not isinstance(obj, GangaObject): raise BoxTypeError( "The Box can only contain Ganga Objects (i.e. Applications, Datasets or Backends). Check that the object is first in box.add(obj,'name')") if obj._category == 'jobs': if hasattr(obj.application, 'is_prepared'): if obj.application.is_prepared is not None and obj.application.is_prepared is not True: logger.debug( 'Adding a prepared job to the box and increasing the shareref counter') obj.application.incrementShareCounter( obj.application.is_prepared.name) if obj._category == 'applications': if hasattr(obj, 'is_prepared'): if obj.is_prepared is not None and obj.is_prepared is not True: logger.debug( 'Adding a prepared application to the box and increasing the shareref counter') obj.incrementShareCounter(obj.is_prepared.name) obj = obj.clone() nobj = BoxMetadataObject() nobj.name = name self._add(obj) self.metadata._add(nobj, self.find(obj)) nobj._setDirty() obj._setDirty()
def __get__(self, obj, cls): # at class level return a helper object (for textual description) if obj is None: # return Schema.make_helper(getattr(getattr(cls, implRef), getName(self))) return getattr(stripProxy(cls), getName(self)) try: val = getattr(stripProxy(obj), getName(self)) except Exception as err: if getName(self) in stripProxy(obj).__dict__.keys(): val = stripProxy(obj).__dict__[getName(self)] else: val = getattr(stripProxy(obj), getName(self)) # wrap proxy item = stripProxy(obj)._schema[getName(self)] if item['proxy_get']: return getattr(stripProxy(obj), item['proxy_get'])() if isType(item, ComponentItem): disguiser = self.disguiseComponentObject else: disguiser = self.disguiseAttribute ## FIXME Add GangaList? if item['sequence'] and isType(val, list): from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList val = makeGangaList(val, disguiser) returnable = disguiser(val) GangaObject = _getGangaObject() if isType(returnable, GangaObject): return addProxy(returnable) else: return returnable
def gangaObjectFactory(attrDict, migration_class=None): # gangaObjectFactory(...) --> (object, migrated, [<list of errors>]) migrated = [False] errors = [] if attrDict['simple']: return (None, migrated[0], errors) if migration_class: cls = migration_class else: try: cls = allPlugins.find(attrDict['category'], attrDict['name']) except PluginManagerError as e: msg = "Plugin Manager Error: %s" % str(e) errors.append(GangaObjectFactoryError(e, msg=msg)) return (EmptyGangaObject(), migrated[0], errors) schema = cls._schema major, minor = attrDict['version'] version = Version(major, minor) if not schema.version.isCompatible(version): v1 = '.'.join(map(str, [major, minor])) v2 = '.'.join(map(str, [schema.version.major, schema.version.minor])) msg = "Incompatible schema versions of plugin %s in the category %s. Current version %s. Repository version %s." % ( attrDict['name'], attrDict['category'], v2, v1) if schema.version.major > version.major: # no forward migration from .MigrationControl import migration if migration.isAllowed(attrDict['category'], attrDict['name'], attrDict['version'], msg=msg): # try if migration provided by the plugin class try: old_cls = cls.getMigrationClass(version) except: old_cls = None if old_cls: old_obj, old_migrated, old_errors = gangaObjectFactory( attrDict, migration_class=old_cls) if old_migrated: migrated[0] = old_migrated if not old_errors: try: obj = cls.getMigrationObject(old_obj) #assert(isinstance(obj, cls)) except Exception as e: msg += ' Error in object migration: ' + str(e) else: obj.__setstate__(obj.__dict__) migrated[0] = True return (obj, migrated[0], errors) else: msg += ' Errors in object migration ' + \ str(map(str, old_errors)) else: msg += ' No migration class is provided.' else: msg += ' Migration was denied in the migration control object.' errors.append(GangaObjectFactoryError(msg=msg)) return (EmptyGangaObject(), migrated[0], errors) obj = super(cls, cls).__new__(cls) obj._data = {} def mapper(attrDict): if attrDict['simple']: val = attrDict['data'] else: val, attr_migrated, attr_errors = gangaObjectFactory(attrDict) if attr_migrated: migrated[0] = attr_migrated for err in attr_errors: if str(err) not in map(str, errors): # don't duplicate the same errors errors.append(err) return val data = attrDict['data'] for attr, item in schema.allItems(): if attr in data: val = data[attr] if item['sequence']: val = makeGangaList(val, mapper, parent=obj) else: val = mapper(val) else: val = schema.getDefaultValue(attr) obj._data[attr] = val obj.__setstate__(obj.__dict__) return (obj, migrated[0], errors)
def __set__(self, obj, val): #self is the attribute we're about to change #obj is the object we're about to make the change in #val is the value we're setting the attribute to. #item is the schema entry of the attribute we're about to change item = obj._impl._schema[self._name] if item['protected']: raise ProtectedAttributeError('"%s" attribute is protected and cannot be modified'%(self._name,)) if obj._impl._readonly() and not (self._name == 'comment' and obj._impl._name == 'Job'): raise ReadOnlyObjectError('object %s is read-only and attribute "%s" cannot be modified now'%(repr(obj),self._name)) #mechanism for locking of preparable attributes if item['preparable'] and obj.is_prepared is not None and obj.is_prepared is not True: raise ProtectedAttributeError('AttributeError: "%s" attribute belongs to a prepared application and so cannot be modified. unprepare() the application or copy the job/application (using j.copy(unprepare=True)) and modify that new instance.'%(self._name,)) #if we set is_prepared to None in the GPI, that should effectively unprepare the application if self._name == 'is_prepared' and val is None and obj.is_prepared is not None: logger.info('Unpreparing application.') obj.unprepare() #Replace is_prepared on an application for another ShareDir object from Ganga.GPIDev.Lib.File import ShareDir if self._name == 'is_prepared' and isType(val, ShareDir) and hasattr(obj._impl,'_getRegistry'): if obj._impl._getRegistry() is not None: logger.debug('Overwriting is_prepared attribute with a ShareDir object') obj.unprepare() #it's safe to unprepare 'not-prepared' applications. from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(val.name) #catch assignment of 'something' to a preparable application if self._name == 'application' and hasattr(obj.application,'is_prepared'): #a=Job(); a.prepare(); a.application=Executable() if obj.application.is_prepared is not None and obj.application.is_prepared is not True and val.is_prepared is None: logger.debug('Overwriting a prepared application with one that is unprepared') obj.application.unprepare() #a=Job(); b=Executable(); b.prepare(); a.application=b elif obj.application.is_prepared is not True and hasattr(val,'is_prepared') and val.is_prepared is not None and val.is_prepared is not True: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug('Overwriting application with a prepared one') obj.application.unprepare() shareref.increase(val.is_prepared.name) #check that the shared directory actually exists before assigning the (prepared) application to a job if hasattr(val, 'is_prepared'): if val.is_prepared is not None and val.is_prepared is not True: if not os.path.isdir(os.path.join(shared_path,val.is_prepared.name)): logger.error('ShareDir directory not found: %s' % val.is_prepared.name) # apply attribute conversion def stripAttribute(v): # just warn #print '**** checking',v,v.__class__, isinstance(val,GPIProxyObject) if isinstance(v,GPIProxyObject): v = v._impl logger.debug('%s property: assigned a component object (_impl used)',self._name) return obj._impl._attribute_filter__set__(self._name,v) # unwrap proxy if item.isA(Schema.ComponentItem): from Filters import allComponentFilters item = obj._impl._schema.getItem(self._name) cfilter = allComponentFilters[item['category']] stripper = lambda v: stripComponentObject(v,cfilter,item) else: stripper = stripAttribute from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList, makeGangaList if item['sequence']: # we need to explicitly check for the list type, because simple values (such as strings) may be iterable if isType(val, (GangaList,list)): #create GangaList val = makeGangaList(val,stripper) else: # val is not iterable if item['strict_sequence']: raise GangaAttributeError('cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)'%(repr(val),obj._impl._schema.name,self._name)) val = makeGangaList(stripper(val)) else: val = stripper(val) # apply attribute filter to component items if item.isA(Schema.ComponentItem): val = stripAttribute(val) self._check_type(obj,val) setattr(obj._impl, self._name, val)
obj = super(cls, cls).__new__(cls) obj._data = {} def mapper(attrDict): if attrDict["simple"]: val = attrDict["data"] else: val, attr_migrated, attr_errors = gangaObjectFactory(attrDict) if attr_migrated: migrated[0] = attr_migrated for err in attr_errors: if str(err) not in map(str, errors): # don't duplicate the same errors errors.append(err) return val data = attrDict["data"] for attr, item in schema.allItems(): if attr in data: val = data[attr] if item["sequence"]: val = makeGangaList(val, mapper, parent=obj) else: val = mapper(val) else: val = schema.getDefaultValue(attr) obj._data[attr] = val obj.__setstate__(obj.__dict__) return (obj, migrated[0], errors)
def _checkoutJobs(self, dir, meta={}): """ Checkout the jobs and return a list of job objects. """ ###logger.debug('checkoutJobs meta=%s', meta) jobs = [] # summary of errors (exception reprs are keys, occurence count is # value) error_summary = {} incomplete_ids = [] # ids of incomplete jobs (schema mismatch etc.) bad_ids = [] # ids of ignored jobs (I/O errors) entries_cnt = 0 master_ids = self._getJobIds(dir, meta) # add a new error entry to error_summary def add_error(e): #re = repr(e) re = str(e) error_summary.setdefault(re, 0) error_summary[re] += 1 # read a job id from dir and append it to jobs list # masterid (if set) is used for reporting purposes only def read_job(dir, id, jobs, masterid=None): def fqid(): if masterid is None: return str(id) else: return '.'.join([str(masterid), str(id)]) try: # read job data and reconstruct the object with open(os.path.join(dir, str(id), 'data')) as data_file: j, errors = from_file(data_file) if errors: # data errors j.status = 'incomplete' for e in errors: add_error(e) incomplete_ids.append(fqid()) jobs.append(j) return j except KeyboardInterrupt: # FIXME: any special cleanup needed? raise except Exception as x: # I/O and parsing errors msg = 'Cannot read job %s: %s' % (fqid(), repr(x)) add_error(msg) bad_ids.append(fqid()) logger.debug(msg) Ganga.Utility.logging.log_user_exception(logger, debug=True) # main loop import time progress = 0 t0 = time.time() for id in master_ids: # progress log if progress % 100 == 0 and progress != 0: logger.info( 'Loaded %d/%d jobs in %d seconds', progress, len(master_ids), time.time() - t0) progress += 1 # read top-level job j = read_job(dir, id, jobs) entries_cnt += 1 if j: # FIXME: hardcoded subjobs handling from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList subjobs_dir = os.path.join(dir, str(id)) if os.path.isdir(subjobs_dir): # get all subjob ids subjob_ids = self._getJobIds(subjobs_dir) entries_cnt += len(subjob_ids) subjobs = [] for sid in subjob_ids: # read-in subjob objects s = read_job(subjobs_dir, sid, subjobs, masterid=id) s._setParent(j) # initialize correctly the subjobs attribute j._data['subjobs'] = makeGangaList(subjobs) j.__setstate__(j.__dict__) # print out reports logger.info( 'Loaded total of %d job entries (including all subjobs)', entries_cnt) logger.info('Loaded total of %d master job entries:', len(master_ids)) logger.info('Load time %d seconds', time.time() - t0) if bad_ids: logger.error( 'Missing job entries due to I/O errors: %d/%d', len(bad_ids), entries_cnt) if incomplete_ids: logger.error( 'Job entries loaded in incomplete state due to data errors: %d/%d', len(incomplete_ids), entries_cnt) if len(incomplete_ids) < 100: logger.error('Incomplete job ids: %s', incomplete_ids) if error_summary: logger.error('Summary of problems:') for re, cnt in error_summary.iteritems(): logger.error(' - %d job entries: %s', cnt, re) return jobs
def __set__(self, obj, val): # self is the attribute we're about to change # obj is the object we're about to make the change in # val is the value we're setting the attribute to. # item is the schema entry of the attribute we're about to change item = stripProxy(obj)._schema[self._name] if item["protected"]: raise ProtectedAttributeError('"%s" attribute is protected and cannot be modified' % (self._name,)) if stripProxy(obj)._readonly(): if not (self._name == "comment" and stripProxy(obj)._name == "Job"): raise ReadOnlyObjectError( 'object %s is read-only and attribute "%s" cannot be modified now' % (repr(obj), self._name) ) # mechanism for locking of preparable attributes if item["preparable"]: if stripProxy(obj).is_prepared is not None: if stripProxy(obj).is_prepared is not True: raise ProtectedAttributeError( 'AttributeError: "%s" attribute belongs to a prepared application and so cannot be modified. unprepare() the application or copy the job/application (using j.copy(unprepare=True)) and modify that new instance.' % (self._name,) ) # if we set is_prepared to None in the GPI, that should effectively # unprepare the application if self._name == "is_prepared": if val is None: if stripProxy(obj).is_prepared is not None: logger.info("Unpreparing application.") stripProxy(obj).unprepare() # Replace is_prepared on an application for another ShareDir object if self._name == "is_prepared": if hasattr(stripProxy(obj), "_getRegistry"): from Ganga.GPIDev.Lib.File import ShareDir if stripProxy(obj)._getRegistry() is not None and isType(val, ShareDir): logger.debug("Overwriting is_prepared attribute with a ShareDir object") # it's safe to unprepare 'not-prepared' applications. stripProxy(obj).unprepare() from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) shareref.increase(val.name) # catch assignment of 'something' to a preparable application if self._name == "application": if hasattr(stripProxy(obj.application), "is_prepared"): # a=Job(); a.prepare(); a.application=Executable() if ( stripProxy(obj.application).is_prepared not in [None, True] and hasattr(stripProxy(val), "is_prepared") and stripProxy(val).is_prepared is None ): logger.debug("Overwriting a prepared application with one that is unprepared") obj.application.unprepare() # a=Job(); b=Executable(); b.prepare(); a.application=b elif stripProxy(obj.application).is_prepared is not True: if hasattr(val, "is_prepared"): if stripProxy(val).is_prepared not in [None, True]: from Ganga.Core.GangaRepository import getRegistry shareref = GPIProxyObjectFactory(getRegistry("prep").getShareRef()) logger.debug("Overwriting application with a prepared one") if stripProxy(obj.application) != val: stripProxy(obj.application).unprepare() shareref.increase(val.is_prepared.name) # check that the shared directory actually exists before # assigning the (prepared) application to a job if hasattr(stripProxy(val), "is_prepared"): if stripProxy(val).is_prepared not in [None, True]: if hasattr(stripProxy(val).is_prepared, "name"): from Ganga.Utility.files import expandfilename Config_conf = getConfig("Configuration") shared_path = os.path.join( expandfilename(Config_conf["gangadir"]), "shared", Config_conf["user"] ) if not os.path.isdir(os.path.join(shared_path, stripProxy(val).is_prepared.name)): logger.error("ShareDir directory not found: %s" % stripProxy(val).is_prepared.name) # unwrap proxy if item.isA(Schema.ComponentItem): from .Filters import allComponentFilters item = getattr(obj, proxyRef)._schema.getItem(self._name) cfilter = allComponentFilters[item["category"]] stripper = lambda v: stripComponentObject(v, cfilter, item) else: stripper = None # stripper = self._stripAttribute if item["sequence"]: # we need to explicitly check for the list type, because simple # values (such as strings) may be iterable from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if isType(val, [getGangaList(), list, type([])]): # create GangaList if stripper is not None: val = makeGangaList(val, stripper) else: val = makeGangaList(self._stripAttribute(obj, val)) else: # val is not iterable if item["strict_sequence"]: raise GangaAttributeError( "cannot assign a simple value %s to a strict sequence attribute %s.%s (a list is expected instead)" % (repr(val), getattr(obj, proxyRef)._schema.name, self._name) ) if stripper is not None: val = makeGangaList(stripper(val)) else: val = makeGangaList(self._stripAttribute(obj, val)) else: if stripper is not None: val = stripper(val) else: val = self._stripAttribute(obj, val) # apply attribute filter to component items if item.isA(Schema.ComponentItem): val = self._stripAttribute(obj, val) self._check_type(obj, val) setattr(stripProxy(obj), self._name, val)
def __set__(self, obj, val): from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList, makeGangaList cs = self._bind_method(obj,self._checkset_name) if cs: cs(val) filter = self._bind_method(obj, self._filter_name) if filter: val = filter(val) #LOCKING obj._getWriteAccess() #self._check_getter() def cloneVal(v): #print 'cloneVal:',self._name,v,item['optional'],item['load_default'], item['defvalue'] if v is None: assert(item['optional']) return None else: assert(isinstance(v, Node)) if isinstance(v, GangaList): catagories = v.getCategory() len_cat = len(catagories) #we pass on empty lists, as the catagory is yet to be defined if (len_cat > 1) or ((len_cat == 1) and (catagories[0] != item['category'])): raise GangaAttributeError('%s: attempt to assign a list containing incompatible objects %s to the property in category "%s"' %(self._name, v,item['category'])) else: if v._category != item['category']: raise GangaAttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s"' %(self._name, v,item['category'])) v = v.clone() v._setParent(obj) return v item = obj._schema[self._name] if item.isA(Schema.ComponentItem): if item['sequence']: ## checklist=filter(lambda x: not implies(x is None,item['optional']) or x._category != item['category'],val) ## if len(checklist) > 0: ## raise AttributeError('%s: attempt to assign incompatible objects %s to the property in category "%s"'%(self._name, str(checklist),item['category'])) if item['preparable']: val = makeGangaList(val,cloneVal, parent = obj, preparable = True) else: val = makeGangaList(val,cloneVal, parent = obj) else: val = cloneVal(val) ## if val is None: ## assert(item['optional']) ## else: ## assert(isinstance(val, Node)) ## if val._category != item['category']: ## raise AttributeError('%s: attempt to assign an incompatible object %s to the property in category "%s"' %(self._name, val,item['category'])) ## val = cloneVal(val) else: if item['sequence']: if item['preparable']: val = makeGangaList(val, parent = obj, preparable = True) else: val = makeGangaList(val, parent = obj) obj._data[self._name] = val obj._setDirty()
def __atomic_set__(self, _obj, _val): ## self: attribute being changed or Ganga.GPIDev.Base.Objects.Descriptor in which case getName(self) gives the name of the attribute being changed ## _obj: parent class which 'owns' the attribute ## _val: value of the attribute which we're about to set #if hasattr(_obj, getName(self)): # if not isinstance(getattr(_obj, getName(self)), GangaObject): # if type( getattr(_obj, getName(self)) ) == type(_val): # object.__setattr__(_obj, getName(self), deepcopy(_val)) # return # # if not isinstance(_obj, GangaObject) and type(_obj) == type(_val): # _obj = deepcopy(_val) # return obj = _obj temp_val = _val from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList if hasattr(obj, '_checkset_name'): checkSet = self._bind_method(obj, self._checkset_name) if checkSet is not None: checkSet(temp_val) if hasattr(obj, '_filter_name'): this_filter = self._bind_method(obj, self._filter_name) if this_filter: val = this_filter(temp_val) else: val = temp_val else: val = temp_val # LOCKING obj._getWriteAccess() #self._check_getter() item = obj._schema[getName(self)] def cloneVal(v): GangaList = _getGangaList() if isinstance(v, (list, tuple, GangaList)): new_v = GangaList() for elem in v: new_v.append(self.__cloneVal(elem, obj)) return new_v else: return self.__cloneVal(v, obj) ## If the item has been defined as a sequence great, let's continue! if item['sequence']: _preparable = True if item['preparable'] else False if len(val) == 0: GangaList = _getGangaList() new_val = GangaList() else: if isinstance(item, Schema.ComponentItem): new_val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable) else: new_val = makeGangaList(val, parent=obj, preparable=_preparable) else: ## Else we need to work out what we've got. if isinstance(item, Schema.ComponentItem): GangaList = _getGangaList() if isinstance(val, (list, tuple, GangaList)): ## Can't have a GangaList inside a GangaList easily so lets not if isinstance(_obj, GangaList): newListObj = [] else: newListObj = GangaList() self.__createNewList(newListObj, val, cloneVal) #for elem in val: # newListObj.append(cloneVal(elem)) new_val = newListObj else: new_val = cloneVal(val) else: new_val = val pass #val = deepcopy(val) if isinstance(new_val, Node): new_val._setParent(obj) obj.setNodeAttribute(getName(self), new_val) obj._setDirty()
def end_element(name): #logger.debug('End element: name=%s', name) # if higher level element had error, ignore the corresponding part # of the XML tree as we go up if self.ignore_count: self.ignore_count -= 1 return # when </attribute> is seen the current object, attribute name and # value should be on top of the stack if name == 'attribute': value = self.stack.pop() aname = self.stack.pop() obj = self.stack[-1] # update the object's attribute try: obj.setSchemaAttribute(aname, value) except: raise GangaException( "ERROR in loading XML, failed to set attribute %s for class %s" % (aname, _getName(obj))) #logger.info("Setting: %s = %s" % (aname, value)) # when </value> is seen the value_construct buffer (CDATA) should # be a python expression (e.g. quoted string) if name == 'value': try: # unescape the special characters s = unescape(self.value_construct) #logger.debug('string value: %s',s) if s not in _cached_eval_strings: # This is ugly and classes which use this are bad, but this needs to be fixed in another PR # TODO Make the scope of objects a lot better than whatever is in the config # This is a dictionary constructed from eval-ing things in the Config. Why does should it do this? # Anyway, lets save the result for speed _cached_eval_strings[s] = eval(s, config_scope) eval_str = _cached_eval_strings[s] if not isinstance(eval_str, str): val = copy.deepcopy(eval_str) else: val = eval_str #logger.debug('evaled value: %s type=%s',repr(val),type(val)) self.stack.append(val) self.value_construct = None except: raise GangaException( "ERROR in loading XML, failed to correctly parse attribute value: \'%s\'" % str(self.value_construct)) # when </sequence> is seen we remove last items from stack (as indicated by sequence_start) # we make a GangaList from these items and put it on stack if name == 'sequence': pos = self.sequence_start.pop() try: alist = makeGangaList(self.stack[pos:]) except: raise GangaException( "ERROR in loading XML, failed to construct a sequence(list) properly" ) del self.stack[pos:] self.stack.append(alist) # when </class> is seen we finish initializing the new object # by setting remaining attributes to their default values # the object stays on the stack (will be removed by </attribute> or # is a root object) if name == 'class': obj = self.stack[-1] cls = obj.__class__ if isinstance(cls, GangaObject): for attr, item in cls._schema.allItems(): if attr not in obj._data: if item.getProperties()['getter'] is None: try: setattr(obj, attr, self._schema.getDefaultValue(attr)) except: raise GangaException( "ERROR in loading XML, failed to set default attribute %s for class %s" % (attr, _getName(obj))) pass
def __set__(self, _obj, _val): obj = stripProxy(_obj) val = stripProxy(_val) from Ganga.GPIDev.Lib.GangaList.GangaList import makeGangaList, GangaList cs = self._bind_method(obj, self._checkset_name) if cs: cs(val) this_filter = self._bind_method(obj, self._filter_name) if this_filter: val = this_filter(val) # LOCKING obj._getWriteAccess() # self._check_getter() item = stripProxy(obj._schema[self._name]) def cloneVal(v): return self.__cloneVal(v, obj) obj_reg = obj._getRegistry() full_reg = obj_reg is not None and hasattr(obj_reg, 'dirty_flush_counter') ### THIS CODE HERE CHANGES THE DIRTY FLUSH COUNTER SUCH THAT GANGALIST OBJECTS ARE WRITTEN TO DISK ATOMICALLY ### THIS COMES WITH A MAJOR PERFORMANCE IMPROVEMENT IN SOME CODE THAT REALLY SHOULDN'T BE SLOW ### Maybe delete this if/when we aren't using XML repo in future as allowing a really dirty repo is probably bad m'kay ## NB Should really tie in the default here to defaults from Core old_count = 10 if hasattr(val, '__len__') and full_reg: old_count = obj_reg.dirty_flush_counter val_len = 2*len(val) + 10 obj_reg.dirty_flush_counter = val_len obj_len = 1 if len(val) > 0: bare_val = stripProxy(val) if hasattr(bare_val, '_schema'): obj_len = len(stripProxy(bare_val._schema).datadict.keys()) obj_len = obj_len*2 val_len = val_len * obj_len if item['sequence']: _preparable = True if item['preparable'] else False if len(val) == 0: val = GangaList() else: if isType(item, Schema.ComponentItem): val = makeGangaList(val, cloneVal, parent=obj, preparable=_preparable) else: val = makeGangaList(val, parent=obj, preparable=_preparable) else: if isType(item, Schema.ComponentItem): val = cloneVal(val) if hasattr(val, '_setParent'): val._setParent(obj) ### RESET DIRTY COUNT if full_reg: obj_reg.dirty_flush_counter = old_count obj._data[self._name] = val obj._setDirty()