def select(self, minid=None, maxid=None, **attrs): import repr from Ganga.GPIDev.Lib.Job.Job import Job if isType(minid, Job): if minid.master: minid = minid.master.id else: minid = minid.id if maxid is None: maxid = minid if isType(maxid, Job): if maxid.master: maxid = maxid.master.id else: maxid = maxid.id logger = getLogger() this_repr = repr.Repr() from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory attrs_str = "" ## Loop through all possible input combinations to constructa string representation of the attrs from possible inputs ## Reuired to flatten the additional arguments into a flat string in attrs_str for a in attrs: from inspect import isclass if isclass(attrs[a]): this_attr = GPIProxyObjectFactory(attrs[a]()) else: from Ganga.GPIDev.Base.Objects import GangaObject if isType(attrs[a], GangaObject): this_attr = GPIProxyObjectFactory(attrs[a]) else: if type(attrs[a]) is str: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject this_attr = getRuntimeGPIObject(attrs[a], True) else: this_attr = attrs[a] full_str = str(this_attr) split_str = full_str.split('\n') for line in split_str: line = line.strip() flat_str = ''.join(split_str) attrs_str += ", %s=\"%s\"" % (str(a), flat_str) logger.debug("Attrs_Str: %s" % str(attrs_str)) logger.debug("Constructing slice: %s" % str("%s.select(minid='%s', maxid='%s'%s)" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str))) this_slice = self.__class__("%s.select(minid='%s', maxid='%s'%s)" % (self.name, this_repr.repr(minid), this_repr.repr(maxid), attrs_str)) def append(id, obj): this_slice.objects[id] = obj self.do_select(append, minid, maxid, **attrs) return this_slice
def safer_eval(_input): try: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject temp_output = getRuntimeGPIObject(_input, True) if temp_output is None: if len(_input) > 0: try: _output = eval(str(_input)) except: _output = str(_input) else: _output = None elif isclass(temp_output): _output = stripProxy(temp_output) else: _output = temp_output except ImportError: if len(_input) > 0: try: _output = eval(str(_input)) except: _output = str(_input) else: _output = None return _output
def safer_eval( _input ): try: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject temp_output = getRuntimeGPIObject(_input, True) if temp_output is None: if len(_input) > 0: try: _output = eval(str(_input)) except: _output = str(_input) else: _output = None elif isclass(temp_output): _output = stripProxy(temp_output) else: _output = temp_output except ImportError: if len(_input) > 0: try: _output = eval(str(_input)) except: _output = str(_input) else: _output = None return _output
def do_select(self, callback, minid=None, maxid=None, **attrs): """Get the slice of jobs. 'minid' and 'maxid' specify optional (inclusive) slice range. The returned slice object has the job registry interface but it is not connected to persistent storage. """ logger = getLogger() ## Loop through attrs to parse possible inputs into instances of a class where appropriate ## Unlike the select method we need to populate this dictionary with instance objects, not str or class for k, v in attrs.iteritems(): if isclass(v): attrs[k] = v() elif type(attrs[k]) is str: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject new_val = getRuntimeGPIObject(attrs[k], True) if new_val is None: continue if isclass(new_val): attrs[k] = new_val() else: attrs[k] = new_val logger.debug("do_select: attrs: %s" % attrs) def select_by_list(this_id): return this_id in ids def select_by_range(this_id): return minid <= this_id <= maxid ids = None if isinstance(minid, collections.Container): ids = minid select = select_by_list else: if minid is None: minid = 0 if maxid is None: maxid = sys.maxsize select = select_by_range for this_id in self.objects.keys(): obj = self.objects[this_id] logger.debug("id, obj: %s, %s" % (this_id, obj)) if select(int(this_id)): logger.debug("Selected: %s" % this_id) selected = True if self.name == 'box': name_str = obj._getRegistry()._getName(obj) else: name_str = '' for a in attrs: if self.name == 'box': attrvalue = attrs[a] if a == 'name': if not fnmatch.fnmatch(name_str, attrvalue): selected = False break elif a == 'application': if hasattr(obj, 'application'): if not getName(obj.application) == attrvalue: selected = False break else: selected = False break elif a == 'type': if not getName(obj) == attrvalue: selected = False break else: from Ganga.GPIDev.Base import GangaAttributeError raise GangaAttributeError( 'undefined select attribute: %s' % a) else: if a == 'ids': if int(this_id) not in attrs['ids']: selected = False break else: try: item = obj._schema.getItem(a) logger.debug("Here: %s, is item: %s" % (a, type(item))) except KeyError as err: from Ganga.GPIDev.Base import GangaAttributeError logger.debug("KeyError getting item: '%s' from schema" % a) raise GangaAttributeError('undefined select attribute: %s' % a) else: attrvalue = attrs[a] if item.isA(ComponentItem): ## TODO we need to distinguish between passing a Class type and a defined class instance ## If we passed a class type to select it should look only for classes which are of this type ## If we pass a class instance a compartison of the internal attributes should be performed from Ganga.GPIDev.Base.Filters import allComponentFilters cfilter = allComponentFilters[item['category']] filtered_value = cfilter(attrs[a], item) from Ganga.GPIDev.Base.Proxy import getName if not filtered_value is None: attrvalue = getName(filtered_value) else: attrvalue = getName(attrvalue) if getName(getattr(obj, a)) != attrvalue: selected = False break else: if isinstance(attrvalue, str): regex = fnmatch.translate(attrvalue) reobj = re.compile(regex) # Compare the type of the attribute # against attrvalue if not reobj.match(str(getattr(obj, a))): selected = False else: if getattr(obj, a) != attrvalue: selected = False break if selected: logger.debug("Actually Selected") callback(this_id, obj) else: logger.debug("NOT Actually Selected") else: logger.debug("NOT Selected: %s" % this_id)
def do_select(self, callback, minid=None, maxid=None, **attrs): """Get the slice of jobs. 'minid' and 'maxid' specify optional (inclusive) slice range. The returned slice object has the job registry interface but it is not connected to persistent storage. """ import sys import fnmatch import re logger = getLogger() from inspect import isclass ## Loop through attrs to parse possible inputs into instances of a class where appropriate ## Unlike the select method we need to populate this dictionary with instance objects, not str or class for k, v in attrs.iteritems(): if isclass(v): attrs[k] = v() elif type(attrs[k]) is str: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject new_val = getRuntimeGPIObject(attrs[k], True) if new_val is None: continue if isclass(new_val): attrs[k] = new_val() else: attrs[k] = new_val logger.debug("do_select: attrs: %s" % str(attrs)) def select_by_list(this_id): return this_id in ids def select_by_range(this_id): return minid <= this_id <= maxid ids = None if isinstance(minid, collections.Container): ids = minid select = select_by_list else: if minid is None: minid = 0 if maxid is None: maxid = sys.maxsize select = select_by_range id_list = self.objects.keys() if not isType(self.objects, SubJobXMLList) else [_id for _id in range(len(self.objects))] for this_id in id_list: obj = self.objects[this_id] logger.debug("id, obj: %s, %s" % (str(this_id), str(obj))) if select(int(this_id)): logger.debug("Selected: %s" % str(this_id)) selected = True for a in attrs: if self.name == 'box': attrvalue = attrs[a] if a == 'name': if not fnmatch.fnmatch(obj._getRegistry()._getName(obj), attrvalue): selected = False break elif a == 'application': if hasattr(obj, 'application'): if not getName(obj.application) == attrvalue: selected = False break else: selected = False break elif a == 'type': if not getName(obj) == attrvalue: selected = False break else: from Ganga.GPIDev.Base import GangaAttributeError raise GangaAttributeError( 'undefined select attribute: %s' % str(a)) else: if a == 'ids': if int(this_id) not in attrs['ids']: selected = False break else: try: item = obj._schema.getItem(a) except KeyError as err: from Ganga.GPIDev.Base import GangaAttributeError logger.debug("KeyError getting item: '%s' from schema" % str(a)) raise GangaAttributeError('undefined select attribute: %s' % str(a)) else: attrvalue = attrs[a] if item.isA(ComponentItem): from Ganga.GPIDev.Base.Filters import allComponentFilters cfilter = allComponentFilters[item['category']] filtered_value = cfilter(attrs[a], item) from Ganga.GPIDev.Base.Proxy import getName if not filtered_value is None: attrvalue = getName(filtered_value) else: attrvalue = getName(attrvalue) if getName(getattr(obj, a)) != attrvalue: selected = False break else: if isinstance(attrvalue, str): regex = fnmatch.translate(attrvalue) reobj = re.compile(regex) # Compare the type of the attribute # against attrvalue if not reobj.match(str(getattr(obj, a))): selected = False else: ## We will want to use this if we ever want to select by: # jobs.select(application=Executable(exe='myExe')) rather than by jobs.select(application=Exectuable()) #if getattr(obj, a) != attrvalue: ## Changed for 6.1.14 rcurrie if not isType(getattr(obj, a), type(attrvalue)): selected = False break if selected: callback(this_id, obj)
def do_select(self, callback, minid=None, maxid=None, **attrs): """Get the slice of jobs. 'minid' and 'maxid' specify optional (inclusive) slice range. The returned slice object has the job registry interface but it is not connected to persistent storage. """ import sys import fnmatch import re logger = getLogger() from inspect import isclass ## Loop through attrs to parse possible inputs into instances of a class where appropriate ## Unlike the select method we need to populate this dictionary with instance objects, not str or class for k, v in attrs.iteritems(): if isclass(v): attrs[k] = v() elif type(attrs[k]) is str: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject new_val = getRuntimeGPIObject(attrs[k], True) if new_val is None: continue if isclass(new_val): attrs[k] = new_val() else: attrs[k] = new_val logger.debug("do_select: attrs: %s" % str(attrs)) def select_by_list(this_id): return this_id in ids def select_by_range(this_id): return minid <= this_id <= maxid ids = None if isinstance(minid, collections.Container): ids = minid select = select_by_list else: if minid is None: minid = 0 if maxid is None: maxid = sys.maxsize select = select_by_range for this_id, obj in self.objects.iteritems(): logger.debug("id, obj: %s, %s" % (str(this_id), str(obj))) if select(int(this_id)): logger.debug("Selected: %s" % str(this_id)) selected = True for a in attrs: if self.name == 'box': attrvalue = attrs[a] if a == 'name': if not fnmatch.fnmatch( obj._getRegistry()._getName(obj), attrvalue): selected = False break elif a == 'application': if hasattr(obj, 'application'): if not getName(obj.application) == attrvalue: selected = False break else: selected = False break elif a == 'type': if not getName(obj) == attrvalue: selected = False break else: from Ganga.GPIDev.Base import GangaAttributeError raise GangaAttributeError( 'undefined select attribute: %s' % str(a)) else: if a == 'ids': if int(this_id) not in attrs['ids']: selected = False break else: try: item = obj._schema.getItem(a) except KeyError as err: from Ganga.GPIDev.Base import GangaAttributeError logger.debug( "KeyError getting item: '%s' from schema" % str(a)) raise GangaAttributeError( 'undefined select attribute: %s' % str(a)) else: attrvalue = attrs[a] if item.isA(ComponentItem): from Ganga.GPIDev.Base.Filters import allComponentFilters cfilter = allComponentFilters[ item['category']] filtered_value = cfilter(attrs[a], item) from Ganga.GPIDev.Base.Proxy import getName if not filtered_value is None: attrvalue = getName(filtered_value) else: attrvalue = getName(attrvalue) if getName(getattr(obj, a)) != attrvalue: selected = False break else: if isinstance(attrvalue, str): regex = fnmatch.translate(attrvalue) reobj = re.compile(regex) # Compare the type of the attribute # against attrvalue if not reobj.match(str(getattr(obj, a))): selected = False else: ## We will want to use this if we ever want to select by: # jobs.select(application=Executable(exe='myExe')) rather than by jobs.select(application=Exectuable()) #if getattr(obj, a) != attrvalue: ## Changed for 6.1.14 rcurrie if not isType(getattr(obj, a), type(attrvalue)): selected = False break if selected: callback(this_id, obj)
def _getDefaultValueInternal(self, attr, val=None, check=False): """ Get the default value of a schema item, both simple and component. If check is True then val is used instead of default value: this is used to check if the val may be used as a default value (e.g. if it is OK to use it as a value in the config file) """ def_name = defaultConfigSectionName(self.name) item = self.getItem(attr) stored_attr_key = def_name + ':' + str(attr) from Ganga.Utility.Config import Config is_finalized = Config._after_bootstrap useDefVal = False try: # Attempt to get the relevant config section config = Config.getConfig(def_name, create=False) if is_finalized and stored_attr_key in _found_attrs and not config.hasModified(): defvalue = _found_attrs[stored_attr_key] else: if attr in config.getEffectiveOptions(): defvalue = config[attr] from Ganga.GPIDev.Base.Proxy import isProxy if isProxy(defvalue): raise GangaException("(1)Proxy found where it shouldn't be in the Config: %s" % stored_attr_key) ## Just in case a developer puts the proxied object into the default value! _found_attrs[stored_attr_key] = defvalue else: useDefVal = True except (KeyError, Config.ConfigError): useDefVal = True if useDefVal: # hidden, protected and sequence values are not represented in config defvalue = item['defvalue'] from Ganga.GPIDev.Base.Proxy import isProxy if isProxy(defvalue): raise GangaException("(2)Proxy found where is shouldn't be in the Config" % stored_attr_key) ## Just in case a developer puts the proxied object into the default value! _found_attrs[stored_attr_key] = defvalue # in the checking mode, use the provided value instead if check is True: defvalue = val if isinstance(item, ComponentItem): # FIXME: limited support for initializing non-empty sequences (i.e. # apps => ['DaVinci','Executable'] is NOT correctly initialized) if not item['sequence']: if defvalue is None: if not item['load_default']: assert(item['optional']) return None # if a defvalue of a component item is an object (not string) just process it as for SimpleItems (useful for FileItems) # otherwise do a lookup via plugin registry category = item['category'] if isinstance(defvalue, str) or defvalue is None: try: config = Config.getConfig(def_name, create=False) has_modified = config.hasModified() except KeyError: has_modified = False if category not in _found_components or has_modified: _found_components[category] = allPlugins.find(category, defvalue) return _found_components[category]() # make a copy of the default value (to avoid strange effects if the # original modified) try: from Ganga.GPIDev.Base.Proxy import isType, getRuntimeGPIObject, stripProxy, getName from Ganga.GPIDev.Base.Objects import Node if isinstance(defvalue, Node): return stripProxy(getRuntimeGPIObject(getName(defvalue))) else: return copy.deepcopy(defvalue) except ImportError: return copy.deepcopy(defvalue)
def _getDefaultValueInternal(self, attr, val=None, check=False): """ Get the default value of a schema item, both simple and component. If check is True then val is used instead of default value: this is used to check if the val may be used as a default value (e.g. if it is OK to use it as a value in the config file) """ def_name = defaultConfigSectionName(self.name) item = self.getItem(attr) stored_attr_key = def_name + ':' + str(attr) from Ganga.Utility.Config import Config from Ganga.Utility.Config import getConfig is_finalized = Config._after_bootstrap config = Ganga.Utility.Config.getConfig(def_name) if is_finalized and stored_attr_key in _found_attrs.keys() and not config.hasModified(): defvalue = _found_attrs[stored_attr_key] else: config.setModified(False) # hidden, protected and sequence values are not represented in config if attr in config: defvalue = config[attr] else: defvalue = item['defvalue'] _found_attrs[stored_attr_key] = defvalue # in the checking mode, use the provided value instead if check: defvalue = val if item.isA(ComponentItem): # FIXME: limited support for initializing non-empty sequences (i.e. # apps => ['DaVinci','Executable'] is NOT correctly initialized) if not item['sequence']: if defvalue is None: if not item['load_default']: assert(item['optional']) return None # if a defvalue of a component item is an object (not string) just process it as for SimpleItems (useful for FileItems) # otherwise do a lookup via plugin registry if isinstance(defvalue, str) or defvalue is None: return allPlugins.find(item['category'], defvalue)() # make a copy of the default value (to avoid strange effects if the # original modified) try: from Ganga.GPIDev.Base.Proxy import isType, getRuntimeGPIObject, stripProxy, getName from Ganga.GPIDev.Base.Objects import Node if isType(defvalue, Node): return stripProxy(getRuntimeGPIObject(getName(defvalue))) else: return copy.deepcopy(defvalue) except ImportError: return copy.deepcopy(defvalue)
def do_select(self, callback, minid=None, maxid=None, **attrs): """Get the slice of jobs. 'minid' and 'maxid' specify optional (inclusive) slice range. The returned slice object has the job registry interface but it is not connected to persistent storage. """ logger = getLogger() ## Loop through attrs to parse possible inputs into instances of a class where appropriate ## Unlike the select method we need to populate this dictionary with instance objects, not str or class for k, v in attrs.iteritems(): if isclass(v): attrs[k] = v() elif type(attrs[k]) is str: from Ganga.GPIDev.Base.Proxy import getRuntimeGPIObject new_val = getRuntimeGPIObject(attrs[k], True) if new_val is None: continue if isclass(new_val): attrs[k] = new_val() else: attrs[k] = new_val logger.debug("do_select: attrs: %s" % attrs) def select_by_list(this_id): return this_id in ids def select_by_range(this_id): return minid <= this_id <= maxid ids = None if isinstance(minid, collections.Container): ids = minid select = select_by_list else: if minid is None: minid = 0 if maxid is None: maxid = sys.maxsize select = select_by_range for this_id in self.objects.keys(): obj = self.objects[this_id] logger.debug("id, obj: %s, %s" % (this_id, obj)) if select(int(this_id)): logger.debug("Selected: %s" % this_id) selected = True if self.name == 'box': name_str = obj._getRegistry()._getName(obj) else: name_str = '' for a in attrs: if self.name == 'box': attrvalue = attrs[a] if a == 'name': if not fnmatch.fnmatch(name_str, attrvalue): selected = False break elif a == 'application': if hasattr(obj, 'application'): if not getName(obj.application) == attrvalue: selected = False break else: selected = False break elif a == 'type': if not getName(obj) == attrvalue: selected = False break else: from Ganga.GPIDev.Base import GangaAttributeError raise GangaAttributeError( 'undefined select attribute: %s' % a) else: if a == 'ids': if int(this_id) not in attrs['ids']: selected = False break else: try: item = obj._schema.getItem(a) logger.debug("Here: %s, is item: %s" % (a, type(item))) except KeyError as err: from Ganga.GPIDev.Base import GangaAttributeError logger.debug( "KeyError getting item: '%s' from schema" % a) raise GangaAttributeError( 'undefined select attribute: %s' % a) else: attrvalue = attrs[a] if item.isA(ComponentItem): ## TODO we need to distinguish between passing a Class type and a defined class instance ## If we passed a class type to select it should look only for classes which are of this type ## If we pass a class instance a compartison of the internal attributes should be performed from Ganga.GPIDev.Base.Filters import allComponentFilters cfilter = allComponentFilters[ item['category']] filtered_value = cfilter(attrs[a], item) from Ganga.GPIDev.Base.Proxy import getName if not filtered_value is None: attrvalue = getName(filtered_value) else: attrvalue = getName(attrvalue) if getName(getattr(obj, a)) != attrvalue: selected = False break else: if isinstance(attrvalue, str): regex = fnmatch.translate(attrvalue) reobj = re.compile(regex) # Compare the type of the attribute # against attrvalue if not reobj.match(str(getattr(obj, a))): selected = False else: if getattr(obj, a) != attrvalue: selected = False break if selected: logger.debug("Actually Selected") callback(this_id, obj) else: logger.debug("NOT Actually Selected") else: logger.debug("NOT Selected: %s" % this_id)