def tree2expr(self, node): if isinstance(node, tuple): (operator, args) = node if operator == 'lookup': assert(len(args) == 2) return self._createVarSource(tree2names(args[0]), tree2names(args[1])) elif operator == 'ref': assert(len(args) == 1) refTypeDefault = 'dataset' if args[0] not in DataParameterSource.datasetsAvailable: refTypeDefault = 'csv' refType = self._paramConfig.get(args[0], 'type', refTypeDefault) if refType == 'dataset': return [DataParameterSource.create(self._paramConfig, args[0])] elif refType == 'csv': return [CSVParameterSource.create(self._paramConfig, args[0])] raise APIError('Unknown reference type: "%s"' % refType) else: args_complete = lchain(imap(self.tree2expr, args)) if operator == '*': return self.combineSources(CrossParameterSource, args_complete) elif operator == '+': return self.combineSources(ChainParameterSource, args_complete) elif operator == ',': return self.combineSources(ZipLongParameterSource, args_complete) raise APIError('Unknown token: "%s"' % operator) elif isinstance(node, int): return [node] else: return self._createVarSource([node], None)
def setDefault(self, entry): curEntry = self.getDefault(entry) if self._read_only and not curEntry: raise APIError('Config container is read-only!') elif curEntry and (curEntry.value != entry.value): raise APIError('Inconsistent default values! (%r != %r)' % (curEntry.value, entry.value)) entry.order = 0 self._content_default.setdefault(entry.option, {}).setdefault(entry.section, entry)
def set_default_entry(self, entry): entry_cur = self.get_default_entry(entry) if entry_cur and (entry_cur.value != entry.value): raise APIError('Inconsistent default values! (%r != %r)' % (entry_cur.value, entry.value)) elif (self._write_mode is False) and not entry_cur: raise APIError('Config container is read-only!') elif self._write_mode is True: entry.order = 0 self._content_default.setdefault(entry.option, {}).setdefault( entry.section, entry)
def _register_enum(cls, name): value = len(cls.enum_name_list) if use_hash: value += int(enum_id, 16) for enum_cls in make_enum.enum_list: if use_hash and (value in enum_cls.enum_value_list) and (enum_cls.enum_id != enum_id): raise APIError('enum value collision detected!') cls.enum_name_list.append(name) cls.enum_value_list.append(value) setattr(cls, name, value) _map_name2value[name.lower()] = value _map_value2name[value] = name if len(_map_name2value) != len(_map_value2name): raise APIError('Invalid enum definition! (%s:%s)' % (_map_name2value, _map_value2name))
def _getDefaultStr(self, default_obj, def2obj, obj2str): # First transform default into string if applicable if default_obj != noDefault: try: if def2obj: default_obj = def2obj(default_obj) except Exception: raise APIError('Unable to convert default object: %s' % repr(default_obj)) try: result = obj2str(default_obj) assert(isinstance(result, str)) return result except Exception: raise APIError('Unable to get string representation of default object: %s' % repr(default_obj)) return noDefault
def _get_tag_tuple(tag_obj): try: config_tag_name = tag_obj.config_tag_name.lower() except Exception: raise APIError('Class %r does not define a valid tag name!' % tag_obj.__class__.__name__) return [(config_tag_name, tag_obj.get_object_name().lower())]
def _formatRequirements(self, reqs): result = ['other.GlueHostNetworkAdapterOutboundIP'] for reqType, arg in reqs: if reqType == WMS.SOFTWARE: result.append( 'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)' % jdlEscape(arg)) elif reqType == WMS.WALLTIME: if arg > 0: result.append( '(other.GlueCEPolicyMaxWallClockTime >= %d)' % int( (arg + 59) / 60)) elif reqType == WMS.CPUTIME: if arg > 0: result.append('(other.GlueCEPolicyMaxCPUTime >= %d)' % int( (arg + 59) / 60)) elif reqType == WMS.MEMORY: if arg > 0: result.append('(other.GlueHostMainMemoryRAMSize >= %d)' % arg) elif reqType == WMS.STORAGE: result.append(self.storageReq(arg)) elif reqType == WMS.SITES: result.append(self.sitesReq(arg)) elif reqType == WMS.CPUS: pass # Handle number of cpus in makeJDL else: raise APIError('Unknown requirement type %s or argument %r' % (WMS.reqTypes[reqType], arg)) return str.join(' && ', ifilter(lambda x: x is not None, result))
def makeEnum(members = None, cls = None, useHash = True): members = members or [] if cls: enumID = md5_hex(str(members) + '!' + cls.__name__)[:4] else: enumID = md5_hex(str(members))[:4] cls = type('Enum_%s_%s' % (enumID, str.join('_', members)), (), {}) def getValue(idx, name): if useHash: return idx + int(enumID, 16) else: return idx values = lsmap(getValue, enumerate(members)) cls.enumNames = members cls.enumValues = values enumMapNV = dict(izip(imap(str.lower, cls.enumNames), cls.enumValues)) enumMapVN = dict(izip(cls.enumValues, cls.enumNames)) if len(enumMapNV) != len(enumMapVN): raise APIError('Invalid enum definition!') def str2enum(cls, value, *args): return enumMapNV.get(value.lower(), *args) cls.enum2str = enumMapVN.get cls.str2enum = classmethod(str2enum) for name, value in izip(cls.enumNames, cls.enumValues): setattr(cls, name, value) return cls
def makeTagTuple(t): try: tagName = t.tagName.lower() except Exception: raise APIError('Class %r does not define a valid tag name!' % t.__class__.__name__) return [(tagName, t.getObjectName().lower())]
def _format_reqs(self, req_list, result): req_string_list = ['other.GlueHostNetworkAdapterOutboundIP'] for req_type, arg in req_list: if req_type == WMS.SOFTWARE: req_string_list.append( 'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)' % self._escape(arg)) elif req_type == WMS.WALLTIME: if arg > 0: req_string_list.append( '(other.GlueCEPolicyMaxWallClockTime >= %d)' % int( (arg + 59) / 60)) elif req_type == WMS.CPUTIME: if arg > 0: req_string_list.append( '(other.GlueCEPolicyMaxCPUTime >= %d)' % int( (arg + 59) / 60)) elif req_type == WMS.MEMORY: if arg > 0: req_string_list.append( '(other.GlueHostMainMemoryRAMSize >= %d)' % arg) elif req_type == WMS.STORAGE: req_string_list.append(self._format_reqs_storage(arg)) elif req_type == WMS.SITES: req_string_list.append(self._format_reqs_sites(arg)) elif req_type == WMS.CPUS: pass # Handle number of cpus in makeJDL else: raise APIError('Unknown requirement type %r or argument %r' % (WMS.enum2str(req_type), arg)) result['Requirements'] = str.join(' && ', ifilter(identity, req_string_list))
def getChoice(self, option, choices, default=noDefault, obj2str=str.__str__, str2obj=str, def2obj=None, **kwargs): default_str = self._getDefaultStr(default, def2obj, obj2str) capDefault = lambda value: utils.QM(value == default_str, value.upper( ), value.lower()) choices_str = str.join('/', imap(capDefault, imap(obj2str, choices))) if (default != noDefault) and (default not in choices): raise APIError('Invalid default choice "%s" [%s]!' % (default, choices_str)) if 'interactive_msg' in kwargs: kwargs['interactive_msg'] += (' [%s]' % choices_str) def checked_str2obj(value): obj = str2obj(value) if obj not in choices: raise ConfigError('Invalid choice "%s" [%s]!' % (value, choices_str)) return obj return self._getInternal('choice', obj2str, checked_str2obj, def2obj, option, default, interactive_msg_append_default=False, **kwargs)
def _tree2expr(self, node, repository): if isinstance(node, int): return node elif isinstance(node, tuple): (operator, args) = node if operator == '[]': psrc_list = [] for output_vn in _tree2names(args[0]): psrc_list.append( ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, output_vn, _tree2names(args[1]))) return ParameterSource.create_psrc_safe( 'CrossParameterSource', self._parameter_config, repository, *psrc_list) elif operator in self._operator_map_raw: return ParameterSource.create_psrc_safe( self._operator_map_raw[operator], self._parameter_config, repository, *args) elif operator in self._operator_map_eval: evaluated_args = lmap( lambda node: self._tree2expr(node, repository), args) return ParameterSource.create_psrc_safe( self._operator_map_eval[operator], self._parameter_config, repository, *evaluated_args) else: return ParameterSource.create_instance( 'InternalAutoParameterSource', self._parameter_config, repository, node) raise APIError('Unable to parse node %s!' % repr(node))
def _set_internal(self, desc, obj2str, option, set_obj, opttype, source, section=None): try: value = obj2str(set_obj) except Exception: raise APIError('Unable to set %s %r - invalid object %s' % (desc, option, repr(set_obj))) try: if section is not None: scoped_iterface = self.change_view( view_class='SimpleConfigView', set_sections=[section]) return scoped_iterface.set(option, value, opttype, source, str.__str__) if not source: source = '<%s by %s>' % (desc, self._get_caller()) entry = self._config_view.set(norm_config_locations(option), value, opttype, source) self._log.log(logging.INFO2, 'Setting %s %s %s ', desc, ConfigEntry.map_opt_type2desc[opttype], entry.format(print_section=True)) return entry except Exception: raise ConfigError('Unable to set %s %r to %r (source: %r)' % (desc, option, value, source))
def __init__(self, message=None, level=logging.INFO, name=None, parent=None): (self.name, self._level, self._message, self._parent, self._children) = (name, level, None, None, []) self._current_thread_name = get_thread_name(get_current_thread()) Activity.lock.acquire() try: self._id = Activity.counter Activity.counter += 1 # search parent: self._cleanup_running() # cleanup list of running activities for parent_candidate in self._iter_possible_parents( self._current_thread_name): if (parent is None) or (parent == parent_candidate.name): self._parent = parent_candidate break if (parent is not None) and (self._parent is None): raise APIError('Invalid parent given!') # set this activity as topmost activity in the current thread Activity.running_by_thread_name.setdefault( self._current_thread_name, []).append(self) finally: Activity.lock.release() self.depth = len(list(self.get_parents())) if message is not None: self.update(message) if self._parent: self._parent.add_child(self)
def _createPSpace(self, args): SubSpaceParameterSource = ParameterSource.getClass('SubSpaceParameterSource') if len(args) == 1: return SubSpaceParameterSource.create(self._paramConfig, args[0]) elif len(args) == 3: return SubSpaceParameterSource.create(self._paramConfig, args[2], args[0]) else: raise APIError('Invalid subspace reference!: %r' % args)
def append(self, entry): if self._write_mode is False: raise APIError('Config container is read-only!') elif self._write_mode is True: self._counter += 1 entry.order = self._counter option_list = self._content.setdefault(entry.option, []) option_list.append(entry)
def getJobInfo(self, jobNum, pNum = None): if pNum is None: pNum = jobNum if jobNum is None: raise APIError('Unable to process jobNum None!') result = {ParameterInfo.ACTIVE: True, ParameterInfo.REQS: []} result['GC_JOB_ID'] = jobNum result['GC_PARAM'] = pNum self._source.fillParameterInfo(pNum, result) return utils.filterDict(result, vF = lambda v: v != '')
def _setInternal(self, desc, obj2str, option, set_obj, opttype, source): if not source: source = '<%s by %s>' % (desc, self._getCaller()) try: value = obj2str(set_obj) except Exception: raise APIError('Unable to get string representation of set value: %s' % repr(set_obj)) entry = self._configView.set(standardConfigForm(option), value, opttype, source) self._log.log(logging.INFO2, 'Setting %s %s %s ', desc, ConfigEntry.OptTypeDesc[opttype], entry.format(printSection = True)) return entry
def get_job_content(self, jobnum, pnum=None): if pnum is None: pnum = jobnum if jobnum is None: raise APIError('Unable to process job number None!') result = {ParameterInfo.ACTIVE: True, ParameterInfo.REQS: []} result['GC_JOB_ID'] = jobnum result['GC_PARAM'] = pnum self._psrc.fill_parameter_content(pnum, result) return filter_dict(result, value_filter=lambda x: x != '')
def _createRef(self, arg): refTypeDefault = 'dataset' DataParameterSource = ParameterSource.getClass('DataParameterSource') if arg not in DataParameterSource.datasetsAvailable: refTypeDefault = 'csv' refType = self._paramConfig.get(arg, 'type', refTypeDefault) if refType == 'dataset': return DataParameterSource.create(self._paramConfig, arg) elif refType == 'csv': return ParameterSource.getClass('CSVParameterSource').create(self._paramConfig, arg) raise APIError('Unknown reference type: "%s"' % refType)
def _get_default_str(self, default_obj, def2obj, obj2str): # First transform default into string if applicable if not unspecified(default_obj): try: if def2obj: default_obj = def2obj(default_obj) except Exception: raise APIError('Unable to convert default object: %s' % repr(default_obj)) try: result = obj2str(default_obj) if not isinstance(result, str): raise APIError( 'Default string representation function returned %r' % result) return result except Exception: raise APIError( 'Unable to get string representation of default object: %s' % repr(default_obj)) return unspecified
def _add_activity(self, parent): Activity.counter += 1 # search parent: self._cleanup_running() # cleanup list of running activities for parent_candidate in self._iter_possible_parents( self._current_thread_name): if (parent is None) or (parent == parent_candidate.name): self._parent = parent_candidate break if (parent is not None) and (self._parent is None): raise APIError('Invalid parent given!') # set this activity as topmost activity in the current thread Activity.running_by_thread_name.setdefault(self._current_thread_name, []).append(self)
def _parse_entry(entry, entry_desc=''): try: obj = str2obj(entry.value) entry.value = obj2str( obj) # Update value of entry with formatted data if not isinstance(entry.value, str): raise APIError( 'String representation function returned %r' % entry.value) return obj except Exception: raise ConfigError( 'Unable to parse %s: %s' % (entry_desc + desc, entry.format(print_section=True)))
def _setInternal(self, desc, obj2str, option, set_obj, opttype, source): mode = { '?=': 'default', '+=': 'append', '^=': 'prepend', '=': 'override' }.get(opttype, 'set') if not source: source = '<%s by %s>' % (desc, self._getCaller()) try: value = obj2str(set_obj) except Exception: raise APIError( 'Unable to get string representation of set value: %s' % repr(set_obj)) entry = self._configView.set(standardConfigForm(option), value, opttype, source) self._log.log(logging.INFO2, 'Setting %s %s %s ', desc, mode, entry.format(printSection=True)) return entry
def get_choice(self, option, choices, default=unspecified, obj2str=str.__str__, str2obj=str, def2obj=None, **kwargs): default_str = self._get_default_str(default, def2obj, obj2str) def _cap_default(value): # capitalize default value if value == default_str: return value.upper() return value.lower() choices_str = str.join('/', imap(_cap_default, imap(obj2str, choices))) if (default not in choices) and not unspecified(default): raise APIError('Invalid default choice "%s" [%s]!' % (default, choices_str)) if 'interactive_msg' in kwargs: kwargs['interactive_msg'] += (' [%s]' % choices_str) def _checked_str2obj(value): obj = str2obj(value) if obj not in choices: raise ConfigError('Invalid choice "%s" [%s]!' % (value, choices_str)) return obj return self._get_internal('choice', obj2str, _checked_str2obj, def2obj, option, default, interactive_msg_append_default=False, **kwargs)
def _format_reqs(self, req_list, result): req_string_list = ['other.GlueHostNetworkAdapterOutboundIP'] for req_type, arg in req_list: if req_type == WMS.SOFTWARE: software_template_str = 'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)' req_string_list.append(software_template_str % self._escape(arg)) elif req_type == WMS.WALLTIME: if arg > 0: req_string_list.append('(other.GlueCEPolicyMaxWallClockTime >= %d)' % int((arg + 59) / 60)) elif req_type == WMS.CPUTIME: if arg > 0: req_string_list.append('(other.GlueCEPolicyMaxCPUTime >= %d)' % int((arg + 59) / 60)) elif req_type == WMS.MEMORY: if arg > 0: req_string_list.append('(other.GlueHostMainMemoryRAMSize >= %d)' % arg) elif req_type == WMS.STORAGE: req_string_list.append(self._format_reqs_storage(arg)) elif req_type == WMS.SITES: req_string_list.append(self._format_reqs_sites(arg)) elif req_type in (WMS.CPUS, WMS.DISKSPACE): pass # Handled outside of "requirement" directive or GlueCE attribute not available else: raise APIError('Unknown requirement type %r or argument %r' % (WMS.enum2str(req_type), arg)) result['Requirements'] = str.join(' && ', iidfilter(req_string_list))
def _tree2expr(self, node): if isinstance(node, tuple): (operator, args) = node if operator == 'lookup': assert(len(args) == 2) return self._createVarSource(tree2names(args[0]), tree2names(args[1])) elif operator == 'ref': assert(len(args) == 1) return self._createRef(args[0]) elif operator == 'pspace': return self._createPSpace(args) else: args_complete = lmap(self._tree2expr, args) if operator == '*': return self._combineSources('CrossParameterSource', args_complete) elif operator == '+': return self._combineSources('ChainParameterSource', args_complete) elif operator == ',': return self._combineSources('ZipLongParameterSource', args_complete) raise APIError('Unknown token: "%s"' % operator) elif isinstance(node, int): return node else: return self._createVarSource([node], None)
def __init__(self, *args, **kwargs): ParameterSource.__init__(self) raise APIError('Redirector class initialized')
def getHash(self): raise APIError('Not yet implemented') # return hash of file content
def append(self, entry): if self._read_only: raise APIError('Config container is read-only!') self._counter += 1 entry.order = self._counter self._content.setdefault(entry.option, []).append(entry)