def tree2expr(self, node):
if isinstance(node, tuple):
(operator, args) = node
if operator == 'lookup':
assert(len(args) == 2)
return self._createVarSource(tree2names(args[0]), tree2names(args[1]))
elif operator == 'ref':
assert(len(args) == 1)
refTypeDefault = 'dataset'
if args[0] not in DataParameterSource.datasetsAvailable:
refTypeDefault = 'csv'
refType = self._paramConfig.get(args[0], 'type', refTypeDefault)
if refType == 'dataset':
return [DataParameterSource.create(self._paramConfig, args[0])]
elif refType == 'csv':
return [CSVParameterSource.create(self._paramConfig, args[0])]
raise APIError('Unknown reference type: "%s"' % refType)
else:
args_complete = lchain(imap(self.tree2expr, args))
if operator == '*':
return self.combineSources(CrossParameterSource, args_complete)
elif operator == '+':
return self.combineSources(ChainParameterSource, args_complete)
elif operator == ',':
return self.combineSources(ZipLongParameterSource, args_complete)
raise APIError('Unknown token: "%s"' % operator)
elif isinstance(node, int):
return [node]
else:
return self._createVarSource([node], None)
def setDefault(self, entry):
curEntry = self.getDefault(entry)
if self._read_only and not curEntry:
raise APIError('Config container is read-only!')
elif curEntry and (curEntry.value != entry.value):
raise APIError('Inconsistent default values! (%r != %r)' % (curEntry.value, entry.value))
entry.order = 0
self._content_default.setdefault(entry.option, {}).setdefault(entry.section, entry)
def set_default_entry(self, entry):
entry_cur = self.get_default_entry(entry)
if entry_cur and (entry_cur.value != entry.value):
raise APIError('Inconsistent default values! (%r != %r)' %
(entry_cur.value, entry.value))
elif (self._write_mode is False) and not entry_cur:
raise APIError('Config container is read-only!')
elif self._write_mode is True:
entry.order = 0
self._content_default.setdefault(entry.option, {}).setdefault(
entry.section, entry)
def _register_enum(cls, name):
value = len(cls.enum_name_list)
if use_hash:
value += int(enum_id, 16)
for enum_cls in make_enum.enum_list:
if use_hash and (value in enum_cls.enum_value_list) and (enum_cls.enum_id != enum_id):
raise APIError('enum value collision detected!')
cls.enum_name_list.append(name)
cls.enum_value_list.append(value)
setattr(cls, name, value)
_map_name2value[name.lower()] = value
_map_value2name[value] = name
if len(_map_name2value) != len(_map_value2name):
raise APIError('Invalid enum definition! (%s:%s)' % (_map_name2value, _map_value2name))
def _getDefaultStr(self, default_obj, def2obj, obj2str):
# First transform default into string if applicable
if default_obj != noDefault:
try:
if def2obj:
default_obj = def2obj(default_obj)
except Exception:
raise APIError('Unable to convert default object: %s' % repr(default_obj))
try:
result = obj2str(default_obj)
assert(isinstance(result, str))
return result
except Exception:
raise APIError('Unable to get string representation of default object: %s' % repr(default_obj))
return noDefault
def _get_tag_tuple(tag_obj):
try:
config_tag_name = tag_obj.config_tag_name.lower()
except Exception:
raise APIError('Class %r does not define a valid tag name!' %
tag_obj.__class__.__name__)
return [(config_tag_name, tag_obj.get_object_name().lower())]
def _formatRequirements(self, reqs):
result = ['other.GlueHostNetworkAdapterOutboundIP']
for reqType, arg in reqs:
if reqType == WMS.SOFTWARE:
result.append(
'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)'
% jdlEscape(arg))
elif reqType == WMS.WALLTIME:
if arg > 0:
result.append(
'(other.GlueCEPolicyMaxWallClockTime >= %d)' % int(
(arg + 59) / 60))
elif reqType == WMS.CPUTIME:
if arg > 0:
result.append('(other.GlueCEPolicyMaxCPUTime >= %d)' % int(
(arg + 59) / 60))
elif reqType == WMS.MEMORY:
if arg > 0:
result.append('(other.GlueHostMainMemoryRAMSize >= %d)' %
arg)
elif reqType == WMS.STORAGE:
result.append(self.storageReq(arg))
elif reqType == WMS.SITES:
result.append(self.sitesReq(arg))
elif reqType == WMS.CPUS:
pass # Handle number of cpus in makeJDL
else:
raise APIError('Unknown requirement type %s or argument %r' %
(WMS.reqTypes[reqType], arg))
return str.join(' && ', ifilter(lambda x: x is not None, result))
def makeEnum(members = None, cls = None, useHash = True):
members = members or []
if cls:
enumID = md5_hex(str(members) + '!' + cls.__name__)[:4]
else:
enumID = md5_hex(str(members))[:4]
cls = type('Enum_%s_%s' % (enumID, str.join('_', members)), (), {})
def getValue(idx, name):
if useHash:
return idx + int(enumID, 16)
else:
return idx
values = lsmap(getValue, enumerate(members))
cls.enumNames = members
cls.enumValues = values
enumMapNV = dict(izip(imap(str.lower, cls.enumNames), cls.enumValues))
enumMapVN = dict(izip(cls.enumValues, cls.enumNames))
if len(enumMapNV) != len(enumMapVN):
raise APIError('Invalid enum definition!')
def str2enum(cls, value, *args):
return enumMapNV.get(value.lower(), *args)
cls.enum2str = enumMapVN.get
cls.str2enum = classmethod(str2enum)
for name, value in izip(cls.enumNames, cls.enumValues):
setattr(cls, name, value)
return cls
def makeTagTuple(t):
try:
tagName = t.tagName.lower()
except Exception:
raise APIError('Class %r does not define a valid tag name!' %
t.__class__.__name__)
return [(tagName, t.getObjectName().lower())]
def _format_reqs(self, req_list, result):
req_string_list = ['other.GlueHostNetworkAdapterOutboundIP']
for req_type, arg in req_list:
if req_type == WMS.SOFTWARE:
req_string_list.append(
'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)'
% self._escape(arg))
elif req_type == WMS.WALLTIME:
if arg > 0:
req_string_list.append(
'(other.GlueCEPolicyMaxWallClockTime >= %d)' % int(
(arg + 59) / 60))
elif req_type == WMS.CPUTIME:
if arg > 0:
req_string_list.append(
'(other.GlueCEPolicyMaxCPUTime >= %d)' % int(
(arg + 59) / 60))
elif req_type == WMS.MEMORY:
if arg > 0:
req_string_list.append(
'(other.GlueHostMainMemoryRAMSize >= %d)' % arg)
elif req_type == WMS.STORAGE:
req_string_list.append(self._format_reqs_storage(arg))
elif req_type == WMS.SITES:
req_string_list.append(self._format_reqs_sites(arg))
elif req_type == WMS.CPUS:
pass # Handle number of cpus in makeJDL
else:
raise APIError('Unknown requirement type %r or argument %r' %
(WMS.enum2str(req_type), arg))
result['Requirements'] = str.join(' && ',
ifilter(identity, req_string_list))
def getChoice(self,
option,
choices,
default=noDefault,
obj2str=str.__str__,
str2obj=str,
def2obj=None,
**kwargs):
default_str = self._getDefaultStr(default, def2obj, obj2str)
capDefault = lambda value: utils.QM(value == default_str, value.upper(
), value.lower())
choices_str = str.join('/', imap(capDefault, imap(obj2str, choices)))
if (default != noDefault) and (default not in choices):
raise APIError('Invalid default choice "%s" [%s]!' %
(default, choices_str))
if 'interactive_msg' in kwargs:
kwargs['interactive_msg'] += (' [%s]' % choices_str)
def checked_str2obj(value):
obj = str2obj(value)
if obj not in choices:
raise ConfigError('Invalid choice "%s" [%s]!' %
(value, choices_str))
return obj
return self._getInternal('choice',
obj2str,
checked_str2obj,
def2obj,
option,
default,
interactive_msg_append_default=False,
**kwargs)
def _tree2expr(self, node, repository):
if isinstance(node, int):
return node
elif isinstance(node, tuple):
(operator, args) = node
if operator == '[]':
psrc_list = []
for output_vn in _tree2names(args[0]):
psrc_list.append(
ParameterSource.create_instance(
'InternalAutoParameterSource',
self._parameter_config, repository, output_vn,
_tree2names(args[1])))
return ParameterSource.create_psrc_safe(
'CrossParameterSource', self._parameter_config, repository,
*psrc_list)
elif operator in self._operator_map_raw:
return ParameterSource.create_psrc_safe(
self._operator_map_raw[operator], self._parameter_config,
repository, *args)
elif operator in self._operator_map_eval:
evaluated_args = lmap(
lambda node: self._tree2expr(node, repository), args)
return ParameterSource.create_psrc_safe(
self._operator_map_eval[operator], self._parameter_config,
repository, *evaluated_args)
else:
return ParameterSource.create_instance(
'InternalAutoParameterSource', self._parameter_config,
repository, node)
raise APIError('Unable to parse node %s!' % repr(node))
def _set_internal(self,
desc,
obj2str,
option,
set_obj,
opttype,
source,
section=None):
try:
value = obj2str(set_obj)
except Exception:
raise APIError('Unable to set %s %r - invalid object %s' %
(desc, option, repr(set_obj)))
try:
if section is not None:
scoped_iterface = self.change_view(
view_class='SimpleConfigView', set_sections=[section])
return scoped_iterface.set(option, value, opttype, source,
str.__str__)
if not source:
source = '<%s by %s>' % (desc, self._get_caller())
entry = self._config_view.set(norm_config_locations(option), value,
opttype, source)
self._log.log(logging.INFO2, 'Setting %s %s %s ', desc,
ConfigEntry.map_opt_type2desc[opttype],
entry.format(print_section=True))
return entry
except Exception:
raise ConfigError('Unable to set %s %r to %r (source: %r)' %
(desc, option, value, source))
def __init__(self,
message=None,
level=logging.INFO,
name=None,
parent=None):
(self.name, self._level, self._message, self._parent,
self._children) = (name, level, None, None, [])
self._current_thread_name = get_thread_name(get_current_thread())
Activity.lock.acquire()
try:
self._id = Activity.counter
Activity.counter += 1
# search parent:
self._cleanup_running() # cleanup list of running activities
for parent_candidate in self._iter_possible_parents(
self._current_thread_name):
if (parent is None) or (parent == parent_candidate.name):
self._parent = parent_candidate
break
if (parent is not None) and (self._parent is None):
raise APIError('Invalid parent given!')
# set this activity as topmost activity in the current thread
Activity.running_by_thread_name.setdefault(
self._current_thread_name, []).append(self)
finally:
Activity.lock.release()
self.depth = len(list(self.get_parents()))
if message is not None:
self.update(message)
if self._parent:
self._parent.add_child(self)
def _createPSpace(self, args):
SubSpaceParameterSource = ParameterSource.getClass('SubSpaceParameterSource')
if len(args) == 1:
return SubSpaceParameterSource.create(self._paramConfig, args[0])
elif len(args) == 3:
return SubSpaceParameterSource.create(self._paramConfig, args[2], args[0])
else:
raise APIError('Invalid subspace reference!: %r' % args)
def append(self, entry):
if self._write_mode is False:
raise APIError('Config container is read-only!')
elif self._write_mode is True:
self._counter += 1
entry.order = self._counter
option_list = self._content.setdefault(entry.option, [])
option_list.append(entry)
def getJobInfo(self, jobNum, pNum = None):
if pNum is None:
pNum = jobNum
if jobNum is None:
raise APIError('Unable to process jobNum None!')
result = {ParameterInfo.ACTIVE: True, ParameterInfo.REQS: []}
result['GC_JOB_ID'] = jobNum
result['GC_PARAM'] = pNum
self._source.fillParameterInfo(pNum, result)
return utils.filterDict(result, vF = lambda v: v != '')
def _setInternal(self, desc, obj2str, option, set_obj, opttype, source):
if not source:
source = '<%s by %s>' % (desc, self._getCaller())
try:
value = obj2str(set_obj)
except Exception:
raise APIError('Unable to get string representation of set value: %s' % repr(set_obj))
entry = self._configView.set(standardConfigForm(option), value, opttype, source)
self._log.log(logging.INFO2, 'Setting %s %s %s ', desc, ConfigEntry.OptTypeDesc[opttype], entry.format(printSection = True))
return entry
def get_job_content(self, jobnum, pnum=None):
if pnum is None:
pnum = jobnum
if jobnum is None:
raise APIError('Unable to process job number None!')
result = {ParameterInfo.ACTIVE: True, ParameterInfo.REQS: []}
result['GC_JOB_ID'] = jobnum
result['GC_PARAM'] = pnum
self._psrc.fill_parameter_content(pnum, result)
return filter_dict(result, value_filter=lambda x: x != '')
def _createRef(self, arg):
refTypeDefault = 'dataset'
DataParameterSource = ParameterSource.getClass('DataParameterSource')
if arg not in DataParameterSource.datasetsAvailable:
refTypeDefault = 'csv'
refType = self._paramConfig.get(arg, 'type', refTypeDefault)
if refType == 'dataset':
return DataParameterSource.create(self._paramConfig, arg)
elif refType == 'csv':
return ParameterSource.getClass('CSVParameterSource').create(self._paramConfig, arg)
raise APIError('Unknown reference type: "%s"' % refType)
def _get_default_str(self, default_obj, def2obj, obj2str):
# First transform default into string if applicable
if not unspecified(default_obj):
try:
if def2obj:
default_obj = def2obj(default_obj)
except Exception:
raise APIError('Unable to convert default object: %s' %
repr(default_obj))
try:
result = obj2str(default_obj)
if not isinstance(result, str):
raise APIError(
'Default string representation function returned %r' %
result)
return result
except Exception:
raise APIError(
'Unable to get string representation of default object: %s'
% repr(default_obj))
return unspecified
def _add_activity(self, parent):
Activity.counter += 1
# search parent:
self._cleanup_running() # cleanup list of running activities
for parent_candidate in self._iter_possible_parents(
self._current_thread_name):
if (parent is None) or (parent == parent_candidate.name):
self._parent = parent_candidate
break
if (parent is not None) and (self._parent is None):
raise APIError('Invalid parent given!')
# set this activity as topmost activity in the current thread
Activity.running_by_thread_name.setdefault(self._current_thread_name,
[]).append(self)
def _parse_entry(entry, entry_desc=''):
try:
obj = str2obj(entry.value)
entry.value = obj2str(
obj) # Update value of entry with formatted data
if not isinstance(entry.value, str):
raise APIError(
'String representation function returned %r' %
entry.value)
return obj
except Exception:
raise ConfigError(
'Unable to parse %s: %s' %
(entry_desc + desc, entry.format(print_section=True)))
def _setInternal(self, desc, obj2str, option, set_obj, opttype, source):
mode = {
'?=': 'default',
'+=': 'append',
'^=': 'prepend',
'=': 'override'
}.get(opttype, 'set')
if not source:
source = '<%s by %s>' % (desc, self._getCaller())
try:
value = obj2str(set_obj)
except Exception:
raise APIError(
'Unable to get string representation of set value: %s' %
repr(set_obj))
entry = self._configView.set(standardConfigForm(option), value,
opttype, source)
self._log.log(logging.INFO2, 'Setting %s %s %s ', desc, mode,
entry.format(printSection=True))
return entry
def get_choice(self,
option,
choices,
default=unspecified,
obj2str=str.__str__,
str2obj=str,
def2obj=None,
**kwargs):
default_str = self._get_default_str(default, def2obj, obj2str)
def _cap_default(value): # capitalize default value
if value == default_str:
return value.upper()
return value.lower()
choices_str = str.join('/', imap(_cap_default, imap(obj2str, choices)))
if (default not in choices) and not unspecified(default):
raise APIError('Invalid default choice "%s" [%s]!' %
(default, choices_str))
if 'interactive_msg' in kwargs:
kwargs['interactive_msg'] += (' [%s]' % choices_str)
def _checked_str2obj(value):
obj = str2obj(value)
if obj not in choices:
raise ConfigError('Invalid choice "%s" [%s]!' %
(value, choices_str))
return obj
return self._get_internal('choice',
obj2str,
_checked_str2obj,
def2obj,
option,
default,
interactive_msg_append_default=False,
**kwargs)
def _format_reqs(self, req_list, result):
req_string_list = ['other.GlueHostNetworkAdapterOutboundIP']
for req_type, arg in req_list:
if req_type == WMS.SOFTWARE:
software_template_str = 'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)'
req_string_list.append(software_template_str % self._escape(arg))
elif req_type == WMS.WALLTIME:
if arg > 0:
req_string_list.append('(other.GlueCEPolicyMaxWallClockTime >= %d)' % int((arg + 59) / 60))
elif req_type == WMS.CPUTIME:
if arg > 0:
req_string_list.append('(other.GlueCEPolicyMaxCPUTime >= %d)' % int((arg + 59) / 60))
elif req_type == WMS.MEMORY:
if arg > 0:
req_string_list.append('(other.GlueHostMainMemoryRAMSize >= %d)' % arg)
elif req_type == WMS.STORAGE:
req_string_list.append(self._format_reqs_storage(arg))
elif req_type == WMS.SITES:
req_string_list.append(self._format_reqs_sites(arg))
elif req_type in (WMS.CPUS, WMS.DISKSPACE):
pass # Handled outside of "requirement" directive or GlueCE attribute not available
else:
raise APIError('Unknown requirement type %r or argument %r' % (WMS.enum2str(req_type), arg))
result['Requirements'] = str.join(' && ', iidfilter(req_string_list))
def _tree2expr(self, node):
if isinstance(node, tuple):
(operator, args) = node
if operator == 'lookup':
assert(len(args) == 2)
return self._createVarSource(tree2names(args[0]), tree2names(args[1]))
elif operator == 'ref':
assert(len(args) == 1)
return self._createRef(args[0])
elif operator == 'pspace':
return self._createPSpace(args)
else:
args_complete = lmap(self._tree2expr, args)
if operator == '*':
return self._combineSources('CrossParameterSource', args_complete)
elif operator == '+':
return self._combineSources('ChainParameterSource', args_complete)
elif operator == ',':
return self._combineSources('ZipLongParameterSource', args_complete)
raise APIError('Unknown token: "%s"' % operator)
elif isinstance(node, int):
return node
else:
return self._createVarSource([node], None)
def __init__(self, *args, **kwargs):
ParameterSource.__init__(self)
raise APIError('Redirector class initialized')
def getHash(self):
raise APIError('Not yet implemented') # return hash of file content
def append(self, entry):
if self._read_only:
raise APIError('Config container is read-only!')
self._counter += 1
entry.order = self._counter
self._content.setdefault(entry.option, []).append(entry)
