def add(dictionary,
iterable,
pattern,
must_match=True,
add_only_keys=None):
""" Updates *dictionary* with items from *iterable* object.
This method modifies/updates *dictionary* with items from *iterable*
object. This object must support ``for something in iterable`` (list,
opened file etc). Only those items in *iterable* are considered, that match
*pattern* (it's a regexp epression). If a particular item does not match,
and *must_match* is True, *ConfigurationError* exception is thrown.
Regexp pattern must return two groups (1 and 2). First group is considered
as a key, and second group is considered to be value. Values must be a
json-parseable strings.
If *add_only_keys* is not None, only those items are added to *dictionary*,
that are in this list.
Existing items in *dictionary* are overwritten with new ones if key already
exists.
One use case to use this method is to populate a dictionary with key-values
from log files.
:param dict dictionary: Dictionary to update in-place.
:param obj iterable: Iterable object (list, opened file name etc).
:param str patter: A regexp pattern for matching items in ``iterable``.
:param bool must_match: Specifies if every element in *iterable* must match\
*pattern*. If True and not match, raises exception.
:param list add_only_keys: If not None, specifies keys that are added into\
*dictionary*. Others are ignored.
:raises ConfigurationError: If *must_match* is True and not match or if value\
is not a json-parseable string.
"""
matcher = re.compile(pattern)
for line in iterable:
match = matcher.match(line)
if not match:
if must_match:
raise ConfigurationError(
"Cannot match key-value from '%s' with pattern '%s'. Must match is set to true"
% (line, pattern))
else:
continue
key = match.group(1).strip()
try:
value = match.group(2).strip()
value = json.loads(value) if len(value) > 0 else None
except ValueError as err:
raise ConfigurationError(
"Cannot parse JSON string '%s' with key '%s' (key-value definition: '%s'). Error is %s"
% (value, key, line, str(err)))
if add_only_keys is None or key in add_only_keys:
dictionary[key] = value
logging.debug(
"Key-value item (%s=%s) has been parsed and added to dictionary",
key, str(value))
def _raise_types_mismatch_config_error(key, dest_val_type,
src_val_type, valid_types):
raise ConfigurationError(
"Configuration update error - expecting value types to be same and one of %s but"
" Dest(key=%s, val_type=%s) <- Source(key=%s, val_type=%s)" %
(valid_types, key, dest_val_type.__name__, key,
src_val_type.__name__))
def remove_info(config):
"""In parameter section of a `config` the function removes parameter info
leaving only their values
Args:
config (dict): A dictionary with configuration section that may contain parameters, variables
and extensions. The `config` is a result of parsing a JSON configuration file.
Returns:
dict: A copy of `config` with info removed. This new dictionary maps parameter name to parameter
value.
"""
clean_config = copy.deepcopy(config)
if 'parameters' in clean_config:
params = clean_config['parameters']
for name in params:
val = params[name]
if isinstance(val, dict):
# This should not generally happen since we deal with it in update_param_info, but just in case
if 'val' not in val:
raise ConfigurationError(
"Parameter info remove error. "
"Parameter that is defined by a dictionary must contain 'val' field that "
"defines its default value. Found this definition: %s=%s"
% (name, val))
params[name] = val['val']
return clean_config
def update(dest, source, is_root=True):
"""Merge **source** dictionary into **dest** dictionary assuming source
and dest are JSON configuration configs or their members.
:param dict dest: Merge data to this dictionary.
:param dict source: Merge data from this dictionary.
:param bool is_root: True if **dest** and *source** are root configuration
objects. False if these objects are members.
"""
def _raise_types_mismatch_config_error(key, dest_val_type, src_val_type, valid_types):
raise ConfigurationError(
"Configuration update error - expecting value types to be same and one of %s but"
" Dest(key=%s, val_type=%s) <- Source(key=%s, val_type=%s)" % (valid_types, key, dest_val_type.__name__, key, src_val_type.__name__)
)
# Types and expected key names. Types must always match, else exception is thrown.
if is_root:
schema = {'types':(dict, list), 'dict':['parameters', 'variables'], 'list':['extensions']}
else:
schema = {'types':(list, basestring, int, float, long)}
for key in source:
# Firstly, check that type of value is expected.
val_type = type(source[key]).__name__
if not isinstance(source[key], schema['types']):
raise ConfigurationError(
"Configuration update error - unexpected type of key value: "
" is_root=%s, key=%s, value type=%s, expected type is one of %s" % \
(str(is_root), key, val_type, str(schema['types']))
)
# So, the type is expected. Warn if key value is suspicious - we can do it only for root.
if is_root and key not in schema[val_type]:
logging.warn("The name of a root key is '%s' but expected is one of '%s'", key, schema[val_type])
if key not in dest:
# The key in source dictionary is not in destination dictionary.
dest[key] = copy.deepcopy(source[key])
else:
# The key from source is in dest.
both_dicts = isinstance(dest[key], dict) and isinstance(source[key], dict)
both_lists = isinstance(dest[key], list) and isinstance(source[key], list)
both_primitive = type(dest[key]) is type(source[key]) and isinstance(dest[key], (basestring, int, float, long))
if is_root:
if not both_dicts and not both_lists:
_raise_types_mismatch_config_error(key, type(dest[key]), type(source[key]), '[dict, list]')
if both_dicts:
ConfigurationLoader.update(dest[key], source[key], is_root=False)
else:
dest[key].extend(source[key])
else:
if not both_lists and not both_primitive:
_raise_types_mismatch_config_error(key, type(dest[key]), type(source[key]), '[list, basestring, int, float, long]')
dest[key] = copy.deepcopy(source[key]) if both_lists else source[key]
def compute_current_variables(self, experiment, computable_variables):
"""Computes all variables in *experiment* that are in *computable_variables*.
:param dict experiment: Current experiment.
:param list computable_variables: Names of variables that need to be computed.
:return: computed (list), partially_computed(list)
:rtype: tuple (list, list)
The computed variables are those that have been computed and their
values can be used. The partially computed variables are those that
contain nested references (`finalized` is set to False for them).
"""
computed = []
partially_computed = []
for var in computable_variables:
is_str = isinstance(experiment[var], basestring)
if not is_str:
computed.append(var)
continue
if is_str and len(self.fwd_index[var]['deps']) > 0:
for ref_var in self.fwd_index[var]['deps']:
replace_pattern = "${%s}" % (ref_var)
if ref_var in experiment:
replace_value = str(experiment[ref_var])
elif ref_var in os.environ:
replace_value = str(os.environ[ref_var])
else:
msg = [
"Variable '%s' not found. This may happen if variable's name depend",
"on other variable that's empty or set to an incorrect value. For instance,",
"the ${${exp.framework}.docker.image} variable depends on ${exp.framework}",
"value. If it's empty, the variable name becomes '.docker.image' what's wrong."
]
raise LogicError(' '.join(msg) % (ref_var))
experiment[var] = experiment[var].replace(
replace_pattern, replace_value)
# Search for computable components
while True:
idx = experiment[var].find('$(')
if idx < 0:
break
end_idx = experiment[var].find(')$', idx + 2)
if end_idx < 0:
raise ConfigurationError(
"Cannot find ')$' in %s. Variable cannot be computed" %
(experiment[var]))
try:
eval_res = eval(experiment[var][idx + 2:end_idx])
except NameError as err:
logging.error("Cannot evaluate python expression: %s",
experiment[var][idx + 2:end_idx])
raise err
logging.debug("\"%s\" -> \"%s\"",
experiment[var][idx + 2:end_idx], str(eval_res))
experiment[var] = experiment[var][:idx] + str(
eval_res) + experiment[var][end_idx + 2:]
if self.fwd_index[var]['finalized'] is True:
computed.append(var)
self.cast_variable(experiment, var)
self.check_variable_value(experiment, var)
else:
partially_computed.append(var)
return (computed, partially_computed)
def update_param_info(param_info, config, is_user_config=False):
"""Update parameter info dictionary based on configuration in `config`
Args:
param_info (dict): A parameter info dictionary that maps parameter name to its description
dictionary that contains such fields as value, help message, type, fieldsaints etc.
config (dict): A dictionary with configuration section that may contain parameters, variables
and extensions. The `config` is a result of parsing a JSON configuration file.
is_user_config (bool): If True, the config object represents user-provided configuration. If False,
this is a system configuration. Based on this flag, we deal with parameters in config that
redefine parameters in existing param_info differently. See comments below.
We are interested here only in parameters section where parameter information
is defined. There are two scenarios this method is used:
1. Load standard configuration. In this case, parameter redefinition is
prohibited. If `parameters` section in `config` redefines existing
parameters in param_info (already loaded params), program terminates.
2. Load user-provided configuration. In this case, we still update parameter
info structure, but deal with it in slightly different way. If parameter in
`config` exists in param_info, it means user has provided their specific
value for this parameter.
Types of user defined parameters are defined either by user in a standard way as
we define types for standard parameters or induced automatically based on JSON
parse result.
"""
if 'parameters' not in config:
return
params = config['parameters']
for name in params:
val = params[name]
if not is_user_config:
# If this is not a user-provided configuration, we disallow parameter redefinition.
if name in param_info:
raise ConfigurationError(
"Parameter info update error."
" Parameter redefinition is not allowed for non-user configuration."
" This is a system configuration error that must not happen."
" Parameter %s=%s, new parameter definition (value) is %s"
% (name, str(param_info[name]), val))
if isinstance(val, dict):
# This is a complete parameter definition with name, value and description.
if 'val' not in val:
raise ConfigurationError(
"Parameter info update error."
" Parameter that is defined by a dictionary must contain 'val' field that"
" defines its default value. Found this definition: %s=%s"
% (name, val))
if name not in param_info:
param_info[name] = copy.deepcopy(
val) # New parameter, set it info object.
# TODO what about parameter type and description?
else:
logging.warn(
" Parameter (%s) entirely redefines existing parameter (%s)."
" Normally, only value needs to be provided."
" We will proceed but you may want to fix this.",
json.dumps(val), json.dumps(param_info[name]))
param_info[name]['val'] = val[
'val'] # Existing parameter from user configuration, update its value
else:
# Just parameter value
val_type = 'str' if isinstance(
val, Six.string_types) or isinstance(
val, list) else type(val).__name__
if name not in param_info:
param_info[name] = {
'val':
val,
'type':
val_type,
'desc':
"No description for this parameter provided "
"(it was automatically converted from its value)."
}
else:
param_info[name]['val'] = val
# Do final validations
if 'type' in param_info[name] and param_info[name]['type'] not in (
'int', 'str', 'float', 'bool'):
raise ConfigurationError(
"Parameter info update error."
" Parameter has invalid type = '%s'."
" Parameter definition is %s = %s" %
(param_info[name]['type'], name, param_info[name]))
if 'type' not in param_info[name] or 'desc' not in param_info[name]:
logging.warn(
"Parameter definition does not contain type ('type') and/or description ('desc')."
" You should fix this. Parameter definition is"
" %s = %s", name, param_info[name])
def compute_current_variables(self, experiment, computable_variables):
"""Computes all variables in `experiment` that are in `computable_variables`.
Args:
experiment (dict): Current experiment.
computable_variables (list): Names of variables that need to be computed.
Returns:
tuple: (computed (list), partially_computed(list))
The computed variables are those that have been computed and their values can be used. The partially computed
variables are those that contain nested references (`finalized` is set to False for them).
"""
computed = []
partially_computed = []
for var in computable_variables:
is_str = isinstance(experiment[var], Six.string_types)
if not is_str:
computed.append(var)
continue
if is_str and len(self.fwd_index[var]['deps']) > 0:
for ref_var in self.fwd_index[var]['deps']:
replace_pattern = "${%s}" % ref_var
if ref_var in experiment:
replace_value = ParamUtils.to_string(
experiment[ref_var])
elif ref_var in os.environ:
replace_value = ParamUtils.to_string(
os.environ[ref_var])
else:
msg = "Cannot determine value of the parameter %s = %s because variable `%s` not found. "\
"Either this variable is not in the list of benchmark parameters, or this may happen "\
"if variable's name depends on other variable that's empty or set to an incorrect "\
"value. For instance, the name of ${${exp.framework}.docker.image} variable depends "\
"on ${exp.framework} value. If it's empty, the variable name becomes '.docker.image' "\
"what's wrong."
raise LogicError(msg % (var, experiment[var], ref_var))
experiment[var] = experiment[var].replace(
replace_pattern, replace_value)
# Search for computable components
while True:
idx = experiment[var].find('$(')
if idx < 0:
break
end_idx = experiment[var].find(')$', idx + 2)
if end_idx < 0:
raise ConfigurationError(
"Cannot find ')$' in %s. Variable cannot be computed" %
(experiment[var]))
try:
eval_res = eval(experiment[var][idx + 2:end_idx])
except NameError as err:
logging.error("Cannot evaluate python expression: %s",
experiment[var][idx + 2:end_idx])
raise err
logging.debug("\"%s\" -> \"%s\"",
experiment[var][idx + 2:end_idx], str(eval_res))
experiment[var] = experiment[var][:idx] + str(
eval_res) + experiment[var][end_idx + 2:]
if self.fwd_index[var]['finalized'] is True:
computed.append(var)
self.cast_variable(experiment, var)
self.check_variable_value(experiment, var)
else:
partially_computed.append(var)
return computed, partially_computed