def coerce_xtrigger(self, value, keys):
"""Coerce a string into an xtrigger function context object.
func_name(*func_args, **func_kwargs)
Checks for legal string templates in arg values too.
"""
label = keys[-1]
value = self.strip_and_unquote(keys, value)
if not value:
raise IllegalValueError("xtrigger", keys, value)
fname = None
args = []
kwargs = {}
match = self._REC_TRIG_FUNC.match(value)
if match is None:
raise IllegalValueError("xtrigger", keys, value)
fname, fargs, intvl = match.groups()
if intvl:
intvl = self.coerce_interval(intvl, keys)
if fargs:
# Extract function args and kwargs.
for farg in fargs.split(r','):
try:
key, val = farg.strip().split(r'=', 1)
except ValueError:
args.append(self._coerce_type(farg.strip()))
else:
kwargs[key.strip()] = self._coerce_type(val.strip())
return SubFuncContext(label, fname, args, kwargs, intvl)
def expand_list(cls, values, keys, type_):
"""Handle multiplier syntax N*VALUE in a list.
Examples:
>>> ParsecValidator.expand_list(['1', '2*3'], None, int)
[1, 3, 3]
"""
lvalues = []
for item in values:
try:
mult, val = item.split('*', 1)
except ValueError:
# too few values to unpack: no multiplier
try:
lvalues.append(type_(item))
except ValueError as exc:
raise IllegalValueError('list', keys, item, exc=exc)
else:
# mult * val
try:
lvalues += int(mult) * [type_(val)]
except ValueError as exc:
raise IllegalValueError('list', keys, item, exc=exc)
return lvalues
def coerce_cycle_point_format(cls, value, keys):
"""Coerce to a cycle point format (either CCYYMM... or %Y%m...)."""
value = cls.strip_and_unquote(keys, value)
if not value:
return None
test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1,
hour_of_day=4, minute_of_hour=30,
second_of_minute=54)
if '/' in value:
raise IllegalValueError('cycle point format', keys, value)
if '%' in value:
try:
TimePointDumper().strftime(test_timepoint, value)
except ValueError:
raise IllegalValueError('cycle point format', keys, value)
return value
if 'X' in value:
for i in range(1, 101):
dumper = TimePointDumper(num_expanded_year_digits=i)
try:
dumper.dump(test_timepoint, value)
except ValueError:
continue
return value
raise IllegalValueError('cycle point format', keys, value)
dumper = TimePointDumper()
try:
dumper.dump(test_timepoint, value)
except ValueError:
raise IllegalValueError('cycle point format', keys, value)
return value
def coerce_cycle_point(cls, value, keys):
"""Coerce value to a cycle point.
Examples:
>>> CylcConfigValidator.coerce_cycle_point('2000', None)
'2000'
>>> CylcConfigValidator.coerce_cycle_point('now', None)
'now'
>>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None)
'next(T-00)'
"""
if not value:
return None
value = cls.strip_and_unquote(keys, value)
if value == 'now':
# Handle this later in config.py when workflow UTC mode is known.
return value
if "next" in value or "previous" in value:
# Handle this later, as for "now".
return value
if value.isdigit():
# Could be an old date-time cycle point format, or integer format.
return value
if "P" not in value and (
value.startswith('-') or value.startswith('+')):
# We don't know the value given for num expanded year digits...
for i in range(1, 101):
try:
TimePointParser(num_expanded_year_digits=i).parse(value)
except IsodatetimeError:
continue
return value
raise IllegalValueError('cycle point', keys, value)
if "P" in value:
# ICP is an offset
parser = DurationParser()
try:
if value.startswith("-"):
# parser doesn't allow negative duration with this setup?
parser.parse(value[1:])
else:
parser.parse(value)
return value
except IsodatetimeError as exc:
raise IllegalValueError('cycle point', keys, value, exc=exc)
try:
TimePointParser().parse(value)
except IsodatetimeError as exc:
if isinstance(exc, ISO8601SyntaxError):
# Don't know cycling mode yet, so override ISO8601-specific msg
details = {'msg': "Invalid cycle point"}
else:
details = {'exc': exc}
raise IllegalValueError('cycle point', keys, value, **details)
return value
def coerce_cycle_point_format(cls, value, keys):
"""Coerce to a cycle point format.
Examples:
>>> CylcConfigValidator.coerce_cycle_point_format(
... 'CCYYMM', None)
'CCYYMM'
>>> CylcConfigValidator.coerce_cycle_point_format(
... '%Y%m', None)
'%Y%m'
"""
value = cls.strip_and_unquote(keys, value)
if not value:
return None
test_timepoint = TimePoint(year=2001,
month_of_year=3,
day_of_month=1,
hour_of_day=4,
minute_of_hour=30,
second_of_minute=54)
if '/' in value:
raise IllegalValueError(
'cycle point format',
keys,
value,
msg=('Illegal character: "/".'
' Datetimes are used in Cylc file paths.'))
if ':' in value:
raise IllegalValueError(
'cycle point format',
keys,
value,
msg=('Illegal character: ":".'
' Datetimes are used in Cylc file paths.'))
if '%' in value:
try:
TimePointDumper().strftime(test_timepoint, value)
except IsodatetimeError:
raise IllegalValueError('cycle point format', keys, value)
return value
if 'X' in value:
for i in range(1, 101):
dumper = TimePointDumper(num_expanded_year_digits=i)
try:
dumper.dump(test_timepoint, value)
except IsodatetimeError:
continue
return value
raise IllegalValueError('cycle point format', keys, value)
dumper = TimePointDumper()
try:
dumper.dump(test_timepoint, value)
except IsodatetimeError:
raise IllegalValueError('cycle point format', keys, value)
return value
def coerce_parameter_list(cls, value, keys):
"""Coerce parameter list.
Args:
value (str):
This can be a list of str values. Each str value must conform
to the same restriction as a task name.
Otherwise, this can be a mixture of int ranges and int values.
keys (list):
Keys in nested dict that represents the raw configuration.
Return (list):
A list of strings or a list of sorted integers.
Raise:
IllegalValueError:
If value has both str and int range or if a str value breaks
the task name restriction.
Examples:
>>> CylcConfigValidator.coerce_parameter_list('1..4, 6', None)
[1, 2, 3, 4, 6]
>>> CylcConfigValidator.coerce_parameter_list('a, b, c', None)
['a', 'b', 'c']
"""
items = []
can_only_be = None # A flag to prevent mixing str and int range
for item in cls.strip_and_unquote_list(keys, value):
values = cls.parse_int_range(item)
if values is not None:
if can_only_be == str:
raise IllegalValueError(
'parameter', keys, value, 'mixing int range and str')
can_only_be = int
items.extend(values)
elif cls._REC_NAME_SUFFIX.match(item):
try:
int(item)
except ValueError:
if can_only_be == int:
raise IllegalValueError(
'parameter', keys, value,
'mixing int range and str')
can_only_be = str
items.append(item)
else:
raise IllegalValueError(
'parameter', keys, value, '%s: bad value' % item)
try:
return [int(item) for item in items]
except ValueError:
return items
def coerce_cycle_point(cls, value, keys):
"""Coerce value to a cycle point.
Examples:
>>> CylcConfigValidator.coerce_cycle_point('2000', None)
'2000'
>>> CylcConfigValidator.coerce_cycle_point('now', None)
'now'
>>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None)
'next(T-00)'
"""
if not value:
return None
value = cls.strip_and_unquote(keys, value)
if value == 'now':
# Handle this later in config.py when the suite UTC mode is known.
return value
if "next" in value or "previous" in value:
# Handle this later, as for "now".
return value
if value.isdigit():
# Could be an old date-time cycle point format, or integer format.
return value
if "P" not in value and (
value.startswith('-') or value.startswith('+')):
# We don't know the value given for num expanded year digits...
for i in range(1, 101):
try:
TimePointParser(num_expanded_year_digits=i).parse(value)
except ValueError:
continue
return value
raise IllegalValueError('cycle point', keys, value)
if "P" in value:
# ICP is an offset
parser = DurationParser()
try:
if value.startswith("-"):
# parser doesn't allow negative duration with this setup?
parser.parse(value[1:])
else:
parser.parse(value)
return value
except ValueError:
raise IllegalValueError("cycle point", keys, value)
try:
TimePointParser().parse(value)
except ValueError:
raise IllegalValueError('cycle point', keys, value)
return value
def coerce_cycle_point_time_zone(cls, value, keys):
"""Coerce value to a cycle point time zone format.
Examples:
>>> CylcConfigValidator.coerce_cycle_point_time_zone(
... 'Z', None)
'Z'
>>> CylcConfigValidator.coerce_cycle_point_time_zone(
... '+13', None)
'+13'
>>> CylcConfigValidator.coerce_cycle_point_time_zone(
... '-0800', None)
'-0800'
"""
value = cls.strip_and_unquote(keys, value)
if not value:
return None
test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1,
hour_of_day=4, minute_of_hour=30,
second_of_minute=54)
dumper = TimePointDumper()
test_timepoint_string = dumper.dump(test_timepoint, 'CCYYMMDDThhmmss')
test_timepoint_string += value
parser = TimePointParser(allow_only_basic=True)
try:
parser.parse(test_timepoint_string)
except ValueError:
raise IllegalValueError(
'cycle point time zone format', keys, value)
return value
def strip_and_unquote(cls, keys, value):
"""Remove leading and trailing spaces and unquote value.
Args:
keys (list):
Keys in nested dict that represents the raw configuration.
value (str):
String value in raw configuration.
Return (str):
Processed value.
"""
for substr, rec in [
["'''", cls._REC_MULTI_LINE_SINGLE],
['"""', cls._REC_MULTI_LINE_DOUBLE],
['"', cls._REC_DQ_VALUE],
["'", cls._REC_SQ_VALUE]]:
if value.startswith(substr):
match = rec.match(value)
if match:
value = match.groups()[0]
else:
raise IllegalValueError("string", keys, value)
break
else:
# unquoted
value = value.split(r'#', 1)[0]
# Note strip() removes leading and trailing whitespace, including
# initial newlines on a multiline string:
return dedent(value).strip()
def validate(self, cfg_root, spec_root):
"""Validate and coerce a nested dict against a parsec spec.
Args:
cfg_root (dict):
A nested dict representing the raw configuration.
spec_root (dict):
A nested dict containing the spec for the configuration.
Raises:
IllegalItemError: on bad configuration items.
IllegalValueError: on bad configuration values.
"""
queue = deque([[cfg_root, spec_root, []]])
while queue:
# Walk items, breadth first
cfg, spec, keys = queue.popleft()
for key, value in cfg.items():
if key not in spec:
if '__MANY__' not in spec:
raise IllegalItemError(keys, key)
else:
# only accept the item if its value is of the same type
# as that of the __MANY__ item, i.e. dict or not-dict.
val_is_dict = isinstance(value, dict)
spc_is_dict = not spec['__MANY__'].is_leaf()
if (
keys != ['scheduling', 'graph'] and
not val_is_dict and
' ' in key
):
# Item names shouldn't have consecutive spaces
# (GitHub #2417)
raise IllegalItemError(
keys, key, 'consecutive spaces')
if not (
(val_is_dict and spc_is_dict)
or (not val_is_dict and not spc_is_dict)
):
raise IllegalItemError(keys, key)
speckey = '__MANY__'
else:
speckey = key
specval = spec[speckey]
if isinstance(value, dict) and not specval.is_leaf():
# Item is dict, push to queue
queue.append([value, specval, keys + [key]])
elif value is not None and specval.is_leaf():
# Item is value, coerce according to value type
cfg[key] = self.coercers[specval.vdr](value, keys + [key])
if specval.options:
voptions = specval.options
if (isinstance(cfg[key], list) and
any(val not in voptions for val in cfg[key]) or
not isinstance(cfg[key], list) and
cfg[key] not in voptions):
raise IllegalValueError(
'option', keys + [key], cfg[key])
def test_illegal_value_error():
value_type = 'ClassA'
keys = ['a,', 'b', 'c']
value = 'a sample value'
error = IllegalValueError(value_type, keys, value)
output = str(error)
expected = "(type=ClassA) [a,][b]c = a sample value"
assert expected == output
def test_illegal_value_error_with_exception():
value_type = 'ClassA'
keys = ['a,', 'b', 'c']
value = 'a sample value'
exc = Exception('test')
error = IllegalValueError(value_type, keys, value, exc)
output = str(error)
expected = "(type=ClassA) [a,][b]c = a sample value - (test)"
assert expected == output
def coerce_int(cls, value, keys):
"""Coerce value to an integer."""
value = cls.strip_and_unquote(keys, value)
if value in ['', None]:
return None
try:
return int(value)
except ValueError:
raise IllegalValueError('int', keys, value)
def coerce_boolean(cls, value, keys):
"""Coerce value to a boolean."""
value = cls.strip_and_unquote(keys, value)
if value in ['True', 'true']:
return True
elif value in ['False', 'false']:
return False
elif value in ['', None]:
return None
else:
raise IllegalValueError('boolean', keys, value)
def coerce_interval(self, value, keys):
"""Coerce an ISO 8601 interval (or number: back-comp) into seconds."""
value = self.strip_and_unquote(keys, value)
if not value:
# Allow explicit empty values.
return None
try:
interval = DurationParser().parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
days, seconds = interval.get_days_and_seconds()
return DurationFloat(
days * Calendar.default().SECONDS_IN_DAY + seconds)
def coerce_int(cls, value, keys):
"""Coerce value to an integer.
Examples:
>>> ParsecValidator.coerce_int('1', None)
1
"""
value = cls.strip_and_unquote(keys, value)
if value in ['', None]:
return None
try:
return int(value)
except ValueError:
raise IllegalValueError('int', keys, value)
def coerce_interval(cls, value, keys):
"""Coerce an ISO 8601 interval into seconds.
Examples:
>>> CylcConfigValidator.coerce_interval('PT1H', None)
3600.0
"""
value = cls.strip_and_unquote(keys, value)
if not value:
# Allow explicit empty values.
return None
try:
interval = DurationParser().parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
days, seconds = interval.get_days_and_seconds()
return DurationFloat(
days * Calendar.default().SECONDS_IN_DAY + seconds)
def coerce_float(cls, value, keys):
"""Coerce value to a float.
Examples:
>>> ParsecValidator.coerce_float('1', None)
1.0
>>> ParsecValidator.coerce_float('1.1', None)
1.1
>>> ParsecValidator.coerce_float('1.1e1', None)
11.0
"""
value = cls.strip_and_unquote(keys, value)
if value in ['', None]:
return None
try:
return float(value)
except ValueError:
raise IllegalValueError('float', keys, value)
def coerce_boolean(cls, value, keys):
"""Coerce value to a boolean.
Examples:
>>> ParsecValidator.coerce_boolean('True', None)
True
>>> ParsecValidator.coerce_boolean('true', None)
True
"""
value = cls.strip_and_unquote(keys, value)
if value in ['True', 'true']:
return True
elif value in ['False', 'false']:
return False
elif value in ['', None]:
return None
else:
raise IllegalValueError('boolean', keys, value)
