def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
print >> sys.stderr, itemstr(parents, key, val)
raise FileParseError(
'ERROR line ' + str(index) + ': already encountered ' +
itemstr(parents))
if key in cfig:
# this item already exists
if (key == 'graph' and (
parents == ['scheduling', 'dependencies'] or
len(parents) == 3 and
parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
if cylc.flags.verbose:
print 'Merging graph strings under ' + itemstr(parents)
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
if cylc.flags.verbose:
print >> sys.stderr, (
'WARNING: overriding ' + itemstr(parents, key))
print >> sys.stderr, ' old value: ' + cfig[key]
print >> sys.stderr, ' new value: ' + val
cfig[key] = val
else:
cfig[key] = val
def _coerce_cycleinterval(value, keys, _):
"""Coerce value to a cycle interval."""
if not value:
return None
value = _strip_and_unquote(keys, value)
if value.isdigit():
# Old runahead limit format.
set_syntax_version(VERSION_PREV,
"integer interval for %s" % itemstr(
keys[:-1], keys[-1], value))
return value
if REC_INTEGER_INTERVAL.match(value):
# New integer cycling format.
set_syntax_version(VERSION_NEW,
"integer interval for %s" % itemstr(
keys[:-1], keys[-1], value))
return value
parser = DurationParser()
try:
parser.parse(value)
except ValueError:
raise IllegalValueError("interval", keys, value)
set_syntax_version(VERSION_NEW,
"ISO 8601 interval for %s" % itemstr(
keys[:-1], keys[-1], value))
return value
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
print >> sys.stderr, itemstr(parents, key, val)
raise FileParseError('ERROR line ' + str(index) +
': already encountered ' + itemstr(parents))
if key in cfig:
# this item already exists
if (key == 'graph' and
(parents == ['scheduling', 'dependencies'] or len(parents) == 3
and parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
if cylc.flags.verbose:
print 'Merging graph strings under ' + itemstr(parents)
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
if cylc.flags.verbose:
print >> sys.stderr, ('WARNING: overriding ' +
itemstr(parents, key))
print >> sys.stderr, ' old value: ' + cfig[key]
print >> sys.stderr, ' new value: ' + val
cfig[key] = val
else:
cfig[key] = val
def coerce_interval(value,
keys,
args,
back_comp_unit_factor=1,
check_syntax_version=True):
"""Coerce an ISO 8601 interval (or number: back-comp) into seconds."""
value = _strip_and_unquote(keys, value)
try:
backwards_compat_value = float(value) * back_comp_unit_factor
except (TypeError, ValueError):
pass
else:
if check_syntax_version:
set_syntax_version(
VERSION_PREV,
"integer interval: %s" % itemstr(keys[:-1], keys[-1], value))
return backwards_compat_value
try:
interval = interval_parser.parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
if check_syntax_version:
try:
set_syntax_version(
VERSION_NEW,
"ISO 8601 interval: %s" % itemstr(keys[:-1], keys[-1], value))
except SyntaxVersionError as exc:
raise Exception(str(exc))
days, seconds = interval.get_days_and_seconds()
seconds += days * Calendar.default().SECONDS_IN_DAY
return seconds
def coerce_interval(value,
keys,
_,
back_comp_unit_factor=1,
check_syntax_version=True):
"""Coerce an ISO 8601 interval (or number: back-comp) into seconds."""
value = _strip_and_unquote(keys, value)
if not value:
# Allow explicit empty values.
return None
try:
backwards_compat_value = float(value) * back_comp_unit_factor
except (TypeError, ValueError):
pass
else:
if check_syntax_version:
set_syntax_version(
VERSION_PREV,
"integer interval: %s" % itemstr(keys[:-1], keys[-1], value))
return DurationFloat(backwards_compat_value)
try:
interval = DURATION_PARSER.parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
if check_syntax_version:
set_syntax_version(
VERSION_NEW,
"ISO 8601 interval: %s" % itemstr(keys[:-1], keys[-1], value))
days, seconds = interval.get_days_and_seconds()
return DurationFloat(days * CALENDAR.SECONDS_IN_DAY + seconds)
def _coerce_cycleinterval(value, keys, _):
"""Coerce value to a cycle interval."""
if not value:
return None
value = _strip_and_unquote(keys, value)
if value.isdigit():
# Old runahead limit format.
set_syntax_version(
VERSION_PREV,
"integer interval for %s" % itemstr(keys[:-1], keys[-1], value))
return value
if REC_INTEGER_INTERVAL.match(value):
# New integer cycling format.
set_syntax_version(
VERSION_NEW,
"integer interval for %s" % itemstr(keys[:-1], keys[-1], value))
return value
parser = DurationParser()
try:
parser.parse(value)
except ValueError:
raise IllegalValueError("interval", keys, value)
set_syntax_version(
VERSION_NEW,
"ISO 8601 interval for %s" % itemstr(keys[:-1], keys[-1], value))
return value
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError(
'line %d: already encountered %s',
index, itemstr(parents, key, val))
if key in cfig:
# this item already exists
if (key == 'graph' and (
parents == ['scheduling', 'dependencies'] or
len(parents) == 3 and
parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
LOG.debug('Merging graph strings under %s', itemstr(parents))
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
LOG.debug(
'overriding %s old value: %s new value: %s',
itemstr(parents, key), cfig[key], val)
cfig[key] = val
else:
cfig[key] = val
def addict(cfig, key, val, parents, index):
"""Add a new [parents...]key=value pair to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if not isinstance(cfig, dict):
# an item of this name has already been encountered at this level
raise FileParseError('ERROR line %d: already encountered %s', index,
itemstr(parents, key, val))
if key in cfig:
# this item already exists
if (key == 'graph' and
(parents == ['scheduling', 'dependencies'] or len(parents) == 3
and parents[-3:-1] == ['scheduling', 'dependencies'])):
# append the new graph string to the existing one
LOG.debug('Merging graph strings under %s', itemstr(parents))
if not isinstance(cfig[key], list):
cfig[key] = [cfig[key]]
cfig[key].append(val)
else:
# otherwise override the existing item
LOG.debug('overriding %s old value: %s new value: %s',
itemstr(parents, key), cfig[key], val)
cfig[key] = val
else:
cfig[key] = val
def coerce_interval(value, keys, _, back_comp_unit_factor=1,
check_syntax_version=True):
"""Coerce an ISO 8601 interval (or number: back-comp) into seconds."""
value = _strip_and_unquote(keys, value)
if not value:
# Allow explicit empty values.
return None
try:
backwards_compat_value = float(value) * back_comp_unit_factor
except (TypeError, ValueError):
pass
else:
if check_syntax_version:
set_syntax_version(
VERSION_PREV,
"integer interval: %s" % itemstr(keys[:-1], keys[-1], value))
return DurationFloat(backwards_compat_value)
try:
interval = DURATION_PARSER.parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
if check_syntax_version:
set_syntax_version(
VERSION_NEW,
"ISO 8601 interval: %s" % itemstr(keys[:-1], keys[-1], value))
days, seconds = interval.get_days_and_seconds()
return DurationFloat(days * CALENDAR.SECONDS_IN_DAY + seconds)
def coerce_interval(value, keys, args, back_comp_unit_factor=1,
check_syntax_version=True):
"""Coerce an ISO 8601 interval (or number: back-comp) into seconds."""
value = _strip_and_unquote(keys, value)
try:
backwards_compat_value = float(value) * back_comp_unit_factor
except (TypeError, ValueError):
pass
else:
if check_syntax_version:
set_syntax_version(VERSION_PREV,
"integer interval: %s" % itemstr(
keys[:-1], keys[-1], value))
return backwards_compat_value
try:
interval = interval_parser.parse(value)
except ValueError:
raise IllegalValueError("ISO 8601 interval", keys, value)
if check_syntax_version:
try:
set_syntax_version(VERSION_NEW,
"ISO 8601 interval: %s" % itemstr(
keys[:-1], keys[-1], value))
except SyntaxVersionError as exc:
raise Exception(str(exc))
days, seconds = interval.get_days_and_seconds()
seconds += days * Calendar.default().SECONDS_IN_DAY
return seconds
def __str__(self):
msg = ''
if self.vtype:
msg += f'(type={self.vtype}) '
if self.key:
msg += itemstr(self.keys, self.key)
elif self.value:
msg += itemstr(self.keys[:-1], self.keys[-1], value=self.value)
if self.msg or self.exc:
msg += (
f' - ({self.exc or ""}'
f'{": " if (self.exc and self.msg) else ""}'
f'{self.msg or ""})'
)
return msg
def _coerce_cycletime( value, keys, args ):
"""Coerce value to a cycle point."""
value = _strip_and_unquote( keys, value )
if re.match(r"\d+$", value):
# Could be an old date-time cycle point format, or integer format.
return value
if value.startswith("-") or value.startswith("+"):
# We don't know the value given for num expanded year digits...
for i in range(1, 101):
parser = TimePointParser(num_expanded_year_digits=i)
try:
parser.parse(value)
except ValueError:
continue
return value
raise IllegalValueError("cycle point", keys, value)
parser = TimePointParser()
try:
parser.parse(value)
except ValueError:
raise IllegalValueError("cycle point", keys, value)
set_syntax_version(VERSION_NEW,
"cycle point: %s" % itemstr(
keys[:-1], keys[-1], value))
return value
def get(self, keys=None, sparse=False):
"""
Retrieve items or sections, sparse or dense, by list of keys:
[sec1,sec2,item] =>
[sec1]
[[sec2]]
item = value
"""
if sparse:
cfg = self.sparse
else:
self.expand()
cfg = self.dense
parents = []
if keys:
for key in keys:
try:
cfg = cfg[key]
except KeyError:
raise ItemNotFoundError(itemstr(parents, key))
else:
parents.append(key)
return cfg
def get(self, keys=None, sparse=False):
"""
Retrieve items or sections, sparse or dense, by list of keys:
[sec1,sec2,item] =>
[sec1]
[[sec2]]
item = value
"""
if sparse:
cfg = self.sparse
else:
self.expand()
cfg = self.dense
parents = []
if keys:
for key in keys:
try:
cfg = cfg[key]
except KeyError:
raise ItemNotFoundError(itemstr(parents, key))
else:
parents.append(key)
return cfg
def _coerce_cycletime(value, keys, _):
"""Coerce value to a cycle point."""
if not value:
return None
if value == "now":
# Handle this later in config.py when the suite UTC mode is known.
return value
value = _strip_and_unquote(keys, value)
if re.match(r"\d+$", value):
# Could be an old date-time cycle point format, or integer format.
return value
if value.startswith("-") or value.startswith("+"):
# We don't know the value given for num expanded year digits...
for i in range(1, 101):
parser = TimePointParser(num_expanded_year_digits=i)
try:
parser.parse(value)
except ValueError:
continue
return value
raise IllegalValueError("cycle point", keys, value)
parser = TimePointParser()
try:
parser.parse(value)
except ValueError:
raise IllegalValueError("cycle point", keys, value)
set_syntax_version(VERSION_NEW,
"cycle point: %s" % itemstr(keys[:-1], keys[-1], value))
return value
def checkspec( self, spec, parents=[] ):
"check that the file spec is a nested dict of validators"
for key, value in spec.items():
pars = parents + [key]
if isinstance( value, dict ):
self.checkspec( value, pars )
else:
if not isinstance( value, validator ):
raise ParsecError( "Illegal file spec item: " + itemstr( pars, repr(value)) )
def checkspec(self, spec, parents=[]):
"check that the file spec is a nested dict of validators"
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
self.checkspec(value, pars)
else:
if not isinstance(value, validator):
raise ParsecError("Illegal file spec item: %s" %
itemstr(pars, repr(value)))
def addsect(cfig, sname, parents):
"""Add a new section to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if sname in cfig:
# this doesn't warrant a warning unless contained items are repeated
LOG.debug('Section already encountered: %s',
itemstr(parents + [sname]))
else:
cfig[sname] = OrderedDictWithDefaults()
def addsect(cfig, sname, parents):
"""Add a new section to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if sname in cfig:
# this doesn't warrant a warning unless contained items are repeated
if cylc.flags.verbose:
print 'Section already encountered: ' + itemstr(parents + [sname])
else:
cfig[sname] = OrderedDictWithDefaults()
def checkspec(spec_root, parents=None):
"""Check that the file spec is a nested dict of specifications"""
stack = [[spec_root, []]]
while stack:
spec, parents = stack.pop()
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
stack.append([value, pars])
elif not isinstance(value, list):
raise ParsecError("Illegal file spec item: %s" %
itemstr(pars, repr(value)))
def checkspec(spec_root, parents=None):
"""Check that the file spec is a nested dict of specifications"""
stack = [[spec_root, []]]
while stack:
spec, parents = stack.pop()
for key, value in spec.items():
pars = parents + [key]
if isinstance(value, dict):
stack.append([value, pars])
elif not isinstance(value, list):
raise ParsecError(
"Illegal file spec item: %s" % itemstr(
pars, repr(value)))
def mdump( self, mkeys=[], sparse=False, pnative=False, prefix='', oneline=False, none_str='' ):
if oneline:
items = []
for keys in mkeys:
item = self.get( keys, sparse )
if isinstance( item, list ) or isinstance( item, dict ):
raise NotSingleItemError(itemstr(keys))
if not item:
item = none_str or "None"
items.append(str(item))
# TODO - quote items if they contain spaces or comment delimiters?
print prefix + ' '.join( items )
else:
for keys in mkeys:
self.dump( keys, sparse, pnative, prefix, none_str )
def mdump( self, mkeys=[], sparse=False, pnative=False, prefix='', oneline=False, none_str='' ):
if oneline:
items = []
for keys in mkeys:
item = self.get( keys, sparse )
if isinstance( item, list ) or isinstance( item, dict ):
raise NotSingleItemError(itemstr(keys))
if not item:
item = none_str or "None"
items.append(str(item))
# TODO - quote items if they contain spaces or comment delimiters?
print prefix + ' '.join( items )
else:
for keys in mkeys:
self.dump( keys, sparse, pnative, prefix, none_str )
def __init__(self, vtype, keys, value, exc=None):
self.msg = 'Illegal %s value: %s' % (vtype, itemstr(keys, value=value))
if exc:
self.msg += ": %s" % exc
def __init__(self, vtype, keys, value):
self.msg = 'Illegal %s value: %s' % (vtype, itemstr(keys, value=value))
def __init__(self, keys, key, msg=None):
if msg is not None:
self.msg = 'Illegal item (%s): %s' % (msg, itemstr(keys, key))
else:
self.msg = 'Illegal item: %s' % itemstr(keys, key)
def __init__(self, keys, value, exc=None):
self.msg = (
"ERROR: names containing commas must be quoted"
" (e.g. 'foo<m,n>'):\n %s" % itemstr(keys, value=value))
if exc:
self.msg += ": %s" % exc
def __init__(self, vtype, keys, value):
msg = "Illegal " + vtype + " value: " + itemstr(keys, value=value)
ValidationError.__init__(self, msg)
def __init__(self, keys, key):
msg = "Illegal item: " + itemstr(keys, key)
ValidationError.__init__(self, msg)
def __init__(self, keys, key, msg=None):
ValidationError.__init__(self)
if msg is not None:
self.msg = 'Illegal item (%s): %s' % (msg, itemstr(keys, key))
else:
self.msg = 'Illegal item: %s' % itemstr(keys, key)
def __init__(self, vtype, keys, value, exc=None):
ValidationError.__init__(self)
self.msg = 'Illegal %s value: %s' % (
vtype, itemstr(keys[:-1], keys[-1], value=value))
if exc:
self.msg += ": %s" % exc
def __init__(self, keys, value, msg='', exc=None):
ValidationError.__init__(self)
self.msg = '%s\n %s' % (
msg, itemstr(keys[:-1], keys[-1], value=value))
if exc:
self.msg += ": %s" % exc
def __init__(self, vtype, keys, value, exc=None):
self.msg = 'Illegal %s value: %s' % (vtype, itemstr(keys, value=value))
if exc:
self.msg += ": %s" % exc
def __init__(self, keys, key):
self.msg = 'Illegal item: %s' % itemstr(keys, key)
def __init__(self, vtype, keys, value):
self.msg = 'Illegal %s value: %s' % (vtype, itemstr(keys, value=value))
def __init__(self, keys, key):
self.msg = 'Illegal item: %s' % itemstr(keys, key)
def __init__(self, keys, value, exc=None):
self.msg = ("ERROR: names containing commas must be quoted"
" (e.g. 'foo<m,n>'):\n %s" % itemstr(keys, value=value))
if exc:
self.msg += ": %s" % exc