def coerce_xtrigger(self, value, keys): """Coerce a string into an xtrigger function context object. func_name(*func_args, **func_kwargs) Checks for legal string templates in arg values too. """ label = keys[-1] value = self.strip_and_unquote(keys, value) if not value: raise IllegalValueError("xtrigger", keys, value) fname = None args = [] kwargs = {} match = self._REC_TRIG_FUNC.match(value) if match is None: raise IllegalValueError("xtrigger", keys, value) fname, fargs, intvl = match.groups() if intvl: intvl = self.coerce_interval(intvl, keys) if fargs: # Extract function args and kwargs. for farg in fargs.split(r','): try: key, val = farg.strip().split(r'=', 1) except ValueError: args.append(self._coerce_type(farg.strip())) else: kwargs[key.strip()] = self._coerce_type(val.strip()) return SubFuncContext(label, fname, args, kwargs, intvl)
def expand_list(cls, values, keys, type_): """Handle multiplier syntax N*VALUE in a list. Examples: >>> ParsecValidator.expand_list(['1', '2*3'], None, int) [1, 3, 3] """ lvalues = [] for item in values: try: mult, val = item.split('*', 1) except ValueError: # too few values to unpack: no multiplier try: lvalues.append(type_(item)) except ValueError as exc: raise IllegalValueError('list', keys, item, exc=exc) else: # mult * val try: lvalues += int(mult) * [type_(val)] except ValueError as exc: raise IllegalValueError('list', keys, item, exc=exc) return lvalues
def coerce_cycle_point_format(cls, value, keys): """Coerce to a cycle point format (either CCYYMM... or %Y%m...).""" value = cls.strip_and_unquote(keys, value) if not value: return None test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1, hour_of_day=4, minute_of_hour=30, second_of_minute=54) if '/' in value: raise IllegalValueError('cycle point format', keys, value) if '%' in value: try: TimePointDumper().strftime(test_timepoint, value) except ValueError: raise IllegalValueError('cycle point format', keys, value) return value if 'X' in value: for i in range(1, 101): dumper = TimePointDumper(num_expanded_year_digits=i) try: dumper.dump(test_timepoint, value) except ValueError: continue return value raise IllegalValueError('cycle point format', keys, value) dumper = TimePointDumper() try: dumper.dump(test_timepoint, value) except ValueError: raise IllegalValueError('cycle point format', keys, value) return value
def coerce_cycle_point(cls, value, keys): """Coerce value to a cycle point. Examples: >>> CylcConfigValidator.coerce_cycle_point('2000', None) '2000' >>> CylcConfigValidator.coerce_cycle_point('now', None) 'now' >>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None) 'next(T-00)' """ if not value: return None value = cls.strip_and_unquote(keys, value) if value == 'now': # Handle this later in config.py when workflow UTC mode is known. return value if "next" in value or "previous" in value: # Handle this later, as for "now". return value if value.isdigit(): # Could be an old date-time cycle point format, or integer format. return value if "P" not in value and ( value.startswith('-') or value.startswith('+')): # We don't know the value given for num expanded year digits... for i in range(1, 101): try: TimePointParser(num_expanded_year_digits=i).parse(value) except IsodatetimeError: continue return value raise IllegalValueError('cycle point', keys, value) if "P" in value: # ICP is an offset parser = DurationParser() try: if value.startswith("-"): # parser doesn't allow negative duration with this setup? parser.parse(value[1:]) else: parser.parse(value) return value except IsodatetimeError as exc: raise IllegalValueError('cycle point', keys, value, exc=exc) try: TimePointParser().parse(value) except IsodatetimeError as exc: if isinstance(exc, ISO8601SyntaxError): # Don't know cycling mode yet, so override ISO8601-specific msg details = {'msg': "Invalid cycle point"} else: details = {'exc': exc} raise IllegalValueError('cycle point', keys, value, **details) return value
def coerce_cycle_point_format(cls, value, keys): """Coerce to a cycle point format. Examples: >>> CylcConfigValidator.coerce_cycle_point_format( ... 'CCYYMM', None) 'CCYYMM' >>> CylcConfigValidator.coerce_cycle_point_format( ... '%Y%m', None) '%Y%m' """ value = cls.strip_and_unquote(keys, value) if not value: return None test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1, hour_of_day=4, minute_of_hour=30, second_of_minute=54) if '/' in value: raise IllegalValueError( 'cycle point format', keys, value, msg=('Illegal character: "/".' ' Datetimes are used in Cylc file paths.')) if ':' in value: raise IllegalValueError( 'cycle point format', keys, value, msg=('Illegal character: ":".' ' Datetimes are used in Cylc file paths.')) if '%' in value: try: TimePointDumper().strftime(test_timepoint, value) except IsodatetimeError: raise IllegalValueError('cycle point format', keys, value) return value if 'X' in value: for i in range(1, 101): dumper = TimePointDumper(num_expanded_year_digits=i) try: dumper.dump(test_timepoint, value) except IsodatetimeError: continue return value raise IllegalValueError('cycle point format', keys, value) dumper = TimePointDumper() try: dumper.dump(test_timepoint, value) except IsodatetimeError: raise IllegalValueError('cycle point format', keys, value) return value
def coerce_parameter_list(cls, value, keys): """Coerce parameter list. Args: value (str): This can be a list of str values. Each str value must conform to the same restriction as a task name. Otherwise, this can be a mixture of int ranges and int values. keys (list): Keys in nested dict that represents the raw configuration. Return (list): A list of strings or a list of sorted integers. Raise: IllegalValueError: If value has both str and int range or if a str value breaks the task name restriction. Examples: >>> CylcConfigValidator.coerce_parameter_list('1..4, 6', None) [1, 2, 3, 4, 6] >>> CylcConfigValidator.coerce_parameter_list('a, b, c', None) ['a', 'b', 'c'] """ items = [] can_only_be = None # A flag to prevent mixing str and int range for item in cls.strip_and_unquote_list(keys, value): values = cls.parse_int_range(item) if values is not None: if can_only_be == str: raise IllegalValueError( 'parameter', keys, value, 'mixing int range and str') can_only_be = int items.extend(values) elif cls._REC_NAME_SUFFIX.match(item): try: int(item) except ValueError: if can_only_be == int: raise IllegalValueError( 'parameter', keys, value, 'mixing int range and str') can_only_be = str items.append(item) else: raise IllegalValueError( 'parameter', keys, value, '%s: bad value' % item) try: return [int(item) for item in items] except ValueError: return items
def coerce_cycle_point(cls, value, keys): """Coerce value to a cycle point. Examples: >>> CylcConfigValidator.coerce_cycle_point('2000', None) '2000' >>> CylcConfigValidator.coerce_cycle_point('now', None) 'now' >>> CylcConfigValidator.coerce_cycle_point('next(T-00)', None) 'next(T-00)' """ if not value: return None value = cls.strip_and_unquote(keys, value) if value == 'now': # Handle this later in config.py when the suite UTC mode is known. return value if "next" in value or "previous" in value: # Handle this later, as for "now". return value if value.isdigit(): # Could be an old date-time cycle point format, or integer format. return value if "P" not in value and ( value.startswith('-') or value.startswith('+')): # We don't know the value given for num expanded year digits... for i in range(1, 101): try: TimePointParser(num_expanded_year_digits=i).parse(value) except ValueError: continue return value raise IllegalValueError('cycle point', keys, value) if "P" in value: # ICP is an offset parser = DurationParser() try: if value.startswith("-"): # parser doesn't allow negative duration with this setup? parser.parse(value[1:]) else: parser.parse(value) return value except ValueError: raise IllegalValueError("cycle point", keys, value) try: TimePointParser().parse(value) except ValueError: raise IllegalValueError('cycle point', keys, value) return value
def coerce_cycle_point_time_zone(cls, value, keys): """Coerce value to a cycle point time zone format. Examples: >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... 'Z', None) 'Z' >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... '+13', None) '+13' >>> CylcConfigValidator.coerce_cycle_point_time_zone( ... '-0800', None) '-0800' """ value = cls.strip_and_unquote(keys, value) if not value: return None test_timepoint = TimePoint(year=2001, month_of_year=3, day_of_month=1, hour_of_day=4, minute_of_hour=30, second_of_minute=54) dumper = TimePointDumper() test_timepoint_string = dumper.dump(test_timepoint, 'CCYYMMDDThhmmss') test_timepoint_string += value parser = TimePointParser(allow_only_basic=True) try: parser.parse(test_timepoint_string) except ValueError: raise IllegalValueError( 'cycle point time zone format', keys, value) return value
def strip_and_unquote(cls, keys, value): """Remove leading and trailing spaces and unquote value. Args: keys (list): Keys in nested dict that represents the raw configuration. value (str): String value in raw configuration. Return (str): Processed value. """ for substr, rec in [ ["'''", cls._REC_MULTI_LINE_SINGLE], ['"""', cls._REC_MULTI_LINE_DOUBLE], ['"', cls._REC_DQ_VALUE], ["'", cls._REC_SQ_VALUE]]: if value.startswith(substr): match = rec.match(value) if match: value = match.groups()[0] else: raise IllegalValueError("string", keys, value) break else: # unquoted value = value.split(r'#', 1)[0] # Note strip() removes leading and trailing whitespace, including # initial newlines on a multiline string: return dedent(value).strip()
def validate(self, cfg_root, spec_root): """Validate and coerce a nested dict against a parsec spec. Args: cfg_root (dict): A nested dict representing the raw configuration. spec_root (dict): A nested dict containing the spec for the configuration. Raises: IllegalItemError: on bad configuration items. IllegalValueError: on bad configuration values. """ queue = deque([[cfg_root, spec_root, []]]) while queue: # Walk items, breadth first cfg, spec, keys = queue.popleft() for key, value in cfg.items(): if key not in spec: if '__MANY__' not in spec: raise IllegalItemError(keys, key) else: # only accept the item if its value is of the same type # as that of the __MANY__ item, i.e. dict or not-dict. val_is_dict = isinstance(value, dict) spc_is_dict = not spec['__MANY__'].is_leaf() if ( keys != ['scheduling', 'graph'] and not val_is_dict and ' ' in key ): # Item names shouldn't have consecutive spaces # (GitHub #2417) raise IllegalItemError( keys, key, 'consecutive spaces') if not ( (val_is_dict and spc_is_dict) or (not val_is_dict and not spc_is_dict) ): raise IllegalItemError(keys, key) speckey = '__MANY__' else: speckey = key specval = spec[speckey] if isinstance(value, dict) and not specval.is_leaf(): # Item is dict, push to queue queue.append([value, specval, keys + [key]]) elif value is not None and specval.is_leaf(): # Item is value, coerce according to value type cfg[key] = self.coercers[specval.vdr](value, keys + [key]) if specval.options: voptions = specval.options if (isinstance(cfg[key], list) and any(val not in voptions for val in cfg[key]) or not isinstance(cfg[key], list) and cfg[key] not in voptions): raise IllegalValueError( 'option', keys + [key], cfg[key])
def test_illegal_value_error(): value_type = 'ClassA' keys = ['a,', 'b', 'c'] value = 'a sample value' error = IllegalValueError(value_type, keys, value) output = str(error) expected = "(type=ClassA) [a,][b]c = a sample value" assert expected == output
def test_illegal_value_error_with_exception(): value_type = 'ClassA' keys = ['a,', 'b', 'c'] value = 'a sample value' exc = Exception('test') error = IllegalValueError(value_type, keys, value, exc) output = str(error) expected = "(type=ClassA) [a,][b]c = a sample value - (test)" assert expected == output
def coerce_int(cls, value, keys): """Coerce value to an integer.""" value = cls.strip_and_unquote(keys, value) if value in ['', None]: return None try: return int(value) except ValueError: raise IllegalValueError('int', keys, value)
def coerce_boolean(cls, value, keys): """Coerce value to a boolean.""" value = cls.strip_and_unquote(keys, value) if value in ['True', 'true']: return True elif value in ['False', 'false']: return False elif value in ['', None]: return None else: raise IllegalValueError('boolean', keys, value)
def coerce_interval(self, value, keys): """Coerce an ISO 8601 interval (or number: back-comp) into seconds.""" value = self.strip_and_unquote(keys, value) if not value: # Allow explicit empty values. return None try: interval = DurationParser().parse(value) except ValueError: raise IllegalValueError("ISO 8601 interval", keys, value) days, seconds = interval.get_days_and_seconds() return DurationFloat( days * Calendar.default().SECONDS_IN_DAY + seconds)
def coerce_int(cls, value, keys): """Coerce value to an integer. Examples: >>> ParsecValidator.coerce_int('1', None) 1 """ value = cls.strip_and_unquote(keys, value) if value in ['', None]: return None try: return int(value) except ValueError: raise IllegalValueError('int', keys, value)
def coerce_interval(cls, value, keys): """Coerce an ISO 8601 interval into seconds. Examples: >>> CylcConfigValidator.coerce_interval('PT1H', None) 3600.0 """ value = cls.strip_and_unquote(keys, value) if not value: # Allow explicit empty values. return None try: interval = DurationParser().parse(value) except ValueError: raise IllegalValueError("ISO 8601 interval", keys, value) days, seconds = interval.get_days_and_seconds() return DurationFloat( days * Calendar.default().SECONDS_IN_DAY + seconds)
def coerce_float(cls, value, keys): """Coerce value to a float. Examples: >>> ParsecValidator.coerce_float('1', None) 1.0 >>> ParsecValidator.coerce_float('1.1', None) 1.1 >>> ParsecValidator.coerce_float('1.1e1', None) 11.0 """ value = cls.strip_and_unquote(keys, value) if value in ['', None]: return None try: return float(value) except ValueError: raise IllegalValueError('float', keys, value)
def coerce_boolean(cls, value, keys): """Coerce value to a boolean. Examples: >>> ParsecValidator.coerce_boolean('True', None) True >>> ParsecValidator.coerce_boolean('true', None) True """ value = cls.strip_and_unquote(keys, value) if value in ['True', 'true']: return True elif value in ['False', 'false']: return False elif value in ['', None]: return None else: raise IllegalValueError('boolean', keys, value)