def _get_list_dicts(dct: Dict[str, Any], key: str) -> List[RawDefinition]:
result: List[RawDefinition] = []
if key not in dct:
raise InternalException(
f'Expected to find key {key} in dict, only found {list(dct)}')
values = dct[key]
if not isinstance(values, list):
raise ValidationException(
f'Invalid value type {type(values)} in key "{key}" '
f'(value "{values}")')
for value in values:
if isinstance(value, dict):
for value_key in value:
if not isinstance(value_key, str):
raise ValidationException(
f'Expected all keys to "{key}" dict to be strings, '
f'but "{value_key}" is a "{type(value_key)}"')
result.append(value)
elif isinstance(value, str):
result.append(value)
else:
raise ValidationException(
f'Invalid value type {type(value)} in key "{key}", expected '
f'dict or str (value: {value}).')
return result
def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
diff_arg: Optional[SelectionSpec] = None
if len(definition) == 1:
key = list(definition)[0]
value = definition[key]
if not isinstance(key, str):
raise ValidationException(
f'Expected definition key to be a "str", got one of type '
f'"{type(key)}" ({key})')
dct = {
'method': key,
'value': value,
}
elif 'method' in definition and 'value' in definition:
dct = definition
if 'exclude' in definition:
diff_arg = _parse_exclusions(definition)
dct = {k: v for k, v in dct.items() if k != 'exclude'}
else:
raise ValidationException(
f'Expected exactly 1 key in the selection definition or "method" '
f'and "value" keys, but got {list(definition)}')
# if key isn't a valid method name, this will raise
base = SelectionCriteria.from_dict(definition, dct)
if diff_arg is None:
return base
else:
return SelectionDifference(components=[base, diff_arg])
def validate_with(schema, data):
try:
schema(data)
except MultipleInvalid as e:
logger.debug(schema)
logger.debug(data)
logger.error(str(e))
raise ValidationException(str(e))
except Invalid as e:
logger.debug(schema)
logger.debug(data)
logger.error(str(e))
raise ValidationException(str(e))
def inner(value: T) -> None:
for arg in args:
if isinstance(arg, type) and isinstance(value, arg):
return
elif value == arg:
return
raise ValidationException(
'Expected value "{}" to be one of {}'.format(
value, ','.join(map(str, args))))
def validate_connection(connection):
try:
connection_contract(connection)
credentials_contract = credentials_mapping.get(connection.get('type'))
credentials_contract(connection.get('credentials'))
except MultipleInvalid as e:
logger.info(e)
raise ValidationException(str(e))
def parse_from_definition(definition: RawDefinition) -> SelectionSpec:
if isinstance(definition, str):
return SelectionCriteria.from_single_spec(definition)
elif 'union' in definition:
return parse_union_definition(definition)
elif 'intersection' in definition:
return parse_intersection_definition(definition)
elif isinstance(definition, dict):
return parse_dict_definition(definition)
else:
raise ValidationException(
f'Expected to find str or dict, instead found '
f'{type(definition)}: {definition}')
def read_profile(profiles_dir):
path = os.path.join(profiles_dir, 'profiles.yml')
contents = None
if os.path.isfile(path):
try:
contents = load_file_contents(path, strip=False)
return load_yaml_text(contents)
except ValidationException as e:
msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
raise ValidationException(msg)
return {}
def parse_from_definition(definition: RawDefinition,
rootlevel=False) -> SelectionSpec:
if (isinstance(definition, dict)
and ('union' in definition or 'intersection' in definition)
and rootlevel and len(definition) > 1):
keys = ",".join(definition.keys())
raise ValidationException(
f"Only a single 'union' or 'intersection' key is allowed "
f"in a root level selector definition; found {keys}.")
if isinstance(definition, str):
return SelectionCriteria.from_single_spec(definition)
elif 'union' in definition:
return parse_union_definition(definition)
elif 'intersection' in definition:
return parse_intersection_definition(definition)
elif isinstance(definition, dict):
return parse_dict_definition(definition)
else:
raise ValidationException(
f'Expected to find union, intersection, str or dict, instead '
f'found {type(definition)}: {definition}')
def validate(self):
"""
Using the SCHEMA property, validate the attributes
of this instance. If any attributes are missing or
invalid, raise a ValidationException.
"""
validator = Draft4Validator(self.SCHEMA)
errors = set() # make errors a set to avoid duplicates
for error in validator.iter_errors(self.serialize()):
errors.add('.'.join(list(map(str, error.path)) + [error.message]))
if errors:
msg = ('Invalid arguments passed to "{}" instance: {}'.format(
type(self).__name__, ', '.join(errors)))
raise ValidationException(msg)
def _parse_include_exclude_subdefs(
definitions: List[RawDefinition]
) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]:
include_parts: List[SelectionSpec] = []
diff_arg: Optional[SelectionSpec] = None
for definition in definitions:
if isinstance(definition, dict) and 'exclude' in definition:
# do not allow multiple exclude: defs at the same level
if diff_arg is not None:
raise ValidationException(
f'Got multiple exclusion definitions in definition list '
f'{definitions}')
diff_arg = _parse_exclusions(definition)
else:
include_parts.append(parse_from_definition(definition))
return (include_parts, diff_arg)
def read_profile(profiles_dir: str) -> Dict[str, Any]:
path = os.path.join(profiles_dir, 'profiles.yml')
contents = None
if os.path.isfile(path):
try:
contents = load_file_contents(path, strip=False)
yaml_content = load_yaml_text(contents)
if not yaml_content:
msg = f'The profiles.yml file at {path} is empty'
raise DbtProfileError(
INVALID_PROFILE_MESSAGE.format(error_string=msg))
return yaml_content
except ValidationException as e:
msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
raise ValidationException(msg) from e
return {}
def validate(self):
"""
Using the SCHEMA property, validate the attributes
of this instance. If any attributes are missing or
invalid, raise a ValidationException.
"""
validator = Draft4Validator(self.SCHEMA)
errors = []
for error in validator.iter_errors(self.serialize()):
errors.append('property "{}", {}'.format(".".join(error.path),
error.message))
if errors:
raise ValidationException(
'Invalid arguments passed to "{}" instance: {}'.format(
type(self).__name__, ", ".join(errors)))
def _parse_include_exclude_subdefs(
definitions: List[RawDefinition]
) -> Tuple[List[SelectionSpec], Optional[SelectionSpec]]:
include_parts: List[SelectionSpec] = []
diff_arg: Optional[SelectionSpec] = None
for definition in definitions:
if isinstance(definition, dict) and 'exclude' in definition:
# do not allow multiple exclude: defs at the same level
if diff_arg is not None:
yaml_sel_cfg = yaml.dump(definition)
raise ValidationException(
f"You cannot provide multiple exclude arguments to the "
f"same selector set operator:\n{yaml_sel_cfg}")
diff_arg = _parse_exclusions(definition)
else:
include_parts.append(parse_from_definition(definition))
return (include_parts, diff_arg)