def read(config_values):
"""Reads an ordered list of configuration values and deep merge the values in reverse order."""
if not config_values:
raise RheaError('Cannot read config_value: `{}`'.format(config_values))
config_values = to_list(config_values)
config = {}
for config_value in config_values:
if not isinstance(config_value, (Mapping, six.string_types)):
raise RheaError(
"Expects Mapping, string, or list of Mapping/string instances, "
"received {} instead".format(type(config_value)))
if isinstance(config_value, Mapping):
config_results = config_value
elif os.path.isfile(config_value):
config_results = _read_from_file(config_value)
else:
# try reading a stream of yaml or json
try:
config_results = _read_from_stream(config_value)
except ScannerError:
raise RheaError('Received non valid yaml stream: `{}`'.format(
config_value))
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
else:
raise RheaError(
'Cannot read config_value: `{}`'.format(config_value))
return config
def _get_typed_list_value(self,
key,
target_type,
type_convert,
is_optional=False,
is_secret=False,
is_local=False,
default=None,
options=None):
"""
Return the value corresponding to the key converted first to list
than each element to the given type.
Args:
key: the dict key.
target_type: The type we expect the variable or key to be in.
type_convert: A lambda expression that converts the key to the desired type.
is_optional: To raise an error if key was not found.
is_secret: If the key is a secret.
is_local: If the key is a local to this service.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
"""
value = self._get_typed_value(key=key,
target_type=list,
type_convert=json.loads,
is_optional=is_optional,
is_secret=is_secret,
is_local=is_local,
default=default,
options=options)
if not value:
return default
raise_type = 'dict' if target_type == Mapping else target_type
if not isinstance(value, list):
raise RheaError("Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, raise_type))
# If we are here the value must be a list
result = []
for v in value:
if isinstance(v, six.string_types):
try:
result.append(type_convert(v))
except ValueError:
raise RheaError(
"Cannot convert value `{}` (found in list key: `{}`) "
"to `{}`".format(v, key, raise_type))
elif isinstance(v, target_type):
result.append(v)
else:
raise RheaError(
"Cannot convert value `{}` (found in list key: `{}`) "
"to `{}`".format(v, key, raise_type))
return result
def _read_from_json(f_path, is_stream=False):
if is_stream:
try:
return json.loads(f_path)
except ValueError as e:
raise RheaError(e)
try:
return json.loads(open(f_path).read())
except ValueError as e:
raise RheaError(e)
def parse_gcs_path(gcs_path):
"""
Parses and validates a google cloud storage url.
Returns:
tuple(bucket_name, blob).
"""
parsed_url = urllib.parse.urlparse(gcs_path)
if not parsed_url.netloc:
raise RheaError('Received an invalid GCS url `{}`'.format(gcs_path))
if parsed_url.scheme != 'gs':
raise RheaError('Received an invalid url GCS `{}`'.format(gcs_path))
blob = parsed_url.path.lstrip('/')
return GCSSpec(parsed_url.netloc, blob)
def _get_typed_value(self,
key,
target_type,
type_convert,
is_optional=False,
is_secret=False,
is_local=False,
default=None,
options=None):
"""
Return the value corresponding to the key converted to the given type.
Args:
key: the dict key.
target_type: The type we expect the variable or key to be in.
type_convert: A lambda expression that converts the key to the desired type.
is_optional: To raise an error if key was not found.
is_secret: If the key is a secret.
is_local: If the key is a local to this service.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
Returns:
The corresponding value of the key converted.
"""
try:
value = self._get(key)
except KeyError:
if not is_optional:
raise RheaError(
'No value was provided for the non optional key `{}`.'.
format(key))
return default
if isinstance(value, six.string_types):
try:
self._add_key(key, is_secret=is_secret, is_local=is_local)
self._check_options(key=key, value=value, options=options)
return type_convert(value)
except ValueError:
raise RheaError("Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, target_type))
if isinstance(value, target_type):
self._add_key(key, is_secret=is_secret, is_local=is_local)
self._check_options(key=key, value=value, options=options)
return value
raise RheaError("Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, target_type))
def parse_uri_spec(uri_spec):
parts = uri_spec.split('@')
if len(parts) != 2:
raise RheaError(
'Received invalid uri_spec `{}`. '
'The uri must be in the format `user:pass@host`'.format(uri_spec))
user_pass, host = parts
user_pass = user_pass.split(':')
if len(user_pass) != 2:
raise RheaError(
'Received invalid uri_spec `{}`. `user:host` is not conform.'
'The uri must be in the format `user:pass@host`'.format(uri_spec))
return UriSpec(user=user_pass[0], password=user_pass[1], host=host)
def convert_to_dict(x):
x = json.loads(x)
if not isinstance(x, Mapping):
raise RheaError(
"Cannot convert value `{}` (key: `{}`) to `dict`".format(
x, key))
return x
def _read_from_yml(f_path, is_stream=False):
try:
if is_stream:
return yaml.safe_load(f_path)
with open(f_path) as f:
return yaml.safe_load(f)
except (ScannerError, ParserError):
raise RheaError('Received non valid yaml: `{}`'.format(f_path))
def check_type(self):
type_check = (self.config_type is None
and not isinstance(self.value,
(Mapping, six.string_types)))
if type_check:
raise RheaError(
"Expects Mapping, string, or list of Mapping/string instances, "
"received {} instead".format(type(self.value)))
def parse_auth_spec(auth_spec):
user_pass = auth_spec.split(':')
if len(user_pass) != 2:
raise RheaError(
'Received invalid uri_spec `{}`. `user:host` is not conform.'
'The uri must be in the format `user:pass`'.format(auth_spec))
return AuthSpec(user=user_pass[0], password=user_pass[1])
def _read_from_file(f_path, file_type):
_, ext = os.path.splitext(f_path)
if ext in ('.yml', '.yaml') or file_type in ('.yml', '.yaml'):
return _read_from_yml(f_path)
elif ext == '.json' or file_type == '.json':
return _read_from_json(f_path)
raise RheaError(
"Expects a file with extension: `.yml`, `.yaml`, or `json`, "
"received instead `{}`".format(ext))
def get_dict(self,
key,
is_list=False,
is_optional=False,
is_secret=False,
is_local=False,
default=None,
options=None):
"""
Get a the value corresponding to the key and converts it to `dict`.
Args:
key: the dict key.
is_list: If this is one element or a list of elements.
is_optional: To raise an error if key was not found.
is_secret: If the key is a secret.
is_local: If the key is a local to this service.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
Returns:
`str`: value corresponding to the key.
"""
def convert_to_dict(x):
x = json.loads(x)
if not isinstance(x, Mapping):
raise RheaError(
"Cannot convert value `{}` (key: `{}`) to `dict`".format(
x, key))
return x
if is_list:
return self._get_typed_list_value(key=key,
target_type=Mapping,
type_convert=convert_to_dict,
is_optional=is_optional,
is_secret=is_secret,
is_local=is_local,
default=default,
options=options)
value = self._get_typed_value(key=key,
target_type=Mapping,
type_convert=convert_to_dict,
is_optional=is_optional,
is_secret=is_secret,
is_local=is_local,
default=default,
options=options)
if not value:
return default
if not isinstance(value, Mapping):
raise RheaError("Cannot convert value `{}` (key: `{}`) "
"to `dict`".format(value, key))
return value
def read(config_values):
"""Reads an ordered list of configuration values and deep merge the values in reverse order."""
if not config_values:
raise RheaError('Cannot read config_value: `{}`'.format(config_values))
config_values = to_list(config_values)
config = {}
for config_value in config_values:
config_value = ConfigSpec.get_from(value=config_value)
config_value.check_type()
config_results = config_value.read()
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
elif config_value.check_if_exists:
raise RheaError(
'Cannot read config_value: `{}`'.format(config_value))
return config
def read(self):
if isinstance(self.value, Mapping):
config_results = self.value
elif os.path.isfile(self.value):
config_results = _read_from_file(self.value, self.config_type)
else:
# try reading a stream of yaml or json
try:
config_results = _read_from_stream(self.value)
except (ScannerError, ParserError):
raise RheaError('Received non valid yaml stream: `{}`'.format(
self.value))
return config_results
def parse_s3_path(s3_path):
"""
Parses and validates an S3 url.
Returns:
tuple(bucket_name, key).
"""
parsed_url = urllib.parse.urlparse(s3_path)
if not parsed_url.netloc:
raise RheaError('Received an invalid S3 url `{}`'.format(s3_path))
else:
bucket_name = parsed_url.netloc
key = parsed_url.path.strip('/')
return S3Spec(bucket_name, key)
def parse_wasbs_path(wasbs_path):
parsed_url = urllib.parse.urlparse(wasbs_path)
match = re.match("([^@]+)@([^.]+)\\.blob\\.core\\.windows\\.net",
parsed_url.netloc)
if match is None:
raise RheaError(
'wasbs url must be of the form <container>@<account>.blob.core.windows.net'
)
container = match.group(1)
storage_account = match.group(2)
path = parsed_url.path
if path.startswith('/'):
path = path[1:]
return WasbsSpec(container, storage_account, path)
def get_dict_of_dicts(self,
key,
is_optional=False,
is_secret=False,
is_local=False,
default=None,
options=None):
"""
Get a the value corresponding to the key and converts it to `dict`.
Add an extra validation that all keys have a dict as values.
Args:
key: the dict key.
is_optional: To raise an error if key was not found.
is_secret: If the key is a secret.
is_local: If the key is a local to this service.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
Returns:
`str`: value corresponding to the key.
"""
value = self.get_dict(
key=key,
is_optional=is_optional,
is_secret=is_secret,
is_local=is_local,
default=default,
options=options,
)
if not value:
return default
for k in value:
if not isinstance(value[k], Mapping):
raise RheaError(
"`{}` must be an object. "
"Received a non valid configuration for key `{}`.".format(
value[k], key))
return value
def _check_options(key, value, options):
if options and value not in options:
raise RheaError('The value `{}` provided for key `{}` '
'is not one of the possible values.'.format(
value, key))
