def from_yaml(self, node):
'''
Implementes a !from_yaml constructor with the following syntax:
!from_yaml filename key
Arguments:
filename: Filename of external YAML document from which to load,
relative to the current YAML file.
key: Key from external YAML document to return,
using a dot-separated syntax for nested keys.
Examples:
!from_yaml external.yml pop
!from_yaml external.yml foo.bar.pop
!from_yaml "another file.yml" "foo bar.snap crackle.pop"
'''
# Load the content from the node, as a scalar
content = self.construct_scalar(node)
# Split on unquoted spaces
try:
parts = shlex.split(content)
except UnicodeEncodeError:
raise yaml.YAMLError(
'Non-ASCII arguments to !from_yaml are unsupported')
if len(parts) != 2:
raise yaml.YAMLError('Two arguments expected to !from_yaml')
filename, key = parts
# path is relative to the current YAML document
path = os.path.join(self._root, filename)
# Load the other YAML document
doc = self._cache.get(path)
if not doc:
with open(path, 'r') as f:
doc = yaml.load(f, self.__class__)
self._cache[path] = doc
# Retrieve the key
try:
cur = doc
# Use a negative look-behind to split the key on non-escaped '.' characters
for k in re.split(r'(?<!\\)\.', key):
cur = cur[k.replace(
'\\.', '.'
)] # Be sure to replace any escaped '.' characters with *just* the '.'
except KeyError:
raise yaml.YAMLError('Key "{}" not found in {}'.format(
key, filename))
return cur
def include_tag(loader, node):
"""
Provides simplistic include support for YAML.
"""
current_file = path(loader.stream.name)
include_file = current_file.parent.join(node.value)
if not include_file.exists:
raise yaml.YAMLError("'%s' does not exist" % include_file)
_include_edges[current_file.absolute].append(include_file.absolute)
for edges in tarjan.tarjan(_include_edges):
if len(edges) > 1:
raise yaml.YAMLError("circular dependency detected between %r" %
edges)
return yaml.load(include_file.open())
def include_merge_constructor(self,
node: yaml.Node = None) -> (list, dict):
"""
!include <file|folder> ...
Merges file or folder contents
This constructor only works if all the files' contents are of same type
and if this type is either list or dict.
"""
paths = node.value
if isinstance(paths, str):
paths = paths.split(" ")
elif not isinstance(paths, list):
raise TypeError(f"paths must be either of type str or list")
paths = [
resolve_path(path,
file_path=os.path.dirname(self.name),
config_dir=self.cfg_folder) for path in paths
]
files = set()
for path in paths:
if os.path.isfile(path):
files.add(path)
elif os.path.isdir(path):
for file in os.listdir(path):
if file.endswith(".yaml"):
files.add(os.path.join(path, file))
loaded_files = [
self.__class__.load(open(file, "r"), cfg_folder=self.cfg_folder)
for file in files
]
if not all(
isinstance(loaded_file, type(loaded_files[0]))
for loaded_file in loaded_files):
raise yaml.YAMLError(f"Cannot join {files}, they are not all "
f"of type {type(loaded_files[0]).__name__}")
if isinstance(loaded_files[0], list):
return list(itertools.chain(*loaded_files))
if isinstance(loaded_files[0], dict):
return dict(
itertools.chain(
*[loaded_file.items() for loaded_file in loaded_files]))
raise yaml.YAMLError(
f"Unmergable type: {type(loaded_files[0]).__name__}")
def _resolve_reuse_reference_constructor(self, loader, node):
import six
if not isinstance(node.value, six.string_types):
raise yaml.YAMLError(
'Cannot resolve reference {} because the value is not '
'a string.'.format(node))
if node.value in self.reusables:
return self.reusables[node.value]
else:
raise yaml.YAMLError(
'Cannot resolve reference {} because no reusable '
'data has been defined with the name "{}". Please double '
'check the reference name or the reusables file "{}".'.format(
node, node.value, self.reusables_path))
def load_yaml(self, config_path: str):
with open(config_path, 'r') as stream:
try:
yaml_config = yaml.safe_load(stream)
assert yaml_config is not None
Utils.logger.info(f'Loaded config {yaml_config}')
Utils.logger.info('Verifying..')
self.verify_yaml(yaml_config)
Utils.logger.info('Verification success.')
return yaml_config
except yaml.YAMLError as e:
Utils.logger.error(
'Failed to load config file. Check its path.')
raise yaml.YAMLError(e)
except KeyError as e:
Utils.logger.error('Config file is invalid. Missing field:')
Utils.logger.error(e)
raise AssertionError(e)
except AssertionError:
Utils.logger.error('Config file has empty fields.')
raise AssertionError
def test_cli_fail():
runner = CliRunner()
with mock.patch("piny.loaders.yaml.load") as loader_mock:
loader_mock.side_effect = yaml.YAMLError("Oops!")
result = runner.invoke(cli, input="password: ${DB_PASSWORD}")
assert result.exit_code == 1
assert type(result.exception) == LoadingError
def multi_constructor_import(loader, tag_suffix, node):
"""
Callback used by PyYAML when a "!import:" tag is encountered.
"""
if '.' not in tag_suffix:
raise yaml.YAMLError("!import: tag suffix contains no '.'")
return try_to_import(tag_suffix)
def test_yaml_load_load_error(self, mock_log, mock_load, mock_open):
mock_load.side_effect = yaml.YAMLError("yaml errmsg")
with self.assertRaisesRegexp(yaml.YAMLError, "yaml errmsg"):
config.yaml_load("fakepath")
assert mock_open.called
assert mock_load.called
assert mock_log.called
def read(path, create=False) -> 'Repository':
"""
Creates an instance of Repository from the given file path.
:param path: The file path from which the repository is read.
:param create: If true, a new repository will be returned if it was not found at the given path.
:return: A new Repository instance.
"""
try:
stream = open(path, 'rt')
except FileNotFoundError:
if create:
_logger.info("New repository created")
return Repository.new()
else:
raise
else:
with stream:
try:
_logger.debug("Loading YAML repository '%s'",
os.path.abspath(path))
r = yaml.load(stream)
except (ValueError, yaml.YAMLError):
raise RuntimeError(
"Repository '{0}' was not valid".format(path))
if not isinstance(r, Repository):
raise yaml.YAMLError(
"YAML file '{0}' did not contain a repository".format(
path))
return r
def __init__(self):
self.checkpoints = dict()
rospack = rospkg.RosPack()
traj_path = rospack.get_path('bender_arm')+'/config/predefined_trajectories/'
traj = listdir(traj_path)
for t in traj:
# genera los checkpoints
pos = t.split('.')
# checkea si existe el nodo, si no, crea uno nuevo
if pos[0] not in self.checkpoints.keys():
self.checkpoints[pos[0]] = Node(pos[0])
traj_data = None
try:
with open(traj_path+t, 'r') as f:
# load all documents
traj_data = yaml.load(f)
if traj_data is None:
raise yaml.YAMLError("Empty files not allowed")
# rellena los checkpoints con las trajectorias predefinidas
# print traj_data['header']
traj = JointTrajectory()
genpy.message.fill_message_args(traj,traj_data)
self.checkpoints[pos[0]].add_trayectory(pos[1],traj)
except yaml.YAMLError as e:
rospy.logerr('Invalid YAML file: %s' % (str(e)))
def query_maintenance_information(self):
ucr.load()
if ucr.is_true('license/extended_maintenance/disable_warning'):
return {'show_warning': False}
version = self.uu.get_ucs_version()
try:
url = 'http://updates.software-univention.de/download/ucs-maintenance/{}.yaml'.format(
version)
response = requests.get(url, timeout=10)
if not response.ok:
response.raise_for_status()
status = yaml.load(response.content)
if not isinstance(status, dict):
raise yaml.YAMLError(repr(status))
# the yaml file contains for maintained either false, true or extended as value.
# yaml.load converts true and false into booleans but extended into string.
_maintained_status = status.get('maintained')
maintenance_extended = _maintained_status == 'extended'
show_warning = maintenance_extended or not _maintained_status
except yaml.YAMLError as exc:
MODULE.error('The YAML format is malformed: %s' % (exc, ))
return {'show_warning': False}
except requests.exceptions.RequestException as exc:
MODULE.error("Querying maintenance information failed: %s" %
(exc, ))
return {'show_warning': False}
return {
'ucs_version': version,
'show_warning': show_warning,
'maintenance_extended': maintenance_extended,
'base_dn': ucr.get('license/base')
}
def get_yaml_val(yaml_file: str, *strings: str) -> dict:
import yaml
from string import punctuation
return_dict = {}
for string in strings:
sep_string = "".join(
[" " if char in punctuation else char for char in string]).split()
with open(yaml_file, encoding="utf-8") as config:
data = yaml.load(config, Loader=yaml.UnsafeLoader)
for key in sep_string:
if key in data:
data = data[key]
if key == sep_string[-1]:
return_dict[key] = data
else:
raise yaml.YAMLError(f'Key "{key}" cannot be found')
return return_dict
def test_broken_schemas(self):
"""
Test failure modes for the schema loading
"""
# Case 1: the file is not present
schema = loader.Schema.from_file('doesnt.exists')
self.assertListEqual(sorted(schema.entities.keys()),
['node', 'service'])
self.assertFalse(schema.has_errors)
# Case 2: broken file (that means the file includes invalid data)
schema = loader.Schema.from_file(
os.path.join(test_base, 'fixtures', 'broken_schema.yaml'))
self.assertListEqual(sorted(schema.entities.keys()),
['node', 'pony', 'service', 'unicorn'])
self.assertTrue(schema.has_errors)
# Case 3: invalid yaml
with mock.patch('conftool.yaml.safe_load') as mocker:
mocker.side_effect = yaml.YAMLError('something unexpected')
schema = loader.Schema.from_file(self.schema_file)
self.assertTrue(schema.has_errors)
# Case 4: generic exception is *not* handled
with mock.patch('conftool.yaml.safe_load') as mocker:
mocker.side_effect = Exception('something unexpected')
self.assertRaises(Exception, loader.Schema.from_file,
self.schema_file)
def test_invalid_run_malformed_input(mock_method, runner):
mock_method.side_effect = yaml.YAMLError('error')
results = runner.invoke(
teflo, ['run', '-s', '../assets/descriptor.yml', '-d', '/tmp']
)
assert 'Error loading scenario data!' in results.output
assert results.exit_code != 0
def _get_tissue_types():
yaml_file_url = SchemaConstants.TISSUE_TYPES_YAML
# Function cache to improve performance
response = schema_manager.make_request_get(yaml_file_url)
if response.status_code == 200:
yaml_file = response.text
try:
tissue_types_dict = yaml.safe_load(response.text)
# We don't need the description here, just a list of tissue types
# Note: dict.keys() returns a dict, need to typecast to list
tissue_types_list = list(tissue_types_dict.keys())
# Add the 'other'
tissue_types_list.append('other')
return tissue_types_list
except yaml.YAMLError as e:
raise yaml.YAMLError(e)
else:
msg = f"Unable to fetch the: {yaml_file_url}"
# Log the full stack trace, prepend a line with our message
logger.exception(msg)
logger.debug("======_get_tissue_types() status code======")
logger.debug(response.status_code)
logger.debug("======_get_tissue_types() response text======")
logger.debug(response.text)
# Also bubble up the error message
raise requests.exceptions.RequestException(response.text)
def test_user_config_value_error(mock_open):
"""Test the behavior when Config base file throws a YAMLError."""
config_file = os.path.join(os.path.dirname(__file__), "rsc", "config.yaml")
mock_open.side_effect = yaml.YAMLError("error")
with pytest.raises(yaml.YAMLError):
Config(config_file)
def _read_yaml_file(file: str) -> dict:
with open(file, 'r') as stream:
try:
return yaml.load(stream, Loader=yaml.FullLoader) or {}
except yaml.YAMLError as exc:
raise yaml.YAMLError('Wrong YAML format for file ' + file + ' : ' +
str(exc))
def _read_json_file(file: str) -> dict:
with open(file, 'r') as stream:
try:
return json.load(stream) or {}
except yaml.YAMLError as exc:
raise yaml.YAMLError('Wrong YAML format for file ' + file + ' : ' +
str(exc))
def yaml_load(f: Union[str, IO[str]]) -> Any:
"""Wrapper over yaml.load using the C loader if possible."""
start = datetime.datetime.now()
# WORKAROUND for https://github.com/yaml/pyyaml/pull/181
with log.py_warning_filter(
category=DeprecationWarning,
message=r"Using or importing the ABCs from 'collections' instead "
r"of from 'collections\.abc' is deprecated.*"):
try:
data = yaml.load(f, Loader=YamlLoader)
except ValueError as e:
if str(e).startswith('could not convert string to float'):
# WORKAROUND for https://github.com/yaml/pyyaml/issues/168
raise yaml.YAMLError(e)
raise # pragma: no cover
end = datetime.datetime.now()
delta = (end - start).total_seconds()
deadline = 10 if 'CI' in os.environ else 2
if delta > deadline: # pragma: no cover
log.misc.warning(
"YAML load took unusually long, please report this at "
"https://github.com/qutebrowser/qutebrowser/issues/2777\n"
"duration: {}s\n"
"PyYAML version: {}\n"
"C extension: {}\n"
"Stack:\n\n"
"{}".format(delta, yaml.__version__, YAML_C_EXT,
''.join(traceback.format_stack())))
return data
def read_articles(fname):
"""
Read a list of articles from the specified directory.
Arguments:
path (Path) : Path to read from.
Raises:
FileNotFoundError: If the file does not exist.
yaml.YAMLError : If the file is not valid YAML, or it isn't in the
appropriate format for Cygnet.
"""
if not fname.is_file():
raise FileNotFoundError(f"The file {fname} does not exist.")
# Read it in
with open(fname, "r") as fp:
article_dicts = list(yaml.safe_load_all(fp))
# Convert to Article instances. This implicitly validates the data.
try:
articles = [Article(**d) for d in article_dicts]
except TypeError:
raise yaml.YAMLError(f"The file {fname} did not contain "
"articles in the correct format.")
return articles
def read_yaml_file(
filepath="/ect/cobbler/settings.yaml") -> Dict[Hashable, Any]:
"""
Reads settings files from ``filepath`` and all paths in `include` (which is read from the settings file) and saves
the content in a dictionary.
Any key may be overwritten in a later loaded settings file. The last loaded file wins.
:param filepath: Settings file path, defaults to "/ect/cobbler/settings.yaml"
:raises FileNotFoundError: In case file does not exist or is a directory.
:raises yaml.YAMLError: In case the file is not a valid YAML file.
:return: The aggregated dict of all settings.
"""
if not os.path.isfile(filepath):
raise FileNotFoundError(
'Given path "%s" does not exist or is a directory.' % filepath)
try:
with open(filepath) as main_settingsfile:
filecontent = yaml.safe_load(main_settingsfile.read())
for ival in filecontent.get("include", []):
for ifile in glob.glob(ival):
with open(ifile, 'r') as extra_settingsfile:
filecontent.update(
yaml.safe_load(extra_settingsfile.read()))
except yaml.YAMLError as error:
traceback.print_exc()
raise yaml.YAMLError('"%s" is not a valid YAML file' %
filepath) from error
return filecontent
def from_data(cls, name: Text, state_data: Dict[Text, Any],
domain_actions: Set[Text]) -> 'StateNode':
"""Validate information from dialogue.yml before creating node object.
Args:
name: name of the state
state_data: A dict containing all the info for the state
domain_actions: A list of action names that are valid in the domain
Returns:
StateNode: class object
"""
if not name:
raise yaml.YAMLError(
"Invalid dialogue yaml file: missing state name in dialogue.yml"
)
for key in state_data.keys():
if key not in [
'conditions', 'actions', 'connections',
'direct_connection', 'rank_score'
]:
raise yaml.YAMLError(
f"There is a key error in state {name}: {key}")
direct_con = state_data.get('direct_connection', False)
if type(direct_con) is not bool:
raise yaml.YAMLError(
f"Error in state {name}: The value of direct_connection must be a bool"
)
try:
score = state_data.get('rank_score', DEFAULT_RANK_SCORE)
except ValueError:
raise yaml.YAMLError(
f"Error in state {name}: Value of rank_score must be an int or float"
)
conditions = state_data.get('conditions', [])
StateNode.check_conditions(name, conditions)
actions = state_data.get('actions', [])
StateNode.check_actions(name, actions, domain_actions)
connections = state_data.get('connections', [])
return cls(name, conditions, actions, direct_con, connections, score)
def from_yaml(self, node):
'''
Implementes a !from_yaml constructor with the following syntax:
!from_yaml filename key
Arguments:
filename: Filename of external YAML document from which to load,
relative to the current YAML file.
key: Key from external YAML document to return,
using a dot-separated syntax for nested keys.
Examples:
!from_yaml external.yml pop
!from_yaml external.yml foo.bar.pop
!from_yaml "another file.yml" "foo bar.snap crackle.pop"
'''
# Load the content from the node, as a scalar
content = self.construct_scalar(node)
# Split on unquoted spaces
try:
parts = shlex.split(content)
except UnicodeEncodeError:
raise yaml.YAMLError(
'Non-ASCII arguments to !from_yaml are unsupported')
if len(parts) != 2:
raise yaml.YAMLError('Two arguments expected to !from_yaml')
filename, key = parts
# path is relative to the current YAML document
path = os.path.join(self._root, filename)
# Load the other YAML document
with open(path, 'r') as f:
doc = yaml.load(f, self.__class__)
# Retrieve the key
try:
cur = doc
for k in key.split('.'):
cur = cur[k]
except KeyError:
raise yaml.YAMLError('Key "{}" not found in {}'.format(
key, filename))
return cur
def load(file):
with open(file, 'r') as stream:
try:
result = yaml.load(stream)
except yaml.YAMLError as exc:
raise yaml.YAMLError(exc)
return result
def get_organ_types_dict(self) -> object:
yaml_file_url = 'https://raw.githubusercontent.com/hubmapconsortium/search-api/master/src/search-schema/data/definitions/enums/organ_types.yaml'
with urllib.request.urlopen(yaml_file_url) as response:
yaml_file = response.read()
try:
return yaml.safe_load(yaml_file)
except yaml.YAMLError as e:
raise yaml.YAMLError(e)
def _read_yaml_file(file: str) -> dict:
with open(file, 'r') as stream:
try:
return yaml.load(stream) or {}
except yaml.YAMLError as exc:
err = 'Wrong YAML format for file ' + file + ' : ' + str(exc)
error(err)
raise yaml.YAMLError(err)
def test_get_user_levels_value_error(mock_open):
"""Test the behavior when Config user file throws a YAMLError."""
user_config_file = os.path.join(os.path.dirname(__file__), "rsc", "user.yaml")
config = Config("")
mock_open.side_effect = yaml.YAMLError("error")
with pytest.raises(yaml.YAMLError):
config.get_user_levels(user_config_file)
def test_bad_reader(self):
"""Test that reader not existing causes an error."""
from satpy.readers import group_files
import yaml
# touch the file so it exists on disk
with mock.patch('yaml.load') as load:
load.side_effect = yaml.YAMLError("Import problems")
self.assertRaises(yaml.YAMLError, group_files, [], reader='abi_l1b')
def multi_constructor_import(loader, tag_suffix, node):
yaml_src = yaml.serialize(node)
mapping = loader.construct_mapping(node)
if '.' not in tag_suffix:
raise yaml.YAMLError("import tag suffix contains no '.'")
else:
rval = try_to_import(tag_suffix)
return rval
def test_reader_load_failed(self):
"""Test that an exception is raised when a reader can't be loaded."""
from satpy.readers import find_files_and_readers
import yaml
# touch the file so it exists on disk
with mock.patch('yaml.load') as load:
load.side_effect = yaml.YAMLError("Import problems")
self.assertRaises(yaml.YAMLError, find_files_and_readers, reader='viirs_sdr')
