def from_yaml(self, node): ''' Implementes a !from_yaml constructor with the following syntax: !from_yaml filename key Arguments: filename: Filename of external YAML document from which to load, relative to the current YAML file. key: Key from external YAML document to return, using a dot-separated syntax for nested keys. Examples: !from_yaml external.yml pop !from_yaml external.yml foo.bar.pop !from_yaml "another file.yml" "foo bar.snap crackle.pop" ''' # Load the content from the node, as a scalar content = self.construct_scalar(node) # Split on unquoted spaces try: parts = shlex.split(content) except UnicodeEncodeError: raise yaml.YAMLError( 'Non-ASCII arguments to !from_yaml are unsupported') if len(parts) != 2: raise yaml.YAMLError('Two arguments expected to !from_yaml') filename, key = parts # path is relative to the current YAML document path = os.path.join(self._root, filename) # Load the other YAML document doc = self._cache.get(path) if not doc: with open(path, 'r') as f: doc = yaml.load(f, self.__class__) self._cache[path] = doc # Retrieve the key try: cur = doc # Use a negative look-behind to split the key on non-escaped '.' characters for k in re.split(r'(?<!\\)\.', key): cur = cur[k.replace( '\\.', '.' )] # Be sure to replace any escaped '.' characters with *just* the '.' except KeyError: raise yaml.YAMLError('Key "{}" not found in {}'.format( key, filename)) return cur
def include_tag(loader, node): """ Provides simplistic include support for YAML. """ current_file = path(loader.stream.name) include_file = current_file.parent.join(node.value) if not include_file.exists: raise yaml.YAMLError("'%s' does not exist" % include_file) _include_edges[current_file.absolute].append(include_file.absolute) for edges in tarjan.tarjan(_include_edges): if len(edges) > 1: raise yaml.YAMLError("circular dependency detected between %r" % edges) return yaml.load(include_file.open())
def include_merge_constructor(self, node: yaml.Node = None) -> (list, dict): """ !include <file|folder> ... Merges file or folder contents This constructor only works if all the files' contents are of same type and if this type is either list or dict. """ paths = node.value if isinstance(paths, str): paths = paths.split(" ") elif not isinstance(paths, list): raise TypeError(f"paths must be either of type str or list") paths = [ resolve_path(path, file_path=os.path.dirname(self.name), config_dir=self.cfg_folder) for path in paths ] files = set() for path in paths: if os.path.isfile(path): files.add(path) elif os.path.isdir(path): for file in os.listdir(path): if file.endswith(".yaml"): files.add(os.path.join(path, file)) loaded_files = [ self.__class__.load(open(file, "r"), cfg_folder=self.cfg_folder) for file in files ] if not all( isinstance(loaded_file, type(loaded_files[0])) for loaded_file in loaded_files): raise yaml.YAMLError(f"Cannot join {files}, they are not all " f"of type {type(loaded_files[0]).__name__}") if isinstance(loaded_files[0], list): return list(itertools.chain(*loaded_files)) if isinstance(loaded_files[0], dict): return dict( itertools.chain( *[loaded_file.items() for loaded_file in loaded_files])) raise yaml.YAMLError( f"Unmergable type: {type(loaded_files[0]).__name__}")
def _resolve_reuse_reference_constructor(self, loader, node): import six if not isinstance(node.value, six.string_types): raise yaml.YAMLError( 'Cannot resolve reference {} because the value is not ' 'a string.'.format(node)) if node.value in self.reusables: return self.reusables[node.value] else: raise yaml.YAMLError( 'Cannot resolve reference {} because no reusable ' 'data has been defined with the name "{}". Please double ' 'check the reference name or the reusables file "{}".'.format( node, node.value, self.reusables_path))
def load_yaml(self, config_path: str): with open(config_path, 'r') as stream: try: yaml_config = yaml.safe_load(stream) assert yaml_config is not None Utils.logger.info(f'Loaded config {yaml_config}') Utils.logger.info('Verifying..') self.verify_yaml(yaml_config) Utils.logger.info('Verification success.') return yaml_config except yaml.YAMLError as e: Utils.logger.error( 'Failed to load config file. Check its path.') raise yaml.YAMLError(e) except KeyError as e: Utils.logger.error('Config file is invalid. Missing field:') Utils.logger.error(e) raise AssertionError(e) except AssertionError: Utils.logger.error('Config file has empty fields.') raise AssertionError
def test_cli_fail(): runner = CliRunner() with mock.patch("piny.loaders.yaml.load") as loader_mock: loader_mock.side_effect = yaml.YAMLError("Oops!") result = runner.invoke(cli, input="password: ${DB_PASSWORD}") assert result.exit_code == 1 assert type(result.exception) == LoadingError
def multi_constructor_import(loader, tag_suffix, node): """ Callback used by PyYAML when a "!import:" tag is encountered. """ if '.' not in tag_suffix: raise yaml.YAMLError("!import: tag suffix contains no '.'") return try_to_import(tag_suffix)
def test_yaml_load_load_error(self, mock_log, mock_load, mock_open): mock_load.side_effect = yaml.YAMLError("yaml errmsg") with self.assertRaisesRegexp(yaml.YAMLError, "yaml errmsg"): config.yaml_load("fakepath") assert mock_open.called assert mock_load.called assert mock_log.called
def read(path, create=False) -> 'Repository': """ Creates an instance of Repository from the given file path. :param path: The file path from which the repository is read. :param create: If true, a new repository will be returned if it was not found at the given path. :return: A new Repository instance. """ try: stream = open(path, 'rt') except FileNotFoundError: if create: _logger.info("New repository created") return Repository.new() else: raise else: with stream: try: _logger.debug("Loading YAML repository '%s'", os.path.abspath(path)) r = yaml.load(stream) except (ValueError, yaml.YAMLError): raise RuntimeError( "Repository '{0}' was not valid".format(path)) if not isinstance(r, Repository): raise yaml.YAMLError( "YAML file '{0}' did not contain a repository".format( path)) return r
def __init__(self): self.checkpoints = dict() rospack = rospkg.RosPack() traj_path = rospack.get_path('bender_arm')+'/config/predefined_trajectories/' traj = listdir(traj_path) for t in traj: # genera los checkpoints pos = t.split('.') # checkea si existe el nodo, si no, crea uno nuevo if pos[0] not in self.checkpoints.keys(): self.checkpoints[pos[0]] = Node(pos[0]) traj_data = None try: with open(traj_path+t, 'r') as f: # load all documents traj_data = yaml.load(f) if traj_data is None: raise yaml.YAMLError("Empty files not allowed") # rellena los checkpoints con las trajectorias predefinidas # print traj_data['header'] traj = JointTrajectory() genpy.message.fill_message_args(traj,traj_data) self.checkpoints[pos[0]].add_trayectory(pos[1],traj) except yaml.YAMLError as e: rospy.logerr('Invalid YAML file: %s' % (str(e)))
def query_maintenance_information(self): ucr.load() if ucr.is_true('license/extended_maintenance/disable_warning'): return {'show_warning': False} version = self.uu.get_ucs_version() try: url = 'http://updates.software-univention.de/download/ucs-maintenance/{}.yaml'.format( version) response = requests.get(url, timeout=10) if not response.ok: response.raise_for_status() status = yaml.load(response.content) if not isinstance(status, dict): raise yaml.YAMLError(repr(status)) # the yaml file contains for maintained either false, true or extended as value. # yaml.load converts true and false into booleans but extended into string. _maintained_status = status.get('maintained') maintenance_extended = _maintained_status == 'extended' show_warning = maintenance_extended or not _maintained_status except yaml.YAMLError as exc: MODULE.error('The YAML format is malformed: %s' % (exc, )) return {'show_warning': False} except requests.exceptions.RequestException as exc: MODULE.error("Querying maintenance information failed: %s" % (exc, )) return {'show_warning': False} return { 'ucs_version': version, 'show_warning': show_warning, 'maintenance_extended': maintenance_extended, 'base_dn': ucr.get('license/base') }
def get_yaml_val(yaml_file: str, *strings: str) -> dict: import yaml from string import punctuation return_dict = {} for string in strings: sep_string = "".join( [" " if char in punctuation else char for char in string]).split() with open(yaml_file, encoding="utf-8") as config: data = yaml.load(config, Loader=yaml.UnsafeLoader) for key in sep_string: if key in data: data = data[key] if key == sep_string[-1]: return_dict[key] = data else: raise yaml.YAMLError(f'Key "{key}" cannot be found') return return_dict
def test_broken_schemas(self): """ Test failure modes for the schema loading """ # Case 1: the file is not present schema = loader.Schema.from_file('doesnt.exists') self.assertListEqual(sorted(schema.entities.keys()), ['node', 'service']) self.assertFalse(schema.has_errors) # Case 2: broken file (that means the file includes invalid data) schema = loader.Schema.from_file( os.path.join(test_base, 'fixtures', 'broken_schema.yaml')) self.assertListEqual(sorted(schema.entities.keys()), ['node', 'pony', 'service', 'unicorn']) self.assertTrue(schema.has_errors) # Case 3: invalid yaml with mock.patch('conftool.yaml.safe_load') as mocker: mocker.side_effect = yaml.YAMLError('something unexpected') schema = loader.Schema.from_file(self.schema_file) self.assertTrue(schema.has_errors) # Case 4: generic exception is *not* handled with mock.patch('conftool.yaml.safe_load') as mocker: mocker.side_effect = Exception('something unexpected') self.assertRaises(Exception, loader.Schema.from_file, self.schema_file)
def test_invalid_run_malformed_input(mock_method, runner): mock_method.side_effect = yaml.YAMLError('error') results = runner.invoke( teflo, ['run', '-s', '../assets/descriptor.yml', '-d', '/tmp'] ) assert 'Error loading scenario data!' in results.output assert results.exit_code != 0
def _get_tissue_types(): yaml_file_url = SchemaConstants.TISSUE_TYPES_YAML # Function cache to improve performance response = schema_manager.make_request_get(yaml_file_url) if response.status_code == 200: yaml_file = response.text try: tissue_types_dict = yaml.safe_load(response.text) # We don't need the description here, just a list of tissue types # Note: dict.keys() returns a dict, need to typecast to list tissue_types_list = list(tissue_types_dict.keys()) # Add the 'other' tissue_types_list.append('other') return tissue_types_list except yaml.YAMLError as e: raise yaml.YAMLError(e) else: msg = f"Unable to fetch the: {yaml_file_url}" # Log the full stack trace, prepend a line with our message logger.exception(msg) logger.debug("======_get_tissue_types() status code======") logger.debug(response.status_code) logger.debug("======_get_tissue_types() response text======") logger.debug(response.text) # Also bubble up the error message raise requests.exceptions.RequestException(response.text)
def test_user_config_value_error(mock_open): """Test the behavior when Config base file throws a YAMLError.""" config_file = os.path.join(os.path.dirname(__file__), "rsc", "config.yaml") mock_open.side_effect = yaml.YAMLError("error") with pytest.raises(yaml.YAMLError): Config(config_file)
def _read_yaml_file(file: str) -> dict: with open(file, 'r') as stream: try: return yaml.load(stream, Loader=yaml.FullLoader) or {} except yaml.YAMLError as exc: raise yaml.YAMLError('Wrong YAML format for file ' + file + ' : ' + str(exc))
def _read_json_file(file: str) -> dict: with open(file, 'r') as stream: try: return json.load(stream) or {} except yaml.YAMLError as exc: raise yaml.YAMLError('Wrong YAML format for file ' + file + ' : ' + str(exc))
def yaml_load(f: Union[str, IO[str]]) -> Any: """Wrapper over yaml.load using the C loader if possible.""" start = datetime.datetime.now() # WORKAROUND for https://github.com/yaml/pyyaml/pull/181 with log.py_warning_filter( category=DeprecationWarning, message=r"Using or importing the ABCs from 'collections' instead " r"of from 'collections\.abc' is deprecated.*"): try: data = yaml.load(f, Loader=YamlLoader) except ValueError as e: if str(e).startswith('could not convert string to float'): # WORKAROUND for https://github.com/yaml/pyyaml/issues/168 raise yaml.YAMLError(e) raise # pragma: no cover end = datetime.datetime.now() delta = (end - start).total_seconds() deadline = 10 if 'CI' in os.environ else 2 if delta > deadline: # pragma: no cover log.misc.warning( "YAML load took unusually long, please report this at " "https://github.com/qutebrowser/qutebrowser/issues/2777\n" "duration: {}s\n" "PyYAML version: {}\n" "C extension: {}\n" "Stack:\n\n" "{}".format(delta, yaml.__version__, YAML_C_EXT, ''.join(traceback.format_stack()))) return data
def read_articles(fname): """ Read a list of articles from the specified directory. Arguments: path (Path) : Path to read from. Raises: FileNotFoundError: If the file does not exist. yaml.YAMLError : If the file is not valid YAML, or it isn't in the appropriate format for Cygnet. """ if not fname.is_file(): raise FileNotFoundError(f"The file {fname} does not exist.") # Read it in with open(fname, "r") as fp: article_dicts = list(yaml.safe_load_all(fp)) # Convert to Article instances. This implicitly validates the data. try: articles = [Article(**d) for d in article_dicts] except TypeError: raise yaml.YAMLError(f"The file {fname} did not contain " "articles in the correct format.") return articles
def read_yaml_file( filepath="/ect/cobbler/settings.yaml") -> Dict[Hashable, Any]: """ Reads settings files from ``filepath`` and all paths in `include` (which is read from the settings file) and saves the content in a dictionary. Any key may be overwritten in a later loaded settings file. The last loaded file wins. :param filepath: Settings file path, defaults to "/ect/cobbler/settings.yaml" :raises FileNotFoundError: In case file does not exist or is a directory. :raises yaml.YAMLError: In case the file is not a valid YAML file. :return: The aggregated dict of all settings. """ if not os.path.isfile(filepath): raise FileNotFoundError( 'Given path "%s" does not exist or is a directory.' % filepath) try: with open(filepath) as main_settingsfile: filecontent = yaml.safe_load(main_settingsfile.read()) for ival in filecontent.get("include", []): for ifile in glob.glob(ival): with open(ifile, 'r') as extra_settingsfile: filecontent.update( yaml.safe_load(extra_settingsfile.read())) except yaml.YAMLError as error: traceback.print_exc() raise yaml.YAMLError('"%s" is not a valid YAML file' % filepath) from error return filecontent
def from_data(cls, name: Text, state_data: Dict[Text, Any], domain_actions: Set[Text]) -> 'StateNode': """Validate information from dialogue.yml before creating node object. Args: name: name of the state state_data: A dict containing all the info for the state domain_actions: A list of action names that are valid in the domain Returns: StateNode: class object """ if not name: raise yaml.YAMLError( "Invalid dialogue yaml file: missing state name in dialogue.yml" ) for key in state_data.keys(): if key not in [ 'conditions', 'actions', 'connections', 'direct_connection', 'rank_score' ]: raise yaml.YAMLError( f"There is a key error in state {name}: {key}") direct_con = state_data.get('direct_connection', False) if type(direct_con) is not bool: raise yaml.YAMLError( f"Error in state {name}: The value of direct_connection must be a bool" ) try: score = state_data.get('rank_score', DEFAULT_RANK_SCORE) except ValueError: raise yaml.YAMLError( f"Error in state {name}: Value of rank_score must be an int or float" ) conditions = state_data.get('conditions', []) StateNode.check_conditions(name, conditions) actions = state_data.get('actions', []) StateNode.check_actions(name, actions, domain_actions) connections = state_data.get('connections', []) return cls(name, conditions, actions, direct_con, connections, score)
def from_yaml(self, node): ''' Implementes a !from_yaml constructor with the following syntax: !from_yaml filename key Arguments: filename: Filename of external YAML document from which to load, relative to the current YAML file. key: Key from external YAML document to return, using a dot-separated syntax for nested keys. Examples: !from_yaml external.yml pop !from_yaml external.yml foo.bar.pop !from_yaml "another file.yml" "foo bar.snap crackle.pop" ''' # Load the content from the node, as a scalar content = self.construct_scalar(node) # Split on unquoted spaces try: parts = shlex.split(content) except UnicodeEncodeError: raise yaml.YAMLError( 'Non-ASCII arguments to !from_yaml are unsupported') if len(parts) != 2: raise yaml.YAMLError('Two arguments expected to !from_yaml') filename, key = parts # path is relative to the current YAML document path = os.path.join(self._root, filename) # Load the other YAML document with open(path, 'r') as f: doc = yaml.load(f, self.__class__) # Retrieve the key try: cur = doc for k in key.split('.'): cur = cur[k] except KeyError: raise yaml.YAMLError('Key "{}" not found in {}'.format( key, filename)) return cur
def load(file): with open(file, 'r') as stream: try: result = yaml.load(stream) except yaml.YAMLError as exc: raise yaml.YAMLError(exc) return result
def get_organ_types_dict(self) -> object: yaml_file_url = 'https://raw.githubusercontent.com/hubmapconsortium/search-api/master/src/search-schema/data/definitions/enums/organ_types.yaml' with urllib.request.urlopen(yaml_file_url) as response: yaml_file = response.read() try: return yaml.safe_load(yaml_file) except yaml.YAMLError as e: raise yaml.YAMLError(e)
def _read_yaml_file(file: str) -> dict: with open(file, 'r') as stream: try: return yaml.load(stream) or {} except yaml.YAMLError as exc: err = 'Wrong YAML format for file ' + file + ' : ' + str(exc) error(err) raise yaml.YAMLError(err)
def test_get_user_levels_value_error(mock_open): """Test the behavior when Config user file throws a YAMLError.""" user_config_file = os.path.join(os.path.dirname(__file__), "rsc", "user.yaml") config = Config("") mock_open.side_effect = yaml.YAMLError("error") with pytest.raises(yaml.YAMLError): config.get_user_levels(user_config_file)
def test_bad_reader(self): """Test that reader not existing causes an error.""" from satpy.readers import group_files import yaml # touch the file so it exists on disk with mock.patch('yaml.load') as load: load.side_effect = yaml.YAMLError("Import problems") self.assertRaises(yaml.YAMLError, group_files, [], reader='abi_l1b')
def multi_constructor_import(loader, tag_suffix, node): yaml_src = yaml.serialize(node) mapping = loader.construct_mapping(node) if '.' not in tag_suffix: raise yaml.YAMLError("import tag suffix contains no '.'") else: rval = try_to_import(tag_suffix) return rval
def test_reader_load_failed(self): """Test that an exception is raised when a reader can't be loaded.""" from satpy.readers import find_files_and_readers import yaml # touch the file so it exists on disk with mock.patch('yaml.load') as load: load.side_effect = yaml.YAMLError("Import problems") self.assertRaises(yaml.YAMLError, find_files_and_readers, reader='viirs_sdr')