def read_yaml(content): fix_yaml_loader() replace_environment_variables() yaml_parser = yaml.YAML(typ="safe") yaml_parser.version = "1.2" yaml_parser.unicode_supplementary = True return yaml_parser.load(content)
def assert_graphql_resp_expected(resp_orig, exp_response_orig, query, resp_hdrs={}, skip_if_err_msg=False): # Prepare actual and respected responses so comparison takes into # consideration only the ordering that we care about: resp = collapse_order_not_selset(resp_orig, query) exp_response = collapse_order_not_selset(exp_response_orig, query) matched = equal_CommentedMap(resp, exp_response) if PytestConf.config.getoption("--accept"): print('skipping assertion since we chose to --accept new output') else: yml = yaml.YAML() # https://yaml.readthedocs.io/en/latest/example.html#output-of-dump-as-a-string : dump_str = StringIO() test_output = { # Keep strict received order when displaying errors: 'response': resp_orig, 'expected': exp_response_orig, 'diff': (lambda diff: "(results differ only in their order of keys)" if diff == {} else diff)(stringify_keys( jsondiff.diff(exp_response, resp))), 'query': query } if 'x-request-id' in resp_hdrs: test_output['request id'] = resp_hdrs['x-request-id'] yml.dump(test_output, stream=dump_str) if not skip_if_err_msg: assert matched, '\n' + dump_str.getvalue() elif matched: return resp, matched else: def is_err_msg(msg): return any(msg.get(x) for x in ['error', 'errors']) def as_list(x): return x if isinstance(x, list) else [x] # If it is a batch GraphQL query, compare each individual response separately for (exp, out) in zip(as_list(exp_response), as_list(resp)): matched_ = equal_CommentedMap(exp, out) if is_err_msg(exp): if not matched_: warnings.warn( "Response does not have the expected error message\n" + dump_str.getvalue()) return resp, matched else: assert matched_, '\n' + dump_str.getvalue() return resp, matched # matched always True unless --accept
def loads(self, data: str) -> MutableMapping[str, Any]: """ Deserialize data using this :class:`~.Serializer`. :param data: """ y = yaml.YAML(typ="safe", pure=True) return y.load(str(data))
def run_local_topology(name=None, env_name=None, time=0, options=None): """Run a topology locally using Flux and `storm jar`.""" name, topology_file = get_topology_definition(name) config = get_config() env_name, env_config = get_env_config(env_name) topology_class = get_topology_from_file(topology_file) set_topology_serializer(env_config, config, topology_class) storm_options = resolve_options(options, env_config, topology_class, name, local_only=True) if storm_options['topology.acker.executors'] != 0: storm_options['topology.acker.executors'] = 1 storm_options['topology.workers'] = 1 # Set parallelism based on env_name if necessary for spec in topology_class.specs: if isinstance(spec.par, dict): spec.par = spec.par.get(env_name) # Check Storm version is the same local_version = local_storm_version() project_version = storm_lib_version() if local_version != project_version: raise ValueError('Local Storm version, {}, is not the same as the ' 'version in your project.clj, {}. The versions must ' 'match.'.format(local_version, project_version)) # Prepare a JAR that has Storm dependencies packaged topology_jar = jar_for_deploy(simple_jar=False) if time <= 0: time = 9223372036854775807 # Max long value in Java # Write YAML file with show('output'): with NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as yaml_file: topology_flux_dict = topology_class.to_flux_dict(name) topology_flux_dict['config'] = storm_options if yaml.version_info < (0, 15): yaml.safe_dump(topology_flux_dict, yaml_file, default_flow_style=False) else: yml = yaml.YAML(typ='safe', pure=True) yml.default_flow_style = False yml.dump(topology_flux_dict, yaml_file) cmd = ( 'storm jar {jar} org.apache.storm.flux.Flux --local --no-splash ' '--sleep {time} {yaml}'.format(jar=topology_jar, time=time, yaml=yaml_file.name)) local(cmd)
def preview_search(args: Namespace) -> None: experiment_config = yaml.safe_load(args.config_file.read()) args.config_file.close() if "searcher" not in experiment_config: print("Experiment configuration must have 'searcher' section") sys.exit(1) r = api.post(args.master, "searcher/preview", body=experiment_config) j = r.json() def count_steps(sequence: str) -> int: return sum(1 for k in sequence if k == "S") def to_full_name(kind: str) -> str: if kind == "S": return "step" elif kind == "V": return "validation" elif kind == "C": return "checkpoint" else: raise ValueError("unexpected kind: {}".format(kind)) def render_sequence(sequence: str) -> str: if not sequence: return "N/A" instructions = [] current = sequence[0] count = 0 for k in sequence: if k != current: instructions.append("{} x {}".format(count, to_full_name(current))) current = k count = 1 else: count += 1 instructions.append("{} x {}".format(count, to_full_name(current))) return ", ".join(instructions) headers = ["Trials", "Steps", "Breakdown"] values = [ (count, count_steps(workloads), render_sequence(workloads)) for workloads, count in j["results"].items() ] print(colored("Using search configuration:", "green")) yml = yaml.YAML() yml.indent(mapping=2, sequence=4, offset=2) yml.dump(experiment_config["searcher"], sys.stdout) print() print( "This search will create a total of {} trial(s) and run {} steps".format( sum(j["results"].values()), sum(count_steps(workloads) * cnt for workloads, cnt in j["results"].items()), ) ) print(tabulate.tabulate(values, headers, tablefmt="presto"), flush=False)
def process(): with open("../calliope/config/defaults.yaml", "r") as f: defaults = yaml.round_trip_load(f) write_csv( "./user/includes/default_essentials.csv", get_section(defaults["techs"]["default_tech"]["essentials"]), ) write_csv( "./user/includes/default_constraints.csv", get_section(defaults["techs"]["default_tech"]["constraints"]), ) write_csv( "./user/includes/default_costs.csv", get_section( defaults["techs"]["default_tech"]["costs"]["default_cost"]), ) write_csv("./user/includes/model_settings.csv", get_section(defaults["model"])) write_csv("./user/includes/run_settings.csv", get_section(defaults["run"])) y = yaml.YAML() for tech_group in defaults["tech_groups"]: this_group_defaults = { "essentials": defaults["tech_groups"][tech_group].get("essentials", {}), "constraints": defaults["tech_groups"][tech_group].get("constraints", {}), "costs": defaults["tech_groups"][tech_group].get("costs", {}), } with open("./user/includes/basetech_{}.yaml".format(tech_group), "w") as f: f.write(yaml.dump(this_group_defaults, Dumper=yaml.RoundTripDumper)) required_allowed = { "required_constraints": y.seq(defaults["tech_groups"][tech_group].get( "required_constraints", [])), "allowed_constraints": y.seq(defaults["tech_groups"][tech_group].get( "allowed_constraints", [])), "allowed_costs": y.seq(defaults["tech_groups"][tech_group].get("allowed_costs", [])), } with open( "./user/includes/required_allowed_{}.yaml".format(tech_group), "w") as f: f.write( yaml.dump(required_allowed, indent=4, Dumper=yaml.RoundTripDumper))
def test_fileformatyaml_pass_with_substitutions(): """Relative path to file should succeed. Strictly speaking not a unit test. """ context = Context({ 'k1': 'v1', 'k2': 'v2', 'k3': 'v3', 'k4': 'v4', 'k5': 'v5', 'fileFormatYamlIn': './tests/testfiles/testsubst.yaml', 'fileFormatYamlOut': './tests/testfiles/out/outsubst.yaml' }) fileformat.run_step(context) assert context, "context shouldn't be None" assert len(context) == 7, "context should have 7 items" assert context['k1'] == 'v1' assert context['fileFormatYamlIn'] == './tests/testfiles/testsubst.yaml' assert context['fileFormatYamlOut'] == ('./tests/testfiles/out/' 'outsubst.yaml') with open('./tests/testfiles/out/outsubst.yaml') as outfile: yaml_loader = yaml.YAML(typ='rt', pure=True) outcontents = yaml_loader.load(outfile) expected = { 'key': 'v1value1 !£$%# *', 'key2v2': 'blah', # there is a comment here 'key3': [ 'l1', # and another '!£$% * v3', 'l2', ['l31v4', { 'l32': ['l321', 'l322v5'] }] ] } assert outcontents == expected # atrociously lazy test clean-up os.remove('./tests/testfiles/out/outsubst.yaml')
def render_configuration(config): ''' Given a config data structure of nested OrderedDicts, render the config as YAML and return it. ''' dumper = yaml.YAML() dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT) rendered = io.StringIO() dumper.dump(config, rendered) return rendered.getvalue()
def test_standard_ruamel(self): """ Standard yaml_tag method for ruamel. """ yaml_this = yamlr.YAML() # in ruamel we register classes to use yaml_tags yaml_this.register_class(StandardYAMLTag) loaded = yaml_this.load(text_has_anchors_tags) cleaned = recursive_clean(loaded) self.assertEqual(cleaned, {'greeting': 'hello mary'})
def object_to_yaml_str(obj, options=None): if options is None: options = {} string_stream = StringIO() y = yaml.YAML() y.indent(mapping=2, sequence=4, offset=4) y.dump(obj, string_stream, **options) output_str = string_stream.getvalue() string_stream.close() return output_str
def get_yaml_parser_safe(): """Create the safe yaml parser object with this factory method. The safe yaml parser does NOT resolve unknown tags. Returns: ruamel.yaml.YAML object with safe loader """ return yamler.YAML(typ='safe', pure=True)
def load_config_file(self, filename: str): with open(filename) as file: y = yaml.YAML() data = y.load(file) if data is None: raise SetupError( 'No setup configuration found in {}.'.format(filename)) self.load_config(data)
def setUp(self): self.specs_path = 'test_load_namespace.specs.yaml' self.namespace_path = 'test_load_namespace.namespace.yaml' # write basically empty specs file to_dump = {'groups': []} with open(self.specs_path, 'w') as tmp: yaml_obj = yaml.YAML(typ='safe', pure=True) yaml_obj.default_flow_style = False yaml_obj.dump(json.loads(json.dumps(to_dump)), tmp)
def person_list_yaml_dumper(f, person_list, use_omap=False): y = yaml.YAML() if not use_omap: y.representer.add_representer( OrderedDict, yaml.representer.RoundTripRepresenter.represent_dict ) p_list = serializers.person_list_serializer(person_list) for name, person in p_list.items(): y.dump({name: person}, f) print("", file=f)
def __init__(self): if yaml.version_info >= (0,15): self.yaml = yaml.YAML(typ='rt') self.yaml.default_flow_style=False self._load = self.yaml.load self._dump = self.yaml.dump else: self.yaml = yaml self._load = self.yaml.round_trip_load self._dump = self.yaml.round_trip_dump
def load_schema(schema_path): """Load the schema into a dictionary. This function allows the schema, like BIDS itself, to be specified in a hierarchy of directories and files. File names (minus extensions) and directory names become keys in the associative array (dict) of entries composed from content of files and entire directories. Parameters ---------- schema_path : str Folder containing yaml files or yaml file. Returns ------- dict Schema in dictionary form. """ _yaml = yaml.YAML(typ="safe", pure=True) schema_path = Path(schema_path) objects_dir = schema_path / "objects/" rules_dir = schema_path / "rules/" if not objects_dir.is_dir() or not rules_dir.is_dir(): raise ValueError( f"Schema path or paths do not exist:\n\t{str(objects_dir)}\n\t{str(rules_dir)}" ) schema = {} schema["objects"] = {} schema["rules"] = {} # Load object definitions. All are present in single files. for object_group_file in sorted(objects_dir.glob("*.yaml")): lgr.debug(f"Loading {object_group_file.stem} objects.") dict_ = _yaml.load(object_group_file.read_text()) schema["objects"][object_group_file.stem] = dereference_yaml( dict_, dict_) # Grab single-file rule groups for rule_group_file in sorted(rules_dir.glob("*.yaml")): lgr.debug(f"Loading {rule_group_file.stem} rules.") dict_ = _yaml.load(rule_group_file.read_text()) schema["rules"][rule_group_file.stem] = dereference_yaml(dict_, dict_) # Load folders of rule subgroups. for rule_group_file in sorted(rules_dir.glob("*/*.yaml")): rule = schema["rules"].setdefault(rule_group_file.parent.name, {}) lgr.debug(f"Loading {rule_group_file.stem} rules.") dict_ = _yaml.load(rule_group_file.read_text()) rule[rule_group_file.stem] = dereference_yaml(dict_, dict_) return schema
def serialize(cls, data): if settings.INDENT_YAML_BLOCKS: f = StringIO() y = yaml.YAML() y.indent(mapping=2, sequence=4, offset=2) y.dump(data, f) text = f.getvalue().strip() + '\n' else: text = yaml.round_trip_dump(data) or "" text = text.replace('- \n', '-\n') return "" if text == "{}\n" else text
def load_settings(): conandir = os.path.expanduser("~/.conan/") if not os.path.exists(conandir): return settings_file = os.path.join(conandir, 'settings.yml') with open(settings_file) as f: txt = f.read() YAML = yaml.YAML() data = YAML.load(txt) settings = odict(data) return settings
def load_yaml(yml_file): # Load YAML data from file using the "safe" loading option. try: yaml_ = yaml.YAML(typ="safe") with open(yml_file, "rt", encoding="utf-8") as stream: return yaml_.load(stream) except yaml.constructor.ConstructorError: with open(yml_file, "rt", encoding="utf-8") as stream: # Ensure that the loader remains backward-compatible with legacy # ruamel.yaml versions (prior to 0.17.0). return yaml.safe_load(stream)
def parse_config(path): yaml_path = path ext = os.path.splitext(yaml_path)[1].lstrip('.') ext = ext.rstrip(' ') assert ext == 'yaml' with open(yaml_path) as file: yml = yaml.YAML() yml.allow_duplicate_keys = True doc = yml.load(file) usr_config = UsrConfigs(doc) return usr_config
def __init__(self, *args, **kwargs): super(YamlPrinter, self).__init__(*args, retain_none_values=True, **kwargs) # pylint:disable=g-import-not-at-top, Delay import for performance. from ruamel import yaml # Use pure=True to only use python implementations. Otherwise, it can load # the the _ruamel_yaml C extension from site packages if # CLOUDSDK_PYTHON_SITEPACKAGES=1 is set. There is no guarantee that the C # extension is compatible with our vendored ruamel.yaml and the python # runtime. self._yaml = yaml.YAML(typ='safe', pure=True) self._yaml.default_flow_style = False self._yaml.old_indent = resource_printer_base.STRUCTURED_INDENTATION self._yaml.allow_unicode = True self._yaml.encoding = log.LOG_FILE_ENCODING null = self.attributes.get('null') version = self.attributes.get('version') # If no version specified, uses ruamel's default (1.2) if version: self._yaml.version = str(version) def _FloatPresenter(unused_dumper, data): return yaml.nodes.ScalarNode( 'tag:yaml.org,2002:float', resource_transform.TransformFloat(data)) def _LiteralLinesPresenter(dumper, data): return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') def _NullPresenter(dumper, unused_data): if null in ('null', None): return dumper.represent_scalar('tag:yaml.org,2002:null', 'null') return dumper.represent_scalar('tag:yaml.org,2002:str', null) def _OrderedDictPresenter(dumper, data): return dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) def _UndefinedPresenter(dumper, data): r = repr(data) if r == '[]': return dumper.represent_list([]) if r == '{}': return dumper.represent_dict({}) dumper.represent_undefined(data) self._yaml.representer.add_representer(float, _FloatPresenter) self._yaml.representer.add_representer(YamlPrinter._LiteralLines, _LiteralLinesPresenter) self._yaml.representer.add_representer(None, _UndefinedPresenter) self._yaml.representer.add_representer(type(None), _NullPresenter) self._yaml.representer.add_representer(collections.OrderedDict, _OrderedDictPresenter)
def save_commented_model_yaml(self, path): """ Save a fully built and commented version of the model to a YAML file at the given ``path``. Comments in the file indicate where values were overridden. This is Calliope's internal representation of a model directly before the model_data xarray.Dataset is built, and can be useful for debugging possible issues in the model formulation. """ if not self._model_run or not self._debug_data: raise KeyError( 'This model does not have the fully built model attached, ' 'so `save_commented_model_yaml` is not available. Likely ' 'reason is that the model was built with a verion of Calliope ' 'prior to 0.6.5.' ) yaml = ruamel_yaml.YAML() model_run_debug = self._model_run.copy() try: del model_run_debug['timeseries_data'] # Can't be serialised! except KeyError: # Possible that timeseries_data is already gone if the model # was read from a NetCDF file pass # Turn sets in model_run into lists for YAML serialization for k, v in model_run_debug.sets.items(): model_run_debug.sets[k] = list(v) debug_comments = self._debug_data['comments'] stream = StringIO() yaml.dump(model_run_debug.as_dict(), stream=stream) debug_yaml = yaml.load(stream.getvalue()) for k in debug_comments.model_run.keys_nested(): v = debug_comments.model_run.get_key(k) if v: keys = k.split('.') apply_to_dict(debug_yaml, keys[:-1], 'yaml_add_eol_comment', (v, keys[-1])) dumper = ruamel_yaml.dumper.RoundTripDumper dumper.ignore_aliases = lambda self, data: True with open(path, 'w') as f: ruamel_yaml.dump( debug_yaml, stream=f, Dumper=dumper, default_flow_style=False )
def _load_yml(self, yml): YAML = yaml.YAML() dump = YAML.load(yml) dump = odict(dump) for i in ('project', 'config', 'flag_aliases'): if dump.get(i) is None: dump[i] = CommentedMap() setattr(self, i, dump[i]) self._dump = dump from . import flags as c4flags self.flag_aliases = c4flags.FlagAliases( yml=dump.get('flag_aliases', CommentedMap()))
def __call__(self, parser, namespace, values, option_string=None): if getattr(namespace, self.dest, None) is None: setattr(namespace, self.dest, {}) # Only doing a copy here because that's what _AppendAction does items = copy.copy(getattr(namespace, self.dest)) key, val = values.split("=", 1) if yaml.version_info < (0, 15): items[key] = yaml.safe_load(val) else: yml = yaml.YAML(typ="safe", pure=True) items[key] = yml.load(val) setattr(namespace, self.dest, items)
def get_config(): with open('config.yaml') as file: config = yaml.YAML().load(file) def convert(item): if isinstance(item, dict): for key, value in item.items(): item[key] = convert(value) return types.SimpleNamespace(**item) return item return convert(config)
def _get_base_image_name(name: str) -> str: """ Get the name of the docker image to use for the project. Args: name (str): Project name. """ yaml_obj = ryml.YAML() with open(_get_project_folder(name) + '/config.yml', 'r') as f: doc = yaml_obj.load(f) base_docker_image = doc['base-image'] return base_docker_image
def load_config(): from ruamel import yaml with open(OORT_CONFIG, 'r') as fp: yaml = yaml.YAML(typ='safe') config = yaml.load(fp) config['verbose'] = config.get('verbose') or 0 config['port'] = config.get('port') or 51826 config['bridge_name'] = config.get('bridge_name') or 'oort' return config
def dump(self, configs, path): ya_data: CommentedMap = CommentedMap() root_comment = getattr(configs, '__doc__', None) if root_comment: ya_data.yaml_set_start_comment(root_comment) ya_data = _to_yaml(ya_data, configs) yml = yaml.YAML() with open(path, 'w') as f: yml.dump(ya_data, f)
def __init__(self, *args, **kwargs): super(YamlPrinter, self).__init__(*args, retain_none_values=True, **kwargs) # pylint:disable=g-import-not-at-top, Delay import for performance. from ruamel import yaml self._yaml = yaml.YAML(typ='safe') self._yaml.default_flow_style = False self._yaml.old_indent = resource_printer_base.STRUCTURED_INDENTATION self._yaml.allow_unicode = True self._yaml.encoding = log.LOG_FILE_ENCODING null = self.attributes.get('null') version = self.attributes.get('version') # If no version specified, uses ruamel's default (1.2) if version: self._yaml.version = str(version) def _FloatPresenter(unused_dumper, data): return yaml.nodes.ScalarNode( 'tag:yaml.org,2002:float', resource_transform.TransformFloat(data)) def _LiteralLinesPresenter(dumper, data): return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') def _NullPresenter(dumper, unused_data): if null in ('null', None): return dumper.represent_scalar('tag:yaml.org,2002:null', 'null') return dumper.represent_scalar('tag:yaml.org,2002:str', null) def _OrderedDictPresenter(dumper, data): return dumper.represent_mapping('tag:yaml.org,2002:map', data.items()) def _UndefinedPresenter(dumper, data): r = repr(data) if r == '[]': return dumper.represent_list([]) if r == '{}': return dumper.represent_dict({}) dumper.represent_undefined(data) self._yaml.representer.add_representer(float, _FloatPresenter) self._yaml.representer.add_representer(YamlPrinter._LiteralLines, _LiteralLinesPresenter) self._yaml.representer.add_representer(None, _UndefinedPresenter) self._yaml.representer.add_representer(type(None), _NullPresenter) self._yaml.representer.add_representer(collections.OrderedDict, _OrderedDictPresenter)
def _edit_yaml(self, yaml_file, field, value): """ """ yaml_parser = yaml.YAML() loaded = self._load_yaml(yaml_parser=yaml_parser, yaml_file=yaml_file) field_name = field.split("/")[-1] self._get_inner_dict_recursively(loaded, field)[field_name] = value self._edit_file(yaml_file=yaml_file, yaml_parser=yaml_parser, data=loaded)