def render(template_path: str, values: dict, engine: Environment) -> str: output = engine.get_template(template_path).render(values) yaml = YAML(typ='safe', pure=True) try: yaml.load_all(output) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as err: raise YamlValidationError(err) from err return output
def parse(data, raw=False, quiet=False): """ Main text parsing function Parameters: data: (string) text data to parse raw: (boolean) output preprocessed JSON if True quiet: (boolean) suppress warning messages if True Returns: List of dictionaries. Raw or processed structured data. """ if not quiet: jc.utils.compatibility(__name__, info.compatible) raw_output = [] if jc.utils.has_data(data): yaml = YAML(typ='safe') for document in yaml.load_all(data): raw_output.append(document) if raw: return raw_output else: return process(raw_output)
def validate_files( schemafile: PathLike, *datafiles: PathLike, encoding: str = "utf-8", ) -> None: r""" Validate the given datafiles against the given schema. :param schemafile: The ``json`` or ``yaml`` formatted schema to validate with. :param \*datafiles: The ``json`` or ``yaml`` files to validate. :param encoding: Encoding to open the files with. .. versionadded:: 0.4.0 """ schemafile = pathlib.Path(schemafile) yaml = YAML(typ="safe", pure=True) schema = yaml.load(schemafile.read_text(encoding=encoding)) for filename in datafiles: for document in yaml.load_all( pathlib.Path(filename).read_text(encoding=encoding)): try: jsonschema.validate(document, schema, format_checker=jsonschema.FormatChecker()) except jsonschema.exceptions.ValidationError as e: e.filename = str(filename) raise e
def load_config(config_file_path): """should raise ScannerError if there is a problem in the yaml file.""" yaml = YAML(typ='unsafe', pure=True) p = Path(config_file_path) parsed = yaml.load_all(p) data = next(parsed) return DefaultBear(None, **data)
def main(): opts = getArgs() for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() elif o in ("-k", "--key"): key_to_update = a elif o in ("-v", "--value"): value_to_update = a else: assert False, "unhandled option" yaml = YAML() yaml.explicit_start = True yaml.allow_unicode = True yaml.width = 300 data_list = key_to_update.split(".") data_to_refer = value_to_update for k in data_list[::-1]: data_to_refer = {k: data_to_refer} result = [] for data in list(yaml.load_all(sys.stdin)): if data is not None: data = update_data(data, data_to_refer) result.append(data) yaml.dump_all(result, sys.stdout)
def main(): yaml = YAML() file_name = sys.argv[1] file_in = open(file_name).read() docs = yaml.load_all(file_in) i = 0 for doc in docs: if i == 0: code_old = doc else: code_new = doc i = i + 1 # 保存变化后的值,由set_map_item设置 delta_map = CommentedMap() # 每一次traversal的递归遍历处理过的key,每一次递归完成回到root层时该key将会清空 follow_keys = list() # 此次对比增加的key add = list() # 此次对比更新的key update = list() traversal(code_old, code_new, follow_keys, delta_map, update, add) yaml.dump(code_old, sys.stdout) split = '------love----you------choerodon----' print(split) yaml.dump(delta_map, sys.stdout) print(split) change_key_map = dict() change_key_map["add"] = add change_key_map["update"] = update yaml.dump(change_key_map, sys.stdout)
def main(): yaml = YAML() file_name = sys.argv[1] file_in = open(file_name).read() docs = yaml.load_all(file_in) i = 0 for doc in docs: if i == 0: code_old = doc else: code_new = doc i = i + 1 delta_map = dict() follow_keys = list() add = list() update = list() traversal(code_old, code_new, follow_keys, delta_map, update, add) yaml.dump(code_old, sys.stdout) split = '------love----you------choerodon----' print(split) yaml.dump(delta_map, sys.stdout) print(split) change_key_map = dict() change_key_map["add"] = add change_key_map["update"] = update yaml.dump(change_key_map, sys.stdout)
def parse(data, raw=False, quiet=False): """ Main text parsing function Parameters: data: (string) text data to parse raw: (boolean) output preprocessed JSON if True quiet: (boolean) suppress warning messages if True Returns: List of Dictionaries representing the YAML documents. """ if not quiet: jc.utils.compatibility(__name__, info.compatible) raw_output = [] if jc.utils.has_data(data): # monkey patch to disable plugins since we don't use them and in # ruamel.yaml versions prior to 0.17.0 the use of __file__ in the # plugin code is incompatible with the pyoxidizer packager YAML.official_plug_ins = lambda a: [] yaml = YAML(typ='safe') for document in yaml.load_all(data): raw_output.append(document) if raw: return raw_output else: return process(raw_output)
def process_complete_schema(config, config_schema_file, defaults_file, meta_info_file): yaml = YAML() config_schema_file = open(config_schema_file, "r") defaults_file = open(defaults_file, "r") meta_info_file = open(meta_info_file, "r") # Config schema has expected schema and all the expandable types config_schema = yaml.load_all(config_schema_file) defaults = yaml.load(defaults_file) meta_info = yaml.load(meta_info_file) expected_params = None expandable_types = {} for yaml_doc in config_schema: for key in yaml_doc: if key == "expected": expected_params = yaml_doc[key] else: expandable_types[key] = yaml_doc[key] if expected_params is None: message = "Config schema needs to have an expected params section" raise Exception(message) return get_final_config(config, expected_params, defaults, meta_info, expandable_types)
def main(): opts = getArgs() for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() elif o in ("-k", "--key"): d_key = a elif o in ("-v", "--value"): d_value = a else: assert False, "unhandled option" yaml = YAML() yaml.explicit_start = True yaml.allow_unicode = True yaml.width = 300 result = [] for data in list(yaml.load_all(sys.stdin)): if data is not None: if (data['kind'] == "ConfigMap") or \ (data['kind'] == "Secret"): # update data: key=value data['data'][d_key] = d_value result.append(data) elif 'kind' in data.keys(): result.append(data) yaml.dump_all(result, sys.stdout)
def load_multi_doc_file (file_data): yaml=YAML(typ='safe') try: multi_doc_load = yaml.load_all(file_data) except yaml.YAMLError as yaml_error: print(yaml_error) sys.exit() return multi_doc_load
def _parse_yaml(self, yaml_content): try: yaml = YAML() resources = list(yaml.load_all(yaml_content)) except Exception as e: raise ParseError(f'Parse manifest failed: \n{e}\n\nManifest content:\n{yaml_content}') else: return resources
async def differ(ctx, path): """Clear the bring cache dir in the relevant locaiont (e.g. '~/.cache/bring' on Linux).""" si = SmartInput(path) content = await si.content_async() yaml = YAML() dict_content_orig = list(yaml.load_all(content)) print(dict_content_orig) new_content = click.edit(content) dict_content_new = list(yaml.load_all(new_content)) dict_diff = diff(dict_content_orig, dict_content_new) dict_diff = list(dict_diff) print(dict_diff)
def safe_load_all(stream): yaml = YAML() yaml.default_flow_style = False docs = list(yaml.load_all(stream)) conv_docs = [] for doc in docs: conv_docs.append(dict_to_objdict(doc)) return conv_docs
def _parse_yaml(self, yaml_content): try: yaml = YAML() resources = list(yaml.load_all(yaml_content)) except Exception as e: raise ParseError(f'Parse manifest failed: \n{e}\n\nManifest content:\n{yaml_content}') else: # ordereddict to dict return json.loads(json.dumps(resources))
def docs(self, path): from ruamel.yaml import YAML tyaml = YAML(typ='safe', pure=True) tyaml.register_class(YAMLData) tyaml.register_class(Python) tyaml.register_class(Output) tyaml.register_class(Assert) return list(tyaml.load_all(path))
def yamale_converter(config_schema_file, yamale_file, lc_schema_files): yaml = YAML() config_schema_file = open(config_schema_file, "r") config_schema = yaml.load_all(config_schema_file) schema = None expandable_types = {} for yaml_doc in config_schema: for key, value in yaml_doc.items(): if key == "expected": schema = value else: expandable_types[key] = value with open(yamale_file, "w") as yamale: schema = translate(schema, expandable_types) yaml.dump(schema, yamale) # Add types defined in schema file for key, value in expandable_types.items(): final = translate({key: value}, expandable_types) if key == "lightweight_component": lc_schema_filenames = [ lc_schema.split("/")[-1].replace(".yaml", "") for lc_schema in lc_schema_files ] lc_schemas = [ "include('{}')".format(lc_schema.replace(".yaml", "")) for lc_schema in lc_schema_filenames ] any_schemas = "any({})".format(", ".join(lc_schemas)) final[key]["config"] = any_schemas yamale.write("---\n") yaml.dump(final, yamale) # Add lc_schemas for lc_schema_file in lc_schema_files: lc_schema = yaml.load(open(lc_schema_file, "r")) lc_schema = lc_schema["expected-from-site-level-config"] lc_schema_name = lc_schema_file.split("/")[-1].replace(".yaml", "") final = translate({lc_schema_name: lc_schema}, expandable_types) yamale.write("---\n") yaml.dump(final, yamale)
async def patch(self, full_path: str, patch_set: Mapping) -> None: si = SmartInput(full_path) content = await si.content_async() yaml = YAML() dict_content = yaml.load_all(content) new_content = patch(patch_set, list(dict_content)) with open(full_path, "w") as f: yaml.dump_all(new_content, f)
def _add_policies(self, doc, file_name): yaml1 = YAML() code = yaml1.load_all(doc) for policy_list in code: if isinstance(policy_list, dict): self._add_policies_to_parse_queue(policy_list.get('items', []), file_name) else: # we got a list of lists, e.g., when combining calico np, gnp and profiles for policy_list_list in policy_list: if isinstance(policy_list_list, dict): self._add_policies_to_parse_queue(policy_list_list.get('items', []), file_name)
def get_dicts_from_yaml(in_f, encoding='utf-8'): """ Читает список словарей из YAML-файл. :param in_f: YAML-файл :param encoding: кодировка YAML-файла :return: генератор списка, содержащего словари """ yaml = YAML(pure=True) with open(Path(in_f), encoding=encoding) as f_in: for item in yaml.load_all(f_in): yield dict(item)
def fetch_ruleset(subreddit, page): raw_content = RedditManagerUtils.RedditManager.getWikiContent( subreddit, page) yaml = YAML(typ='safe') output = list(yaml.load_all(raw_content)) new_ruleset = RulesManager._parse_rules(output, subreddit) RulesManager.__subreddit_rulesets[subreddit] = new_ruleset
def scan_kustomization_for_images(kust_dir): """Scan kustomization folder and produce a list of images Args: kust_dir (str): Path where the kustomize application resides. """ yaml = YAML() yaml.block_seq_indent = 0 # Load kustomization with open(path.join(kust_dir, "kustomization.yaml")) as f: try: kustomization = yaml.load(f) except Exception as e: log.error("Error loading kustomization in %s: %s", kust_dir, e) raise (e) # Get current image list from kustomization img_list = kustomization.get("images", []) # Get local resource files (_, _, filenames) = next(walk(kust_dir)) filenames = [ filename for filename in filenames if filename != "kustomization.yaml" and filename != "params.yaml" and filename.endswith(".yaml") ] for filename in filenames: with open(path.join(kust_dir, filename)) as f: resources = list(yaml.load_all(f)) for r in resources: if not isinstance(r, Mapping): continue if r.get("kind", "").lower() in accepted_kinds: try: containers = r["spec"]["template"]["spec"]["containers"] except KeyError: continue for c in containers: try: img_str = c["image"] except KeyError: continue new_img = image_from_string(img_str) append_or_update(img_list, new_img) if img_list: kustomization["images"] = img_list with open(path.join(kust_dir, "kustomization.yaml"), "w") as f: yaml.dump(kustomization, f)
def test_multi_document_load(self, tmpdir): """this went wrong on 3.7 because of StopIteration, PR 37 and Issue 211""" from ruamel.yaml import YAML fn = Path(str(tmpdir)) / 'test.yaml' fn.write_text( textwrap.dedent(u"""\ --- - a --- - b ... """)) yaml = YAML() assert list(yaml.load_all(fn)) == [['a'], ['b']]
def main(): yaml = YAML() yaml.explicit_start = True yaml.allow_unicode = True yaml.width = 300 result = [] for data in list(yaml.load_all(sys.stdin)): if data is not None: if data['kind'] == 'Secret': for k, v in data['data'].items(): data['data'][k] = base64.b64encode( v.encode('utf-8')).decode('utf-8') result.append(data) elif 'kind' in data.keys(): result.append(data) yaml.dump_all(result, sys.stdout)
def from_yaml(file): """Creates a new bibliography from a YAML source file. Args: file (file): path to YAML file from which to load database. Returns: An OrderedDict containing the bibliography as per the provided YAML file. """ yaml = YAML() bib = OrderedDict() LOGGER.debug('Loading YAML data from file: %s.', file) for entry in yaml.load_all(file): for label, data in entry.items(): bib[label] = Entry(label, data) return bib
def main(): opts = getArgs() for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() elif o in ("-t", "--type"): c_type = a elif o in ("-n", "--name"): c_name = a elif o in ("-i", "--image"): c_image = a else: assert False, "unhandled option" yaml = YAML() yaml.explicit_start = True yaml.allow_unicode = True yaml.width = 300 result = [] for data in list(yaml.load_all(sys.stdin)): if data is not None: if (data['kind'] == "Deployment") or \ (data['kind'] == "DaemonSet") or \ (data['kind'] == "StatefulSet"): # update image container_ls = [ c['name'] for c in data['spec']['template']['spec'][c_type] ] for n, c in enumerate(container_ls): if c == c_name: data['spec']['template']['spec'][c_type][n][ 'image'] = c_image result.append(data) elif data['kind'] == "CronJob": # update image container_ls = [ c['name'] for c in data['spec']['jobTemplate']['spec'] ['template']['spec'][c_type] ] for n, c in enumerate(container_ls): if c == c_name: data['spec']['jobTemplate']['spec']['template'][ 'spec'][c_type][n]['image'] = c_image result.append(data) elif 'kind' in data.keys(): result.append(data) yaml.dump_all(result, sys.stdout)
def __init__(self, path = "./configs/tune/example.yaml"): """Loads and prepares the specified config file. Keyword Arguments: path {str} -- Yaml file path to the to be loaded config (default: {"./configs/tune/search.yaml"}) """ # Load the config file stream = open(path, "r") yaml = YAML() yaml_args = yaml.load_all(stream) # Final contents of the config file will be added to a dictionary self._config = {} # Prepare data for data in yaml_args: self._config = dict(data)
def setup(protocol: str = 'protocol.yml'): # TODO: Check Correctness # needs: Transition to end, no unreachable nodes, no unknown task types # Build database # Handle api calls and predictions automaton = AnnotationAutomaton() with open(protocol) as f: yaml = YAML(typ='safe') protocol: dict = next(yaml.load_all(f)) for state, val in protocol.items(): for transition in val['transitions']: for trigger in transition: target = transition[trigger]['target'] actions = (transition[trigger]['actions'] if 'actions' in transition[trigger] else None) automaton.add_transition(trigger, state, target, before=actions) del val['transitions'] on_exit = ['save'] # on_exit.extend(['save'] if 'column' in val else []) automaton.add_state( State(name=state, meta=val, on_exit=on_exit)) automaton.add_transition('fail', '*', 'failure') automaton.add_transition('to_start', '*', 'start') # print(automaton.states) # print(automaton.get_transitions()) illegal = automaton.check_machine_validity() if len(illegal['unreachable']) != 0 and len(illegal['undefined']) != 0: raise error_handler.AutomatonError( f'Unreachable states: {illegal["unreachable"]} and Undefined states: {illegal["undefined"]}' ) elif len(illegal['unreachable']) != 0: raise error_handler.AutomatonError( f'Unreachable states: {illegal["unreachable"]}') elif len(illegal['undefined']) != 0: raise error_handler.AutomatonError( f'Undefined states: {illegal["undefined"]}') return automaton
def run(): args = _parse() keys = args.keys if not keys: keys = ["kind", "metadata.name"], yaml = YAML() input_docs = yaml.load_all(sys.stdin) unique_docs = dict() for doc in input_docs: # Determine ID of earch supplied document # based on the keys ids = [] for key in keys: # Split each key into the hierarchy of subkeys key_parts = key.split(".") try: # Start iterating with the top-most subkey value = doc[key_parts[0]] # Go deeper into the lower nested subkeys for key_part in key_parts[1:]: # until values is obtainer value = value[key_part] ids.append(str(value)) except KeyError as ke: print( f"Supplied document does not have required key {ke}", file=sys.stderr, ) print("Failed document is:", file=sys.stderr) yaml.dump(doc, sys.stderr) exit(1) # If document with such ID does not yet exist in our # collection, then add it id = "-".join(ids) if id not in unique_docs: unique_docs[id] = doc yaml.dump_all(unique_docs.values(), sys.stdout)
def main(): opts = getArgs() for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() elif o in ("-t", "--tag"): tag = a else: assert False, "unhandled option" yaml = YAML() yaml.explicit_start = True yaml.allow_unicode = True yaml.width = 300 result = [] for data in list(yaml.load_all(sys.stdin)): if data is not None: if (data['kind'] == "Deployment") or \ (data['kind'] == "DaemonSet") or \ (data['kind'] == "StatefulSet"): # add label: tag data['spec']['template']['metadata']['labels']['tag'] = tag # add label: imageName.n, imageVersion.n image_ls = [c['image'] for c in data['spec']['template']['spec']['containers']] for n,i in enumerate(image_ls): i_name = i.split(':')[0][i.split(':')[0].rfind('/')+1:] i_version = i.split(':')[1] data['spec']['template']['metadata']['labels']['imageName.' + str(n)] = i_name data['spec']['template']['metadata']['labels']['imageVersion.' + str(n)] = i_version result.append(data) elif data['kind'] == "CronJob": # add label: tag data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['tag'] = tag # add label: imageName.n, imageVersion.n image_ls = [c['image'] for c in data['spec']['jobTemplate']['spec']['template']['spec']['containers']] for n,i in enumerate(image_ls): i_name = i.split(':')[0][i.split(':')[0].rfind('/')+1:] i_version = i.split(':')[1] data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['imageName.' + str(n)] = i_name data['spec']['jobTemplate']['spec']['template']['metadata']['labels']['imageVersion.' + str(n)] = i_version result.append(data) elif 'kind' in data.keys(): result.append(data) yaml.dump_all(result, sys.stdout)