def main(): subprocess.run(['git', 'checkout', '--', 'zuul.d/projects.yaml']) yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) projects = yaml.load(open('zuul.d/projects.yaml', 'r')) for project in projects: if project['project']['name'].split('/')[1].startswith('networking-'): if 'templates' not in project['project']: continue templates = project['project']['templates'] for template in ('openstack-python-jobs', 'openstack-python35-jobs'): if template in templates: new_name = template + '-neutron' templates[templates.index(template)] = new_name yaml.dump(projects, open('zuul.d/projects.yaml', 'w')) # Strip the extra 2 spaces that ruamel.yaml appends because we told it # to indent an extra 2 spaces. Because the top level entry is a list it # applies that indentation at the top. It doesn't indent the comment lines # extra though, so don't do them. with open('zuul.d/projects.yaml', 'r') as main_in: main_content = main_in.readlines() with open('zuul.d/projects.yaml', 'w') as main_out: for line in main_content: if '#' in line: main_out.write(line) else: if line.startswith(' - project'): main_out.write('\n') main_out.write(line[2:])
def pretty_dump(obj): yaml = ruamel.yaml.YAML(typ='rt') yaml.indent(mapping=2, sequence=2, offset=0) yaml.width = 99999 def make_good_strings(obj): if type(obj) == list: return list(map(make_good_strings, obj)) elif type(obj) == dict: return dict( map(lambda items: ( items[0], make_good_strings(items[1]), ), obj.items())) elif type(obj) == str: if obj.count('\n') > 0: return ruamel.yaml.scalarstring.LiteralScalarString('\n'.join( map(lambda x: x.rstrip(), obj.splitlines()))) else: return obj else: return obj with StringIO() as strio: yaml.dump(make_good_strings(obj), stream=strio) return strio.getvalue()
def create(self): yaml = ruamel.yaml.YAML() yaml.version = (1, 2) yaml.indent(mapping=3, sequence=2, offset=0) yaml.allow_duplicate_keys = False return yaml
def round_trip_dump_all( data, stream=None, # *, indent=None, block_seq_indent=None, default_flow_style=unset, top_level_colon_align=None, prefix_colon=None, explicit_start=None, explicit_end=None, version=None, allow_unicode=None, ): import ruamel.yaml # NOQA yaml = ruamel.yaml.YAML() yaml.indent(mapping=indent, sequence=indent, offset=block_seq_indent) if default_flow_style is not unset: yaml.default_flow_style = default_flow_style yaml.top_level_colon_align = top_level_colon_align yaml.prefix_colon = prefix_colon yaml.explicit_start = explicit_start yaml.explicit_end = explicit_end yaml.version = version yaml.allow_unicode = allow_unicode if stream is not None: yaml.dump(data, stream=stream) return buf = io.StringIO() yaml.dump_all(data, stream=buf) return buf.getvalue()
def create_settings_file(comments : bool = True): """ Creates a new settings config. `comments` : If true, will add useful tips for every config option """ yaml = ruamel.yaml.YAML() data = ruamel.yaml.comments.CommentedMap() # Empty Data for index, config_option in enumerate(_DEFAULT_CONFIGS, start=0): data.insert( index, config_option['varname'], config_option['default'], comment = config_option['comment'] if comments else None ) with open('./config.yml', 'w+') as ymlfile: yaml.indent( # QOL Indenting <3 offset = 2, mapping = 2, sequence= 4 ) yaml.dump( data, ymlfile ) return load_settings_file(create_if_not_exists=False, include_defaults=False)
def standard_to_any(path, schema): if list(path)[-1] != '/': path += '/' with open(schema) as f: json_data = json.load(f) list_standards = get_standards(schema) os.chdir(path) for file in glob.glob('**/*.yml', recursive=True): print('[INFO] Processing {}'.format(file)) with open(file) as f: yaml_file = ruamel.yaml.round_trip_load(f, preserve_quotes=True) for element in yaml_file['checks']: try: add_standard(element['compliance'], list_standards, json_data) except: pass with open(file, 'w') as f: yaml = ruamel.yaml.YAML() yaml.width = 4096 yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict) yaml.Representer.add_representer(FlowList, represent_flow_seq) yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(yaml_file, f)
def delete_standard(path, standard): if list(path)[-1] != '/': path += '/' os.chdir(path) for file in glob.glob('**/*.yml', recursive=True): changed = False with open(file) as f: yaml_file = ruamel.yaml.round_trip_load(f, preserve_quotes=True) for element in yaml_file['checks']: try: for index, key in enumerate(element['compliance']): for k in key: if k == standard: element['compliance'].pop(index) changed = True except: pass with open(file, 'w') as f: yaml = ruamel.yaml.YAML() yaml.width = 4096 yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict) yaml.Representer.add_representer(FlowList, represent_flow_seq) yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(yaml_file, f) if changed: print('[DELETE] Deleted {} in file {}'.format(standard, file))
def test_issue_290a(self): import sys from ruamel.yaml.compat import StringIO from ruamel.yaml import YAML yamldoc = dedent("""\ --- aliases: # Folded-element comment # for a multi-line value - &FoldedEntry > THIS IS A FOLDED, MULTI-LINE VALUE # Literal-element comment # for a multi-line value - &literalEntry | THIS IS A LITERAL, MULTI-LINE VALUE # Plain-element comment - &plainEntry Plain entry """) yaml = YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.explicit_start = True yaml.preserve_quotes = True yaml.width = sys.maxsize data = yaml.load(yamldoc) buf = StringIO() yaml.dump(data, buf) assert buf.getvalue() == yamldoc
def makeComposeFile(): """Creates an docker-compose.yaml file for the stella app. Returns: bool: True if successful """ systems = System.query.filter_by().all() compose = {'version': '3', 'networks': {'stella-shared': {'external': {'name': 'stella-server_default'}}}, 'services': { 'app': { 'build': './app', 'volumes': ['/var/run/docker.sock/:/var/run/docker.sock', './app/log:/app/log'], 'ports': ["8080:8000"], 'depends_on': [system.name for system in systems], 'networks': ['stella-shared'] } } } for system in systems: compose['services'][str(system.name)] = { 'build': system.url, 'container_name': system.name, 'volumes': ['./data/:/data/'], 'networks': ['stella-shared'] } yaml = ruamel.yaml.YAML() yaml.indent(sequence=4, offset=4) with open('docker-compose.yml', 'w') as file: yaml.dump(compose, file) return True
def test_issue_288a(self): import sys from ruamel.yaml.compat import StringIO from ruamel.yaml import YAML yamldoc = dedent("""\ --- # Reusable values aliases: # First-element comment - &firstEntry First entry # Second-element comment - &secondEntry Second entry # Third-element comment is # a multi-line value - &thirdEntry Third entry # EOF Comment """) yaml = YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.explicit_start = True yaml.preserve_quotes = True yaml.width = sys.maxsize data = yaml.load(yamldoc) buf = StringIO() yaml.dump(data, buf) assert buf.getvalue() == yamldoc
def _to_nice_yaml(self, a, indent=4, *args, **kw): """Make verbose, human readable yaml.""" yaml = ruamel.yaml.YAML() yaml.indent(mapping=indent, sequence=(indent * 2), offset=indent) stream = ruamel.yaml.compat.StringIO() yaml.dump(a, stream, **kw) return stream.getvalue().rstrip()
def save_yaml(fname: str, data: JSON_TYPE) -> None: """Save a YAML file.""" yaml = YAML(typ='rt') yaml.indent(sequence=4, offset=2) tmp_fname = fname + "__TEMP__" try: try: file_stat = os.stat(fname) except OSError: file_stat = stat_result( (0o644, -1, -1, -1, -1, -1, -1, -1, -1, -1)) with open(os.open(tmp_fname, O_WRONLY | O_CREAT | O_TRUNC, file_stat.st_mode), 'w', encoding='utf-8') \ as temp_file: yaml.dump(data, temp_file) os.replace(tmp_fname, fname) if hasattr(os, 'chown') and file_stat.st_ctime > -1: try: os.chown(fname, file_stat.st_uid, file_stat.st_gid) except OSError: pass except YAMLError as exc: _LOGGER.error(str(exc)) raise HomeAssistantError(exc) except OSError as exc: _LOGGER.exception('Saving YAML file %s failed: %s', fname, exc) raise WriteError(exc) finally: if os.path.exists(tmp_fname): try: os.remove(tmp_fname) except OSError as exc: # If we are cleaning up then something else went wrong, so # we should suppress likely follow-on errors in the cleanup _LOGGER.error("YAML replacement cleanup failed: %s", exc)
def get_ruamel_yaml_instance() -> Any: # Import the ruamel.yaml module locally so that the common_util module is still usable outside # of any virtualenv. import ruamel.yaml # type: ignore yaml = ruamel.yaml.YAML() yaml.indent(sequence=4, offset=2) return yaml
def add_bloxberg_config(fname): bloxberg_config = { "name": "bloxberg (Bloxberg)", "id": "bloxberg", "chainid": 8995, "host": "https://core.bloxberg.org", "explorer": "https://blockexplorer.bloxberg.org/api", } config, ind, bsi = ruamel.yaml.util.load_yaml_guess_indent(open(fname)) data = config is_bloxberg_added = False for config in data["live"]: if config["name"] == "Ethereum": for network in config["networks"]: if "bloxberg" in network["name"]: is_bloxberg_added = True if json.loads(json.dumps(network)) == bloxberg_config: log(f"## bloxberg config is already added into {fname}") else: network["name"] = bloxberg_config["name"] network["id"] = bloxberg_config["id"] network["chainid"] = bloxberg_config["chainid"] network["host"] = bloxberg_config["host"] network["explorer"] = bloxberg_config["explorer"] if not is_bloxberg_added: config["networks"].append(bloxberg_config) yaml = ruamel.yaml.YAML() yaml.indent(mapping=ind, sequence=ind, offset=bsi) with open(fname, "w") as fp: yaml.dump(data, fp)
def dump(self, data): '''return yaml string from python dict''' yaml = YAML() yaml.indent(mapping=4, sequence=6, offset=3) stream = StringIO() ruamel.yaml.safe_dump(data, stream, default_flow_style=False) return stream.getvalue()
def file_output(dataMap, dir): yaml = ruamel.yaml.YAML() yaml.indent(mapping=4) yaml.preserve_quotes = True # allow_unicode = True yaml对中文的处理 with open(dir + '/result.yaml', 'w+', encoding='utf8') as loadfile: yaml.dump(dataMap, loadfile)
def _save(self, filename: str): ''' Saves off the current dictionary state in Plover to a file. :param filename: The file path of the dictionary to save to. ''' # Group dictionary by value, sorted alphabetically data = {} for strokes, translation in sorted(self._dict.items(), key=lambda x: x[1].casefold()): # Need to join the multi-stroke entries into one stroke string first stroke = STROKE_DELIMITER.join(strokes) data.setdefault(translation, []).append(stroke) data[translation] = sorted(data[translation]) # Write out the data yaml = ruamel.yaml.YAML(typ='safe') yaml.allow_unicode = True yaml.default_flow_style = False yaml.indent(sequence=4, offset=2) with open(filename, 'w', encoding='utf-8') as out_file: yaml.dump(data, out_file)
def create_conda_env_yaml(folder_name, conda_env_name, extra_conda_channels, language, extra_conda_packages, extra_pip_packages): extra_conda_channels = [x.strip() for x in extra_conda_channels.split(',')] extra_conda_packages = [x.strip() for x in extra_conda_packages.split(',')] extra_pip_packages = [x.strip() for x in extra_pip_packages.split(',')] conda_env_name = re.sub('([A-Z]{1})', r'\1', conda_env_name).lower() conda_env_name.replace(" ", "_") path_for_yaml = os.environ['KERNEL_FOLDER'] + '/' + folder_name extra_conda_channels.insert(0, "anaconda") #python 3 and 2 require different versions of same packages for a kernel if language == "python3": print("xxxxxxxxxxx----python3") extra_conda_packages.insert(0, "ipython") extra_conda_packages.insert(0, "future") extra_conda_packages.insert(0, "pycryptodomex") extra_conda_packages.insert(0, "jupyter_client") extra_conda_packages.insert(0, "python=3.7") extra_pip_packages.insert(0, "ipykernel") elif language == "python2": print("xxxxxxxxxxx----python2") extra_pip_packages.insert(0, "IPython==5.8.0") extra_conda_packages.insert(0, "jupyter_client=5.3.1") extra_conda_packages.insert(0, "pycryptodomex") extra_conda_packages.insert(0, "future") extra_conda_packages.insert(0, "python=2.7") extra_pip_packages.insert(0, "ipykernel==4.10.0") extra_conda_packages.append({'pip' : extra_pip_packages}) extra_pip_packages = list(filter(len, extra_pip_packages)) extra_conda_channels = list(filter(len, extra_conda_channels)) extra_conda_packages = list(filter(len, extra_conda_packages)) yaml_prep = {"name": conda_env_name, "channels": extra_conda_channels, "dependencies": extra_conda_packages} yaml = ruamel.yaml.YAML() yaml.default_flow_style = False yaml.default_style = '"' yaml.Representer.represent_key = non_quoted_key yaml.indent(mapping=2, sequence=4, offset=2) try: with open(path_for_yaml + '/' + conda_env_name + '/' + conda_env_name + '.yml', 'w') as f: yaml.dump(yaml_prep, f) except OSError: print("Creation of the yml file for te Conda environment failed") return False else: print("Successfully created yml file for the Conda environment") return True
def dict_to_yaml(d: dict, output_file: Union[str, pathlib.Path]) -> None: yaml = ruamel.yaml.YAML() yaml.indent(sequence=4, offset=2) with open(output_file, 'w') as f: yaml.dump(d, f)
def save(self): """Save the current configuration to file.""" with Config.__lock: yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) with open(self._config_file, 'w+') as f: yaml.dump(self._user, f)
def _save_yaml(mod_yaml, file_name: str): yaml = ruamel.yaml.YAML(typ='rt') yaml.indent(sequence=4, offset=2) yaml.preserve_quotes = True tmp_file = file_name + '.new' with open(tmp_file, 'w') as wf: yaml.dump(mod_yaml, wf) os.rename(tmp_file, file_name)
def setup_yaml(): # Setup the ruamel.yaml parser yaml = ruamel.yaml.YAML(typ='rt') yaml.preserve_quotes = True # This is what's used in the template, currently ~36 KEPs have drifted yaml.indent(mapping=2, sequence=4, offset=2) yaml.width = MAX_WIDTH return yaml
def get_tasks(target_dir, *scenarios): tasks = [] for scenario in scenarios: task_dir = target_dir / "tests" / "integration" / "targets" / scenario / "tasks" for _file in task_dir.glob("*"): yaml = ruamel.yaml.YAML() yaml.indent(sequence=4, offset=2) tasks += yaml.load(_file.open()) return tasks
def format_yaml(unformatted: str, _info_str: str) -> str: yaml = ruamel.yaml.YAML() # Make sure to always have `sequence >= offset + 2` yaml.indent(mapping=2, sequence=2, offset=0) parsed = yaml.load(unformatted) dump_stream = io.StringIO() yaml.dump(parsed, stream=dump_stream) return dump_stream.getvalue()
def test_02(self): yaml = YAML() yaml.indent(mapping=5, sequence=6, offset=3) inp = """ a: b: - 1 - [1, 2] """ yaml.round_trip(inp)
def test_01(self): yaml = YAML() yaml.indent(sequence=6) yaml.indent(offset=3) inp = """ a: - 1 - {b: 3} """ yaml.round_trip(inp)
def update_model_configs(models, descriptions, mode): diffs = update_model_descriptions(models, descriptions, mode) if mode == 'update': for name in diffs: model = models[name] yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.width = 80 with model[0].open("w", encoding="utf-8") as file: yaml.dump(model[1], file)
def dumps(obj): """Dumps yaml content into a string.""" yaml = ruamel.yaml.YAML() yaml.width = 66 stream = ruamel.yaml.compat.StringIO() yaml.explicit_start = True yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(obj, stream) return stream.getvalue()
def fix(repo_root): # repo_root = "/home/tamal/go/src/stash.appscode.dev/stash" directory = os.path.join(repo_root, ".github", "workflows") for filename in os.listdir(directory): if not filename.endswith(".yml"): continue with open(os.path.join(directory, filename), 'r+') as f: yaml = ruamel.yaml.YAML(typ='rt') yaml.preserve_quotes = True yaml.width = 4096 data = yaml.load(f) for job in data['jobs']: for i, step in enumerate(data['jobs'][job]['steps']): if 'name' in data['jobs'][job]['steps'][i].keys() and data['jobs'][job]['steps'][i]['name'] == 'Available platforms': data['jobs'][job]['steps'].pop(i) break for i, step in enumerate(data['jobs'][job]['steps']): if 'id' in step.keys() and step['id'] == 'buildx': data['jobs'][job]['steps'].pop(i) # add blank line before data['jobs'][job]['steps'][i].ca.comment = [ None, [CT('\n', CommentMark(0), None)]] e1 = CM({ 'name': 'Set up Docker Buildx', 'uses': 'docker/setup-buildx-action@v1' }) # e1.yaml_set_start_comment('\n') e1.ca.comment = [ None, [CT('\n', CommentMark(0), None)]] data['jobs'][job]['steps'].insert(i, e1) e2 = CM({ 'name': 'Available platforms', 'run': 'echo ${{steps.qemu.outputs.platforms}}' }) e2.ca.comment = [ None, [CT('\n', CommentMark(0), None)]] data['jobs'][job]['steps'].insert(i, e2) data['jobs'][job]['steps'].insert(i, CM({ 'name': 'Set up QEMU', 'id': 'qemu', 'uses': 'docker/setup-qemu-action@v1' })) break f.seek(0) f.truncate(0) yaml.indent(mapping=2, sequence=4, offset=2) yaml.dump(data, f)
def test_04(self): yaml = YAML() yaml.indent(mapping=5, sequence=6) inp = """ a: b: - 1 - [1, 2] - {d: 3.14} """ yaml.round_trip(inp)
def object_to_yaml(data: JSON_TYPE) -> str: """Create yaml string from object.""" yaml = YAML(typ='rt') yaml.indent(sequence=4, offset=2) stream = StringIO() try: yaml.dump(data, stream) result = stream.getvalue() # type: str return result except YAMLError as exc: _LOGGER.error("YAML error: %s", exc) raise HomeAssistantError(exc)
#!/usr/bin/env python import sys try: import ruamel.yaml except ImportError: sys.exit('Error, ruamel library not found. Try: pip3 install ruamel.yaml') yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.default_flow_style = False yaml.explicit_start = True yaml.preserve_quotes = True # Represent null values as ~ instead of blanks # https://stackoverflow.com/a/44314840 def custom_null(self, data): return self.represent_scalar(u'tag:yaml.org,2002:null', u'~') ruamel.yaml.RoundTripRepresenter.add_representer(type(None), custom_null) yaml.dump(yaml.load(sys.stdin), sys.stdout)
def add_projects(): yaml = ruamel.yaml.YAML() yaml.indent(mapping=2, sequence=4, offset=2) all_projects = yaml.load(open('gerrit/projects.yaml', 'r')) zuul_main = yaml.load(open('zuul/main.yaml', 'r')) existing_projects = set() gerrit = None for tenant in zuul_main: if tenant['tenant']['name'] == 'openstack': gerrit = tenant['tenant']['source']['gerrit'] break # Find the point in the list where we want to add things and save the # comment text from it so that we can re-add it sorted_index = None saved_comment = None for idx, token in gerrit['untrusted-projects'].ca.items.items(): text = get_comment_text(token) if text.startswith('# After this point'): sorted_index = idx saved_comment = token break # Get the list of things above the marker comment for project_type in ('config-projects', 'untrusted-projects'): for idx, project in enumerate(gerrit[project_type]): if idx == sorted_index: break if isinstance(project, dict): project = get_single_key(project) existing_projects.add(project) new_projects = [] for project in all_projects: name = project['project'] # It's in the file, already - it's taken care of if name in existing_projects: continue # Skip or remove retired projects is_retired = name.split('/')[0].endswith('-attic') in_attic = project.get('acl-config', '').endswith('/retired.config') if is_retired or in_attic: if name in gerrit['untrusted-projects']: del gerrit['untrusted-projects'][name] continue new_projects.append(name) # Pop things off the end of the list until we're down at the length # indicated by the saved index position. We have to do this weirdly # with passing the index to pop because it's not a real list for idx in reversed(range(sorted_index, len(gerrit['untrusted-projects']))): gerrit['untrusted-projects'].pop(idx) # Toss in a sorted just to make sure - the source list should be sorted # but why tempt fate right? gerrit['untrusted-projects'].extend(sorted(new_projects)) gerrit['untrusted-projects'].ca.items[sorted_index] = saved_comment yaml.dump(zuul_main, open('zuul/main.yaml', 'w')) # Strip the extra 2 spaces that ruamel.yaml appends because we told it # to indent an extra 2 spaces. Because the top level entry is a list it # applies that indentation at the top. It doesn't indent the comment lines # extra though, so don't do them. with open('zuul/main.yaml', 'r') as main_in: main_content = main_in.readlines() with open('zuul/main.yaml', 'w') as main_out: for line in main_content: if '#' in line: main_out.write(line) else: main_out.write(line[2:])