def dict_to_yaml(data, width=None, sort=False): """ Convert dictionary into yaml """ output = StringIO() # Set formatting options yaml = YAML() yaml.indent(mapping=4, sequence=4, offset=2) yaml.default_flow_style = False yaml.allow_unicode = True yaml.encoding = 'utf-8' yaml.width = width # Make sure that multiline strings keep the formatting data = copy.deepcopy(data) scalarstring.walk_tree(data) # Sort the data https://stackoverflow.com/a/40227545 if sort: sorted_data = CommentedMap() for key in sorted(data): sorted_data[key] = data[key] data = sorted_data yaml.dump(data, output) return output.getvalue()
def publicize(self, resource_id, public_data): walk_tree(public_data) path = str(Path(resource_id.path).with_suffix('.public.yaml')) ensure_dir(path) if path.endswith('.yaml'): with open(path, 'wb') as out: yaml.dump(public_data, out) else: with open(path, 'w') as out: out.write(public_data)
def to_disk(self, resource_id, resource): resource_data = resource.to_disk(resource_id) walk_tree(resource_data) # Make a backup of the local version backed_up = self.backup_resource(resource_id, resource_data) if backed_up: log("Backed up file: ", resource_id.path) ensure_dir(resource_id.path) if resource_id.path.endswith('.yaml'): with open(resource_id.path, 'wb') as out: yaml.dump(resource_data, out) else: with open(resource_id.path, 'w') as out: out.write(resource_data)
def test_basic(self): from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import walk_tree data = CommentedMap() data[1] = 'a' data[2] = 'with\nnewline\n' walk_tree(data) exp = """\ 1: a 2: | with newline """ assert round_trip_dump(data) == dedent(exp)
def write_yaml_file(file_path, dict_like): """write a dict_like object to a file using the given file path. Parameters ---------- file_path: str path with file name and extension dict_like: dict or OrderedDict the data to be written to file """ with open(file_path, 'w', encoding='utf8') as fpt: if USE_RUAMEL: yaml_obj = YAML() yaml_obj.indent(mapping=4, sequence=4, offset=2) scalarstring.walk_tree(dict_like) yaml_obj.dump(dict_like, fpt) else: yaml.dump(dict_like, fpt)
def update_bank(cls, course, question_name, bank_source, new_question): course.backup_bank(bank_source) course_cache = cls.CACHE[course.course_name] course_cache[question_name] = new_question # Grab the names of the old questions kept_question_names = [] with open(bank_source) as bank_file: questions = yaml.load(bank_file) for question in questions: question_name = question['question_name'] kept_question_names.append(['question_name']) # Get the actual up-to-date questions questions = [ course_cache[name].to_disk(force=True) for name in kept_question_names ] # Dump them back into the file walk_tree(questions) with open(bank_source, 'w') as bank_file: yaml.dump(questions, out)
def safe_dump_yaml(tool_path: str, tool_json: dict): """ Dump dict to a yaml file without overriding existing files (add _#_ prefix) :param tool_path: expected path :param tool_json: tool dict :return: path where file is dumped """ counter = 1 basedir = os.path.dirname(tool_path) basename = os.path.basename(tool_path) # Append and increment prefix while there is a conflict while os.path.exists(tool_path): tool_path = os.path.join(basedir, '_{}_{}'.format(counter, basename)) counter += 1 # Ensure any missing path directories are created os.makedirs(os.path.dirname(os.path.abspath(tool_path)), exist_ok=True) # Dump yaml to the path y = YAML() walk_tree(tool_json) with open(tool_path, 'w') as f: y.dump(tool_json, f) return tool_path
def from_json(self): # use roundtrip to preserve order errors = 0 docs = [] dumper = ruamel.yaml.RoundTripDumper for file_name in self._args.file: if not self._args.write and file_name == '-': inp = sys.stdin.read() else: inp = open(file_name).read() loader = ruamel.yaml.Loader # RoundTripLoader data = ruamel.yaml.load(inp, loader) data = commentify(data) if self._args.write: yaml_file_name = os.path.splitext(file_name)[0] + '.yaml' with open(yaml_file_name, 'w') as fp: ruamel.yaml.dump( data, fp, Dumper=dumper, default_flow_style=self._args.flow, allow_unicode=True, ) else: docs.append(data) if self._args.write: return 1 if errors else 0 if self._args.literal: from ruamel.yaml.scalarstring import walk_tree for doc in docs: walk_tree(doc) print( ruamel.yaml.dump_all( docs, Dumper=dumper, default_flow_style=self._args.flow, allow_unicode=True, )) return 1 if errors else 0
def test_map(self): from ruamel.yaml.compat import ordereddict from ruamel.yaml.comments import CommentedMap from ruamel.yaml.scalarstring import walk_tree, preserve_literal from ruamel.yaml.scalarstring import DoubleQuotedScalarString as dq from ruamel.yaml.scalarstring import SingleQuotedScalarString as sq data = CommentedMap() data[1] = 'a' data[2] = 'with\nnew : line\n' data[3] = '${abc}' data[4] = 'almost:mapping' m = ordereddict([('\n', preserve_literal), ('${', sq), (':', dq)]) walk_tree(data, map=m) exp = """\ 1: a 2: | with new : line 3: '${abc}' 4: "almost:mapping" """ assert round_trip_dump(data) == dedent(exp)
def breakdown_wf_local( wf_path: str, installed_apps: dict = None, nested_wf_json: dict = None, # use if is_main is false steps_dir: str = None, # use if is_main is false is_main: bool = True): """ Recursively walk through all the steps (tools and nested wfs) and install them in steps folder. Reference them in the main workflow. :param wf_path: Path where to dump the tool/workflow :param installed_apps: Dict containing already installed apps. :param nested_wf_json: None in main call, dict in recursive calls :param steps_dir: None in main call, path in recursive calls :param is_main: True in main call, False in recursive calls :return: (Workflow path, Installed apps dictionary) """ msg = ("Decomposing workflow '{}' and" " installing parts in 'steps' folder. " "This may take a minute or two.\n" "Set log level to INFO" " to track decomposing progress.").format(os.path.abspath(wf_path)) logger.info(msg) print(colored(msg, 'green')) wf_path = os.path.abspath(wf_path) installed_apps = installed_apps or dict() base_dir = os.path.dirname(wf_path) updated_wf_path = '.'.join(wf_path.split('.')[:-1]) if is_main: updated_wf_path += '_decomposed.cwl' else: updated_wf_path += '.cwl' # Resolve main workflow or use provided json for nested wf if is_main and not nested_wf_json: wf_json = resolve_cwl(wf_path) else: wf_json = nested_wf_json # Make steps dir steps_dir = steps_dir or os.path.join(base_dir, 'steps') if not os.path.exists(steps_dir): os.mkdir(steps_dir) wf_json['steps'] = cwl_ensure_dict(wf_json['steps'], 'id') for step_id, step in wf_json['steps'].items(): app_hash = calc_json_hash(step['run']) if app_hash in installed_apps: wf_json['steps'][step_id]['run'] = os.path.relpath( installed_apps[app_hash], base_dir) else: tool_path = os.path.join(steps_dir, step_id + '.cwl') if step['run']['class'] in ['CommandLineTool', 'ExpressionTool']: # Dump run contents to file tool_path = safe_dump_yaml(tool_path, step['run']) # Add to installed apps to avoid duplicates installed_apps[app_hash] = os.path.abspath(tool_path) # Add a relative path to wf_json wf_json['steps'][step_id]['run'] = os.path.relpath( tool_path, base_dir) elif step['run']['class'] == 'Workflow': nested_wf, installed_apps = breakdown_wf_local( tool_path, installed_apps=installed_apps, nested_wf_json=step['run'], is_main=False, steps_dir=steps_dir) wf_json['steps'][step_id]['run'] = os.path.relpath( nested_wf, base_dir) if is_main: y = YAML() with open(updated_wf_path, 'w') as f: walk_tree(wf_json) y.dump(wf_json, f) msg = ("Rewiring done. " "New tools are now connected" " in the workflow {}.").format(os.path.abspath(updated_wf_path)) logger.info(msg) print(colored(msg, 'green')) else: wf_hash = calc_json_hash(wf_json) if wf_hash in installed_apps: return installed_apps[wf_hash], installed_apps else: safe_dump_yaml(updated_wf_path, wf_json) installed_apps[wf_hash] = os.path.abspath(updated_wf_path) return os.path.abspath(updated_wf_path), installed_apps
vault_pw = VaultSecret(pw) vault_pw.load() except FileNotFoundError: print("Password file not found") sys.exit(1) else: vault_pw = PromptVaultSecret(prompt_formats=["password: "]) vault_pw.load() vl = VaultLib(secrets=[ (None, vault_pw) ]) def to_yaml(representer, node): return representer.represent_scalar('!vault', node, style='|') yaml = YAML() yaml.indent(mapping=2, sequence=4, offset=2) yaml.representer.add_representer(LiteralScalarString, to_yaml) with open(sys.argv[1], 'r') as orig: y = yaml.load(orig) for value in y: y[value] = vl.encrypt(y[value], vault_pw).decode('utf-8') scalarstring.walk_tree(y) with open(sys.argv[2], 'w') as dest: yaml.dump(y, dest)
def dict_to_yaml(data: dict, out_path: str): y = YAML() data = deepcopy(data) walk_tree(data) with open(out_path, 'w') as outfile: y.dump(data, outfile)