def decrypt(config, output=False): stage = config['stage'] check_encryption_required_fields(config['stages'][stage]) region = config['stages'][stage]['keyRegion'] kms = boto3.client('kms', region_name=region) enc_config = get_secret_config(config, stage) if isinstance(enc_config, string_types): # This is the old-style secretConfig, when everything was encrypted # into a single string. stage_cfg = base64.b64decode(enc_config) resp = kms.decrypt(CiphertextBlob=stage_cfg) plain = json.loads(resp['Plaintext']) if output: print('Decrypted config for stage {}:\n\n{}'.format( stage, yaml.round_trip_dump(plain))) return plain elif isinstance(enc_config, dict): # This is the new way, where all config items are encrypted separately. plain = {} for key, value in enc_config.items(): if is_value_already_encrypted(value): ciphertext_blob = base64.b64decode( value[len(ENCRYPTED_PREFIX):]) resp = kms.decrypt(CiphertextBlob=ciphertext_blob) plain[key] = resp['Plaintext'] else: raise Exception('Found unencrypted item in secretConfig: ' '{}'.format(key)) if output: print('Decrypted config for stage {}:\n\n{}'.format( stage, yaml.round_trip_dump(plain))) return plain
def sorted_boskos_config(): """Get the sorted boskos configuration.""" with open(test_infra('boskos/resources.yaml'), 'r') as fp: configs = yaml.round_trip_load(fp, preserve_quotes=True) for rtype in configs['resources']: rtype["names"] = sorted(rtype["names"]) output = cStringIO.StringIO() yaml.round_trip_dump( configs, output, default_flow_style=False, width=float("inf")) return output
def sorted_prow_config(prow_config_path=None): """Get the sorted Prow configuration.""" with open(prow_config_path, 'r') as fp: configs = yaml.round_trip_load(fp, preserve_quotes=True) configs['periodics'] = sorted_seq(configs['periodics']) configs['presubmits'] = sorted_map(configs['presubmits']) configs['postsubmits'] = sorted_map(configs['postsubmits']) output = cStringIO.StringIO() yaml.round_trip_dump( configs, output, default_flow_style=False, width=float("inf")) return output
def put(self, path, value): ''' put path, value into a dict ''' try: entry = Yedit.get_entry(self.yaml_dict, path, self.separator) except KeyError: entry = None if entry == value: return (False, self.yaml_dict) # deepcopy didn't work # Try to use ruamel.yaml and fallback to pyyaml try: tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) except AttributeError: tmp_copy = copy.deepcopy(self.yaml_dict) # set the format attributes if available try: tmp_copy.fa.set_block_style() except AttributeError: pass result = Yedit.add_entry(tmp_copy, path, value, self.separator) if not result: return (False, self.yaml_dict) self.yaml_dict = tmp_copy return (True, self.yaml_dict)
def generate(self, legacy_config): action_id = 1 actions = { 'actions': {} } for project in legacy_config: if project == '.defaults' or project == '.regex': continue unit = legacy_config[project]['delete']['unit'] count = legacy_config[project]['delete']['count'] raw_regex = legacy_config[project][self.RAW_REGEX] if raw_regex: regex = project else: regex = self.format_regex(project) action = self.generate_action(unit, count, regex, exclude=False) actions['actions'][action_id] = action action_id += 1 actions['actions'][action_id] = self.generate_defaults(legacy_config) with open(self.actions_file, 'w') as f: yaml.round_trip_dump(actions, f, default_flow_style=False, explicit_start=True)
def normalize_docker_compose_command(): """ Sub-command, see main() """ parser = argparse.ArgumentParser( description='Normalize the input docker-compose file, then write it in the output') parser.add_argument('-i', '--input', type=str, help='<Required> Path to the input yaml file', required=True) parser.add_argument('-o', '--output', type=str, help='Path to the output file, or stdout by default') args = parser.parse_args(sys.argv[2:]) input_file = open(args.input, 'r') content = input_file.read() input_file.close() output_data = normalize_docker_compose(content) output_file = open(args.output, 'w') if args.output else sys.stdout round_trip_dump(output_data, output_file) output_file.close()
def write_template(self, output, filename=None): if not filename: filename = 'swagger.yml' swagger_file = os.path.join(self.project_dir, filename) _, ext = os.path.splitext(filename) with open(swagger_file, 'w') as fh: # Could be `.yaml` or `.yml` :/ if '.y' in ext: fh.write(yaml.round_trip_dump(output)) elif '.json' in ext: fh.write(json.dumps(output)) return swagger_file
def _process_section(self, section_value, callback=None, templar=None): if not templar: templar = self._templar processed = ordereddict() for key, value in section_value.items(): if isinstance(value, string_types): # strings can be templated processed[key] = templar.template(value) if isinstance(processed[key], AnsibleUnsafeText): processed[key] = str(processed[key]) elif isinstance(value, (list, dict)): # if it's a dimensional structure, it's cheaper just to serialize # it, treat it like a template, and then deserialize it again buffer = BytesIO() # use bytes explicitly, not unicode yaml.round_trip_dump(value, buffer) processed[key] = yaml.round_trip_load( templar.template(buffer.getvalue())) else: # ints, booleans, etc. processed[key] = value if callback: callback(processed) return processed
def install_extras(mkdocs_yml, theme=None): with open(mkdocs_yml) as f: config, indent, bsi = load_yaml_guess_indent(f) if theme is None and 'theme' not in config: raise ValueError('no theme specified in mkdocs.yml; pass ' + '--theme instead') theme_dir = get_theme_dir(config.get('theme', theme)) docs_dir = config.get('docs_dir', 'docs') for path, prop in [('css', 'extra_css'), ('js', 'extra_javascript')]: files = os.listdir(os.path.join(theme_dir, path)) if not files: continue extras = config.setdefault(prop, []) for f in files: relpath = os.path.join(path, f) shutil.copyfile(os.path.join(theme_dir, relpath), os.path.join(docs_dir, relpath)) if relpath not in extras: extras.append(relpath) yaml.round_trip_dump(config, open(mkdocs_yml, 'w'), indent=indent, block_seq_indent=bsi)
def delete_command(): """ Sub-command, see main() """ parser = argparse.ArgumentParser( description='Delete one item from the input yaml file') parser.add_argument('path_to_key', type=str, nargs='+', help='<Required> Yaml item to be deleted, e.g. "foo 0 bar"') parser.add_argument('-i', '--input', type=str, help='<Required> Path to the input yaml files', required=True) parser.add_argument('-o', '--output', type=str, help='Path to the output file, or stdout by default') args = parser.parse_args(sys.argv[2:]) input_file = open(args.input, 'r') data = round_trip_load(input_file.read(), preserve_quotes=True) input_file.close() output_data, _ = delete_yaml_item(data, args.path_to_key, True) output_file = open(args.output, 'w') if args.output else sys.stdout round_trip_dump(output_data, output_file) output_file.close()
def update_generated_config(path, latest_version): print("Updating test_config.yaml...") with open(path, 'r') as f: config = yaml.round_trip_load(f) v = latest_version suffixes = ['beta', 'stable1', 'stable2', 'stable3'] for i, s in enumerate(suffixes): vs = "%d.%d" % (v[0], v[1] + 1 - i) markers = config['k8sVersions'][s] markers['version'] = vs for j, arg in enumerate(markers['args']): markers['args'][j] = re.sub( r'latest(-\d+\.\d+)?', 'latest-%s' % vs, arg) node = config['nodeK8sVersions'][s] for k, arg in enumerate(node['args']): node['args'][k] = re.sub( r'master|release-\d+\.\d+', 'release-%s' % vs, arg) node['prowImage'] = node['prowImage'].rpartition('-')[0] + '-' + vs with open(path, 'w') as f: yaml.round_trip_dump(config, f)
def create(self, path, value): ''' create a yaml file ''' if not self.file_exists(): # deepcopy didn't work tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) # pylint: disable=no-member if hasattr(self.yaml_dict, 'fa'): tmp_copy.fa.set_block_style() result = Yedit.add_entry(tmp_copy, path, value, self.separator) if result: self.yaml_dict = tmp_copy return (True, self.yaml_dict) return (False, self.yaml_dict)
def create(self, path, value): ''' create a yaml file ''' if not self.file_exists(): # deepcopy didn't work tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), # noqa: E501 yaml.RoundTripLoader) # pylint: disable=no-member if hasattr(self.yaml_dict, 'fa'): tmp_copy.fa.set_block_style() result = Yedit.add_entry(tmp_copy, path, value, self.separator) if result: self.yaml_dict = tmp_copy return (True, self.yaml_dict) return (False, self.yaml_dict)
def _process_section(self, section_value, callback=None, templar=None): if not templar: templar = self._templar processed = yaml.compat.ordereddict() for key, value in section_value.items(): if isinstance(value, basestring): # strings can be templated processed[key] = templar.template(value) if isinstance(processed[key], AnsibleUnsafeText): processed[key] = str(processed[key]) elif isinstance(value, (list, dict)): # if it's a dimensional structure, it's cheaper just to serialize # it, treat it like a template, and then deserialize it again buffer = BytesIO() # use bytes explicitly, not unicode yaml.round_trip_dump(value, buffer) processed[key] = yaml.round_trip_load( templar.template(buffer.getvalue()) ) else: # ints, booleans, etc. processed[key] = value if callback: callback(processed) return processed
def edit_soa_configs(filename, instance, mem, cpu): with open(filename, 'r') as fi: yams = fi.read() yams = yams.replace('cpus: .', 'cpus: 0.') data = yaml.round_trip_load(yams, preserve_quotes=True) instdict = data[instance] if mem: instdict['mem'] = mem else: instdict['cpus'] = cpu out = yaml.round_trip_dump(data, width=10000) with open(filename, 'w') as fi: fi.write(out)
def export_config(args, output_file=None): if not output_file: output_file = sys.stdout if os.path.exists(args.config): with open(args.config, 'rt') as f: config = yaml.round_trip_load(f.read()) STATE['stages'] = config['stages'] config['config'] = _decrypt_dict(config['config']) else: config = { 'stages': { env['name']: { 'environment': env['name'], 'key': 'enter-key-name-here' } for env in STATE['awscreds'].environments }, 'config': {} } if args.json: output_file.write(json.dumps(config, indent=4)) elif config: yaml.round_trip_dump(config, output_file)
def export_config(args, output_file=None): if not output_file: output_file = sys.stdout if os.path.exists(args.config): with open(args.config, "rt") as f: config = yaml.round_trip_load(f.read()) STATE["stages"] = config["stages"] config["config"] = _decrypt_dict(config["config"]) else: config = { "stages": { env["name"]: { "environment": env["name"], "key": "enter-key-name-here" } for env in STATE["awscreds"].environments }, "config": {}, } if args.json: output_file.write(json.dumps(config, indent=4)) elif config: yaml.round_trip_dump(config, output_file)
def merge_command(): """ Sub-command, see main() """ parser = argparse.ArgumentParser( description='Merge two or more yaml files and preserve the comments') parser.add_argument('-i', '--inputs', nargs='+', type=str, help='<Required> List of input yaml files, merged from the last to the first', required=True) parser.add_argument('-o', '--output', type=str, help='Path to the output file, or stdout by default') args = parser.parse_args(sys.argv[2:]) file_contents = [] for f in args.inputs: file = open(f, 'r') file_contents.append(file.read()) file.close() out_content = successive_merge(file_contents) output_file = open(args.output, 'w') if args.output else sys.stdout round_trip_dump(out_content, output_file) output_file.close()
def update_yaml_from_filter(filename, data, dir_path=DATA_PATH, ind=None, bsi=None): ''' 更新yaml文件 :param filename: 文件名称 :param data: 写入数据 :param dir_path: 文件路径 :param ind: :param bsi: :return: ''' file_url = os.path.join(dir_path, filename) try: with open(file_url, 'w', encoding='utf-8') as fp: round_trip_dump(data=data, stream=fp, block_seq_indent=bsi, indent=ind) except Exception as error: print('异常') raise error
def write_configuration(config_filename, config, mode=0o600): ''' Given a target config filename and a config data structure of nested OrderedDicts, write out the config to file as YAML. Create any containing directories as needed. ''' if os.path.exists(config_filename): raise FileExistsError('{} already exists. Aborting.'.format(config_filename)) try: os.makedirs(os.path.dirname(config_filename), mode=0o700) except (FileExistsError, FileNotFoundError): pass with open(config_filename, 'w') as config_file: config_file.write(yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)) os.chmod(config_filename, mode)
def decrypt(config, output=False): stage = config['stage'] enc_config = get_secret_config(config, stage) if not isinstance(enc_config, basestring): raise Exception('Secret config for stage {} is not a ' 'string! Did you forget to encrypt ' 'first?'.format(stage)) stage_cfg = base64.b64decode(enc_config) region = config['stages'][stage]['keyRegion'] kms = boto3.client('kms', region_name=region) resp = kms.decrypt(CiphertextBlob=bytes(stage_cfg)) plain = json.loads(resp['Plaintext']) if output: print('Decrypted config for stage {}:\n\n{}'.format( stage, yaml.round_trip_dump(plain))) return plain
def dump(self): """Dump this Project Schema to YAML.""" my_dict = yaml.comments.CommentedMap() for key in REQUIRED_KEYS: if key in ['files', 'sources']: continue my_dict[key] = getattr(self, key) for key in OPTIONAL_KEYS: try: val = getattr(self, key) if val: my_dict[key] = val except KeyError: pass my_dict['sources'] = self.sources my_dict['files'] = self.files return yaml.round_trip_dump(my_dict)
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, submit_runner_ram=0, name=None): packed = packed_workflow(arvRunner, tool) adjustDirObjs(job_order, trim_listing) adjustFileObjs(job_order, trim_anonymous_location) adjustDirObjs(job_order, trim_anonymous_location) main = [p for p in packed["$graph"] if p["id"] == "#main"][0] for inp in main["inputs"]: sn = shortname(inp["id"]) if sn in job_order: inp["default"] = job_order[sn] if not name: name = tool.tool.get("label", os.path.basename(tool.tool["id"])) upload_dependencies(arvRunner, name, tool.doc_loader, packed, tool.tool["id"], False) # TODO nowhere for submit_runner_ram to go. body = { "workflow": { "name": name, "description": tool.tool.get("doc", ""), "definition": yaml.round_trip_dump(packed) } } if project_uuid: body["workflow"]["owner_uuid"] = project_uuid if uuid: call = arvRunner.api.workflows().update(uuid=uuid, body=body) else: call = arvRunner.api.workflows().create(body=body) return call.execute(num_retries=arvRunner.num_retries)["uuid"]
def put(self, path, value): ''' put path, value into a dict ''' try: entry = Yedit.get_entry(self.yaml_dict, path, self.separator) except KeyError: entry = None if entry == value: return (False, self.yaml_dict) # deepcopy didn't work # Try to use ruamel.yaml and fallback to pyyaml try: tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) except AttributeError: tmp_copy = copy.deepcopy(self.yaml_dict) # set the format attributes if available try: tmp_copy.fa.set_block_style() except AttributeError: pass result = Yedit.add_entry(tmp_copy, path, value, self.separator) if result is None: return (False, self.yaml_dict) # When path equals "" it is a special case. # "" refers to the root of the document # Only update the root path (entire document) when its a list or dict if path == '': if isinstance(result, list) or isinstance(result, dict): self.yaml_dict = result return (True, self.yaml_dict) return (False, self.yaml_dict) self.yaml_dict = tmp_copy return (True, self.yaml_dict)
def best_practice_comment_round_trip(yaml_path): """Round-trips the YAML document. If comments (#) are incorrectly escaped or not using literal style they will affect tools parsing the content. Also catches indentation changes, extra spaces in unneeded areas, other minor stylistic changes.""" with open(yaml_path, mode="r", encoding="utf-8") as f: yaml_str1 = f.read() yaml_object = yaml.round_trip_load(yaml_str1, preserve_quotes=True) yaml_str2 = yaml.round_trip_dump(yaml_object, width=300) results = list(difflib.Differ().compare( yaml_str1.splitlines(keepends=True), yaml_str2.splitlines(keepends=True))) for item in results: if item.startswith(("+ ", "- ", "? ")): pprint.pprint(item) assert yaml_str1 == yaml_str2
def serialize(cls, data): from ruamel import yaml yaml.representer.RoundTripRepresenter.add_representer( types.List, yaml.representer.RoundTripRepresenter.represent_list) yaml.representer.RoundTripRepresenter.add_representer( types.Dict, yaml.representer.RoundTripRepresenter.represent_dict) if settings.INDENT_YAML_BLOCKS: f = StringIO() y = yaml.YAML() y.indent(mapping=2, sequence=4, offset=2) y.dump(data, f) text = f.getvalue().strip() + '\n' else: text = yaml.round_trip_dump(data) or "" if text == "{}\n": return "" return text.replace('- \n', '-\n')
def print_parsed_obj(cwlyml, target_path=None, ofd=sys.stdout): if cwlyml.target['class'] == 'Workflow': cwlyml.target = update_workflow(cwlyml.target) else: cwlyml.load_target_as_ruamel_obj(target_path) cwlyml.target = update_clt(cwlyml.target) root_keys = [ 'class', 'cwlVersion', 'doc', 'requirements', 'hints', 'inputs', 'steps', 'expression', 'outputs', 'baseCommand', 'arguments', 'stdin', 'stdout' ] # print(ryaml.round_trip_dump(cwlyml.target, indent=2, explicit_end=False)) st = ryaml.round_trip_dump(OrderedDict( sorted(cwlyml.target.items(), key=lambda t: root_keys.index(t[0]))), indent=2, explicit_end=False, width=10000) final = re.sub('^-', '', re.sub('\n\n', '\n', st)).replace('!!omap\n-', '') ofd.write(re.sub('\n-', '\n', final))
def to_yaml(message, stream=None): """Returns the YAML of the given protobuf message. Args: Message (obj): The protobuf message to be converted Returns: str: the resulting YAML """ from google.protobuf.json_format import MessageToDict data = MessageToDict(message) from ruamel.yaml import YAML, round_trip_dump yaml = YAML(typ="safe") yaml.default_flow_style = False stream.write("---\n") return round_trip_dump(data, stream)
def put(self, path, value): ''' put path, value into a dict ''' try: entry = Yedit.get_entry(self.yaml_dict, path, self.separator) except KeyError as _: entry = None if entry == value: return (False, self.yaml_dict) # deepcopy didn't work tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) # pylint: disable=no-member if hasattr(self.yaml_dict, 'fa'): tmp_copy.fa.set_block_style() result = Yedit.add_entry(tmp_copy, path, value, self.separator) if not result: return (False, self.yaml_dict) self.yaml_dict = tmp_copy return (True, self.yaml_dict)
def run(self) -> Sequence[nodes.Node]: # type: ignore """ Process the content of the directive. """ plugin_name = self.options.get("plugin-name", self.env.config.github_repository) flake8_version = self.options.get("flake8-version", "3.8.4") config = { "repo": "https://gitlab.com/pycqa/flake8", "rev": flake8_version, "hooks": [{ "id": "flake8", "additional_dependencies": [f"{plugin_name}=={self.arguments[0]}"] }] } targetid = f'pre-commit-{self.env.new_serialno("pre-commit"):d}' targetnode = nodes.section(ids=[targetid]) yaml_output = yaml.round_trip_dump([config], default_flow_style=False) if not yaml_output: return [] content = f".. code-block:: yaml\n\n{indent(yaml_output, ' ')}\n\n" view = StringList(content.split('\n')) pre_commit_node = nodes.paragraph(rawsource=content) self.state.nested_parse(view, self.content_offset, pre_commit_node) pre_commit_f8_node_purger.add_node(self.env, pre_commit_node, targetnode, self.lineno) return [pre_commit_node]
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None, submit_runner_ram=0, name=None): packed = packed_workflow(arvRunner, tool) adjustDirObjs(job_order, trim_listing) adjustFileObjs(job_order, trim_anonymous_location) adjustDirObjs(job_order, trim_anonymous_location) main = [p for p in packed["$graph"] if p["id"] == "#main"][0] for inp in main["inputs"]: sn = shortname(inp["id"]) if sn in job_order: inp["default"] = job_order[sn] if not name: name = tool.tool.get("label", os.path.basename(tool.tool["id"])) upload_dependencies(arvRunner, name, tool.doc_loader, packed, tool.tool["id"], False) # TODO nowhere for submit_runner_ram to go. body = { "workflow": { "name": name, "description": tool.tool.get("doc", ""), "definition":yaml.round_trip_dump(packed) }} if project_uuid: body["workflow"]["owner_uuid"] = project_uuid if uuid: call = arvRunner.api.workflows().update(uuid=uuid, body=body) else: call = arvRunner.api.workflows().create(body=body) return call.execute(num_retries=arvRunner.num_retries)["uuid"]
def dump(self, configs, path): cfg = configparser.ConfigParser(allow_no_value=True) leftovers = [] root_comment = getattr(configs, '__doc__', '') if root_comment: cfg.setdefault(self.root_name, {}) for comment in root_comment.split(os.linesep): cfg.set(self.root_name, f'# {comment}', None) for p, value, prop in configs.get_prop_paths(): if '.' in p: top = '.'.join(p.split('.')[:-1]) else: top = self.root_name cfg.setdefault(top, {}) if isinstance(value, Nestable): # Put it before the first value if prop.comment: leftovers = prop.comment.split(os.linesep) continue if prop.comment: for c in itertools.chain( *[leftovers, prop.comment.split(os.linesep)]): cfg.set(top, f'# {c}', None) leftovers = [] if isinstance(value, list): cfg[top][prop.name] = yaml.round_trip_dump(value) else: cfg[top][prop.name] = str(value) with open(path, 'w') as f: cfg.write(f)
def create_proj(self): yml = CommentedMap() yml['project'] = CommentedMap() # def _add(name): items = getattr(self, name) #if BuildItem.trivial_item(items): # yml['project'][name] = "_default_" #elif BuildItem.no_flags_in_collection(items): if BuildItem.no_flags_in_collection(items): out = [] for s in items: out.append(s.name) yml['project'][name] = out else: out = [] for s in items: cm = CommentedMap() cm[s.name] = CommentedMap() s.save_config(cm[s.name]) out.append(cm) yml['project'][name] = out # _add('systems') _add('architectures') _add('compilers') _add('build_types') _add('variants') txt = yaml.round_trip_dump(yml) fn = self.kwargs['output_file'] if not os.path.isabs(fn): fn = os.path.join(self.root_dir, fn) with open(fn, "w") as f: f.write(txt)
def put(self, path, value): ''' put path, value into a dict ''' try: entry = Yedit.get_entry(self.yaml_dict, path, self.separator) except KeyError: entry = None if entry == value: return (False, self.yaml_dict) # deepcopy didn't work tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) # pylint: disable=no-member if hasattr(self.yaml_dict, 'fa'): tmp_copy.fa.set_block_style() result = Yedit.add_entry(tmp_copy, path, value, self.separator) if not result: return (False, self.yaml_dict) self.yaml_dict = tmp_copy return (True, self.yaml_dict)
def get_pattern_review(): global pattern, pattern_list, pattern_iri_prefix, repo_name, gh_paths yaml_content = pattern_list[pattern] res = "" for line in yaml.round_trip_dump(yaml_content, indent=5, block_seq_indent=3).splitlines(True): res += line[3:] iri = str(yaml_content['pattern_iri']) path_gh = "https://github.com/{}/blob/master/{}".format(repo_name,gh_paths[pattern]) gh_issue = get_issue(pattern) return html.Div([ html.H2('Pattern review for {}'.format(pattern)), html.Div([html.H4("Name: "), html.Div(str(yaml_content['pattern_name']),id="pname")]), html.Div([html.H4("IRI: "), html.Div(html.A(href=path_gh, children=iri, target="_blank"))]), html.Div([html.H4("GitHub discussion: "), html.Div(html.A(href=gh_issue, children="{}".format(gh_issue), target="_blank"))]), html.Div([html.H4("Description: "),html.Div(get_val(yaml_content,'description'))]), html.Div([html.H4("Classes: "),html.Div(get_val(yaml_content,'classes',True))]), html.Div([html.H4("Relations: "),html.Div(get_val(yaml_content,'relations',True))]), html.Div([html.H4("Annotation Properties: "),html.Span(get_val(yaml_content,'annotationProperties',True))]), html.Div([html.H4("Variables: "),html.Div(get_val(yaml_content,'vars'))]), html.Div([html.H4("Phenotype name: "),html.Div(get_val(yaml_content,'name'))]), html.Div([html.H4("Phenotype annotations: "),html.Div(get_val(yaml_content,'annotations'))]), html.Div([html.H4("Phenotype definition: "),html.Div(get_val(yaml_content,'def'))]), html.Div([html.H4("EQ: "),html.Div(get_val(yaml_content,'equivalentTo'))]), dcc.RadioItems( id='radio-sign', options = [ {'label': 'Sign', 'value': 'sign'}, {'label': 'Unsign', 'value': 'unsign'}, ], value = "", labelStyle={'display': 'inline-block'} ), html.Button('Submit', id='sign-pattern'), html.Div("",id='signed'), ])
def create(self, path, value): ''' create a yaml file ''' if not self.file_exists(): # deepcopy didn't work # Try to use ruamel.yaml and fallback to pyyaml try: tmp_copy = yaml.load(yaml.round_trip_dump(self.yaml_dict, default_flow_style=False), yaml.RoundTripLoader) except AttributeError: tmp_copy = copy.deepcopy(self.yaml_dict) # set the format attributes if available try: tmp_copy.fa.set_block_style() except AttributeError: pass result = Yedit.add_entry(tmp_copy, path, value, self.separator) if result: self.yaml_dict = tmp_copy return (True, self.yaml_dict) return (False, self.yaml_dict)
def main(argsl=None, # type: List[str] args=None, # type: argparse.Namespace job_order_object=None, # type: MutableMapping[Text, Any] stdin=sys.stdin, # type: IO[Any] stdout=None, # type: Union[TextIO, StreamWriter] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] logger_handler=None, # custom_schema_callback=None, # type: Callable[[], None] executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] loadingContext=None, # type: LoadingContext runtimeContext=None # type: RuntimeContext ): # type: (...) -> int if not stdout: # force UTF-8 even if the console is configured differently if (hasattr(sys.stdout, "encoding") # type: ignore and sys.stdout.encoding != 'UTF-8'): # type: ignore if PY3 and hasattr(sys.stdout, "detach"): stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') else: stdout = getwriter('utf-8')(sys.stdout) # type: ignore else: stdout = cast(TextIO, sys.stdout) # type: ignore _logger.removeHandler(defaultStreamHandler) if logger_handler is not None: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) # pre-declared for finally block workflowobj = None prov_log_handler = None # type: Optional[logging.StreamHandler] try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) if args.record_container_id: if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) else: runtimeContext = runtimeContext.copy() # If on Windows platform, a default Docker Container is used if not # explicitely provided by user if onWindows() and not runtimeContext.default_container: # This docker image is a minimal alpine image with bash installed # (size 6 mb). source: https://github.com/frol/docker-alpine-bash runtimeContext.default_container = windows_default_container_id # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. for key, val in iteritems(get_default_args()): if not hasattr(args, key): setattr(args, key, val) # Configure logging rdflib_logger = logging.getLogger("rdflib.term") rdflib_logger.addHandler(stderr_handler) rdflib_logger.setLevel(logging.ERROR) if args.quiet: # Silence STDERR, not an eventual provenance log file stderr_handler.setLevel(logging.WARN) if runtimeContext.debug: # Increase to debug for both stderr and provenance log file _logger.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) formatter = None # type: Optional[logging.Formatter] if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) ## if args.version: print(versionfunc()) return 0 _logger.info(versionfunc()) if args.print_supported_versions: print("\n".join(supported_cwl_versions(args.enable_dev))) return 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error("CWL document required, no input file was provided") arg_parser().print_help() return 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback is not None: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") if args.provenance: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") return 1 ro = ResearchObject( temp_prefix_ro=args.tmpdir_prefix, orcid=args.orcid, full_name=args.cwl_full_name) runtimeContext.research_obj = ro log_file_io = ro.open_log_file_for_activity(ro.engine_uuid) prov_log_handler = logging.StreamHandler(log_file_io) class ProvLogFormatter(logging.Formatter): """Enforce ISO8601 with both T and Z.""" def __init__(self): # type: () -> None super(ProvLogFormatter, self).__init__( "[%(asctime)sZ] %(message)s") def formatTime(self, record, datefmt=None): # type: (logging.LogRecord, str) -> str record_time = time.gmtime(record.created) formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", record_time) with_msecs = "%s,%03d" % (formatted_time, record.msecs) return with_msecs prov_log_handler.setFormatter(ProvLogFormatter()) _logger.addHandler(prov_log_handler) _logger.debug(u"[provenance] Logging to %s", log_file_io) if argsl is not None: # Log cwltool command line options to provenance file _logger.info("[cwltool] %s %s", sys.argv[0], u" ".join(argsl)) _logger.debug(u"[cwltool] Arguments: %s", args) if loadingContext is None: loadingContext = LoadingContext(vars(args)) else: loadingContext = loadingContext.copy() loadingContext.loader = default_loader(loadingContext.fetcher_constructor) loadingContext.research_obj = runtimeContext.research_obj loadingContext.disable_js_validation = \ args.disable_js_validation or (not args.do_validate) loadingContext.construct_tool_object = getdefault( loadingContext.construct_tool_object, workflow.default_make_tool) loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver) loadingContext.do_update = not (args.pack or args.print_subgraph) uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) try_again_msg = "" if args.debug else ", try again with --debug for more information" try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, loadingContext.fetcher_constructor, loadingContext.overrides_list, tool_file_uri) if args.overrides: loadingContext.overrides_list.extend(load_overrides( file_uri(os.path.abspath(args.overrides)), tool_file_uri)) loadingContext, workflowobj, uri = fetch_document( uri, loadingContext) if args.print_deps and loadingContext.loader: printdeps(workflowobj, loadingContext.loader, stdout, args.relative_deps, uri) return 0 loadingContext, uri \ = resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only=(args.print_pre or args.pack), skip_schemas=args.skip_schemas) if loadingContext.loader is None: raise Exception("Impossible code path.") processobj, metadata = loadingContext.loader.resolve_ref(uri) processobj = cast(CommentedMap, processobj) if args.pack: stdout.write(print_pack(loadingContext.loader, processobj, uri, metadata)) return 0 if args.provenance and runtimeContext.research_obj: # Can't really be combined with args.pack at same time runtimeContext.research_obj.packed_workflow( print_pack(loadingContext.loader, processobj, uri, metadata)) if args.print_pre: stdout.write(json_dumps(processobj, indent=4, sort_keys=True, separators=(',', ': '))) return 0 tool = make_tool(uri, loadingContext) if args.make_template: def my_represent_none(self, data): # pylint: disable=unused-argument """Force clean representation of 'null'.""" return self.represent_scalar(u'tag:yaml.org,2002:null', u'null') yaml.RoundTripRepresenter.add_representer(type(None), my_represent_none) yaml.round_trip_dump( generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return 0 if args.validate: print("{} is valid CWL.".format(args.workflow)) return 0 if args.print_rdf: stdout.write(printrdf(tool, loadingContext.loader.ctx, args.rdf_serializer)) return 0 if args.print_dot: printdot(tool, loadingContext.loader.ctx, stdout) return 0 if args.print_targets: for f in ("outputs", "steps", "inputs"): if tool.tool[f]: _logger.info("%s%s targets:", f[0].upper(), f[1:-1]) stdout.write(" "+"\n ".join([shortname(t["id"]) for t in tool.tool[f]])+"\n") return 0 if args.target: if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: extracted = get_subgraph([tool.tool["id"] + "/" + r for r in args.target], tool) else: extracted = get_subgraph([loadingContext.loader.fetcher.urljoin(tool.tool["id"], "#" + r) for r in args.target], tool) else: _logger.error("Can only use --target on Workflows") return 1 loadingContext.loader.idx[extracted["id"]] = extracted tool = make_tool(extracted["id"], loadingContext) if args.print_subgraph: if "name" in tool.tool: del tool.tool["name"] stdout.write(json_dumps(tool.tool, indent=4, sort_keys=True, separators=(',', ': '))) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s", try_again_msg, exc if not args.debug else "", exc_info=args.debug) return 1 if isinstance(tool, int): return tool # If on MacOS platform, TMPDIR must be set to be under one of the # shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmp_outdir_prefix = default_mac_path if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmpdir_prefix = default_mac_path for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(runtimeContext, dirprefix) and getattr(runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \ else "" setattr(runtimeContext, dirprefix, os.path.abspath(getattr(runtimeContext, dirprefix)) + sl) if not os.path.exists(os.path.dirname(getattr(runtimeContext, dirprefix))): try: os.makedirs(os.path.dirname(getattr(runtimeContext, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": runtimeContext.move_outputs = "copy" runtimeContext.tmp_outdir_prefix = args.cachedir runtimeContext.secret_store = getdefault(runtimeContext.secret_store, SecretStore()) runtimeContext.make_fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess) try: initialized_job_order_object = init_job_order( job_order_object, args, tool, jobloader, stdout, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, make_fs_access=runtimeContext.make_fs_access, input_basedir=input_basedir, secret_store=runtimeContext.secret_store) except SystemExit as err: return err.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() runtimeContext.select_resources = executor.select_resources else: executor = SingleJobExecutor() try: runtimeContext.basedir = input_basedir del args.workflow del args.job_order conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text if conf_file or use_conda_dependencies: runtimeContext.job_script_provider = DependenciesConfiguration(args) else: runtimeContext.find_default_container = functools.partial( find_default_container, default_container=runtimeContext.default_container, use_biocontainers=args.beta_use_biocontainers) (out, status) = executor(tool, initialized_job_order_object, runtimeContext, logger=_logger) if out is not None: if runtimeContext.research_obj is not None: runtimeContext.research_obj.create_job( out, None, True) def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"]) visit_class(out, ("File", "Directory"), loc_to_path) # Unsetting the Generation from final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, string_types): stdout.write(out) else: stdout.write(json_dumps(out, indent=4, # type: ignore ensure_ascii=False)) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() # type: ignore if status != "success": _logger.warning(u"Final process status is %s", status) return 1 _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error( u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error( u"Workflow error%s:\n%s", try_again_msg, strip_dup_lineno(Text(exc)), exc_info=args.debug) return 1 except Exception as exc: # pylint: disable=broad-except _logger.error( u"Unhandled error%s:\n %s", try_again_msg, exc, exc_info=args.debug) return 1 finally: if args and runtimeContext and runtimeContext.research_obj \ and workflowobj and loadingContext: research_obj = runtimeContext.research_obj if loadingContext.loader is not None: research_obj.generate_snapshot(prov_deps( workflowobj, loadingContext.loader, uri)) else: _logger.warning("Unable to generate provenance snapshot " " due to missing loadingContext.loader.") if prov_log_handler is not None: # Stop logging so we won't half-log adding ourself to RO _logger.debug(u"[provenance] Closing provenance log file %s", prov_log_handler) _logger.removeHandler(prov_log_handler) # Ensure last log lines are written out prov_log_handler.flush() # Underlying WritableBagFile will add the tagfile to the manifest prov_log_handler.stream.close() prov_log_handler.close() research_obj.close(args.provenance) _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
def show_pipeline(self, dependencies, format_): report = "" p = Profile() for element in dependencies: line = format_ key = element._get_display_key() dim_keys = not key.strict line = p.fmt_subst(line, "name", element._get_full_name(), fg="blue", bold=True) line = p.fmt_subst(line, "key", key.brief, fg="yellow", dim=dim_keys) line = p.fmt_subst(line, "full-key", key.full, fg="yellow", dim=dim_keys) try: if not element._has_all_sources_resolved(): line = p.fmt_subst(line, "state", "no reference", fg="red") else: if element.get_kind() == "junction": line = p.fmt_subst(line, "state", "junction", fg="magenta") elif not element._can_query_cache(): line = p.fmt_subst(line, "state", "waiting", fg="blue") elif element._cached_failure(): line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_success(): line = p.fmt_subst(line, "state", "cached", fg="magenta") elif not element._can_query_source_cache(): line = p.fmt_subst(line, "state", "waiting", fg="blue") elif element._fetch_needed(): line = p.fmt_subst(line, "state", "fetch needed", fg="red") elif element._buildable(): line = p.fmt_subst(line, "state", "buildable", fg="green") else: line = p.fmt_subst(line, "state", "waiting", fg="blue") except BstError as e: # Provide context to plugin error e.args = ("Failed to determine state for {}: {}".format( element._get_full_name(), str(e)), ) raise e # Element configuration if "%{config" in format_: line = p.fmt_subst( line, "config", yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True), ) # Variables if "%{vars" in format_: variables = dict(element._Element__variables) line = p.fmt_subst( line, "vars", yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True)) # Environment if "%{env" in format_: environment = element._Element__environment line = p.fmt_subst( line, "env", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) # Public if "%{public" in format_: environment = element._Element__public line = p.fmt_subst( line, "public", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) # Workspaced if "%{workspaced" in format_: line = p.fmt_subst( line, "workspaced", "(workspaced)" if element._get_workspace() else "", fg="yellow") # Workspace-dirs if "%{workspace-dirs" in format_: workspace = element._get_workspace() if workspace is not None: path = workspace.get_absolute_path() if path.startswith("~/"): path = os.path.join(os.getenv("HOME", "/root"), path[2:]) line = p.fmt_subst(line, "workspace-dirs", "Workspace: {}".format(path)) else: line = p.fmt_subst(line, "workspace-dirs", "") # Dependencies if "%{deps" in format_: deps = [ e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False) ] line = p.fmt_subst( line, "deps", yaml.safe_dump(deps, default_style=None).rstrip("\n")) # Build Dependencies if "%{build-deps" in format_: build_deps = [ e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False) ] line = p.fmt_subst( line, "build-deps", yaml.safe_dump(build_deps, default_style=False).rstrip("\n")) # Runtime Dependencies if "%{runtime-deps" in format_: runtime_deps = [ e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False) ] line = p.fmt_subst( line, "runtime-deps", yaml.safe_dump(runtime_deps, default_style=False).rstrip("\n")) report += line + "\n" return report.rstrip("\n")
def dump_yaml_rt(data): """Dump round-tripped YAML.""" return yaml.round_trip_dump(data, indent=4, width=999)
def _render_configuration(config): ''' Given a config data structure of nested OrderedDicts, render the config as YAML and return it. ''' return yaml.round_trip_dump(config, indent=INDENT, block_seq_indent=INDENT)
def format_repo_url(repo): return f'[{repo}](https://github.com/{repo})' if __name__ == '__main__': remote_config_file = 'https://raw.githubusercontent.com/NCAR/xdev/xdevbot/xdevbot.yaml' config_file = 'xdevbot.yaml' resp = urllib.request.urlopen(remote_config_file) original_config = yaml.safe_load(resp) new_config, old_config, repos, error_messages_to_report = configure( original_config) if new_config != old_config: with open(config_file, 'w') as file_obj: yaml.round_trip_dump(new_config, file_obj, indent=2, block_seq_indent=2) added_hooks = {} removed_hooks = {} loop = asyncio.get_event_loop() add_tasks = [ loop.create_task(install_repo_webhook(repo, added_hooks)) for repo in repos['add'] ] remove_tasks = [ loop.create_task(delete_repo_webhook(repo, removed_hooks)) for repo in repos['remove'] ] loop.run_until_complete(asyncio.gather(*add_tasks)) loop.run_until_complete(asyncio.gather(*remove_tasks))
def fake_mpi_conf(tmp_path_factory: Any) -> Generator[str, None, None]: """ Make a super simple mpirun-alike for applications that don't actually use MPI. It just runs the command multiple times (in serial). Then create a plaform MPI config YAML file that should make it work for the testing examples. """ mpirun_text = """#!{interpreter} import argparse import sys import subprocess from io import StringIO from typing import List def make_parser(): p = argparse.ArgumentParser() p.add_argument("--num", type=int, help="number of times to run the application") p.add_argument( "--no-fail", help="add this flag to actually work", action="store_true" ) p.add_argument( "progargs", nargs=argparse.REMAINDER, help="The program and its arguments" ) return p class Runner: def __init__(self): if sys.stdin.isatty(): self.indata = None else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) def run_once(self, args: List[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() def run_many(self, n: int, args: List[str]): for i in range(n): self.run_once(args) if __name__ == "__main__": args = make_parser().parse_args() assert args.no_fail == True, "Didn't set the --no-fail flag" r = Runner() r.run_many(args.num, args.progargs) """.format( interpreter=sys.executable ) mpitmp = tmp_path_factory.mktemp("fake_mpi") mpirun_file = mpitmp / "fake_mpirun" mpirun_file.write_text(mpirun_text) mpirun_file.chmod(0o755) plat_conf = { "runner": str(mpirun_file), "nproc_flag": "--num", "extra_flags": ["--no-fail"], "env_set": {"TEST_MPI_FOO": "bar"}, "env_pass": ["USER"], } plat_conf_file = mpitmp / "plat_mpi.yml" plat_conf_file.write_text(yaml.round_trip_dump(plat_conf)) yield str(plat_conf_file) plat_conf_file.unlink() mpirun_file.unlink() mpitmp.rmdir()
def main(argsl=None, # type: List[str] args=None, # type: argparse.Namespace job_order_object=None, # type: MutableMapping[Text, Any] stdin=sys.stdin, # type: IO[Any] stdout=None, # type: Union[TextIO, StreamWriter] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] logger_handler=None, # custom_schema_callback=None, # type: Callable[[], None] executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] loadingContext=None, # type: LoadingContext runtimeContext=None # type: RuntimeContext ): # type: (...) -> int if not stdout: # force UTF-8 even if the console is configured differently if (hasattr(sys.stdout, "encoding") # type: ignore and sys.stdout.encoding != 'UTF-8'): # type: ignore if PY3 and hasattr(sys.stdout, "detach"): stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') else: stdout = getwriter('utf-8')(sys.stdout) # type: ignore else: stdout = cast(TextIO, sys.stdout) # type: ignore _logger.removeHandler(defaultStreamHandler) if logger_handler is not None: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) # pre-declared for finally block workflowobj = None prov_log_handler = None # type: Optional[logging.StreamHandler] try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) if args.record_container_id: if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) else: runtimeContext = runtimeContext.copy() # If on Windows platform, a default Docker Container is used if not # explicitely provided by user if onWindows() and not runtimeContext.default_container: # This docker image is a minimal alpine image with bash installed # (size 6 mb). source: https://github.com/frol/docker-alpine-bash runtimeContext.default_container = windows_default_container_id # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. for key, val in iteritems(get_default_args()): if not hasattr(args, key): setattr(args, key, val) # Configure logging rdflib_logger = logging.getLogger("rdflib.term") rdflib_logger.addHandler(stderr_handler) rdflib_logger.setLevel(logging.ERROR) if args.quiet: # Silence STDERR, not an eventual provenance log file stderr_handler.setLevel(logging.WARN) if runtimeContext.debug: # Increase to debug for both stderr and provenance log file _logger.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) formatter = None # type: Optional[logging.Formatter] if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) ## if args.version: print(versionfunc()) return 0 _logger.info(versionfunc()) if args.print_supported_versions: print("\n".join(supported_cwl_versions(args.enable_dev))) return 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error("CWL document required, no input file was provided") arg_parser().print_help() return 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback is not None: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") if args.provenance: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") return 1 ro = ResearchObject( temp_prefix_ro=args.tmpdir_prefix, orcid=args.orcid, full_name=args.cwl_full_name) runtimeContext.research_obj = ro log_file_io = ro.open_log_file_for_activity(ro.engine_uuid) prov_log_handler = logging.StreamHandler(log_file_io) class ProvLogFormatter(logging.Formatter): """Enforce ISO8601 with both T and Z.""" def __init__(self): # type: () -> None super(ProvLogFormatter, self).__init__( "[%(asctime)sZ] %(message)s") def formatTime(self, record, datefmt=None): # type: (logging.LogRecord, str) -> str record_time = time.gmtime(record.created) formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", record_time) with_msecs = "%s,%03d" % (formatted_time, record.msecs) return with_msecs prov_log_handler.setFormatter(ProvLogFormatter()) _logger.addHandler(prov_log_handler) _logger.debug(u"[provenance] Logging to %s", log_file_io) if argsl is not None: # Log cwltool command line options to provenance file _logger.info("[cwltool] %s %s", sys.argv[0], u" ".join(argsl)) _logger.debug(u"[cwltool] Arguments: %s", args) if loadingContext is None: loadingContext = LoadingContext(vars(args)) else: loadingContext = loadingContext.copy() loadingContext.loader = default_loader(loadingContext.fetcher_constructor) loadingContext.research_obj = runtimeContext.research_obj loadingContext.disable_js_validation = \ args.disable_js_validation or (not args.do_validate) loadingContext.construct_tool_object = getdefault( loadingContext.construct_tool_object, workflow.default_make_tool) loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver) loadingContext.do_update = not (args.pack or args.print_subgraph) uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) try_again_msg = "" if args.debug else ", try again with --debug for more information" try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, loadingContext.fetcher_constructor, loadingContext.overrides_list, tool_file_uri) if args.overrides: loadingContext.overrides_list.extend(load_overrides( file_uri(os.path.abspath(args.overrides)), tool_file_uri)) loadingContext, workflowobj, uri = fetch_document( uri, loadingContext) assert loadingContext.loader is not None if args.print_deps: printdeps(workflowobj, loadingContext.loader, stdout, args.relative_deps, uri) return 0 loadingContext, uri \ = resolve_and_validate_document(loadingContext, workflowobj, uri, preprocess_only=(args.print_pre or args.pack), skip_schemas=args.skip_schemas) assert loadingContext.loader is not None processobj, metadata = loadingContext.loader.resolve_ref(uri) processobj = cast(CommentedMap, processobj) if args.pack: stdout.write(print_pack(loadingContext.loader, processobj, uri, metadata)) return 0 if args.provenance and runtimeContext.research_obj: # Can't really be combined with args.pack at same time runtimeContext.research_obj.packed_workflow( print_pack(loadingContext.loader, processobj, uri, metadata)) if args.print_pre: stdout.write(json_dumps(processobj, indent=4, sort_keys=True, separators=(',', ': '))) return 0 tool = make_tool(uri, loadingContext) if args.make_template: def my_represent_none(self, data): # pylint: disable=unused-argument """Force clean representation of 'null'.""" return self.represent_scalar(u'tag:yaml.org,2002:null', u'null') yaml.RoundTripRepresenter.add_representer(type(None), my_represent_none) yaml.round_trip_dump( generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return 0 if args.validate: print("{} is valid CWL.".format(args.workflow)) return 0 if args.print_rdf: stdout.write(printrdf(tool, loadingContext.loader.ctx, args.rdf_serializer)) return 0 if args.print_dot: printdot(tool, loadingContext.loader.ctx, stdout) return 0 if args.print_targets: for f in ("outputs", "steps", "inputs"): if tool.tool[f]: _logger.info("%s%s targets:", f[0].upper(), f[1:-1]) stdout.write(" "+"\n ".join([shortname(t["id"]) for t in tool.tool[f]])+"\n") return 0 if args.target: if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: extracted = get_subgraph([tool.tool["id"] + "/" + r for r in args.target], tool) else: extracted = get_subgraph([loadingContext.loader.fetcher.urljoin(tool.tool["id"], "#" + r) for r in args.target], tool) else: _logger.error("Can only use --target on Workflows") return 1 loadingContext.loader.idx[extracted["id"]] = extracted tool = make_tool(extracted["id"], loadingContext) if args.print_subgraph: if "name" in tool.tool: del tool.tool["name"] stdout.write(json_dumps(tool.tool, indent=4, sort_keys=True, separators=(',', ': '))) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s", try_again_msg, exc if not args.debug else "", exc_info=args.debug) return 1 if isinstance(tool, int): return tool # If on MacOS platform, TMPDIR must be set to be under one of the # shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmp_outdir_prefix = default_mac_path if runtimeContext.tmpdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmpdir_prefix = default_mac_path for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(runtimeContext, dirprefix) and getattr(runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \ else "" setattr(runtimeContext, dirprefix, os.path.abspath(getattr(runtimeContext, dirprefix)) + sl) if not os.path.exists(os.path.dirname(getattr(runtimeContext, dirprefix))): try: os.makedirs(os.path.dirname(getattr(runtimeContext, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": runtimeContext.move_outputs = "copy" runtimeContext.tmp_outdir_prefix = args.cachedir runtimeContext.secret_store = getdefault(runtimeContext.secret_store, SecretStore()) runtimeContext.make_fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess) try: initialized_job_order_object = init_job_order( job_order_object, args, tool, jobloader, stdout, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, make_fs_access=runtimeContext.make_fs_access, input_basedir=input_basedir, secret_store=runtimeContext.secret_store) except SystemExit as err: return err.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() runtimeContext.select_resources = executor.select_resources else: executor = SingleJobExecutor() assert executor is not None try: runtimeContext.basedir = input_basedir del args.workflow del args.job_order conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text if conf_file or use_conda_dependencies: runtimeContext.job_script_provider = DependenciesConfiguration(args) else: runtimeContext.find_default_container = functools.partial( find_default_container, default_container=runtimeContext.default_container, use_biocontainers=args.beta_use_biocontainers) (out, status) = executor(tool, initialized_job_order_object, runtimeContext, logger=_logger) if out is not None: if runtimeContext.research_obj is not None: runtimeContext.research_obj.create_job( out, None, True) def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"]) visit_class(out, ("File", "Directory"), loc_to_path) # Unsetting the Generation from final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, string_types): stdout.write(out) else: stdout.write(json_dumps(out, indent=4, # type: ignore ensure_ascii=False)) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() # type: ignore if status != "success": _logger.warning(u"Final process status is %s", status) return 1 _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error( u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error( u"Workflow error%s:\n%s", try_again_msg, strip_dup_lineno(Text(exc)), exc_info=args.debug) return 1 except Exception as exc: # pylint: disable=broad-except _logger.error( u"Unhandled error%s:\n %s", try_again_msg, exc, exc_info=args.debug) return 1 finally: if args and runtimeContext and runtimeContext.research_obj \ and workflowobj: research_obj = runtimeContext.research_obj assert loadingContext is not None assert loadingContext.loader is not None prov_dependencies = prov_deps(workflowobj, loadingContext.loader, uri) research_obj.generate_snapshot(prov_dependencies) if prov_log_handler is not None: # Stop logging so we won't half-log adding ourself to RO _logger.debug(u"[provenance] Closing provenance log file %s", prov_log_handler) _logger.removeHandler(prov_log_handler) # Ensure last log lines are written out prov_log_handler.flush() # Underlying WritableBagFile will add the tagfile to the manifest prov_log_handler.stream.close() prov_log_handler.close() research_obj.close(args.provenance) _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
def save(self): if not self.mutable: raise RuntimeError("You may not modify a defaults file at runtime - check the mutable attribute!") with open(self.path, "w") as fh: yaml.round_trip_dump(self.data, fh)
def comment_yaml_item(data, path_to_key, data_contains_list=True): """ (EXPERIMENTAL) Comment a yaml item given its path_to_key (e.g. [foo 0 bar]), with comment preservation Inspired from https://stackoverflow.com/a/43927974 @cherrot """ if data_contains_list: path_to_key = list(map(str_or_int_map, path_to_key)) parent = data.mlget(path_to_key[:-1], list_ok=data_contains_list) if len(path_to_key) > 1 else data item_key = path_to_key[-1] deleted_item = item_key next_key = None if isinstance(parent, CommentedMap): if item_key not in parent: raise KeyError("the key \'{}\' does not exist".format(item_key)) # don't just pop the value for item_key that way you lose comments # in the original YAML, instead deepcopy and delete what is not needed block_copy = deepcopy(parent) found = False keys = [k for k in parent.keys()] for key in reversed(keys): if key == item_key: found = True else: if not found: next_key = key del block_copy[key] # now delete the key and its value, but preserve its preceding comments preceding_comments = parent.ca.items.get(item_key, [None, None, None, None])[1] if next_key is None: if parent.ca.comment is None: parent.ca.comment = [None, []] if parent.ca.comment[1] is None: parent.ca.comment[1] = [] comment_list = parent.ca.comment[1] else: comment_list = parent.ca.items.get(next_key, [None, None, None, None])[1] if comment_list is None: parent.ca.items[next_key] = [None, [], None, None] comment_list = parent.ca.items.get(next_key)[1] if preceding_comments is not None: for c in reversed(preceding_comments): comment_list.insert(0, c) del parent[item_key] elif isinstance(parent, CommentedSeq): if not is_int(item_key) or item_key >= len(parent): raise RuntimeError("the key \'{}\' is not an integer or exceeds its parent's length".format(item_key)) else: block_copy = deepcopy(parent) for i in reversed(range(len(parent))): if i != item_key: del block_copy[i] next_key = item_key preceding_comments = deepcopy(parent.ca.items.get(item_key, [None, None, None, None])[1]) parent.pop(item_key) # CommentedSet.pop(idx) automatically shifts all ca.items' indexes ! if len(parent) == 1 or next_key == len(parent): comment_list = parent.ca.end # TODO: fix this, the appended comments don't show up in some case else: comment_list = parent.ca.items.get(next_key, [None, None, None, None])[1] if comment_list is None: parent.ca.items[next_key] = [None, [], None, None] comment_list = parent.ca.items.get(next_key)[1] if preceding_comments is not None: for c in reversed(preceding_comments): comment_list.insert(0, c) else: raise RuntimeError("Couldn't reach the last item following the path_to_key " + str(path_to_key)) key_dept = len(path_to_key) - 1 if is_int(path_to_key[-1]) and key_dept > 0: key_dept = key_dept - 1 comment_list_copy = deepcopy(comment_list) del comment_list[:] start_mark = StreamMark(None, None, None, 2 * key_dept) skip = True for line in round_trip_dump(block_copy).splitlines(True): if skip: if line.strip(' ').startswith('#'): # and deleted_item not in line: continue skip = False comment_list.append(CommentToken('#' + line, start_mark, None)) comment_list.extend(comment_list_copy) return data
for x in dirs_set: if not x.endswith("/"): # Dirs must end with "/" to be created x += "/" num = slash_sort(x) if num not in dir_ordering: dir_ordering[num] = [] dir_ordering[num].append(x) # Finally, pretty things up final_dirs = [] final_files = sorted(list(files_set)) for key in sorted(dir_ordering.keys()): final_dirs += sorted(dir_ordering[key]) final_list = final_dirs + sorted(final_files) # Store our new list of files and write it to the yaml file data["files"] = final_list try: with open(filename, "w") as fh: yaml.round_trip_dump(data, fh, default_flow_style=False) except Exception as e: print("Failed to dump YAML data: {}".format(e)) exit(1)