def add_examples(self, input: str, output: str, explanation: str): example = CommentedMap( input=LiteralScalarString(input), output=LiteralScalarString(output), explanation=FoldedScalarString(explanation), ) self._content["examples"].append(example)
def exportScriptYAMLObj(scriptId): try: print("Try exportScriptYAMLObj()") script = PythonScript.objects.get(pk=scriptId) print("Fetched script obj") supported_file_types = ['*'] try: supported_file_types = json.loads(script.supported_file_types) except Exception as e: logger.error( f"Error trying to parse supported_file_types for script ID #{script.id}: {e}" ) env_variables = {'test': 'test'} try: env_variables = json.loads(script.env_variables) except Exception as e: logger.error( f"Error trying to parse env_variables for script ID #{script.id}: {e}" ) print("Start data") data = { 'id': script.id, 'data_file': f"{script.data_file.uuid}" if script.data_file else None, 'name': script.name, 'human_name': script.human_name, 'description': LiteralScalarString(script.description), 'type': script.type, 'supported_file_types': supported_file_types, 'script': LiteralScalarString(script.script), 'required_packages': LiteralScalarString(script.required_packages), 'setup_script': LiteralScalarString(script.setup_script), 'env_variables': env_variables, 'schema': LiteralScalarString(script.schema) } print(f"data: {data}") return data except Exception as e: print(f"Error exporting script YAML: {e}") return {}
def _render_training_examples_as_text(examples: List[Dict]) -> List[Text]: def render(example: Dict) -> Text: return TrainingDataWriter.generate_list_item( example[KEY_INTENT_TEXT]) return LiteralScalarString("".join( [render(example) for example in examples]))
def clean(task_spec: TaskSpec) -> Dict: """Strip unnecessary fields and serialize non-primitive ones to create a dict that a user would like to see in a Task Spec file""" unset_fields = (field for field, value in task_spec.dict().items() if value is None) exclusions = (*FIELD_NAMES_EXCLUDED_FROM_CLEANED_TASK_SPEC, *unset_fields) task_spec_sans_exclusions: Dict = { k: v for k, v in task_spec.dict().items() if k not in exclusions } steps_as_literal_scalar_string: Dict = { "steps": LiteralScalarString( STEP_SEPARATOR.join(step.prompt for step in task_spec.steps).rstrip()) } context_as_list_of_mappings: Dict[str, List[Union[Dict, str]]] = { "context": [{ item.var_name: item.prompt } if item.is_complex else item.var_name for item in task_spec.context] } return { **task_spec_sans_exclusions, **steps_as_literal_scalar_string, **context_as_list_of_mappings, }
def exportPipelineNodeToYAMLObj(pythonNodeId): try: node = PipelineNode.objects.select_related('script').get( pk=pythonNodeId) step_settings = {} try: step_settings = json.loads(node.step_settings) except Exception as e: logger.error( f"Error trying to parse step_settings for node ID #{node.id}: {e}" ) data = { 'id': node.id, 'name': node.name, 'script': node.script.id if node.script else None, 'type': node.type, 'input_transform': LiteralScalarString(node.input_transform), 'step_settings': step_settings, 'x_coord': node.x_coord, 'y_coord': node.y_coord } return data except Exception as e: print(f"Error exporting PythonNode to YAML: {e}") return {}
def base64_decode_secrets(content: str) -> str: """ Base64 decode a Kubernetes Secret yaml file :param content: The content of the yaml file :return str: The base64 decoded version of the yaml file """ ruamel_yaml = YAML() secrets = ruamel_yaml.load(content) data = secrets["data"] for key, value in data.items(): if value is not None: value = base64decode(value) value = normalize_line_endings(value) if "\n" in value: # If there's a line break in the value we want to dump it using # the literal syntax. This will use the pipe symbol (|) to # display for example PEM keys on multiple lines in the final # file rather than as one long string containing "\n". value = LiteralScalarString(value) data[key] = value stream = StringIO() ruamel_yaml.dump(secrets, stream) return stream.getvalue().rstrip() + "\n"
def process_user_utterance(user_utterance: UserUttered) -> OrderedDict: """Converts a single user utterance into an ordered dict. Args: user_utterance: Original user utterance object. Returns: Dict with a user utterance. """ result = CommentedMap() result[KEY_USER_INTENT] = user_utterance.intent["name"] if hasattr(user_utterance, "inline_comment"): result.yaml_add_eol_comment(user_utterance.inline_comment(), KEY_USER_INTENT) if (YAMLStoryWriter._text_is_real_message(user_utterance) and user_utterance.text): result[KEY_USER_MESSAGE] = LiteralScalarString(user_utterance.text) if len(user_utterance.entities): entities = [] for entity in user_utterance.entities: if entity["value"]: entities.append( OrderedDict([(entity["entity"], entity["value"])])) else: entities.append(entity["entity"]) result[KEY_ENTITIES] = entities return result
def test_ya(): dic = {'slogan': BULLSHIT} f = NamedTemporaryFile() yadu(dic, f) f.seek(0) assert yalo(f) == dic multiline_content = {'so': LiteralScalarString('so\nlong')} s = yadu(multiline_content) # should dump multiline string in readable format assert ': |' in s
def run_facet(path, args=None): """ Run a facet and return the results as a Python object """ # Package single string args up as list if args and not isinstance(args, list): args = [args] args = list(map(str, args or [])) if not os.access(path, os.X_OK): sys.stderr.write("ERROR: Facet is not executable: %s" % path) return "WARNING: Skipping non-executable facet: %s" % path try: process = subprocess.Popen([path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() out_decoded = out.decode("utf-8").strip() if err: combined = out_decoded + err.decode("utf-8") else: combined = out_decoded if not out_decoded: # If no stdout, then just return stderr return LiteralScalarString(combined) try: # Try to parse JSON return json.loads(out_decoded) # NOTE: We're throwing away stderr if present except json.decoder.JSONDecodeError as e: # maybe it's a multiline JSON try: return [json.loads(line) for line in out_decoded.split('\n')] except ValueError as e: try: # If that fails, try to parse YAML return yaml.load(out_decoded) # NOTE: We're throwing away stderr if present except ScannerError as e: # If that fails, return raw output return LiteralScalarString(out_decoded) except subprocess.CalledProcessError as e: return LiteralScalarString("ERROR: Problem running %s: %e" % (path, e))
def write(procedure: Procedure, force: bool = False): """Output the YAML representation of a Procedure object to a file in destination_dir""" procedure.path.touch(exist_ok=force) writable_procedure: Dict = procedure.dict(exclude={"path"}, exclude_defaults=True) writable_procedure["steps"] = LiteralScalarString( STEP_SEPARATOR.join(step for step in procedure.steps).rstrip()) yaml.dump(writable_procedure, procedure.path)
def update(self, issue_id, data): relpath = self.relpath(issue_id) fullpath = os.path.join(self.repo_path, relpath) now = datetime.now(tz=timezone.utc) with update_yaml(fullpath) as y: issue = dict(y.data, **data) issue['body'] = LiteralScalarString(issue['body']) issue['updated_at'] = now.isoformat() y.data = issue commit_id = self.do_commit([relpath]) return issue, commit_id
def exportPipelineToYAMLObj(pipelineId): try: pipeline = Pipeline.objects.select_related('root_node').get( pk=pipelineId) nodes = [] edges = [] scripts = {} for node in PipelineNode.objects.filter(parent_pipeline=pipeline): print("Node:") print(node) if node.script: scripts[node.script.id] = exportScriptYAMLObj(node.script.id) scripts = list(scripts.values()) for edge in Edge.objects.filter(parent_pipeline=pipeline): edges.append(exportPipelineEdgeToYAMLObj(edge.id)) for node in PipelineNode.objects.filter(parent_pipeline=pipeline): nodes.append(exportPipelineNodeToYAMLObj(node.id)) pipeline_meta = { 'name': pipeline.name, 'description': LiteralScalarString(pipeline.description), 'root_node': pipeline.root_node.id, 'scale': pipeline.scale, 'x_offset': pipeline.x_offset, 'y_offset': pipeline.y_offset, 'input_json_schema': pipeline.input_json_schema } data = { 'pipeline': pipeline_meta, 'scripts': scripts, 'edges': edges, 'nodes': nodes, } return data except Exception as e: print(f"Error exporting Pipeline Edge to YAML: {e}") return {}
class Test_enums_YAMLValueFormats(): """Tests for the YAMLValueFormats enumeration.""" def test_get_names(self): assert YAMLValueFormats.get_names() == [ "BARE", "BOOLEAN", "DEFAULT", "DQUOTE", "FLOAT", "FOLDED", "INT", "LITERAL", "SQUOTE", ] @pytest.mark.parametrize("input,output", [ ("BARE", YAMLValueFormats.BARE), ("BOOLEAN", YAMLValueFormats.BOOLEAN), ("DEFAULT", YAMLValueFormats.DEFAULT), ("DQUOTE", YAMLValueFormats.DQUOTE), ("FLOAT", YAMLValueFormats.FLOAT), ("FOLDED", YAMLValueFormats.FOLDED), ("INT", YAMLValueFormats.INT), ("LITERAL", YAMLValueFormats.LITERAL), ("SQUOTE", YAMLValueFormats.SQUOTE), ]) def test_from_str(self, input, output): assert output == YAMLValueFormats.from_str(input) def test_from_str_nameerror(self): with pytest.raises(NameError): YAMLValueFormats.from_str("NO SUCH NAME") @pytest.mark.parametrize("input,output", [ (FoldedScalarString(""), YAMLValueFormats.FOLDED), (LiteralScalarString(""), YAMLValueFormats.LITERAL), (DoubleQuotedScalarString(''), YAMLValueFormats.DQUOTE), (SingleQuotedScalarString(""), YAMLValueFormats.SQUOTE), (PlainScalarString(""), YAMLValueFormats.BARE), (ScalarBoolean(False), YAMLValueFormats.BOOLEAN), (ScalarFloat(1.01), YAMLValueFormats.FLOAT), (ScalarInt(10), YAMLValueFormats.INT), (None, YAMLValueFormats.DEFAULT), ]) def test_from_node(self, input, output): assert output == YAMLValueFormats.from_node(input)
def process_user_utterance( user_utterance: UserUttered, is_test_story: bool = False ) -> OrderedDict: """Converts a single user utterance into an ordered dict. Args: user_utterance: Original user utterance object. is_test_story: Identifies if the user utterance should be added to the final YAML or not. Returns: Dict with a user utterance. """ result = CommentedMap() if user_utterance.intent_name and not user_utterance.use_text_for_featurization: result[KEY_USER_INTENT] = user_utterance.intent_name if hasattr(user_utterance, "inline_comment"): result.yaml_add_eol_comment( user_utterance.inline_comment(), KEY_USER_INTENT ) if user_utterance.text and ( # We only print the utterance text if it was an end-to-end prediction user_utterance.use_text_for_featurization # or if we want to print a conversation test story. or is_test_story ): result[KEY_USER_MESSAGE] = LiteralScalarString( rasa.shared.core.events.format_message( user_utterance.text, user_utterance.intent_name, user_utterance.entities, ) ) if len(user_utterance.entities) and not is_test_story: entities = [] for entity in user_utterance.entities: if entity["value"]: entities.append(OrderedDict([(entity["entity"], entity["value"])])) else: entities.append(entity["entity"]) result[KEY_ENTITIES] = entities return result
def rotate_cred(vault, args, new_access_key, new_secret_key): """rotates an IAM credential""" # pylint: disable=too-many-arguments # rotate secret LOGGER.info("Begin rotate creds") if args.encryptiontype == 'file': vault_file = get_vault_encrypted_file(vault, args.vaultfile) vault_file[args.secretkeyname] = new_secret_key with open(args.vaultfile, "w") as outfile: outfile.write(vault.encrypt(vault_file)) else: vault_file = yaml.safe_load(open(args.vaultfile).read()) vault_id = None if args.vaultid: vault_id = args.vaultid new_secret_vaulted = vault.encrypt(new_secret_key.strip(), None, vault_id) new_secret_key_local = new_secret_vaulted vault_file[args.secretkeyname] = LiteralScalarString(format_vault_encrypted_secret(new_secret_key_local)) if args.accessfile == args.vaultfile: vault_file[args.accesskeyname] = new_access_key file_to_write = yaml.round_trip_dump(vault_file) # TODO: This is still hacky; but less so? Has to be a more elegant way.... file_to_write = "---\n" + file_to_write file_to_write = file_to_write.replace(args.secretkeyname + ":", args.secretkeyname + ": !vault") with io.open(args.vaultfile, "w", encoding="utf-8") as outfile: outfile.write(file_to_write.decode('utf-8')) # if the access key is in a different location, we need to update that too if args.accessfile != args.vaultfile: try: current_access_key = yaml.safe_load(open(args.accessfile).read()) # pylint: disable=broad-except except Exception as error: LOGGER.error("An error occured opening current access key file: %s", error.message) return None current_access_key[args.accesskeyname] = new_access_key file_to_write = yaml.round_trip_dump(current_access_key) file_to_write = "---\n" + file_to_write with open(args.accessfile, "w") as outfile: outfile.write(file_to_write.decode('utf-8')) LOGGER.info("End rotate creds") return "Rotated"
def exportPipelineEdgeToYAMLObj(pythonEdgeId): try: edge = Edge.objects.get(pk=pythonEdgeId) data = { 'id': edge.id, 'label': edge.label, 'start_node': edge.start_node.id, 'end_node': edge.end_node.id, 'transform_script': LiteralScalarString(edge.transform_script) } return data except Exception as e: print(f"Error exporting Pipeline Edge to YAML: {e}") return {}
def update_comment(self, issue_id, comment_id, data): relpath = self.relpath(issue_id) fullpath = os.path.join(self.repo_path, relpath) now = datetime.now(tz=timezone.utc) with update_yaml(fullpath) as y: issue = y.data if 'comments' in issue: for c in issue['comments']: if c['id'] == comment_id: c['body'] = LiteralScalarString(data['body']) issue['updated_at'] = now.isoformat() y.data = issue # TODO: raise error when nothing updated commit_id = self.do_commit([relpath]) return issue, commit_id
def add_comment(self, issue_id, data): relpath = self.relpath(issue_id) fullpath = os.path.join(self.repo_path, relpath) comment = dict(data, id=str(uuid4()), body=LiteralScalarString(data['body'])) now = datetime.now(tz=timezone.utc) with update_yaml(fullpath) as y: issue = y.data if 'comments' not in issue: issue['comments'] = [] issue['comments'].append(comment) issue['updated_at'] = now.isoformat() y.data = issue commit_id = self.do_commit([relpath]) return issue, commit_id
def create(self, data): now = datetime.now(tz=timezone.utc) issue = dict(data, id=str(uuid4()), body=LiteralScalarString(data['body']), created_at=now.isoformat(), updated_at=now.isoformat(), comments=[]) relpath = self.relpath(issue['id']) fullpath = os.path.join(self.repo_path, relpath) os.makedirs(os.path.dirname(fullpath), exist_ok=True) yaml = YAML() with open(fullpath, 'wb') as f: yaml.dump(issue, f) commit_id = self.do_commit([relpath]) return issue, commit_id
def bio_to_yaml(): nlu_dict = defaultdict(list) with open(args.bio, "r") as f_bio: for line in f_bio.readlines(): example, target, intent = line.strip().split("\t") example_list = example.split(" ") target_list = target.split(" ") entity_list = [] for i, t in enumerate(target_list): if "B" == t[0]: entity = t.replace("B-", "") pos_start = i for j in range(i + 1, len(target_list), 1): if f"I-{entity}" == target_list[j]: pos_end = j else: entity_list.append([pos_start, pos_end, entity]) example_list[ pos_start] = "[" + example_list[pos_start] example_list[ pos_end] = f"{example_list[pos_end]}]({entity})" break nlu_dict[intent].append("".join(example_list)) save_dict = {"version": "2.0", "nlu": []} for intent, examples in nlu_dict.items(): save_dict["nlu"].append({ "intent": intent, "examples": LiteralScalarString("- " + "\n- ".join(examples) + "\n") }) with open(args.yaml, "w") as f_yaml: yaml.dump(save_dict, f_yaml) print("convert success!")
def process_training_examples_by_key( training_examples: Dict, key_name: Text, key_examples: Text, example_extraction_predicate=lambda x: x, ) -> List[OrderedDict]: from ruamel.yaml.scalarstring import LiteralScalarString result = [] for entity_key, examples in training_examples.items(): converted_examples = [ TrainingDataWriter.generate_list_item( example_extraction_predicate(example).strip(STRIP_SYMBOLS)) for example in examples ] next_item = OrderedDict() next_item[key_name] = entity_key next_item[key_examples] = LiteralScalarString( "".join(converted_examples)) result.append(next_item) return result
def get_html_content_block(copy): return { 'jcr:primaryType': 'hippostd:html', 'hippostd:content': LiteralScalarString(copy) }
def render(example: Dict) -> Dict: text = example[KEY_INTENT_TEXT] example[KEY_INTENT_TEXT] = LiteralScalarString(text + "\n") return example
def LS(s): return LiteralScalarString(textwrap.dedent(s))
def to_preserialization_data(cls, python_value, *, default_to_skip=None): data = super().to_preserialization_data(python_value).strip() if "\n" in data: return LiteralScalarString(data + "\n") return data
def to_preserialization_data(cls, python_value, *, skip=Missing): data = super().to_preserialization_data(python_value).strip() if '\n' in data: return LiteralScalarString(data + '\n') return data
def render(example: Dict) -> Dict: text = example[KEY_INTENT_TEXT] example[KEY_INTENT_TEXT] = LiteralScalarString( TrainingDataWriter.generate_string_item(text)) return example
def definition(self, definition): self._definition = LiteralScalarString(definition) if not isinstance( definition, LiteralScalarString) else definition
def read_definition(self, filename): self.definition_file = filename with open(self.definition_file, 'r') as f: self.definition = LiteralScalarString(f.read())
def exportPipelineToZip(pipelineId): try: usingS3 = (settings.DEFAULT_FILE_STORAGE == "gremlin.utils.storages.MediaRootS3Boto3Storage") pipeline = Pipeline.objects.select_related('root_node').get( pk=pipelineId) print("Fetched pipeline obj...") nodes = [] edges = [] scripts = {} script_dict = {} zip_bytes = io.BytesIO() myYaml = io.StringIO() yaml = YAML() yaml.preserve_quotes = False zip_file = zipfile.ZipFile(zip_bytes, mode='w', compression=zipfile.ZIP_DEFLATED) print("Zip file created") for node in PipelineNode.objects.filter(parent_pipeline=pipeline): print(f"Look over {node}") if node.script: print(f"Node ID {node.script.id} has script: {node.script}") print(f"Scripts was: {script_dict}") script_dict[node.script.id] = exportScriptYAMLObj( node.script.id) print(f"Script YAML created...") if node.script.data_file and node.script.data_file.data_file: print("Script had a data file") # THe precise field with the valid filename / path depends on the storage adapter, so handle accordingly if usingS3: filename = node.script.data_file.data_file.name else: filename = node.script.data_file.data_file.path data_file_bytes = default_storage.open(filename, mode='rb').read() zip_file.writestr( f"/data/{node.script.data_file.uuid}.zip", data_file_bytes) scripts = list(script_dict.values()) print(f"Scripts are: {scripts}") for edge in Edge.objects.filter(parent_pipeline=pipeline): edges.append(exportPipelineEdgeToYAMLObj(edge.id)) print("Edges converted to YAML") for node in PipelineNode.objects.filter(parent_pipeline=pipeline): nodes.append(exportPipelineNodeToYAMLObj(node.id)) print("Nodes converted to YAML") pipeline_meta = { 'name': pipeline.name, 'description': LiteralScalarString(pipeline.description), 'input_json_schema': pipeline.input_json_schema, 'root_node': pipeline.root_node.id, 'scale': pipeline.scale, 'x_offset': pipeline.x_offset, 'y_offset': pipeline.y_offset, } data = { 'pipeline': pipeline_meta, 'scripts': scripts, 'edges': edges, 'nodes': nodes, } print("Dump YAML") yaml.dump(data, myYaml) zip_file.writestr("pipeline_export.yaml", myYaml.getvalue()) zip_file.close() zip_bytes.seek(io.SEEK_SET) print("Done with zip_bytes... returning") return zip_bytes except Exception as e: print(f"Error exporting Pipeline to archive: {e}") return None