def add_examples(self, input: str, output: str, explanation: str):
example = CommentedMap(
input=LiteralScalarString(input),
output=LiteralScalarString(output),
explanation=FoldedScalarString(explanation),
)
self._content["examples"].append(example)
def exportScriptYAMLObj(scriptId):
try:
print("Try exportScriptYAMLObj()")
script = PythonScript.objects.get(pk=scriptId)
print("Fetched script obj")
supported_file_types = ['*']
try:
supported_file_types = json.loads(script.supported_file_types)
except Exception as e:
logger.error(
f"Error trying to parse supported_file_types for script ID #{script.id}: {e}"
)
env_variables = {'test': 'test'}
try:
env_variables = json.loads(script.env_variables)
except Exception as e:
logger.error(
f"Error trying to parse env_variables for script ID #{script.id}: {e}"
)
print("Start data")
data = {
'id': script.id,
'data_file':
f"{script.data_file.uuid}" if script.data_file else None,
'name': script.name,
'human_name': script.human_name,
'description': LiteralScalarString(script.description),
'type': script.type,
'supported_file_types': supported_file_types,
'script': LiteralScalarString(script.script),
'required_packages': LiteralScalarString(script.required_packages),
'setup_script': LiteralScalarString(script.setup_script),
'env_variables': env_variables,
'schema': LiteralScalarString(script.schema)
}
print(f"data: {data}")
return data
except Exception as e:
print(f"Error exporting script YAML: {e}")
return {}
def _render_training_examples_as_text(examples: List[Dict]) -> List[Text]:
def render(example: Dict) -> Text:
return TrainingDataWriter.generate_list_item(
example[KEY_INTENT_TEXT])
return LiteralScalarString("".join(
[render(example) for example in examples]))
def clean(task_spec: TaskSpec) -> Dict:
"""Strip unnecessary fields and serialize non-primitive ones to create
a dict that a user would like to see in a Task Spec file"""
unset_fields = (field for field, value in task_spec.dict().items()
if value is None)
exclusions = (*FIELD_NAMES_EXCLUDED_FROM_CLEANED_TASK_SPEC, *unset_fields)
task_spec_sans_exclusions: Dict = {
k: v
for k, v in task_spec.dict().items() if k not in exclusions
}
steps_as_literal_scalar_string: Dict = {
"steps":
LiteralScalarString(
STEP_SEPARATOR.join(step.prompt
for step in task_spec.steps).rstrip())
}
context_as_list_of_mappings: Dict[str, List[Union[Dict, str]]] = {
"context": [{
item.var_name: item.prompt
} if item.is_complex else item.var_name for item in task_spec.context]
}
return {
**task_spec_sans_exclusions,
**steps_as_literal_scalar_string,
**context_as_list_of_mappings,
}
def exportPipelineNodeToYAMLObj(pythonNodeId):
try:
node = PipelineNode.objects.select_related('script').get(
pk=pythonNodeId)
step_settings = {}
try:
step_settings = json.loads(node.step_settings)
except Exception as e:
logger.error(
f"Error trying to parse step_settings for node ID #{node.id}: {e}"
)
data = {
'id': node.id,
'name': node.name,
'script': node.script.id if node.script else None,
'type': node.type,
'input_transform': LiteralScalarString(node.input_transform),
'step_settings': step_settings,
'x_coord': node.x_coord,
'y_coord': node.y_coord
}
return data
except Exception as e:
print(f"Error exporting PythonNode to YAML: {e}")
return {}
def base64_decode_secrets(content: str) -> str:
"""
Base64 decode a Kubernetes Secret yaml file
:param content: The content of the yaml file
:return str: The base64 decoded version of the yaml file
"""
ruamel_yaml = YAML()
secrets = ruamel_yaml.load(content)
data = secrets["data"]
for key, value in data.items():
if value is not None:
value = base64decode(value)
value = normalize_line_endings(value)
if "\n" in value:
# If there's a line break in the value we want to dump it using
# the literal syntax. This will use the pipe symbol (|) to
# display for example PEM keys on multiple lines in the final
# file rather than as one long string containing "\n".
value = LiteralScalarString(value)
data[key] = value
stream = StringIO()
ruamel_yaml.dump(secrets, stream)
return stream.getvalue().rstrip() + "\n"
def process_user_utterance(user_utterance: UserUttered) -> OrderedDict:
"""Converts a single user utterance into an ordered dict.
Args:
user_utterance: Original user utterance object.
Returns:
Dict with a user utterance.
"""
result = CommentedMap()
result[KEY_USER_INTENT] = user_utterance.intent["name"]
if hasattr(user_utterance, "inline_comment"):
result.yaml_add_eol_comment(user_utterance.inline_comment(),
KEY_USER_INTENT)
if (YAMLStoryWriter._text_is_real_message(user_utterance)
and user_utterance.text):
result[KEY_USER_MESSAGE] = LiteralScalarString(user_utterance.text)
if len(user_utterance.entities):
entities = []
for entity in user_utterance.entities:
if entity["value"]:
entities.append(
OrderedDict([(entity["entity"], entity["value"])]))
else:
entities.append(entity["entity"])
result[KEY_ENTITIES] = entities
return result
def test_ya():
dic = {'slogan': BULLSHIT}
f = NamedTemporaryFile()
yadu(dic, f)
f.seek(0)
assert yalo(f) == dic
multiline_content = {'so': LiteralScalarString('so\nlong')}
s = yadu(multiline_content)
# should dump multiline string in readable format
assert ': |' in s
def run_facet(path, args=None):
""" Run a facet and return the results as a Python object """
# Package single string args up as list
if args and not isinstance(args, list):
args = [args]
args = list(map(str, args or []))
if not os.access(path, os.X_OK):
sys.stderr.write("ERROR: Facet is not executable: %s" % path)
return "WARNING: Skipping non-executable facet: %s" % path
try:
process = subprocess.Popen([path] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
out_decoded = out.decode("utf-8").strip()
if err:
combined = out_decoded + err.decode("utf-8")
else:
combined = out_decoded
if not out_decoded:
# If no stdout, then just return stderr
return LiteralScalarString(combined)
try:
# Try to parse JSON
return json.loads(out_decoded)
# NOTE: We're throwing away stderr if present
except json.decoder.JSONDecodeError as e:
# maybe it's a multiline JSON
try:
return [json.loads(line) for line in out_decoded.split('\n')]
except ValueError as e:
try:
# If that fails, try to parse YAML
return yaml.load(out_decoded)
# NOTE: We're throwing away stderr if present
except ScannerError as e:
# If that fails, return raw output
return LiteralScalarString(out_decoded)
except subprocess.CalledProcessError as e:
return LiteralScalarString("ERROR: Problem running %s: %e" % (path, e))
def write(procedure: Procedure, force: bool = False):
"""Output the YAML representation of a Procedure object
to a file in destination_dir"""
procedure.path.touch(exist_ok=force)
writable_procedure: Dict = procedure.dict(exclude={"path"},
exclude_defaults=True)
writable_procedure["steps"] = LiteralScalarString(
STEP_SEPARATOR.join(step for step in procedure.steps).rstrip())
yaml.dump(writable_procedure, procedure.path)
def update(self, issue_id, data):
relpath = self.relpath(issue_id)
fullpath = os.path.join(self.repo_path, relpath)
now = datetime.now(tz=timezone.utc)
with update_yaml(fullpath) as y:
issue = dict(y.data, **data)
issue['body'] = LiteralScalarString(issue['body'])
issue['updated_at'] = now.isoformat()
y.data = issue
commit_id = self.do_commit([relpath])
return issue, commit_id
def exportPipelineToYAMLObj(pipelineId):
try:
pipeline = Pipeline.objects.select_related('root_node').get(
pk=pipelineId)
nodes = []
edges = []
scripts = {}
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
print("Node:")
print(node)
if node.script:
scripts[node.script.id] = exportScriptYAMLObj(node.script.id)
scripts = list(scripts.values())
for edge in Edge.objects.filter(parent_pipeline=pipeline):
edges.append(exportPipelineEdgeToYAMLObj(edge.id))
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
nodes.append(exportPipelineNodeToYAMLObj(node.id))
pipeline_meta = {
'name': pipeline.name,
'description': LiteralScalarString(pipeline.description),
'root_node': pipeline.root_node.id,
'scale': pipeline.scale,
'x_offset': pipeline.x_offset,
'y_offset': pipeline.y_offset,
'input_json_schema': pipeline.input_json_schema
}
data = {
'pipeline': pipeline_meta,
'scripts': scripts,
'edges': edges,
'nodes': nodes,
}
return data
except Exception as e:
print(f"Error exporting Pipeline Edge to YAML: {e}")
return {}
class Test_enums_YAMLValueFormats():
"""Tests for the YAMLValueFormats enumeration."""
def test_get_names(self):
assert YAMLValueFormats.get_names() == [
"BARE",
"BOOLEAN",
"DEFAULT",
"DQUOTE",
"FLOAT",
"FOLDED",
"INT",
"LITERAL",
"SQUOTE",
]
@pytest.mark.parametrize("input,output", [
("BARE", YAMLValueFormats.BARE),
("BOOLEAN", YAMLValueFormats.BOOLEAN),
("DEFAULT", YAMLValueFormats.DEFAULT),
("DQUOTE", YAMLValueFormats.DQUOTE),
("FLOAT", YAMLValueFormats.FLOAT),
("FOLDED", YAMLValueFormats.FOLDED),
("INT", YAMLValueFormats.INT),
("LITERAL", YAMLValueFormats.LITERAL),
("SQUOTE", YAMLValueFormats.SQUOTE),
])
def test_from_str(self, input, output):
assert output == YAMLValueFormats.from_str(input)
def test_from_str_nameerror(self):
with pytest.raises(NameError):
YAMLValueFormats.from_str("NO SUCH NAME")
@pytest.mark.parametrize("input,output", [
(FoldedScalarString(""), YAMLValueFormats.FOLDED),
(LiteralScalarString(""), YAMLValueFormats.LITERAL),
(DoubleQuotedScalarString(''), YAMLValueFormats.DQUOTE),
(SingleQuotedScalarString(""), YAMLValueFormats.SQUOTE),
(PlainScalarString(""), YAMLValueFormats.BARE),
(ScalarBoolean(False), YAMLValueFormats.BOOLEAN),
(ScalarFloat(1.01), YAMLValueFormats.FLOAT),
(ScalarInt(10), YAMLValueFormats.INT),
(None, YAMLValueFormats.DEFAULT),
])
def test_from_node(self, input, output):
assert output == YAMLValueFormats.from_node(input)
def process_user_utterance(
user_utterance: UserUttered, is_test_story: bool = False
) -> OrderedDict:
"""Converts a single user utterance into an ordered dict.
Args:
user_utterance: Original user utterance object.
is_test_story: Identifies if the user utterance should be added
to the final YAML or not.
Returns:
Dict with a user utterance.
"""
result = CommentedMap()
if user_utterance.intent_name and not user_utterance.use_text_for_featurization:
result[KEY_USER_INTENT] = user_utterance.intent_name
if hasattr(user_utterance, "inline_comment"):
result.yaml_add_eol_comment(
user_utterance.inline_comment(), KEY_USER_INTENT
)
if user_utterance.text and (
# We only print the utterance text if it was an end-to-end prediction
user_utterance.use_text_for_featurization
# or if we want to print a conversation test story.
or is_test_story
):
result[KEY_USER_MESSAGE] = LiteralScalarString(
rasa.shared.core.events.format_message(
user_utterance.text,
user_utterance.intent_name,
user_utterance.entities,
)
)
if len(user_utterance.entities) and not is_test_story:
entities = []
for entity in user_utterance.entities:
if entity["value"]:
entities.append(OrderedDict([(entity["entity"], entity["value"])]))
else:
entities.append(entity["entity"])
result[KEY_ENTITIES] = entities
return result
def rotate_cred(vault, args, new_access_key, new_secret_key):
"""rotates an IAM credential"""
# pylint: disable=too-many-arguments
# rotate secret
LOGGER.info("Begin rotate creds")
if args.encryptiontype == 'file':
vault_file = get_vault_encrypted_file(vault, args.vaultfile)
vault_file[args.secretkeyname] = new_secret_key
with open(args.vaultfile, "w") as outfile:
outfile.write(vault.encrypt(vault_file))
else:
vault_file = yaml.safe_load(open(args.vaultfile).read())
vault_id = None
if args.vaultid:
vault_id = args.vaultid
new_secret_vaulted = vault.encrypt(new_secret_key.strip(), None, vault_id)
new_secret_key_local = new_secret_vaulted
vault_file[args.secretkeyname] = LiteralScalarString(format_vault_encrypted_secret(new_secret_key_local))
if args.accessfile == args.vaultfile:
vault_file[args.accesskeyname] = new_access_key
file_to_write = yaml.round_trip_dump(vault_file)
# TODO: This is still hacky; but less so? Has to be a more elegant way....
file_to_write = "---\n" + file_to_write
file_to_write = file_to_write.replace(args.secretkeyname + ":", args.secretkeyname + ": !vault")
with io.open(args.vaultfile, "w", encoding="utf-8") as outfile:
outfile.write(file_to_write.decode('utf-8'))
# if the access key is in a different location, we need to update that too
if args.accessfile != args.vaultfile:
try:
current_access_key = yaml.safe_load(open(args.accessfile).read())
# pylint: disable=broad-except
except Exception as error:
LOGGER.error("An error occured opening current access key file: %s", error.message)
return None
current_access_key[args.accesskeyname] = new_access_key
file_to_write = yaml.round_trip_dump(current_access_key)
file_to_write = "---\n" + file_to_write
with open(args.accessfile, "w") as outfile:
outfile.write(file_to_write.decode('utf-8'))
LOGGER.info("End rotate creds")
return "Rotated"
def exportPipelineEdgeToYAMLObj(pythonEdgeId):
try:
edge = Edge.objects.get(pk=pythonEdgeId)
data = {
'id': edge.id,
'label': edge.label,
'start_node': edge.start_node.id,
'end_node': edge.end_node.id,
'transform_script': LiteralScalarString(edge.transform_script)
}
return data
except Exception as e:
print(f"Error exporting Pipeline Edge to YAML: {e}")
return {}
def update_comment(self, issue_id, comment_id, data):
relpath = self.relpath(issue_id)
fullpath = os.path.join(self.repo_path, relpath)
now = datetime.now(tz=timezone.utc)
with update_yaml(fullpath) as y:
issue = y.data
if 'comments' in issue:
for c in issue['comments']:
if c['id'] == comment_id:
c['body'] = LiteralScalarString(data['body'])
issue['updated_at'] = now.isoformat()
y.data = issue
# TODO: raise error when nothing updated
commit_id = self.do_commit([relpath])
return issue, commit_id
def add_comment(self, issue_id, data):
relpath = self.relpath(issue_id)
fullpath = os.path.join(self.repo_path, relpath)
comment = dict(data,
id=str(uuid4()),
body=LiteralScalarString(data['body']))
now = datetime.now(tz=timezone.utc)
with update_yaml(fullpath) as y:
issue = y.data
if 'comments' not in issue:
issue['comments'] = []
issue['comments'].append(comment)
issue['updated_at'] = now.isoformat()
y.data = issue
commit_id = self.do_commit([relpath])
return issue, commit_id
def create(self, data):
now = datetime.now(tz=timezone.utc)
issue = dict(data,
id=str(uuid4()),
body=LiteralScalarString(data['body']),
created_at=now.isoformat(),
updated_at=now.isoformat(),
comments=[])
relpath = self.relpath(issue['id'])
fullpath = os.path.join(self.repo_path, relpath)
os.makedirs(os.path.dirname(fullpath), exist_ok=True)
yaml = YAML()
with open(fullpath, 'wb') as f:
yaml.dump(issue, f)
commit_id = self.do_commit([relpath])
return issue, commit_id
def bio_to_yaml():
nlu_dict = defaultdict(list)
with open(args.bio, "r") as f_bio:
for line in f_bio.readlines():
example, target, intent = line.strip().split("\t")
example_list = example.split(" ")
target_list = target.split(" ")
entity_list = []
for i, t in enumerate(target_list):
if "B" == t[0]:
entity = t.replace("B-", "")
pos_start = i
for j in range(i + 1, len(target_list), 1):
if f"I-{entity}" == target_list[j]:
pos_end = j
else:
entity_list.append([pos_start, pos_end, entity])
example_list[
pos_start] = "[" + example_list[pos_start]
example_list[
pos_end] = f"{example_list[pos_end]}]({entity})"
break
nlu_dict[intent].append("".join(example_list))
save_dict = {"version": "2.0", "nlu": []}
for intent, examples in nlu_dict.items():
save_dict["nlu"].append({
"intent":
intent,
"examples":
LiteralScalarString("- " + "\n- ".join(examples) + "\n")
})
with open(args.yaml, "w") as f_yaml:
yaml.dump(save_dict, f_yaml)
print("convert success!")
def process_training_examples_by_key(
training_examples: Dict,
key_name: Text,
key_examples: Text,
example_extraction_predicate=lambda x: x,
) -> List[OrderedDict]:
from ruamel.yaml.scalarstring import LiteralScalarString
result = []
for entity_key, examples in training_examples.items():
converted_examples = [
TrainingDataWriter.generate_list_item(
example_extraction_predicate(example).strip(STRIP_SYMBOLS))
for example in examples
]
next_item = OrderedDict()
next_item[key_name] = entity_key
next_item[key_examples] = LiteralScalarString(
"".join(converted_examples))
result.append(next_item)
return result
def get_html_content_block(copy):
return {
'jcr:primaryType': 'hippostd:html',
'hippostd:content': LiteralScalarString(copy)
}
def render(example: Dict) -> Dict:
text = example[KEY_INTENT_TEXT]
example[KEY_INTENT_TEXT] = LiteralScalarString(text + "\n")
return example
def LS(s):
return LiteralScalarString(textwrap.dedent(s))
def to_preserialization_data(cls, python_value, *, default_to_skip=None):
data = super().to_preserialization_data(python_value).strip()
if "\n" in data:
return LiteralScalarString(data + "\n")
return data
def to_preserialization_data(cls, python_value, *, skip=Missing):
data = super().to_preserialization_data(python_value).strip()
if '\n' in data:
return LiteralScalarString(data + '\n')
return data
def render(example: Dict) -> Dict:
text = example[KEY_INTENT_TEXT]
example[KEY_INTENT_TEXT] = LiteralScalarString(
TrainingDataWriter.generate_string_item(text))
return example
def definition(self, definition):
self._definition = LiteralScalarString(definition) if not isinstance(
definition, LiteralScalarString) else definition
def read_definition(self, filename):
self.definition_file = filename
with open(self.definition_file, 'r') as f:
self.definition = LiteralScalarString(f.read())
def exportPipelineToZip(pipelineId):
try:
usingS3 = (settings.DEFAULT_FILE_STORAGE ==
"gremlin.utils.storages.MediaRootS3Boto3Storage")
pipeline = Pipeline.objects.select_related('root_node').get(
pk=pipelineId)
print("Fetched pipeline obj...")
nodes = []
edges = []
scripts = {}
script_dict = {}
zip_bytes = io.BytesIO()
myYaml = io.StringIO()
yaml = YAML()
yaml.preserve_quotes = False
zip_file = zipfile.ZipFile(zip_bytes,
mode='w',
compression=zipfile.ZIP_DEFLATED)
print("Zip file created")
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
print(f"Look over {node}")
if node.script:
print(f"Node ID {node.script.id} has script: {node.script}")
print(f"Scripts was: {script_dict}")
script_dict[node.script.id] = exportScriptYAMLObj(
node.script.id)
print(f"Script YAML created...")
if node.script.data_file and node.script.data_file.data_file:
print("Script had a data file")
# THe precise field with the valid filename / path depends on the storage adapter, so handle accordingly
if usingS3:
filename = node.script.data_file.data_file.name
else:
filename = node.script.data_file.data_file.path
data_file_bytes = default_storage.open(filename,
mode='rb').read()
zip_file.writestr(
f"/data/{node.script.data_file.uuid}.zip",
data_file_bytes)
scripts = list(script_dict.values())
print(f"Scripts are: {scripts}")
for edge in Edge.objects.filter(parent_pipeline=pipeline):
edges.append(exportPipelineEdgeToYAMLObj(edge.id))
print("Edges converted to YAML")
for node in PipelineNode.objects.filter(parent_pipeline=pipeline):
nodes.append(exportPipelineNodeToYAMLObj(node.id))
print("Nodes converted to YAML")
pipeline_meta = {
'name': pipeline.name,
'description': LiteralScalarString(pipeline.description),
'input_json_schema': pipeline.input_json_schema,
'root_node': pipeline.root_node.id,
'scale': pipeline.scale,
'x_offset': pipeline.x_offset,
'y_offset': pipeline.y_offset,
}
data = {
'pipeline': pipeline_meta,
'scripts': scripts,
'edges': edges,
'nodes': nodes,
}
print("Dump YAML")
yaml.dump(data, myYaml)
zip_file.writestr("pipeline_export.yaml", myYaml.getvalue())
zip_file.close()
zip_bytes.seek(io.SEEK_SET)
print("Done with zip_bytes... returning")
return zip_bytes
except Exception as e:
print(f"Error exporting Pipeline to archive: {e}")
return None