def insert_script_to_yml(self, script_type, yml_unified, yml_data):
script_path = self.get_code_file(script_type)
with io.open(script_path, mode='r', encoding='utf-8') as script_file:
script_code = script_file.read()
# Check if the script imports an API module. If it does,
# the API module code will be pasted in place of the import.
module_import, module_name = self.check_api_module_imports(script_code)
if module_import:
script_code = self.insert_module_code(script_code, module_import,
module_name)
if script_type == '.py':
clean_code = self.clean_python_code(script_code)
if script_type == '.ps1':
clean_code = self.clean_pwsh_code(script_code)
if self.is_script_package:
if yml_data.get('script', '') not in ('', '-'):
print_warning(
f'Script section is not empty in package {self.package_path}.'
f'It should be blank or a dash(-).')
yml_unified['script'] = FoldedScalarString(clean_code)
else:
if yml_data['script'].get('script', '') not in ('', '-'):
print_warning(
f'Script section is not empty in package {self.package_path}.'
f'It should be blank or a dash(-).')
yml_unified['script']['script'] = FoldedScalarString(clean_code)
return yml_unified, script_path
def fix_script_in_unified_yml(yml_content):
if 'script' in yml_content:
if isinstance(yml_content.get('script'), str):
if yml_content.get('script') not in ('-', ''):
yml_content['script'] = FoldedScalarString(
yml_content['script'])
elif yml_content.get('script').get('script') not in ('-', ''):
yml_content['script']['script'] = FoldedScalarString(
yml_content['script']['script'])
def rewrite_yml(file_path, yml_content, new_to_version):
yml_content['toversion'] = new_to_version
check_dockerimage45(yml_content, new_to_version)
if 'script' in yml_content:
if isinstance(yml_content.get('script'), str):
if yml_content.get('script') not in ('-', ''):
yml_content['script'] = FoldedScalarString(yml_content.get('script'))
elif yml_content.get('script').get('script') not in ('-', ''):
yml_content['script']['script'] = FoldedScalarString(yml_content.get('script').get('script'))
with open(file_path, mode='w', encoding='utf-8') as f:
ryaml.dump(yml_content, f)
print(f" - Updating {file_path}")
def add_examples(self, input: str, output: str, explanation: str):
example = CommentedMap(
input=LiteralScalarString(input),
output=LiteralScalarString(output),
explanation=FoldedScalarString(explanation),
)
self._content["examples"].append(example)
def add_from_version_to_yml(self,
file_path=None,
yml_content=None,
save_yml=True):
if self.no_fromversion:
return {}
ryaml = YAML()
ryaml.preserve_quotes = True
ryaml.width = 50000 # make sure long lines will not break (relevant for code section)
if not yml_content:
with open(file_path, 'r') as yml_file:
yml_content = ryaml.load(yml_file)
if parse_version(yml_content.get(
'toversion', '99.99.99')) > parse_version(
LATEST_SUPPORTED_VERSION) > parse_version(
yml_content.get('fromversion', '0.0.0')):
yml_content['fromversion'] = LATEST_SUPPORTED_VERSION
if 'detaileddescription' in yml_content:
yml_content['detaileddescription'] = FoldedScalarString(
yml_content['detaileddescription'])
self.fix_script_in_unified_yml(yml_content)
if save_yml:
with open(file_path, mode='w', encoding='utf-8') as f:
ryaml.dump(yml_content, f)
return yml_content
def update_model_descriptions(models, descriptions, mode):
update_models = []
missed_models = []
for name, desc in descriptions.items():
model = models.get(name, None)
if model is None:
logging.error('For description file {}.md no model found'.format(name))
missed_models.append(name)
continue
if not model[1].get('description', None):
logging.error('No description found in {} for {} model'.format(model[0], name))
missed_models.append(name)
continue
model = model[1]
if model.get('description', '') != desc:
if mode == 'update':
model['description'] = FoldedScalarString(desc)
else:
logging.debug('Found diff for {} model'.format(name))
logging.debug('\n{:12s}{}\n\tvs\n{:12s}{}'
.format('In config:', model['description'], 'In readme:', desc))
update_models.append(name)
if mode == 'update':
msg = 'Description updated for {} models, missed for {} models.'
msg_model_list = 'UPDATED:\n\t{}'
else:
msg = 'Description differs for {} models, missed for {} models.'
msg_model_list = 'DIFFERENCE:\n\t{}'
logging.info(msg.format(len(update_models), len(missed_models)))
if len(update_models) > 0:
logging.info(msg_model_list.format("\n\t".join(update_models)))
if len(missed_models) > 0:
logging.info('FAILED:\n\t{}'.format("\n\t".join(missed_models)))
return update_models
def insert_description_to_yml(self, yml_data, yml_unified):
desc_data, found_desc_path = self.get_data(self.package_path,
'*_description.md')
if yml_data.get('detaileddescription') and self.use_force is False:
raise ValueError(
'Please move the detailed description from the yml to a description file (.md)'
f' in the package: {self.package_path}')
detailed_description = ''
if desc_data:
detailed_description = FoldedScalarString(
desc_data.decode('utf-8'))
integration_doc_link = ''
if '[View Integration Documentation]' not in detailed_description:
integration_doc_link = self.get_integration_doc_link(yml_data)
if integration_doc_link:
if detailed_description:
detailed_description += '\n\n---\n' + integration_doc_link
else:
detailed_description += integration_doc_link
if detailed_description:
yml_unified['detaileddescription'] = detailed_description
return yml_unified, found_desc_path
def toYamlText(val):
if isinstance(val, (ScalarString, sensitive)):
return val
# convert or copy string (copy to deal with things like AnsibleUnsafeText)
val = str(val)
if "\n" in val:
return FoldedScalarString(val)
return val
def insert_description_to_yml(self, yml_data, yml_unified):
desc_data, found_desc_path = self.get_data(self.package_path, '*_description.md')
if yml_data.get('detaileddescription') and self.use_force is False:
raise ValueError('Please move the detailed description from the yml to a description file (.md)'
f' in the package: {self.package_path}')
if desc_data:
yml_unified['detaileddescription'] = FoldedScalarString(desc_data.decode('utf-8'))
return yml_unified, found_desc_path
def to_yaml_text(
val: object
) -> Union[sensitive, ScalarString, FoldedScalarString, str]:
if isinstance(val, (ScalarString, sensitive)):
return val
# convert or copy string (copy to deal with things like AnsibleUnsafeText)
val = str(val)
if "\n" in val:
return FoldedScalarString(val)
return val
def insert_script_to_yml(self, script_type, yml_unified, yml_data):
script_path = self.get_code_file(script_type)
with io.open(script_path, mode='r', encoding='utf-8') as script_file:
script_code = script_file.read()
# Check if the script imports an API module. If it does,
# the API module code will be pasted in place of the import.
module_import, module_name = self.check_api_module_imports(script_code)
if module_import:
script_code = self.insert_module_code(script_code, module_import,
module_name)
if script_type == '.py':
clean_code = self.clean_python_code(script_code)
if 'CommonServer' not in yml_data['name']:
# CommonServerPython has those line hard-coded so there is no need to add them here.
clean_code = f"register_module_line('{yml_data['name']}', 'start', __line__())\n" \
f'{clean_code}\n' \
f"register_module_line('{yml_data['name']}', 'end', __line__())\n"
elif script_type == '.ps1':
clean_code = self.clean_pwsh_code(script_code)
else:
# for JS scripts
clean_code = script_code
if self.is_script_package:
if yml_data.get('script', '') not in ('', '-'):
print_warning(
f'Script section is not empty in package {self.package_path}.'
f'It should be blank or a dash(-).')
yml_unified['script'] = FoldedScalarString(clean_code)
else:
if yml_data['script'].get('script', '') not in ('', '-'):
print_warning(
f'Script section is not empty in package {self.package_path}.'
f'It should be blank or a dash(-).')
yml_unified['script']['script'] = FoldedScalarString(clean_code)
return yml_unified, script_path
def rewrite_yml(file_path: str, yml_content: Dict, new_to_version: str):
"""Updating yml file to new toversion"""
yml_content['toversion'] = new_to_version
check_dockerimage45(yml_content, new_to_version)
if 'script' in yml_content:
if isinstance(yml_content.get('script'), str):
if yml_content.get('script') not in ('-', ''):
yml_content['script'] = FoldedScalarString(yml_content.get('script', ''))
elif yml_content.get('script', {}).get('script', '') not in ('-', ''):
yml_content['script']['script'] = FoldedScalarString(yml_content.get('script', {}).get('script', ''))
# resetting the tests associated with each yml
yml_content['tests'] = ['No test']
with open(file_path, mode='w', encoding='utf-8') as f:
ryaml.dump(yml_content, f)
print(f" - Updating {file_path}")
def rewrite_yml(file_path, yml_content, new_from_version):
# if there is no fromVersion in json or the fromversion is lower than the new from version then update
if ('fromversion' in yml_content and parse_version(yml_content.get('fromversion')) < parse_version(new_from_version)) \
or ('fromversion' not in yml_content):
yml_content['fromversion'] = new_from_version
check_dockerimage45(yml_content, new_from_version)
if 'script' in yml_content:
if isinstance(yml_content.get('script'), str):
if yml_content.get('script') not in ('-', ''):
yml_content['script'] = FoldedScalarString(
yml_content.get('script'))
elif yml_content.get('script').get('script') not in ('-', ''):
yml_content['script']['script'] = FoldedScalarString(
yml_content.get('script').get('script'))
with open(file_path, mode='w', encoding='utf-8') as f:
ryaml.dump(yml_content, f)
print(f" - Updating {file_path}")
def __init__(self, script: str, type_: str, subtype: str, dockerimage: str, isfetch: bool,
commands: list = None):
if script and isinstance(script, str):
self.script = FoldedScalarString(script)
else:
self.script = script
self.type = type_
self.subtype = subtype
self.dockerimage = dockerimage
self.isfetch = isfetch
if commands:
self.commands = commands
def test_eyaml_block_parsing(self):
eyaml = """\
ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw
DQYJKoZIhvcNAQEBBQAEggEAqcl1NXkwXvasks+WFoWfKjydHk9WuA/NI97u
of6Gi61pvf6i2ZO45hqUSQVn0Lbz3nH7Juv3kcXGFZlrvi5/UK1jcT7gqOlb
EWLS+zXFmf26Ou9j4shXiMQPpvPFeqV9X8t+gO84lzJFTgovzgkLNeCD1tIx
EMqDO8zYXxzXPXT21L2e79XL+qiVWtjsPe2q6TPDJN4BD+lq1laki8CZa5cE
wNBlWjPtlUVUk8ptSM9xaZMK5q9NRrJ/0uGIeBUAGdM2Po5BfNpfc9YyHFeH
xbvzUmgkQYVXuvWp0XSM9vRPBZWQtnmqwh2VA22+CgD+mE3drmohraX3hCC1
UdC6wDA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBD39+PUE1rNQN3rtk9l
589EgBBFMsMP2N1a4alw3UvIDYRu]
"""
output = "ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw\u0007\nDQYJKoZIhvcNAQEBBQAEggEAqcl1NXkwXvasks+WFoWfKjydHk9WuA/NI97u\u0007\nof6Gi61pvf6i2ZO45hqUSQVn0Lbz3nH7Juv3kcXGFZlrvi5/UK1jcT7gqOlb\u0007\nEWLS+zXFmf26Ou9j4shXiMQPpvPFeqV9X8t+gO84lzJFTgovzgkLNeCD1tIx\u0007\nEMqDO8zYXxzXPXT21L2e79XL+qiVWtjsPe2q6TPDJN4BD+lq1laki8CZa5cE\u0007\nwNBlWjPtlUVUk8ptSM9xaZMK5q9NRrJ/0uGIeBUAGdM2Po5BfNpfc9YyHFeH\u0007\nxbvzUmgkQYVXuvWp0XSM9vRPBZWQtnmqwh2VA22+CgD+mE3drmohraX3hCC1\u0007\nUdC6wDA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBD39+PUE1rNQN3rtk9l\u0007\n589EgBBFMsMP2N1a4alw3UvIDYRu]\n"
result = eyamladd.parse_eyaml_block(eyaml)
assert result == FoldedScalarString(output)
class Test_enums_YAMLValueFormats():
"""Tests for the YAMLValueFormats enumeration."""
def test_get_names(self):
assert YAMLValueFormats.get_names() == [
"BARE",
"BOOLEAN",
"DEFAULT",
"DQUOTE",
"FLOAT",
"FOLDED",
"INT",
"LITERAL",
"SQUOTE",
]
@pytest.mark.parametrize("input,output", [
("BARE", YAMLValueFormats.BARE),
("BOOLEAN", YAMLValueFormats.BOOLEAN),
("DEFAULT", YAMLValueFormats.DEFAULT),
("DQUOTE", YAMLValueFormats.DQUOTE),
("FLOAT", YAMLValueFormats.FLOAT),
("FOLDED", YAMLValueFormats.FOLDED),
("INT", YAMLValueFormats.INT),
("LITERAL", YAMLValueFormats.LITERAL),
("SQUOTE", YAMLValueFormats.SQUOTE),
])
def test_from_str(self, input, output):
assert output == YAMLValueFormats.from_str(input)
def test_from_str_nameerror(self):
with pytest.raises(NameError):
YAMLValueFormats.from_str("NO SUCH NAME")
@pytest.mark.parametrize("input,output", [
(FoldedScalarString(""), YAMLValueFormats.FOLDED),
(LiteralScalarString(""), YAMLValueFormats.LITERAL),
(DoubleQuotedScalarString(''), YAMLValueFormats.DQUOTE),
(SingleQuotedScalarString(""), YAMLValueFormats.SQUOTE),
(PlainScalarString(""), YAMLValueFormats.BARE),
(ScalarBoolean(False), YAMLValueFormats.BOOLEAN),
(ScalarFloat(1.01), YAMLValueFormats.FLOAT),
(ScalarInt(10), YAMLValueFormats.INT),
(None, YAMLValueFormats.DEFAULT),
])
def test_from_node(self, input, output):
assert output == YAMLValueFormats.from_node(input)
def _insert_samples(self):
samples_dir = os.path.join(os.path.dirname(self.package_path),
SAMPLES_DIR)
if os.path.isdir(samples_dir):
samples = defaultdict(list)
for sample_file in os.listdir(samples_dir):
with io.open(os.path.join(samples_dir, sample_file),
mode='r',
encoding='utf-8') as samples_file_object:
sample = json.loads(samples_file_object.read())
if self.yml_data.get('id') in sample.get('rules', []):
samples[
f'{sample.get("vendor")}_{sample.get("product")}'].extend(
sample.get('samples'))
if samples:
self.yml_data['samples'] = FoldedScalarString(
json.dumps(samples, indent=4))
click.echo(f'Added {len(samples)} samples.')
else:
click.echo('Did not find matching samples.')
def deep_update(data, new_data):
"""Update a dictionary with new data. Does nested updates. Each update is
overwrite-only, this does not append / remove from lists.
Args:
data (dict): Input dict.
new_data (dict): Dict containing updated data.
Returns:
dict: Input dict, updated with new data.
"""
for k, v in new_data.items():
if k.startswith('_'): # computed fields, ignore
continue
if isinstance(v, Mapping):
data[k] = deep_update(data[k], v)
elif isinstance(data[k], FoldedScalarString):
data[k] = FoldedScalarString(v)
else:
data[k] = v
return data
def parse_eyaml_block(block):
"""
This ugly mess is to get ruamel.yaml to handle the block output from
eyaml as proper folded style block scalars. By doing it this way we can
freely use the return value from this function in yaml files without
thinking about formatting when dumping back to the eyaml file.
Basically the following is happening:
1. Remove superfluous spaces and line breaks in every line of the output
2. Join the lines using the BEL (\a) character, which is the way ruamel
yaml represents block folds in the FoldedScalarString object
3. Ensure the last line has a newline to avoid the block being
represented as block style with the block chomping indicator set (>-).
4. Then ensure that blob of text is a FoldedScalarString
It is also possible to just return a long string instead of a folded
style block scalar, but it is much less readable when the strings are
getting long.
"""
lines = [line.strip() for line in block.splitlines()]
return FoldedScalarString('{}\n'.format('\a\n'.join(lines)))
def mockencrypt(data, public_key):
return FoldedScalarString('encrypted')
def _insert_rules(self):
rules_path = Path(self.yml_path).with_suffix('.xif')
with io.open(rules_path, mode='r', encoding='utf-8') as rules_file:
rules = rules_file.read()
self.yml_data['rules'] = FoldedScalarString(rules)
def test_debug_noisy(self, capsys):
args = SimpleNamespace(verbose=False, quiet=False, debug=True)
logger = ConsolePrinter(args)
anchoredkey = PlainScalarString("TestKey", anchor="KeyAnchor")
anchoredval = PlainScalarString("TestVal", anchor="Anchor")
foldedstr = "123456789 123456789 123456789"
foldedstrfolds = [10, 20]
foldedval = FoldedScalarString(foldedstr)
foldedval.fold_pos = foldedstrfolds
logger.debug(anchoredval)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: (&Anchor)TestVal",
]) + "\n" == console.out
logger.debug(["test", anchoredval])
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [0]test<class 'str'>",
"DEBUG: [1](&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, anchoredkey: anchoredval})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [TestKey](&KeyAnchor,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, anchoredkey: "non-anchored value"})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [TestKey](&KeyAnchor,_)non-anchored value<class 'str'>",
]) + "\n" == console.out
logger.debug({"ichi": 1, "non-anchored-key": anchoredval})
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [ichi]1<class 'int'>",
"DEBUG: [non-anchored-key](_,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>",
]) + "\n" == console.out
tagged_value = "value"
tagged_value_node = TaggedScalar(tagged_value, tag="!tag")
tagged_sequence = CommentedSeq(["a", "b"])
tagged_sequence.yaml_set_tag("!raz")
selfref_value = "self_referring"
selfref_value_node = TaggedScalar(selfref_value, tag="!self_referring")
logger.debug(
"test_wrappers_consoleprinter:",
prefix="test_debug_noisy: ",
header="--- HEADER ---",
footer="=== FOOTER ===",
data_header="+++ DATA HEADER +++",
data_footer="::: DATA FOOTER :::",
data=CommentedMap({
"key": "value",
"tagged": tagged_value_node,
tagged_value_node: "untagged value",
selfref_value_node: selfref_value_node,
"array": ["ichi", "ni", "san"],
"tagged_array": tagged_sequence,
"aoh": [{"id": 1},{"id": 2},{"id": 3}],
"aoa": [[True, True], [True, False], [False, True]],
"dod": {"a": {"b": {"c": "d"}}},
"set": CommentedSet(["one", "two"]),
})
)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: test_debug_noisy: --- HEADER ---",
"DEBUG: test_debug_noisy: test_wrappers_consoleprinter:",
"DEBUG: test_debug_noisy: +++ DATA HEADER +++",
"DEBUG: test_debug_noisy: [key]value<class 'str'>",
"DEBUG: test_debug_noisy: [tagged]<_,!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
"DEBUG: test_debug_noisy: [value]<!tag,_>untagged value<class 'str'>",
"DEBUG: test_debug_noisy: [self_referring]<!self_referring,!self_referring>self_referring<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
"DEBUG: test_debug_noisy: [array][0]ichi<class 'str'>",
"DEBUG: test_debug_noisy: [array][1]ni<class 'str'>",
"DEBUG: test_debug_noisy: [array][2]san<class 'str'>",
"DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[0]a<class 'str'>",
"DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[1]b<class 'str'>",
"DEBUG: test_debug_noisy: [aoh][0][id]1<class 'int'>",
"DEBUG: test_debug_noisy: [aoh][1][id]2<class 'int'>",
"DEBUG: test_debug_noisy: [aoh][2][id]3<class 'int'>",
"DEBUG: test_debug_noisy: [aoa][0][0]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][0][1]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][1][0]True<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][1][1]False<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][2][0]False<class 'bool'>",
"DEBUG: test_debug_noisy: [aoa][2][1]True<class 'bool'>",
"DEBUG: test_debug_noisy: [dod][a][b][c]d<class 'str'>",
"DEBUG: test_debug_noisy: [set]{one}<class 'str'>",
"DEBUG: test_debug_noisy: [set]{two}<class 'str'>",
"DEBUG: test_debug_noisy: ::: DATA FOOTER :::",
"DEBUG: test_debug_noisy: === FOOTER ===",
]) + "\n" == console.out
logger.debug(tagged_value_node)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: <!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)",
])
logger.debug(tagged_sequence)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: [tagged_array]<!raz>[0]a<class 'str'>",
"DEBUG: [tagged_array]<!raz>[1]b<class 'str'>",
])
nc = NodeCoords(
"value",
dict(key="value"),
"key",
YAMLPath("doc_root.key"),
[ (dict(doc_root=dict(key="value")), "doc_root"),
(dict(key="value"), "key")],
(PathSegmentTypes.KEY, "key")
)
logger.debug(
"A node coordinate:", prefix="test_debug_noisy: ", data=nc)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: test_debug_noisy: A node coordinate:",
"DEBUG: test_debug_noisy: (path)doc_root.key",
"DEBUG: test_debug_noisy: (segment)[0]PathSegmentTypes.KEY<enum 'PathSegmentTypes'>",
"DEBUG: test_debug_noisy: (segment)[1]key<class 'str'>",
"DEBUG: test_debug_noisy: (node)value",
"DEBUG: test_debug_noisy: (parent)[key]value<class 'str'>",
"DEBUG: test_debug_noisy: (parentref)key",
"DEBUG: test_debug_noisy: (ancestry)[0][0][doc_root][key]value<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[0][1]doc_root<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[1][0][key]value<class 'str'>",
"DEBUG: test_debug_noisy: (ancestry)[1][1]key<class 'str'>",
]) + "\n" == console.out
logger.debug(foldedval)
console = capsys.readouterr()
assert "\n".join([
"DEBUG: {}<class 'ruamel.yaml.scalarstring.FoldedScalarString'>,folded@{}".format(foldedstr, foldedstrfolds)
])
def set_statement(self, statement: str, input: str, output: str,
constraints: str):
self._content["statement"] = FoldedScalarString(statement)
self._content["input"] = FoldedScalarString(input)
self._content["output"] = FoldedScalarString(output)
self._content["constraints"] = FoldedScalarString(constraints)
