def insert_script_to_yml(self, script_type, yml_unified, yml_data): script_path = self.get_code_file(script_type) with io.open(script_path, mode='r', encoding='utf-8') as script_file: script_code = script_file.read() # Check if the script imports an API module. If it does, # the API module code will be pasted in place of the import. module_import, module_name = self.check_api_module_imports(script_code) if module_import: script_code = self.insert_module_code(script_code, module_import, module_name) if script_type == '.py': clean_code = self.clean_python_code(script_code) if script_type == '.ps1': clean_code = self.clean_pwsh_code(script_code) if self.is_script_package: if yml_data.get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script'] = FoldedScalarString(clean_code) else: if yml_data['script'].get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script']['script'] = FoldedScalarString(clean_code) return yml_unified, script_path
def fix_script_in_unified_yml(yml_content): if 'script' in yml_content: if isinstance(yml_content.get('script'), str): if yml_content.get('script') not in ('-', ''): yml_content['script'] = FoldedScalarString( yml_content['script']) elif yml_content.get('script').get('script') not in ('-', ''): yml_content['script']['script'] = FoldedScalarString( yml_content['script']['script'])
def rewrite_yml(file_path, yml_content, new_to_version): yml_content['toversion'] = new_to_version check_dockerimage45(yml_content, new_to_version) if 'script' in yml_content: if isinstance(yml_content.get('script'), str): if yml_content.get('script') not in ('-', ''): yml_content['script'] = FoldedScalarString(yml_content.get('script')) elif yml_content.get('script').get('script') not in ('-', ''): yml_content['script']['script'] = FoldedScalarString(yml_content.get('script').get('script')) with open(file_path, mode='w', encoding='utf-8') as f: ryaml.dump(yml_content, f) print(f" - Updating {file_path}")
def add_examples(self, input: str, output: str, explanation: str): example = CommentedMap( input=LiteralScalarString(input), output=LiteralScalarString(output), explanation=FoldedScalarString(explanation), ) self._content["examples"].append(example)
def add_from_version_to_yml(self, file_path=None, yml_content=None, save_yml=True): if self.no_fromversion: return {} ryaml = YAML() ryaml.preserve_quotes = True ryaml.width = 50000 # make sure long lines will not break (relevant for code section) if not yml_content: with open(file_path, 'r') as yml_file: yml_content = ryaml.load(yml_file) if parse_version(yml_content.get( 'toversion', '99.99.99')) > parse_version( LATEST_SUPPORTED_VERSION) > parse_version( yml_content.get('fromversion', '0.0.0')): yml_content['fromversion'] = LATEST_SUPPORTED_VERSION if 'detaileddescription' in yml_content: yml_content['detaileddescription'] = FoldedScalarString( yml_content['detaileddescription']) self.fix_script_in_unified_yml(yml_content) if save_yml: with open(file_path, mode='w', encoding='utf-8') as f: ryaml.dump(yml_content, f) return yml_content
def update_model_descriptions(models, descriptions, mode): update_models = [] missed_models = [] for name, desc in descriptions.items(): model = models.get(name, None) if model is None: logging.error('For description file {}.md no model found'.format(name)) missed_models.append(name) continue if not model[1].get('description', None): logging.error('No description found in {} for {} model'.format(model[0], name)) missed_models.append(name) continue model = model[1] if model.get('description', '') != desc: if mode == 'update': model['description'] = FoldedScalarString(desc) else: logging.debug('Found diff for {} model'.format(name)) logging.debug('\n{:12s}{}\n\tvs\n{:12s}{}' .format('In config:', model['description'], 'In readme:', desc)) update_models.append(name) if mode == 'update': msg = 'Description updated for {} models, missed for {} models.' msg_model_list = 'UPDATED:\n\t{}' else: msg = 'Description differs for {} models, missed for {} models.' msg_model_list = 'DIFFERENCE:\n\t{}' logging.info(msg.format(len(update_models), len(missed_models))) if len(update_models) > 0: logging.info(msg_model_list.format("\n\t".join(update_models))) if len(missed_models) > 0: logging.info('FAILED:\n\t{}'.format("\n\t".join(missed_models))) return update_models
def insert_description_to_yml(self, yml_data, yml_unified): desc_data, found_desc_path = self.get_data(self.package_path, '*_description.md') if yml_data.get('detaileddescription') and self.use_force is False: raise ValueError( 'Please move the detailed description from the yml to a description file (.md)' f' in the package: {self.package_path}') detailed_description = '' if desc_data: detailed_description = FoldedScalarString( desc_data.decode('utf-8')) integration_doc_link = '' if '[View Integration Documentation]' not in detailed_description: integration_doc_link = self.get_integration_doc_link(yml_data) if integration_doc_link: if detailed_description: detailed_description += '\n\n---\n' + integration_doc_link else: detailed_description += integration_doc_link if detailed_description: yml_unified['detaileddescription'] = detailed_description return yml_unified, found_desc_path
def toYamlText(val): if isinstance(val, (ScalarString, sensitive)): return val # convert or copy string (copy to deal with things like AnsibleUnsafeText) val = str(val) if "\n" in val: return FoldedScalarString(val) return val
def insert_description_to_yml(self, yml_data, yml_unified): desc_data, found_desc_path = self.get_data(self.package_path, '*_description.md') if yml_data.get('detaileddescription') and self.use_force is False: raise ValueError('Please move the detailed description from the yml to a description file (.md)' f' in the package: {self.package_path}') if desc_data: yml_unified['detaileddescription'] = FoldedScalarString(desc_data.decode('utf-8')) return yml_unified, found_desc_path
def to_yaml_text( val: object ) -> Union[sensitive, ScalarString, FoldedScalarString, str]: if isinstance(val, (ScalarString, sensitive)): return val # convert or copy string (copy to deal with things like AnsibleUnsafeText) val = str(val) if "\n" in val: return FoldedScalarString(val) return val
def insert_script_to_yml(self, script_type, yml_unified, yml_data): script_path = self.get_code_file(script_type) with io.open(script_path, mode='r', encoding='utf-8') as script_file: script_code = script_file.read() # Check if the script imports an API module. If it does, # the API module code will be pasted in place of the import. module_import, module_name = self.check_api_module_imports(script_code) if module_import: script_code = self.insert_module_code(script_code, module_import, module_name) if script_type == '.py': clean_code = self.clean_python_code(script_code) if 'CommonServer' not in yml_data['name']: # CommonServerPython has those line hard-coded so there is no need to add them here. clean_code = f"register_module_line('{yml_data['name']}', 'start', __line__())\n" \ f'{clean_code}\n' \ f"register_module_line('{yml_data['name']}', 'end', __line__())\n" elif script_type == '.ps1': clean_code = self.clean_pwsh_code(script_code) else: # for JS scripts clean_code = script_code if self.is_script_package: if yml_data.get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script'] = FoldedScalarString(clean_code) else: if yml_data['script'].get('script', '') not in ('', '-'): print_warning( f'Script section is not empty in package {self.package_path}.' f'It should be blank or a dash(-).') yml_unified['script']['script'] = FoldedScalarString(clean_code) return yml_unified, script_path
def rewrite_yml(file_path: str, yml_content: Dict, new_to_version: str): """Updating yml file to new toversion""" yml_content['toversion'] = new_to_version check_dockerimage45(yml_content, new_to_version) if 'script' in yml_content: if isinstance(yml_content.get('script'), str): if yml_content.get('script') not in ('-', ''): yml_content['script'] = FoldedScalarString(yml_content.get('script', '')) elif yml_content.get('script', {}).get('script', '') not in ('-', ''): yml_content['script']['script'] = FoldedScalarString(yml_content.get('script', {}).get('script', '')) # resetting the tests associated with each yml yml_content['tests'] = ['No test'] with open(file_path, mode='w', encoding='utf-8') as f: ryaml.dump(yml_content, f) print(f" - Updating {file_path}")
def rewrite_yml(file_path, yml_content, new_from_version): # if there is no fromVersion in json or the fromversion is lower than the new from version then update if ('fromversion' in yml_content and parse_version(yml_content.get('fromversion')) < parse_version(new_from_version)) \ or ('fromversion' not in yml_content): yml_content['fromversion'] = new_from_version check_dockerimage45(yml_content, new_from_version) if 'script' in yml_content: if isinstance(yml_content.get('script'), str): if yml_content.get('script') not in ('-', ''): yml_content['script'] = FoldedScalarString( yml_content.get('script')) elif yml_content.get('script').get('script') not in ('-', ''): yml_content['script']['script'] = FoldedScalarString( yml_content.get('script').get('script')) with open(file_path, mode='w', encoding='utf-8') as f: ryaml.dump(yml_content, f) print(f" - Updating {file_path}")
def __init__(self, script: str, type_: str, subtype: str, dockerimage: str, isfetch: bool, commands: list = None): if script and isinstance(script, str): self.script = FoldedScalarString(script) else: self.script = script self.type = type_ self.subtype = subtype self.dockerimage = dockerimage self.isfetch = isfetch if commands: self.commands = commands
def test_eyaml_block_parsing(self): eyaml = """\ ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw DQYJKoZIhvcNAQEBBQAEggEAqcl1NXkwXvasks+WFoWfKjydHk9WuA/NI97u of6Gi61pvf6i2ZO45hqUSQVn0Lbz3nH7Juv3kcXGFZlrvi5/UK1jcT7gqOlb EWLS+zXFmf26Ou9j4shXiMQPpvPFeqV9X8t+gO84lzJFTgovzgkLNeCD1tIx EMqDO8zYXxzXPXT21L2e79XL+qiVWtjsPe2q6TPDJN4BD+lq1laki8CZa5cE wNBlWjPtlUVUk8ptSM9xaZMK5q9NRrJ/0uGIeBUAGdM2Po5BfNpfc9YyHFeH xbvzUmgkQYVXuvWp0XSM9vRPBZWQtnmqwh2VA22+CgD+mE3drmohraX3hCC1 UdC6wDA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBD39+PUE1rNQN3rtk9l 589EgBBFMsMP2N1a4alw3UvIDYRu] """ output = "ENC[PKCS7,MIIBeQYJKoZIhvcNAQcDoIIBajCCAWYCAQAxggEhMIIBHQIBADAFMAACAQEw\u0007\nDQYJKoZIhvcNAQEBBQAEggEAqcl1NXkwXvasks+WFoWfKjydHk9WuA/NI97u\u0007\nof6Gi61pvf6i2ZO45hqUSQVn0Lbz3nH7Juv3kcXGFZlrvi5/UK1jcT7gqOlb\u0007\nEWLS+zXFmf26Ou9j4shXiMQPpvPFeqV9X8t+gO84lzJFTgovzgkLNeCD1tIx\u0007\nEMqDO8zYXxzXPXT21L2e79XL+qiVWtjsPe2q6TPDJN4BD+lq1laki8CZa5cE\u0007\nwNBlWjPtlUVUk8ptSM9xaZMK5q9NRrJ/0uGIeBUAGdM2Po5BfNpfc9YyHFeH\u0007\nxbvzUmgkQYVXuvWp0XSM9vRPBZWQtnmqwh2VA22+CgD+mE3drmohraX3hCC1\u0007\nUdC6wDA8BgkqhkiG9w0BBwEwHQYJYIZIAWUDBAEqBBD39+PUE1rNQN3rtk9l\u0007\n589EgBBFMsMP2N1a4alw3UvIDYRu]\n" result = eyamladd.parse_eyaml_block(eyaml) assert result == FoldedScalarString(output)
class Test_enums_YAMLValueFormats(): """Tests for the YAMLValueFormats enumeration.""" def test_get_names(self): assert YAMLValueFormats.get_names() == [ "BARE", "BOOLEAN", "DEFAULT", "DQUOTE", "FLOAT", "FOLDED", "INT", "LITERAL", "SQUOTE", ] @pytest.mark.parametrize("input,output", [ ("BARE", YAMLValueFormats.BARE), ("BOOLEAN", YAMLValueFormats.BOOLEAN), ("DEFAULT", YAMLValueFormats.DEFAULT), ("DQUOTE", YAMLValueFormats.DQUOTE), ("FLOAT", YAMLValueFormats.FLOAT), ("FOLDED", YAMLValueFormats.FOLDED), ("INT", YAMLValueFormats.INT), ("LITERAL", YAMLValueFormats.LITERAL), ("SQUOTE", YAMLValueFormats.SQUOTE), ]) def test_from_str(self, input, output): assert output == YAMLValueFormats.from_str(input) def test_from_str_nameerror(self): with pytest.raises(NameError): YAMLValueFormats.from_str("NO SUCH NAME") @pytest.mark.parametrize("input,output", [ (FoldedScalarString(""), YAMLValueFormats.FOLDED), (LiteralScalarString(""), YAMLValueFormats.LITERAL), (DoubleQuotedScalarString(''), YAMLValueFormats.DQUOTE), (SingleQuotedScalarString(""), YAMLValueFormats.SQUOTE), (PlainScalarString(""), YAMLValueFormats.BARE), (ScalarBoolean(False), YAMLValueFormats.BOOLEAN), (ScalarFloat(1.01), YAMLValueFormats.FLOAT), (ScalarInt(10), YAMLValueFormats.INT), (None, YAMLValueFormats.DEFAULT), ]) def test_from_node(self, input, output): assert output == YAMLValueFormats.from_node(input)
def _insert_samples(self): samples_dir = os.path.join(os.path.dirname(self.package_path), SAMPLES_DIR) if os.path.isdir(samples_dir): samples = defaultdict(list) for sample_file in os.listdir(samples_dir): with io.open(os.path.join(samples_dir, sample_file), mode='r', encoding='utf-8') as samples_file_object: sample = json.loads(samples_file_object.read()) if self.yml_data.get('id') in sample.get('rules', []): samples[ f'{sample.get("vendor")}_{sample.get("product")}'].extend( sample.get('samples')) if samples: self.yml_data['samples'] = FoldedScalarString( json.dumps(samples, indent=4)) click.echo(f'Added {len(samples)} samples.') else: click.echo('Did not find matching samples.')
def deep_update(data, new_data): """Update a dictionary with new data. Does nested updates. Each update is overwrite-only, this does not append / remove from lists. Args: data (dict): Input dict. new_data (dict): Dict containing updated data. Returns: dict: Input dict, updated with new data. """ for k, v in new_data.items(): if k.startswith('_'): # computed fields, ignore continue if isinstance(v, Mapping): data[k] = deep_update(data[k], v) elif isinstance(data[k], FoldedScalarString): data[k] = FoldedScalarString(v) else: data[k] = v return data
def parse_eyaml_block(block): """ This ugly mess is to get ruamel.yaml to handle the block output from eyaml as proper folded style block scalars. By doing it this way we can freely use the return value from this function in yaml files without thinking about formatting when dumping back to the eyaml file. Basically the following is happening: 1. Remove superfluous spaces and line breaks in every line of the output 2. Join the lines using the BEL (\a) character, which is the way ruamel yaml represents block folds in the FoldedScalarString object 3. Ensure the last line has a newline to avoid the block being represented as block style with the block chomping indicator set (>-). 4. Then ensure that blob of text is a FoldedScalarString It is also possible to just return a long string instead of a folded style block scalar, but it is much less readable when the strings are getting long. """ lines = [line.strip() for line in block.splitlines()] return FoldedScalarString('{}\n'.format('\a\n'.join(lines)))
def mockencrypt(data, public_key): return FoldedScalarString('encrypted')
def _insert_rules(self): rules_path = Path(self.yml_path).with_suffix('.xif') with io.open(rules_path, mode='r', encoding='utf-8') as rules_file: rules = rules_file.read() self.yml_data['rules'] = FoldedScalarString(rules)
def test_debug_noisy(self, capsys): args = SimpleNamespace(verbose=False, quiet=False, debug=True) logger = ConsolePrinter(args) anchoredkey = PlainScalarString("TestKey", anchor="KeyAnchor") anchoredval = PlainScalarString("TestVal", anchor="Anchor") foldedstr = "123456789 123456789 123456789" foldedstrfolds = [10, 20] foldedval = FoldedScalarString(foldedstr) foldedval.fold_pos = foldedstrfolds logger.debug(anchoredval) console = capsys.readouterr() assert "\n".join([ "DEBUG: (&Anchor)TestVal", ]) + "\n" == console.out logger.debug(["test", anchoredval]) console = capsys.readouterr() assert "\n".join([ "DEBUG: [0]test<class 'str'>", "DEBUG: [1](&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>", ]) + "\n" == console.out logger.debug({"ichi": 1, anchoredkey: anchoredval}) console = capsys.readouterr() assert "\n".join([ "DEBUG: [ichi]1<class 'int'>", "DEBUG: [TestKey](&KeyAnchor,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>", ]) + "\n" == console.out logger.debug({"ichi": 1, anchoredkey: "non-anchored value"}) console = capsys.readouterr() assert "\n".join([ "DEBUG: [ichi]1<class 'int'>", "DEBUG: [TestKey](&KeyAnchor,_)non-anchored value<class 'str'>", ]) + "\n" == console.out logger.debug({"ichi": 1, "non-anchored-key": anchoredval}) console = capsys.readouterr() assert "\n".join([ "DEBUG: [ichi]1<class 'int'>", "DEBUG: [non-anchored-key](_,&Anchor)TestVal<class 'ruamel.yaml.scalarstring.PlainScalarString'>", ]) + "\n" == console.out tagged_value = "value" tagged_value_node = TaggedScalar(tagged_value, tag="!tag") tagged_sequence = CommentedSeq(["a", "b"]) tagged_sequence.yaml_set_tag("!raz") selfref_value = "self_referring" selfref_value_node = TaggedScalar(selfref_value, tag="!self_referring") logger.debug( "test_wrappers_consoleprinter:", prefix="test_debug_noisy: ", header="--- HEADER ---", footer="=== FOOTER ===", data_header="+++ DATA HEADER +++", data_footer="::: DATA FOOTER :::", data=CommentedMap({ "key": "value", "tagged": tagged_value_node, tagged_value_node: "untagged value", selfref_value_node: selfref_value_node, "array": ["ichi", "ni", "san"], "tagged_array": tagged_sequence, "aoh": [{"id": 1},{"id": 2},{"id": 3}], "aoa": [[True, True], [True, False], [False, True]], "dod": {"a": {"b": {"c": "d"}}}, "set": CommentedSet(["one", "two"]), }) ) console = capsys.readouterr() assert "\n".join([ "DEBUG: test_debug_noisy: --- HEADER ---", "DEBUG: test_debug_noisy: test_wrappers_consoleprinter:", "DEBUG: test_debug_noisy: +++ DATA HEADER +++", "DEBUG: test_debug_noisy: [key]value<class 'str'>", "DEBUG: test_debug_noisy: [tagged]<_,!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)", "DEBUG: test_debug_noisy: [value]<!tag,_>untagged value<class 'str'>", "DEBUG: test_debug_noisy: [self_referring]<!self_referring,!self_referring>self_referring<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)", "DEBUG: test_debug_noisy: [array][0]ichi<class 'str'>", "DEBUG: test_debug_noisy: [array][1]ni<class 'str'>", "DEBUG: test_debug_noisy: [array][2]san<class 'str'>", "DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[0]a<class 'str'>", "DEBUG: test_debug_noisy: [tagged_array]<_,!raz>[1]b<class 'str'>", "DEBUG: test_debug_noisy: [aoh][0][id]1<class 'int'>", "DEBUG: test_debug_noisy: [aoh][1][id]2<class 'int'>", "DEBUG: test_debug_noisy: [aoh][2][id]3<class 'int'>", "DEBUG: test_debug_noisy: [aoa][0][0]True<class 'bool'>", "DEBUG: test_debug_noisy: [aoa][0][1]True<class 'bool'>", "DEBUG: test_debug_noisy: [aoa][1][0]True<class 'bool'>", "DEBUG: test_debug_noisy: [aoa][1][1]False<class 'bool'>", "DEBUG: test_debug_noisy: [aoa][2][0]False<class 'bool'>", "DEBUG: test_debug_noisy: [aoa][2][1]True<class 'bool'>", "DEBUG: test_debug_noisy: [dod][a][b][c]d<class 'str'>", "DEBUG: test_debug_noisy: [set]{one}<class 'str'>", "DEBUG: test_debug_noisy: [set]{two}<class 'str'>", "DEBUG: test_debug_noisy: ::: DATA FOOTER :::", "DEBUG: test_debug_noisy: === FOOTER ===", ]) + "\n" == console.out logger.debug(tagged_value_node) console = capsys.readouterr() assert "\n".join([ "DEBUG: <!tag>value<class 'ruamel.yaml.comments.TaggedScalar'>(<class 'str'>)", ]) logger.debug(tagged_sequence) console = capsys.readouterr() assert "\n".join([ "DEBUG: [tagged_array]<!raz>[0]a<class 'str'>", "DEBUG: [tagged_array]<!raz>[1]b<class 'str'>", ]) nc = NodeCoords( "value", dict(key="value"), "key", YAMLPath("doc_root.key"), [ (dict(doc_root=dict(key="value")), "doc_root"), (dict(key="value"), "key")], (PathSegmentTypes.KEY, "key") ) logger.debug( "A node coordinate:", prefix="test_debug_noisy: ", data=nc) console = capsys.readouterr() assert "\n".join([ "DEBUG: test_debug_noisy: A node coordinate:", "DEBUG: test_debug_noisy: (path)doc_root.key", "DEBUG: test_debug_noisy: (segment)[0]PathSegmentTypes.KEY<enum 'PathSegmentTypes'>", "DEBUG: test_debug_noisy: (segment)[1]key<class 'str'>", "DEBUG: test_debug_noisy: (node)value", "DEBUG: test_debug_noisy: (parent)[key]value<class 'str'>", "DEBUG: test_debug_noisy: (parentref)key", "DEBUG: test_debug_noisy: (ancestry)[0][0][doc_root][key]value<class 'str'>", "DEBUG: test_debug_noisy: (ancestry)[0][1]doc_root<class 'str'>", "DEBUG: test_debug_noisy: (ancestry)[1][0][key]value<class 'str'>", "DEBUG: test_debug_noisy: (ancestry)[1][1]key<class 'str'>", ]) + "\n" == console.out logger.debug(foldedval) console = capsys.readouterr() assert "\n".join([ "DEBUG: {}<class 'ruamel.yaml.scalarstring.FoldedScalarString'>,folded@{}".format(foldedstr, foldedstrfolds) ])
def set_statement(self, statement: str, input: str, output: str, constraints: str): self._content["statement"] = FoldedScalarString(statement) self._content["input"] = FoldedScalarString(input) self._content["output"] = FoldedScalarString(output) self._content["constraints"] = FoldedScalarString(constraints)