def _split_yaml_4_5_0(self, dest_dir: Path) -> List[Path]: """Split YAMLContentUnfiedObject in destination dir. Args: dest_dir: Destination directory. Returns: List[Path]: List of new created files. Notes: 1. If object contain docker_image_4_5 key with value -> should split to: a. <original_file> b. <original_file_name>_4_5.yml TODO: 1. Add Exception raising in unify module. 2. Verbosity to quiet mode option in unify module. """ # Directory configuration - Integrations or Scripts unify_dir = ENTITY_TYPE_TO_DIR[self._content_type.value] # Split step unifier = IntegrationScriptUnifier(input=str(self.path.parent), dir_name=unify_dir, output=str(dest_dir / self.path.name), force=True) yaml_dict = self.to_dict() yaml_dict_copy = copy.deepcopy(yaml_dict) script_object = self.script created_files: List[str] = unifier.write_yaml_with_docker(yaml_dict_copy, yaml_dict, script_object).keys() # Validate that split succeed - there is not exception raised in unify module. if not created_files: raise exc.ContentDumpError(self, self.path, "Unable to split object") return [Path(path) for path in created_files]
def test_unify_integration__detailed_description_with_special_char(self): """ - """ description = ''' some test with special chars שלום hello 你好 ''' create_test_package( test_dir=self.test_dir_path, package_name=self.package_name, base_yml='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage.yml', script_code=TEST_VALID_CODE, image_file='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage_image.png', detailed_description=description, ) unifier = IntegrationScriptUnifier(self.export_dir_path, output=self.test_dir_path) yml_files = unifier.unify() export_yml_path = yml_files[0] assert export_yml_path == self.expected_yml_path actual_yml = get_yaml(export_yml_path) expected_yml = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/' 'integration-SampleIntegPackageDescSpecialChars.yml') assert expected_yml == actual_yml assert actual_yml['detaileddescription'] == description
def test_unify_integration(self): """ sanity test of merge_script_package_to_yml of integration """ create_test_package( test_dir=self.test_dir_path, package_name=self.package_name, base_yml='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage.yml', script_code=TEST_VALID_CODE, detailed_description=TEST_VALID_DETAILED_DESCRIPTION, image_file='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage_image.png', ) unifier = IntegrationScriptUnifier(input=self.export_dir_path, output=self.test_dir_path) yml_files = unifier.unify() export_yml_path = yml_files[0] assert export_yml_path == self.expected_yml_path comment = '# this is a comment text inside a file 033dab25fd9655480dbec3a4c579a0e6' with open(export_yml_path) as file_: unified_content = file_.read() assert comment in unified_content actual_yml = get_yaml(export_yml_path) expected_yml = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/' 'integration-SampleIntegPackageSanity.yml') assert expected_yml == actual_yml
def build( self, code: Optional[str] = None, yml: Optional[dict] = None, readme: Optional[str] = None, description: Optional[str] = None, changelog: Optional[str] = None, image: Optional[bytes] = None ): """Writes not None objects to files. """ if code is not None: self.code.write(code) if yml is not None: self.yml.write_dict(yml) if readme is not None: self.readme.write(readme) if description is not None: self.description.write(description) if changelog is not None: self.changelog.write(changelog) if image is not None: self.image.write_bytes(image) if self.create_unified: unifier = IntegrationScriptUnifier(input=self.path, output=os.path.dirname(self._tmpdir_integration_path)) yml_path = unifier.unify()[0] readme_path = unifier.move_readme_next_to_unified(yml_path) shutil.rmtree(self._tmpdir_integration_path) self.yml.path = yml_path self.readme.path = readme_path
def test_unify_script__docker45(self): """ sanity test of merge_script_package_to_yml of script """ create_test_package( test_dir=self.test_dir_path, package_name=self.package_name, base_yml='demisto_sdk/tests/test_files/Unifier/SampleScriptPackage/SampleScriptPackageDocker45.yml', script_code=TEST_VALID_CODE, ) unifier = IntegrationScriptUnifier(input=self.export_dir_path, output=self.test_dir_path) yml_files = unifier.unify() assert len(yml_files) == 2 export_yml_path = yml_files[0] export_yml_path_45 = yml_files[1] assert export_yml_path == self.expected_yml_path assert export_yml_path_45 == self.expected_yml_path.replace('.yml', '_45.yml') actual_yml = get_yaml(export_yml_path) expected_yml = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleScriptPackage/' 'script-SampleScriptPackageSanityDocker45.yml') assert expected_yml == actual_yml actual_yml_45 = get_yaml(export_yml_path_45) expected_yml_45 = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleScriptPackage/' 'script-SampleScriptPackageSanityDocker45_45.yml') assert expected_yml_45 == actual_yml_45
def _unify(self, dest_dir: Path) -> List[Path]: """Unify YAMLContentUnfiedObject in destination dir. Args: dest_dir: Destination directory. Returns: List[Path]: List of new created files. TODO: 1. Add Exception raising in unify module. 2. Verbosity to quiet mode option in unify module. """ # Directory configuration - Integrations or Scripts unify_dir = ENTITY_TYPE_TO_DIR[self._content_type.value] # Unify step unifier: Union[IntegrationScriptUnifier, RuleUnifier] if self._content_type in [FileType.SCRIPT, FileType.INTEGRATION]: unifier = IntegrationScriptUnifier(input=str(self.path.parent), dir_name=unify_dir, output=dest_dir, force=True, yml_modified_data=self.to_dict()) elif self._content_type in [FileType.PARSING_RULE, FileType.MODELING_RULE]: unifier = RuleUnifier(input=str(self.path.parent), output=dest_dir, force=True) created_files: List[str] = unifier.unify() # Validate that unify succeed - there is not exception raised in unify module. if not created_files: raise exc.ContentDumpError(self, self.path, "Unable to unify object") return [Path(path) for path in created_files]
def test_get_code_file_case_insensative(tmp_path): # Create an integration dir with some files integration_dir = tmp_path / "TestDummyInt" os.makedirs(integration_dir) open(integration_dir / "Dummy.ps1", 'a') open(integration_dir / "ADummy.tests.ps1", 'a') # a test file which is named such a way that it comes up first unifier = IntegrationScriptUnifier(str(integration_dir)) assert unifier.get_code_file(".ps1") == str(integration_dir / "Dummy.ps1")
def test_get_code_file(): # Test integration case unifier = IntegrationScriptUnifier(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/") assert unifier.get_code_file(".py") == f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB.py" unifier = IntegrationScriptUnifier(f"{git_path()}/demisto_sdk/tests/test_files/Unifier/SampleNoPyFile") with pytest.raises(Exception): unifier.get_code_file(".py") # Test script case unifier = IntegrationScriptUnifier(f"{git_path()}/demisto_sdk/tests/test_files/CalculateGeoDistance/") assert unifier.get_code_file(".py") == f"{git_path()}/demisto_sdk/tests/test_files/CalculateGeoDistance/" \ f"CalculateGeoDistance.py"
def test_clean_python_code(repo): pack = repo.create_pack('PackName') integration = pack.create_integration('integration', 'bla', INTEGRATION_YAML) unifier = IntegrationScriptUnifier(str(integration.path)) script_code = "import demistomock as demisto\nfrom CommonServerPython import * # test comment being removed\n" \ "from CommonServerUserPython import *\nfrom __future__ import print_function" # Test remove_print_future is False script_code = unifier.clean_python_code(script_code, remove_print_future=False) assert script_code == "\n\n\nfrom __future__ import print_function" # Test remove_print_future is True script_code = unifier.clean_python_code(script_code) assert script_code.strip() == ""
def test_empty_yml(tmp_path): """ Given: - An empty unified yml When: - calling the add_custom_section when using the -t flag Then: - Check that the function will not raise any errors. """ unifier = IntegrationScriptUnifier(str(tmp_path)) unifier.add_custom_section({})
def test_insert_module_code__verify_offsets(mocker): """ When: replacing ApiModule code Given: a script with an ApiModule import Then: verify the wrapper of the section line numbers are correct. """ mocker.patch.object(IntegrationScriptUnifier, '_get_api_module_code', return_value=DUMMY_MODULE) import_name = 'from MicrosoftApiModule import * # noqa: E402' before_api_import, after_api_import = DUMMY_SCRIPT.split(import_name, 1) module_name = 'MicrosoftApiModule' code = IntegrationScriptUnifier.insert_module_code(DUMMY_SCRIPT, import_name, module_name) # get only the generated ApiModule code code = code[len(before_api_import):-len(after_api_import)] # we expect the start wrapper will have a negative number so adding it to the regex search start_offset = re.search(rf"register_module_line\('{module_name}', 'start', __line__\(\), wrapper=-(\d+)\)\n", code) end_offset = re.search(rf"register_module_line\('{module_name}', 'end', __line__\(\), wrapper=(\d+)\)\n", code) assert start_offset # the number of lines before the register start match the wrapper value assert int(start_offset.group(1)) == len(code[:start_offset.span()[0]].splitlines()) assert end_offset # the number of lines after the register end match the wrapper value assert int(end_offset.group(1)) == len(code[end_offset.span()[1]:].splitlines())
def test_insert_description_to_yml_doc_link_exist(tmp_path, mocker): """ Given: - integration which have a detailed description with "View Integration Documentation" doc link When: - Getting integration doc link Then: - Verify get_integration_doc_link function is not called """ detailed_desc = tmp_path / 'integration_description.md' detailed_desc.write_text('[View Integration Documentation]' '(https://xsoar.pan.dev/docs/reference/integrations/some-integration-id)') mock_func = mocker.patch.object(IntegrationScriptUnifier, 'get_integration_doc_link', return_result='') unifier = IntegrationScriptUnifier(str(tmp_path)) yml_unified, _ = unifier.insert_description_to_yml({'commonfields': {'id': 'some integration id'}}, {}) assert mock_func.call_count == 0
def test_insert_description_to_yml_with_no_detailed_desc(tmp_path): """ Given: - Integration with empty detailed description and with non-empty README When: - Inserting detailed description to the unified integration YAML Then: - Verify the integration doc markdown link is inserted to the detailed description """ readme = tmp_path / 'README.md' readme.write_text('README') detailed_desc = tmp_path / 'integration_description.md' detailed_desc.write_text('') unifier = IntegrationScriptUnifier(str(tmp_path)) yml_unified, _ = unifier.insert_description_to_yml({'commonfields': {'id': 'some integration id'}}, {}) assert '[View Integration Documentation](https://xsoar.pan.dev/docs/reference/integrations/some-integration-id)'\ == yml_unified['detaileddescription']
def remove_integration_documentation(self, detailed_description): if "[View Integration Documentation]" in detailed_description: normalized_integration_id = IntegrationScriptUnifier.normalize_integration_id(self.yml_data['commonfields']['id']) integration_doc_link = INTEGRATIONS_DOCS_REFERENCE + normalized_integration_id documentation = f'[View Integration Documentation]({integration_doc_link})' if '\n\n---\n' + documentation in detailed_description: detailed_description = detailed_description.replace('\n\n---\n' + documentation, "") elif documentation in detailed_description: detailed_description = detailed_description.replace(documentation, "") return detailed_description
def test_unify_default_output_script(self): """ Given - DummyScript script. - No output path. When - Running Unify on it. Then - Ensure Unify script works with default output. """ input_path_script = TESTS_DIR + '/test_files/Packs/DummyPack/Scripts/DummyScript' unifier = IntegrationScriptUnifier(input_path_script) yml_files = unifier.unify() export_yml_path = yml_files[0] expected_yml_path = TESTS_DIR + '/test_files/Packs/DummyPack/Scripts/DummyScript/script-DummyScript.yml' assert export_yml_path == expected_yml_path os.remove(expected_yml_path)
def test_get_integration_doc_link_negative(tmp_path): """ Given: - Case A: integration which does not have README in the integration dir - Case B: integration with empty README in the integration dir When: - Getting integration doc link Then: - Verify an empty string is returned """ unifier = IntegrationScriptUnifier(str(tmp_path)) integration_doc_link = unifier.get_integration_doc_link({'commonfields': {'id': 'Integration With No README'}}) assert integration_doc_link == '' readme = tmp_path / 'README.md' readme.write_text('') integration_doc_link = unifier.get_integration_doc_link({'commonfields': {'id': 'Integration With Empty README'}}) assert integration_doc_link == ''
def test_unify_default_output_integration(self): """ Given - UploadTest integration. - No output path. When - Running Unify on it. Then - Ensure Unify command works with default output. """ input_path_integration = TESTS_DIR + '/test_files/Packs/DummyPack/Integrations/UploadTest' unifier = IntegrationScriptUnifier(input_path_integration) yml_files = unifier.unify() export_yml_path = yml_files[0] expected_yml_path = TESTS_DIR + '/test_files/Packs/DummyPack/Integrations/UploadTest/integration-UploadTest.yml' assert export_yml_path == expected_yml_path os.remove(expected_yml_path)
def test_insert_image_to_yml_without_image(tmp_path): """ Given: - Integration without image png file When: - Inserting image to unified YAML Then: - Ensure the insertion does not crash - Verify no image path is returned """ integration_dir = tmp_path / 'Integrations' integration_dir.mkdir() integration_yml = integration_dir / 'SomeIntegration.yml' integration_obj = {'id': 'SomeIntegration'} yaml.dump(integration_obj, integration_yml.open('w')) unifier = IntegrationScriptUnifier(str(integration_dir)) yml_unified, found_img_path = unifier.insert_image_to_yml(integration_obj, integration_obj) assert yml_unified == integration_obj assert not found_img_path
def test_add_custom_section(tmp_path): ''' Given: - an Integration to unify When: - the --custom flag is True Then: - Add a "Test" to the name/display/id of the integration if the yml exsits. ''' unifier = IntegrationScriptUnifier(str(tmp_path), custom='Test') unified_yml = { 'display': 'Integration display', 'commonfields': {'id': 'Integration id'}, 'name': 'Integration name' } unified = unifier.add_custom_section(unified_yml) assert unified.get('display') == 'Integration display - Test' assert unified.get('name') == 'Integration name - Test' assert unified.get('commonfields').get('id') == 'Integration id - Test'
def test_get_script_or_integration_package_data(): unifier = IntegrationScriptUnifier(f"{git_path()}/demisto_sdk/tests/test_files/Unifier/SampleNoPyFile") with pytest.raises(Exception): unifier.get_script_or_integration_package_data() unifier = IntegrationScriptUnifier(f"{git_path()}/demisto_sdk/tests/test_files/CalculateGeoDistance") with open(f"{git_path()}/demisto_sdk/tests/test_files/CalculateGeoDistance/CalculateGeoDistance.py", "r") as \ code_file: code = code_file.read() yml_path, code_data = unifier.get_script_or_integration_package_data() assert yml_path == f"{git_path()}/demisto_sdk/tests/test_files/CalculateGeoDistance/CalculateGeoDistance.yml" assert code_data == code
def test_unify_integration__detailed_description_with_yml_structure(self): """ - """ description = ''' this is a regular line some test with special chars hello key: - subkey: hello subkey2: hi keys: "some more values" asd - hello hi: 'dsfsd' final test: hi ''' create_test_package( test_dir=self.test_dir_path, package_name=self.package_name, base_yml='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage.yml', script_code=TEST_VALID_CODE, image_file='demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/SampleIntegPackage_image.png', detailed_description=description, ) unifier = IntegrationScriptUnifier(self.export_dir_path, output=self.test_dir_path) yml_files = unifier.unify() export_yml_path = yml_files[0] assert export_yml_path == self.expected_yml_path actual_yml = get_yaml(export_yml_path) expected_yml = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleIntegPackage/' 'integration-SampleIntegPackageDescAsYML.yml') assert expected_yml == actual_yml assert actual_yml['detaileddescription'] == description
def test_unify_script(self): """ sanity test of merge_script_package_to_yml of script """ create_test_package( test_dir=self.test_dir_path, package_name=self.package_name, base_yml='demisto_sdk/tests/test_files/Unifier/SampleScriptPackage/SampleScriptPackage.yml', script_code=TEST_VALID_CODE, ) unifier = IntegrationScriptUnifier(input=self.export_dir_path, output=self.test_dir_path) yml_files = unifier.unify() export_yml_path = yml_files[0] assert export_yml_path == self.expected_yml_path actual_yml = get_yaml(export_yml_path) expected_yml = get_yaml('demisto_sdk/tests/test_files/Unifier/SampleScriptPackage/' 'script-SampleScriptPackageSanity.yml') assert expected_yml == actual_yml
def test_unify_default_output_integration_for_relative_current_dir_input(self, mocker): """ Given - Input path of '.'. - UploadTest integration. When - Running Unify on it. Then - Ensure Unify command works with default output given relative path to current directory. """ from demisto_sdk.commands.unify.integration_script_unifier import \ IntegrationScriptUnifier abs_path_mock = mocker.patch('demisto_sdk.commands.unify.integration_script_unifier.os.path.abspath') abs_path_mock.return_value = TESTS_DIR + '/test_files/Packs/DummyPack/Integrations/UploadTest' input_path_integration = '.' unifier = IntegrationScriptUnifier(input_path_integration) yml_files = unifier.unify() export_yml_path = yml_files[0] expected_yml_path = TESTS_DIR + '/test_files/Packs/DummyPack/Integrations/UploadTest/integration-UploadTest.yml' assert export_yml_path == expected_yml_path os.remove(expected_yml_path)
def test_add_contributors_support(tmp_path): """ Given: - partner integration which have (Partner Contribution) in the integration display name When: - Adding contribution support to display name Then: - Verify CONTRIBUTOR_DISPLAY_NAME is not added twice """ unifier = IntegrationScriptUnifier(str(tmp_path)) unified_yml = { 'display': 'Test Integration (Partner Contribution)', 'commonfields': {'id': 'Test Integration'} } unifier.add_contributors_support( unified_yml=unified_yml, contributor_type='partner', contributor_email='', contributor_url='', ) assert unified_yml["display"] == 'Test Integration (Partner Contribution)'
def test_get_integration_doc_link_positive(tmp_path): """ Given: - Cortex XDR - IOC integration with README When: - Getting integration doc link Then: - Verify the expected integration doc markdown link is returned - Verify the integration doc URL exists and reachable """ readme = tmp_path / 'README.md' readme.write_text('README') unifier = IntegrationScriptUnifier(str(tmp_path)) integration_doc_link = unifier.get_integration_doc_link({'commonfields': {'id': 'Cortex XDR - IOC'}}) assert integration_doc_link == \ '[View Integration Documentation](https://xsoar.pan.dev/docs/reference/integrations/cortex-xdr---ioc)' link = re.findall(r'\(([^)]+)\)', integration_doc_link)[0] try: r = requests.get(link, verify=False, timeout=10) r.raise_for_status() except requests.HTTPError as ex: raise Exception(f'Failed reaching to integration doc link {link} - {ex}')
def test_insert_module_code(mocker, import_name): mocker.patch.object(IntegrationScriptUnifier, '_get_api_module_code', return_value=DUMMY_MODULE) module_name = 'MicrosoftApiModule' module_code = f'\n### GENERATED CODE ###' \ f': {import_name}\n' \ f'# This code was inserted in place of an API module.\n' \ f"register_module_line('MicrosoftApiModule', 'start', __line__(), wrapper=-3)\n" \ f'{DUMMY_MODULE}\n' \ f"register_module_line('MicrosoftApiModule', 'end', __line__(), wrapper=1)\n" \ f'### END GENERATED CODE ###' new_code = DUMMY_SCRIPT.replace(import_name, module_code) code = IntegrationScriptUnifier.insert_module_code(DUMMY_SCRIPT, import_name, module_name) assert code == new_code
def test_insert_script_to_yml_exceptions(package_path, dir_name, file_path): with patch.object(IntegrationScriptUnifier, "__init__", lambda a, b, c, d, e: None): unifier = IntegrationScriptUnifier("", None, None, None) unifier.package_path = package_path unifier.dir_name = dir_name unifier.is_script_package = dir_name == 'Scripts' with open(file_path + ".yml", "r") as yml: test_yml_data = yaml.load(yml) if dir_name == "Scripts": test_yml_data['script'] = 'blah' else: test_yml_data['script']['script'] = 'blah' unifier.insert_script_to_yml(".py", {'script': {}}, test_yml_data)
def test_get_data(): with patch.object(IntegrationScriptUnifier, "__init__", lambda a, b, c, d, e: None): unifier = IntegrationScriptUnifier('', None, None, None) unifier.package_path = f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/" unifier.is_script_package = False with open(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_image.png", "rb") as image_file: image = image_file.read() data, found_data_path = unifier.get_data(unifier.package_path, "*png") assert data == image assert found_data_path == f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_image.png" unifier.is_script_package = True data, found_data_path = unifier.get_data(unifier.package_path, "*png") assert data is None assert found_data_path is None
def test_insert_image_to_yml(): with patch.object(IntegrationScriptUnifier, "__init__", lambda a, b, c, d, e: None): unifier = IntegrationScriptUnifier('', None, None, None) unifier.package_path = f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/" unifier.dir_name = "Integrations" unifier.is_script_package = False unifier.image_prefix = "data:image/png;base64," with open(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_image.png", "rb") as image_file: image_data = image_file.read() image_data = unifier.image_prefix + base64.b64encode(image_data).decode('utf-8') with open(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB.yml", mode="r", encoding="utf-8") \ as yml_file: yml_unified_test = yaml.load(yml_file) with open(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB.yml", "r") as yml: yml_data = yaml.load(yml) yml_unified, found_img_path = unifier.insert_image_to_yml(yml_data, yml_unified_test) yml_unified_test['image'] = image_data assert found_img_path == f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_image.png" assert yml_unified == yml_unified_test
def test_insert_description_to_yml(): with patch.object(IntegrationScriptUnifier, "__init__", lambda a, b, c, d, e: None): unifier = IntegrationScriptUnifier('', None, None, None) unifier.package_path = f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/" unifier.dir_name = "Integrations" unifier.is_script_package = False with open(f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_description.md", "rb") as desc_file: desc_data = desc_file.read().decode("utf-8") integration_doc_link = '\n\n---\n[View Integration Documentation]' \ '(https://xsoar.pan.dev/docs/reference/integrations/vuln-db)' yml_unified, found_data_path = unifier.insert_description_to_yml( {'commonfields': {'id': 'VulnDB'}}, {} ) assert found_data_path == f"{git_path()}/demisto_sdk/tests/test_files/VulnDB/VulnDB_description.md" assert (desc_data + integration_doc_link) == yml_unified['detaileddescription']