def is_file_has_used_id(self, file_path): """Check if the ID of the given file already exist in the system. Args: file_path (string): Path to the file. Returns: bool. Whether the ID of the given file already exist in the system or not. """ is_used = False is_json_file = False if self.is_circle: if re.match(constants.TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): obj_type = self.TEST_PLAYBOOK_SECTION obj_id = collect_ids(file_path) obj_data = get_playbook_data(file_path) elif re.match(constants.PACKS_SCRIPT_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.TEST_SCRIPT_REGEX, file_path, re.IGNORECASE): obj_type = self.SCRIPTS_SECTION obj_id = get_script_or_integration_id(file_path) obj_data = get_script_data(file_path) elif re.match(constants.PACKS_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.PACKS_INTEGRATION_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE): obj_type = self.INTEGRATION_SECTION obj_id = get_script_or_integration_id(file_path) obj_data = get_integration_data(file_path) elif re.match(constants.PLAYBOOK_REGEX, file_path, re.IGNORECASE): obj_type = self.PLAYBOOK_SECTION obj_id = collect_ids(file_path) obj_data = get_playbook_data(file_path) elif re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.PACKS_SCRIPT_PY_REGEX, file_path, re.IGNORECASE): unifier = Unifier(os.path.dirname(os.path.dirname(file_path))) yml_path, code = unifier.get_script_or_integration_package_data( ) obj_data = get_script_data(yml_path, script_code=code) obj_type = self.SCRIPTS_SECTION obj_id = get_script_or_integration_id(yml_path) else: # In case of a json file is_json_file = True if not is_json_file: is_used = self.is_id_duplicated(obj_id, obj_data, obj_type) return is_used
def is_file_valid_in_set(self, file_path): """Check if the file is represented correctly in the id_set Args: file_path (string): Path to the file. Returns: bool. Whether the file is represented correctly in the id_set or not. """ is_valid = True if self.is_circle: # No need to check on local env because the id_set will contain this info after the commit if re.match(constants.PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_data = get_playbook_data(file_path) is_valid = self.is_valid_in_id_set(file_path, playbook_data, self.playbook_set) elif re.match(constants.TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_data = get_playbook_data(file_path) is_valid = self.is_valid_in_id_set(file_path, playbook_data, self.test_playbook_set) elif re.match(constants.TEST_SCRIPT_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.PACKS_SCRIPT_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE): script_data = get_script_data(file_path) is_valid = self.is_valid_in_id_set(file_path, script_data, self.script_set) elif re.match(constants.PACKS_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.PACKS_INTEGRATION_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE): integration_data = get_integration_data(file_path) is_valid = self.is_valid_in_id_set(file_path, integration_data, self.integration_set) elif re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \ re.match(constants.PACKS_SCRIPT_PY_REGEX, file_path, re.IGNORECASE): unifier = Unifier(os.path.dirname(file_path)) yml_path, code = unifier.get_script_or_integration_package_data( ) script_data = get_script_data(yml_path, script_code=code) is_valid = self.is_valid_in_id_set(yml_path, script_data, self.script_set) return is_valid
def test_get_playbook_data(): """ Test for getting the playbook data """ file_path = TESTS_DIR + '/test_files/DummyPack/Playbooks/DummyPlaybook.yml' data = get_playbook_data(file_path)['Dummy Playbook'] assert IsEqualFunctions.is_dicts_equal(data, TestPlaybooks.PLAYBOOK_DATA)
def test_get_playbook_data(self): """ Test for getting the playbook data """ file_path = TESTS_DIR + '/test_files/DummyPack/Playbooks/DummyPlaybook.yml' data = get_playbook_data(file_path)['Dummy Playbook'] self.assertEqual(data['name'], PLAYBOOK_DATA['name']) self.assertEqual(data['file_path'], PLAYBOOK_DATA['file_path']) self.assertEqual(data['fromversion'], PLAYBOOK_DATA['fromversion']) self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests']) self.assertSetEqual(set(data['implementing_playbooks']), set(PLAYBOOK_DATA['implementing_playbooks'])) self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests']) self.assertDictEqual(data['command_to_integration'], PLAYBOOK_DATA['command_to_integration'])
def test_get_playbook_data(self): """ Test for getting the playbook data """ file_path = f'{git_path()}/demisto_sdk/tests/test_files/CortexXDR/Playbooks/Cortex_XDR_Incident_Handling.yml' file_path = os.path.normpath(file_path) data = get_playbook_data(file_path)['Cortex XDR Incident Handling'] PLAYBOOK_DATA["file_path"] = file_path self.assertEqual(data['name'], PLAYBOOK_DATA['name']) self.assertEqual(data['file_path'], PLAYBOOK_DATA['file_path']) self.assertEqual(data['fromversion'], PLAYBOOK_DATA['fromversion']) self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests']) self.assertSetEqual(set(data['implementing_playbooks']), set(PLAYBOOK_DATA['implementing_playbooks'])) self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests']) self.assertDictEqual(data['command_to_integration'], PLAYBOOK_DATA['command_to_integration'])
def is_file_valid_in_set(self, file_path, file_type, ignored_errors=None): """Check if the file is valid in the id_set Args: file_path (string): Path to the file. file_type (string): The file type. ignored_errors (list): a list of ignored errors for the specific file Returns: bool. Whether the file is valid in the id_set or not. """ self.ignored_errors = ignored_errors is_valid = True if self.is_circle: # No need to check on local env because the id_set will contain this info after the commit click.echo(f"id set validations for: {file_path}") if re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path, re.IGNORECASE): unifier = Unifier(os.path.dirname(file_path)) yml_path, code = unifier.get_script_or_integration_package_data( ) script_data = get_script_data(yml_path, script_code=code) is_valid = self._is_non_real_command_found(script_data) elif file_type == constants.FileType.INCIDENT_TYPE: incident_type_data = OrderedDict( get_incident_type_data(file_path)) is_valid = self._is_incident_type_default_playbook_found( incident_type_data) elif file_type == constants.FileType.INTEGRATION: integration_data = get_integration_data(file_path) is_valid = self._is_integration_classifier_and_mapper_found( integration_data) elif file_type == constants.FileType.CLASSIFIER: classifier_data = get_classifier_data(file_path) is_valid = self._is_classifier_incident_types_found( classifier_data) elif file_type == constants.FileType.MAPPER: mapper_data = get_mapper_data(file_path) is_valid = self._is_mapper_incident_types_found(mapper_data) elif file_type == constants.FileType.PLAYBOOK: playbook_data = get_playbook_data(file_path) is_valid = self._are_playbook_entities_versions_valid( playbook_data, file_path) return is_valid
def test_get_playbook_data_bad_graph_2(): """ Given - A playbook file called playbook-invalid-bad-graph_2.yml: - starttaskid=5 but task 5 does not exist When - parsing playbook files Then - parsing flow graph from file successfully (no actual tasks will be in the graph) """ test_file_path = os.path.join(TESTS_DIR, 'test_files', 'playbook-invalid-bad-graph_2.yml') result = get_playbook_data(test_file_path) playbook_data = result.get('InvalidPlaybook-BadGraph', {}) assert playbook_data.get('name') == 'InvalidPlaybook-BadGraph' assert playbook_data.get('command_to_integration', {}).get('ip') == '' assert playbook_data.get('command_to_integration', {}).get('domain') == '' # domain task is marked as skippable so it will be included regardless to the graph. assert 'domain' in playbook_data.get('skippable_tasks', []) assert len(playbook_data.get('skippable_tasks', [])) == 1
def test_get_playbook_data_bad_graph(): """ Given - A playbook file called playbook-invalid-bad-graph.yml: - task 1 point to non-existing task - task 2 is not connected When - parsing playbook files Then - parsing flow graph from file successfully (only tasks 0 and 1 will be in the graph) """ test_file_path = os.path.join(TESTS_DIR, 'test_files', 'playbook-invalid-bad-graph.yml') result = get_playbook_data(test_file_path) playbook_data = result.get('InvalidPlaybook-BadGraph', {}) assert playbook_data.get('name') == 'InvalidPlaybook-BadGraph' assert playbook_data.get('command_to_integration', {}).get('ip') == '' assert playbook_data.get('command_to_integration', {}).get('domain') == '' assert 'domain' in playbook_data.get('skippable_tasks', []) assert 'ip' not in playbook_data.get('skippable_tasks', [])
def test_get_playbook_data_2(): """ Given - A playbook file called playbook-with-incident-fields.yml When - parsing playbook files Then - parsing all the data from file successfully """ test_dir = f'{git_path()}/demisto_sdk/commands/create_id_set/tests/test_data/playbook-with-incident-fields.yml' result = get_playbook_data(test_dir) result = result.get('Arcsight - Get events related to the Case') assert 'name' in result.keys() assert 'file_path' in result.keys() assert 'implementing_scripts' in result.keys() assert 'command_to_integration' in result.keys() assert 'tests' in result.keys() assert 'incident_fields' in result.keys() assert 'indicator_fields' in result.keys()