def is_file_has_used_id(self, file_path):
"""Check if the ID of the given file already exist in the system.
Args:
file_path (string): Path to the file.
Returns:
bool. Whether the ID of the given file already exist in the system or not.
"""
is_used = False
is_json_file = False
if self.is_circle:
if re.match(constants.TEST_PLAYBOOK_REGEX, file_path,
re.IGNORECASE):
obj_type = self.TEST_PLAYBOOK_SECTION
obj_id = collect_ids(file_path)
obj_data = get_playbook_data(file_path)
elif re.match(constants.PACKS_SCRIPT_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.TEST_SCRIPT_REGEX, file_path, re.IGNORECASE):
obj_type = self.SCRIPTS_SECTION
obj_id = get_script_or_integration_id(file_path)
obj_data = get_script_data(file_path)
elif re.match(constants.PACKS_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.PACKS_INTEGRATION_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE):
obj_type = self.INTEGRATION_SECTION
obj_id = get_script_or_integration_id(file_path)
obj_data = get_integration_data(file_path)
elif re.match(constants.PLAYBOOK_REGEX, file_path, re.IGNORECASE):
obj_type = self.PLAYBOOK_SECTION
obj_id = collect_ids(file_path)
obj_data = get_playbook_data(file_path)
elif re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.PACKS_SCRIPT_PY_REGEX, file_path, re.IGNORECASE):
unifier = Unifier(os.path.dirname(os.path.dirname(file_path)))
yml_path, code = unifier.get_script_or_integration_package_data(
)
obj_data = get_script_data(yml_path, script_code=code)
obj_type = self.SCRIPTS_SECTION
obj_id = get_script_or_integration_id(yml_path)
else: # In case of a json file
is_json_file = True
if not is_json_file:
is_used = self.is_id_duplicated(obj_id, obj_data, obj_type)
return is_used
def is_file_valid_in_set(self, file_path):
"""Check if the file is represented correctly in the id_set
Args:
file_path (string): Path to the file.
Returns:
bool. Whether the file is represented correctly in the id_set or not.
"""
is_valid = True
if self.is_circle: # No need to check on local env because the id_set will contain this info after the commit
if re.match(constants.PLAYBOOK_REGEX, file_path, re.IGNORECASE):
playbook_data = get_playbook_data(file_path)
is_valid = self.is_valid_in_id_set(file_path, playbook_data,
self.playbook_set)
elif re.match(constants.TEST_PLAYBOOK_REGEX, file_path,
re.IGNORECASE):
playbook_data = get_playbook_data(file_path)
is_valid = self.is_valid_in_id_set(file_path, playbook_data,
self.test_playbook_set)
elif re.match(constants.TEST_SCRIPT_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.PACKS_SCRIPT_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE):
script_data = get_script_data(file_path)
is_valid = self.is_valid_in_id_set(file_path, script_data,
self.script_set)
elif re.match(constants.PACKS_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.PACKS_INTEGRATION_NON_SPLIT_YML_REGEX, file_path, re.IGNORECASE):
integration_data = get_integration_data(file_path)
is_valid = self.is_valid_in_id_set(file_path, integration_data,
self.integration_set)
elif re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(constants.PACKS_SCRIPT_PY_REGEX, file_path, re.IGNORECASE):
unifier = Unifier(os.path.dirname(file_path))
yml_path, code = unifier.get_script_or_integration_package_data(
)
script_data = get_script_data(yml_path, script_code=code)
is_valid = self.is_valid_in_id_set(yml_path, script_data,
self.script_set)
return is_valid
def test_get_playbook_data():
"""
Test for getting the playbook data
"""
file_path = TESTS_DIR + '/test_files/DummyPack/Playbooks/DummyPlaybook.yml'
data = get_playbook_data(file_path)['Dummy Playbook']
assert IsEqualFunctions.is_dicts_equal(data, TestPlaybooks.PLAYBOOK_DATA)
def test_get_playbook_data(self):
"""
Test for getting the playbook data
"""
file_path = TESTS_DIR + '/test_files/DummyPack/Playbooks/DummyPlaybook.yml'
data = get_playbook_data(file_path)['Dummy Playbook']
self.assertEqual(data['name'], PLAYBOOK_DATA['name'])
self.assertEqual(data['file_path'], PLAYBOOK_DATA['file_path'])
self.assertEqual(data['fromversion'], PLAYBOOK_DATA['fromversion'])
self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests'])
self.assertSetEqual(set(data['implementing_playbooks']), set(PLAYBOOK_DATA['implementing_playbooks']))
self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests'])
self.assertDictEqual(data['command_to_integration'], PLAYBOOK_DATA['command_to_integration'])
def test_get_playbook_data(self):
"""
Test for getting the playbook data
"""
file_path = f'{git_path()}/demisto_sdk/tests/test_files/CortexXDR/Playbooks/Cortex_XDR_Incident_Handling.yml'
file_path = os.path.normpath(file_path)
data = get_playbook_data(file_path)['Cortex XDR Incident Handling']
PLAYBOOK_DATA["file_path"] = file_path
self.assertEqual(data['name'], PLAYBOOK_DATA['name'])
self.assertEqual(data['file_path'], PLAYBOOK_DATA['file_path'])
self.assertEqual(data['fromversion'], PLAYBOOK_DATA['fromversion'])
self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests'])
self.assertSetEqual(set(data['implementing_playbooks']),
set(PLAYBOOK_DATA['implementing_playbooks']))
self.assertListEqual(data['tests'], PLAYBOOK_DATA['tests'])
self.assertDictEqual(data['command_to_integration'],
PLAYBOOK_DATA['command_to_integration'])
def is_file_valid_in_set(self, file_path, file_type, ignored_errors=None):
"""Check if the file is valid in the id_set
Args:
file_path (string): Path to the file.
file_type (string): The file type.
ignored_errors (list): a list of ignored errors for the specific file
Returns:
bool. Whether the file is valid in the id_set or not.
"""
self.ignored_errors = ignored_errors
is_valid = True
if self.is_circle: # No need to check on local env because the id_set will contain this info after the commit
click.echo(f"id set validations for: {file_path}")
if re.match(constants.PACKS_SCRIPT_YML_REGEX, file_path,
re.IGNORECASE):
unifier = Unifier(os.path.dirname(file_path))
yml_path, code = unifier.get_script_or_integration_package_data(
)
script_data = get_script_data(yml_path, script_code=code)
is_valid = self._is_non_real_command_found(script_data)
elif file_type == constants.FileType.INCIDENT_TYPE:
incident_type_data = OrderedDict(
get_incident_type_data(file_path))
is_valid = self._is_incident_type_default_playbook_found(
incident_type_data)
elif file_type == constants.FileType.INTEGRATION:
integration_data = get_integration_data(file_path)
is_valid = self._is_integration_classifier_and_mapper_found(
integration_data)
elif file_type == constants.FileType.CLASSIFIER:
classifier_data = get_classifier_data(file_path)
is_valid = self._is_classifier_incident_types_found(
classifier_data)
elif file_type == constants.FileType.MAPPER:
mapper_data = get_mapper_data(file_path)
is_valid = self._is_mapper_incident_types_found(mapper_data)
elif file_type == constants.FileType.PLAYBOOK:
playbook_data = get_playbook_data(file_path)
is_valid = self._are_playbook_entities_versions_valid(
playbook_data, file_path)
return is_valid
def test_get_playbook_data_bad_graph_2():
"""
Given
- A playbook file called playbook-invalid-bad-graph_2.yml:
- starttaskid=5 but task 5 does not exist
When
- parsing playbook files
Then
- parsing flow graph from file successfully (no actual tasks will be in the graph)
"""
test_file_path = os.path.join(TESTS_DIR, 'test_files', 'playbook-invalid-bad-graph_2.yml')
result = get_playbook_data(test_file_path)
playbook_data = result.get('InvalidPlaybook-BadGraph', {})
assert playbook_data.get('name') == 'InvalidPlaybook-BadGraph'
assert playbook_data.get('command_to_integration', {}).get('ip') == ''
assert playbook_data.get('command_to_integration', {}).get('domain') == ''
# domain task is marked as skippable so it will be included regardless to the graph.
assert 'domain' in playbook_data.get('skippable_tasks', [])
assert len(playbook_data.get('skippable_tasks', [])) == 1
def test_get_playbook_data_bad_graph():
"""
Given
- A playbook file called playbook-invalid-bad-graph.yml:
- task 1 point to non-existing task
- task 2 is not connected
When
- parsing playbook files
Then
- parsing flow graph from file successfully (only tasks 0 and 1 will be in the graph)
"""
test_file_path = os.path.join(TESTS_DIR, 'test_files', 'playbook-invalid-bad-graph.yml')
result = get_playbook_data(test_file_path)
playbook_data = result.get('InvalidPlaybook-BadGraph', {})
assert playbook_data.get('name') == 'InvalidPlaybook-BadGraph'
assert playbook_data.get('command_to_integration', {}).get('ip') == ''
assert playbook_data.get('command_to_integration', {}).get('domain') == ''
assert 'domain' in playbook_data.get('skippable_tasks', [])
assert 'ip' not in playbook_data.get('skippable_tasks', [])
def test_get_playbook_data_2():
"""
Given
- A playbook file called playbook-with-incident-fields.yml
When
- parsing playbook files
Then
- parsing all the data from file successfully
"""
test_dir = f'{git_path()}/demisto_sdk/commands/create_id_set/tests/test_data/playbook-with-incident-fields.yml'
result = get_playbook_data(test_dir)
result = result.get('Arcsight - Get events related to the Case')
assert 'name' in result.keys()
assert 'file_path' in result.keys()
assert 'implementing_scripts' in result.keys()
assert 'command_to_integration' in result.keys()
assert 'tests' in result.keys()
assert 'incident_fields' in result.keys()
assert 'indicator_fields' in result.keys()
