def setup_id_set():
integration1 = {
'Integration1':
OrderedDict([('name', 'Integration1'),
('commands', ['test-command_1', 'test-command'])])
}
integration2 = {
'Integration2':
OrderedDict([('name', 'Integration2'),
('commands', ['test-command', 'test-command_2'])])
}
playbook1 = {
'Playbook1':
OrderedDict([('name', 'Playbook1'),
('command_to_integration', {
'test-command': "",
'test-command_1': ""
})])
}
playbook2 = {
'Playbook2':
OrderedDict([('name', 'Playbook2'),
('command_to_integration', {
'test-command': "",
'test-command_2': ""
})])
}
id_set_creator = IDSetCreator(print_logs=False)
id_set_creator.id_set["integrations"] = [integration1, integration2]
id_set_creator.id_set["playbooks"] = [playbook1, playbook2]
return id_set_creator
def test_create_id_set_on_specific_empty_pack(self, repo, mocker):
"""
Given
- an empty pack to create from it ID set
When
- create ID set on this pack
Then
- ensure that an ID set is created and no error is returned
- ensure output id_set is empty
"""
pack = repo.create_pack()
repo.add_pack_metadata_file(pack.path, json.dumps(METADATA))
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'should_skip_item_by_mp', return_value=False)
id_set_creator = IDSetCreator(self.file_path, pack.path)
id_set_creator.create_id_set()
with open(self.file_path, 'r') as id_set_file:
private_id_set = json.load(id_set_file)
for content_entity, content_entity_value_list in private_id_set.items(
):
if content_entity != 'Packs':
assert len(content_entity_value_list) == 0
else:
assert len(content_entity_value_list) == 1
def test_create_id_set_on_specific_pack(self, repo):
"""
Given
- two packs with integrations to create an ID set from
When
- create ID set on one of the packs
Then
- ensure there is only one integration in the ID set integrations list
- ensure output id_set contains only the pack on which created the ID set on
- ensure output id_set does not contain the second pack
"""
packs = repo.packs
pack_to_create_id_set_on = repo.create_pack('pack_to_create_id_set_on')
pack_to_create_id_set_on.create_integration(yml={
'commonfields': {
'id': 'id1'
},
'category': '',
'name': 'integration to create id set',
'script': {
'type': 'python'
}
},
name='integration1')
packs.append(pack_to_create_id_set_on)
pack_to_not_create_id_set_on = repo.create_pack(
'pack_to_not_create_id_set_on')
pack_to_not_create_id_set_on.create_integration(yml={
'commonfields': {
'id2': 'id'
},
'category':
'',
'name':
'integration to not create id set'
},
name='integration2')
packs.append(pack_to_not_create_id_set_on)
id_set_creator = IDSetCreator(self.file_path,
pack_to_create_id_set_on.path)
id_set_creator.create_id_set()
with open(self.file_path, 'r') as id_set_file:
private_id_set = json.load(id_set_file)
assert len(private_id_set['integrations']) == 1
assert private_id_set['integrations'][0].get('id1', {}).get(
'name', '') == 'integration to create id set'
assert private_id_set['integrations'][0].get('id2',
{}).get('name', '') == ''
assert private_id_set['Packs']['pack_to_create_id_set_on'][
'ContentItems']['integrations'] == ['id1']
def test_create_id_set_on_specific_pack_output(self):
"""
Given
- input - specific pack to create from it ID set
- output - path to return the created ID set
When
- create ID set on this pack
Then
- ensure that the created ID set is in the path of the output
"""
id_set_creator = IDSetCreator(self.file_path, input='Packs/AMP')
id_set_creator.create_id_set()
assert os.path.exists(self.file_path)
def test_create_id_set_no_output(self):
id_set_creator = IDSetCreator()
id_set = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
assert 'scripts' in id_set.keys()
assert 'integrations' in id_set.keys()
assert 'playbooks' in id_set.keys()
assert 'TestPlaybooks' in id_set.keys()
assert 'Classifiers' in id_set.keys()
assert 'Dashboards' in id_set.keys()
assert 'IncidentFields' in id_set.keys()
assert 'IncidentTypes' in id_set.keys()
assert 'IndicatorFields' in id_set.keys()
assert 'Layouts' in id_set.keys()
assert 'Reports' in id_set.keys()
assert 'Widgets' in id_set.keys()
def test_create_id_set_flow(repo, mocker):
# Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test
mocker.patch.dict(os.environ,
{'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'})
number_of_packs_to_create = 10
repo.setup_content_repo(number_of_packs_to_create)
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
id_set_content = repo.id_set.read_json_as_dict()
assert not IsEqualFunctions.is_dicts_equal(id_set_content, {})
assert set(id_set_content.keys()) == set(ID_SET_ENTITIES + ['Packs'])
for id_set_entity in ID_SET_ENTITIES:
if id_set_entity in [
'ParsingRules', 'ModelingRules', 'CorrelationRules',
'XSIAMDashboards', 'XSIAMReports', 'Triggers'
]:
continue
entity_content_in_id_set = id_set_content.get(id_set_entity)
assert entity_content_in_id_set, f'ID set for {id_set_entity} is empty'
factor = 1
if id_set_entity in {'Layouts', 'TestPlaybooks', 'Jobs'}:
'''
Layouts: The folder contains both layouts and layoutcontainers
TestPlaybooks: each integration and script has a test playbook
Jobs: The default test suite pack has two jobs (is_feed=true, is_feed=false), and a playbook
'''
factor = 2
elif id_set_entity == 'playbooks':
'''
One playbook is generated for every pack,
And one more is created for each of the 2 Job objects that are automatically created in every pack.
'''
factor = 3
assert len(
entity_content_in_id_set) == factor * number_of_packs_to_create
def test_create_id_set_no_output(self, mocker):
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'cpu_count', return_value=1)
id_set_creator = IDSetCreator(output=None)
id_set = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
assert 'scripts' in id_set.keys()
assert 'integrations' in id_set.keys()
assert 'playbooks' in id_set.keys()
assert 'TestPlaybooks' in id_set.keys()
assert 'Classifiers' in id_set.keys()
assert 'Dashboards' in id_set.keys()
assert 'IncidentFields' in id_set.keys()
assert 'IncidentTypes' in id_set.keys()
assert 'IndicatorFields' in id_set.keys()
assert 'IndicatorTypes' in id_set.keys()
assert 'Layouts' in id_set.keys()
assert 'Reports' in id_set.keys()
assert 'Widgets' in id_set.keys()
assert 'Mappers' in id_set.keys()
def test_create_id_set_no_output(self, mocker):
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'cpu_count', return_value=1)
id_set_creator = IDSetCreator(output=None)
id_set, _, _ = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
keys = set(id_set.keys())
expected_keys = {
'scripts', 'playbooks', 'integrations', 'TestPlaybooks',
'Classifiers', 'Dashboards', 'IncidentFields', 'IncidentTypes',
'IndicatorFields', 'IndicatorTypes', 'Layouts', 'Reports',
'Widgets', 'Mappers', 'Packs', 'GenericTypes', 'GenericFields',
'GenericModules', 'GenericDefinitions', 'Lists', 'Jobs',
'ParsingRules', 'ModelingRules', 'CorrelationRules',
'XSIAMDashboards', 'XSIAMReports', 'Triggers', 'Wizards'
}
assert keys == expected_keys, f'missing keys: {expected_keys.difference(keys)}\n' \
f' unexpected keys: {keys.difference(expected_keys)}'
def test_create_id_set_flow(repo):
number_of_packs_to_create = 10
repo.setup_content_repo(number_of_packs_to_create)
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
id_set_content = repo.id_set.read_json_as_dict()
assert not IsEqualFunctions.is_dicts_equal(id_set_content, {})
assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()),
ID_SET_ENTITIES)
for id_set_entity in ID_SET_ENTITIES:
entity_content_in_id_set = id_set_content.get(id_set_entity)
assert entity_content_in_id_set
# Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects
if id_set_entity != 'Layouts':
assert len(entity_content_in_id_set) == number_of_packs_to_create
else:
assert len(
entity_content_in_id_set) == number_of_packs_to_create * 2
def find_dependencies(pack_name,
id_set_path='',
exclude_ignored_dependencies=True,
update_pack_metadata=True,
silent_mode=False,
debug_file_path=''):
"""
Main function for dependencies search and pack metadata update.
Args:
pack_name (str): pack id, currently pack folder name is in use.
id_set_path (str): id set json.
debug_file_path (str): path to dependency explanations file.
silent_mode (bool): Determines whether to echo the dependencies or not.
update_pack_metadata (bool): Determines whether to update to pack metadata or not.
exclude_ignored_dependencies (bool): Determines whether to include unsupported dependencies or not.
Returns:
Dict: first level dependencies of a given pack.
"""
if not id_set_path or not os.path.isfile(id_set_path):
id_set = IDSetCreator(print_logs=False).create_id_set()
else:
with open(id_set_path, 'r') as id_set_file:
id_set = json.load(id_set_file)
with VerboseFile(debug_file_path) as verbose_file:
dependency_graph = PackDependencies.build_dependency_graph(
pack_id=pack_name,
id_set=id_set,
verbose_file=verbose_file,
exclude_ignored_dependencies=exclude_ignored_dependencies)
first_level_dependencies, _ = parse_for_pack_metadata(
dependency_graph, pack_name)
if update_pack_metadata:
update_pack_metadata_with_dependencies(pack_name,
first_level_dependencies)
if not silent_mode:
# print the found pack dependency results
click.echo(
click.style(f"Found dependencies result for {pack_name} pack:",
bold=True))
dependency_result = json.dumps(first_level_dependencies, indent=4)
click.echo(click.style(dependency_result, bold=True))
return first_level_dependencies
def test_generic_command_that_uses_a_specific_brand(repo):
"""
Given
- A playbook with a generic command (send-notification, etc.) that uses a specific brand
When
- Updating the commands_to_integrations fields in playbooks
Then
- Do not update 'command_to_integration' fields with any other brands
"""
integrations = [
{
'Slack':
OrderedDict([
('name', 'Slack'),
('commands', ['send-notification']),
])
},
{
'Syslog':
OrderedDict([
('name', 'Syslog'),
('commands', ['send-notification']),
])
},
]
playbooks = [{
'Playbook':
OrderedDict([
('name', 'Playbook'),
('command_to_integration', {
'send-notification': 'Slack',
}),
]),
}]
id_set_creator = IDSetCreator(print_logs=False)
id_set_creator.id_set["integrations"] = integrations
id_set_creator.id_set["playbooks"] = playbooks
id_set_creator.add_command_to_implementing_integrations_mapping()
playbook = id_set_creator.id_set['playbooks'][0]['Playbook']
assert playbook['command_to_integration'][
'send-notification'] == 'Slack'
def generate_script_doc(input,
examples,
output: str = None,
permissions: str = None,
limitations: str = None,
insecure: bool = False,
verbose: bool = False):
try:
doc = []
errors = []
used_in = []
example_section = []
if not output: # default output dir will be the dir of the input file
output = os.path.dirname(os.path.realpath(input))
if examples:
if not examples.startswith('!'):
examples = f'!{examples}'
example_dict, build_errors = build_example_dict([examples],
insecure)
script_name = examples.split(' ')[0][1:]
example_section, example_errors = generate_script_example(
script_name, example_dict)
errors.extend(build_errors)
errors.extend(example_errors)
else:
errors.append(
f'Note: Script example was not provided. For a more complete documentation,run with the -e '
f'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".'
)
script = get_yaml(input)
# get script data
secript_info = get_script_info(input)
script_id = script.get('commonfields')['id']
# get script dependencies
dependencies, _ = get_depends_on(script)
# get the script usages by the id set
id_set_creator = IDSetCreator()
id_set = id_set_creator.create_id_set()
used_in = get_used_in(id_set, script_id)
description = script.get('comment', '')
deprecated = script.get('deprecated', False)
# get inputs/outputs
inputs, inputs_errors = get_inputs(script)
outputs, outputs_errors = get_outputs(script)
errors.extend(inputs_errors)
errors.extend(outputs_errors)
if not description:
errors.append(
'Error! You are missing description for the playbook')
if deprecated:
doc.append('`Deprecated`')
doc.append(description)
doc.extend(generate_table_section(secript_info, 'Script Data'))
if dependencies:
doc.extend(
generate_list_section(
'Dependencies',
dependencies,
True,
text='This script uses the following commands and scripts.'
))
# Script global permissions
if permissions == 'general':
doc.extend(generate_section('Permissions', ''))
if used_in:
doc.extend(
generate_list_section(
'Used In',
used_in,
True,
text=
'This script is used in the following playbooks and scripts.'
))
doc.extend(
generate_table_section(inputs, 'Inputs',
'There are no inputs for this script.'))
doc.extend(
generate_table_section(outputs, 'Outputs',
'There are no outputs for this script.'))
if example_section:
doc.extend(example_section)
# Known limitations
if limitations:
doc.extend(
generate_numbered_section('Known Limitations', limitations))
doc_text = '\n'.join(doc)
save_output(output, 'README.md', doc_text)
if errors:
print_warning('Possible Errors:')
for error in errors:
print_warning(error)
except Exception as ex:
if verbose:
raise
else:
print_error(f'Error: {str(ex)}')
return
def id_set_command(**kwargs):
id_set_creator = IDSetCreator(**kwargs)
id_set_creator.create_id_set()
def test_do_not_modify_specific_brand(repo):
"""
Given
- playbook with a command using a specific brand
- playbook with a command using a specific brand
When
- updating the commands_to_integrations fields in playbooks
Then
- only update commands that don't have a specific brand
"""
integrations = [
{
'MainInteg':
OrderedDict([
('name', 'MainInteg'),
('commands', ['generic-command']),
])
},
{
'SecondaryInteg':
OrderedDict([
('name', 'SecondaryInteg'),
('commands', ['generic-command', 'specific-command']),
])
},
]
playbooks = [
{
'Playbook1':
OrderedDict([
('name', 'Playbook1'),
('command_to_integration', {
'specific-command': "",
'generic-command': "",
}),
]),
},
{
'Playbook2':
OrderedDict([
('name', 'Playbook2'),
('command_to_integration', {
'generic-command': 'MainInteg',
'no-integration': '',
}),
]),
},
]
id_set_creator = IDSetCreator(print_logs=False)
id_set_creator.id_set["integrations"] = integrations
id_set_creator.id_set["playbooks"] = playbooks
id_set_creator.add_command_to_implementing_integrations_mapping()
playbook1 = id_set_creator.id_set['playbooks'][0]['Playbook1']
playbook2 = id_set_creator.id_set['playbooks'][1]['Playbook2']
assert playbook1['command_to_integration']['specific-command'] == [
'SecondaryInteg'
]
assert playbook1['command_to_integration']['generic-command'] == [
'MainInteg', 'SecondaryInteg'
]
assert playbook2['command_to_integration'][
'generic-command'] == 'MainInteg'
assert playbook2['command_to_integration']['no-integration'] == ''
