def test_create_id_set_flow(repo, mocker):
# Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test
mocker.patch.dict(os.environ,
{'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'})
number_of_packs_to_create = 10
repo.setup_content_repo(number_of_packs_to_create)
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
id_set_content = repo.id_set.read_json_as_dict()
assert not IsEqualFunctions.is_dicts_equal(id_set_content, {})
assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()),
ID_SET_ENTITIES)
for id_set_entity in ID_SET_ENTITIES:
entity_content_in_id_set = id_set_content.get(id_set_entity)
assert entity_content_in_id_set
# Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects
if id_set_entity != 'Layouts':
assert len(entity_content_in_id_set) == number_of_packs_to_create
else:
assert len(
entity_content_in_id_set) == number_of_packs_to_create * 2
def test_create_id_set_on_specific_empty_pack(self, repo, mocker):
"""
Given
- an empty pack to create from it ID set
When
- create ID set on this pack
Then
- ensure that an ID set is created and no error is returned
- ensure output id_set is empty
"""
pack = repo.create_pack()
repo.add_pack_metadata_file(pack.path, json.dumps(METADATA))
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'should_skip_item_by_mp', return_value=False)
id_set_creator = IDSetCreator(self.file_path, pack.path)
id_set_creator.create_id_set()
with open(self.file_path, 'r') as id_set_file:
private_id_set = json.load(id_set_file)
for content_entity, content_entity_value_list in private_id_set.items(
):
if content_entity != 'Packs':
assert len(content_entity_value_list) == 0
else:
assert len(content_entity_value_list) == 1
def test_create_id_set_on_specific_pack(self, repo):
"""
Given
- two packs with integrations to create an ID set from
When
- create ID set on one of the packs
Then
- ensure there is only one integration in the ID set integrations list
- ensure output id_set contains only the pack on which created the ID set on
- ensure output id_set does not contain the second pack
"""
packs = repo.packs
pack_to_create_id_set_on = repo.create_pack('pack_to_create_id_set_on')
pack_to_create_id_set_on.create_integration(yml={
'commonfields': {
'id': 'id1'
},
'category': '',
'name': 'integration to create id set',
'script': {
'type': 'python'
}
},
name='integration1')
packs.append(pack_to_create_id_set_on)
pack_to_not_create_id_set_on = repo.create_pack(
'pack_to_not_create_id_set_on')
pack_to_not_create_id_set_on.create_integration(yml={
'commonfields': {
'id2': 'id'
},
'category':
'',
'name':
'integration to not create id set'
},
name='integration2')
packs.append(pack_to_not_create_id_set_on)
id_set_creator = IDSetCreator(self.file_path,
pack_to_create_id_set_on.path)
id_set_creator.create_id_set()
with open(self.file_path, 'r') as id_set_file:
private_id_set = json.load(id_set_file)
assert len(private_id_set['integrations']) == 1
assert private_id_set['integrations'][0].get('id1', {}).get(
'name', '') == 'integration to create id set'
assert private_id_set['integrations'][0].get('id2',
{}).get('name', '') == ''
assert private_id_set['Packs']['pack_to_create_id_set_on'][
'ContentItems']['integrations'] == ['id1']
def test_create_id_set_on_specific_pack_output(self):
"""
Given
- input - specific pack to create from it ID set
- output - path to return the created ID set
When
- create ID set on this pack
Then
- ensure that the created ID set is in the path of the output
"""
id_set_creator = IDSetCreator(self.file_path, input='Packs/AMP')
id_set_creator.create_id_set()
assert os.path.exists(self.file_path)
def test_create_id_set_flow(repo, mocker):
# Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test
mocker.patch.dict(os.environ,
{'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'})
number_of_packs_to_create = 10
repo.setup_content_repo(number_of_packs_to_create)
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
id_set_content = repo.id_set.read_json_as_dict()
assert not IsEqualFunctions.is_dicts_equal(id_set_content, {})
assert set(id_set_content.keys()) == set(ID_SET_ENTITIES + ['Packs'])
for id_set_entity in ID_SET_ENTITIES:
if id_set_entity in [
'ParsingRules', 'ModelingRules', 'CorrelationRules',
'XSIAMDashboards', 'XSIAMReports', 'Triggers'
]:
continue
entity_content_in_id_set = id_set_content.get(id_set_entity)
assert entity_content_in_id_set, f'ID set for {id_set_entity} is empty'
factor = 1
if id_set_entity in {'Layouts', 'TestPlaybooks', 'Jobs'}:
'''
Layouts: The folder contains both layouts and layoutcontainers
TestPlaybooks: each integration and script has a test playbook
Jobs: The default test suite pack has two jobs (is_feed=true, is_feed=false), and a playbook
'''
factor = 2
elif id_set_entity == 'playbooks':
'''
One playbook is generated for every pack,
And one more is created for each of the 2 Job objects that are automatically created in every pack.
'''
factor = 3
assert len(
entity_content_in_id_set) == factor * number_of_packs_to_create
def test_create_id_set_flow(repo):
number_of_packs_to_create = 10
repo.setup_content_repo(number_of_packs_to_create)
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
id_set_content = repo.id_set.read_json_as_dict()
assert not IsEqualFunctions.is_dicts_equal(id_set_content, {})
assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()),
ID_SET_ENTITIES)
for id_set_entity in ID_SET_ENTITIES:
entity_content_in_id_set = id_set_content.get(id_set_entity)
assert entity_content_in_id_set
# Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects
if id_set_entity != 'Layouts':
assert len(entity_content_in_id_set) == number_of_packs_to_create
else:
assert len(
entity_content_in_id_set) == number_of_packs_to_create * 2
def test_create_id_set_no_output(self):
id_set_creator = IDSetCreator()
id_set = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
assert 'scripts' in id_set.keys()
assert 'integrations' in id_set.keys()
assert 'playbooks' in id_set.keys()
assert 'TestPlaybooks' in id_set.keys()
assert 'Classifiers' in id_set.keys()
assert 'Dashboards' in id_set.keys()
assert 'IncidentFields' in id_set.keys()
assert 'IncidentTypes' in id_set.keys()
assert 'IndicatorFields' in id_set.keys()
assert 'Layouts' in id_set.keys()
assert 'Reports' in id_set.keys()
assert 'Widgets' in id_set.keys()
def test_create_id_set_no_output(self, mocker):
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'cpu_count', return_value=1)
id_set_creator = IDSetCreator(output=None)
id_set = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
assert 'scripts' in id_set.keys()
assert 'integrations' in id_set.keys()
assert 'playbooks' in id_set.keys()
assert 'TestPlaybooks' in id_set.keys()
assert 'Classifiers' in id_set.keys()
assert 'Dashboards' in id_set.keys()
assert 'IncidentFields' in id_set.keys()
assert 'IncidentTypes' in id_set.keys()
assert 'IndicatorFields' in id_set.keys()
assert 'IndicatorTypes' in id_set.keys()
assert 'Layouts' in id_set.keys()
assert 'Reports' in id_set.keys()
assert 'Widgets' in id_set.keys()
assert 'Mappers' in id_set.keys()
def test_create_id_set_no_output(self, mocker):
import demisto_sdk.commands.common.update_id_set as uis
mocker.patch.object(uis, 'cpu_count', return_value=1)
id_set_creator = IDSetCreator(output=None)
id_set, _, _ = id_set_creator.create_id_set()
assert not os.path.exists(self.file_path)
assert id_set is not None
keys = set(id_set.keys())
expected_keys = {
'scripts', 'playbooks', 'integrations', 'TestPlaybooks',
'Classifiers', 'Dashboards', 'IncidentFields', 'IncidentTypes',
'IndicatorFields', 'IndicatorTypes', 'Layouts', 'Reports',
'Widgets', 'Mappers', 'Packs', 'GenericTypes', 'GenericFields',
'GenericModules', 'GenericDefinitions', 'Lists', 'Jobs',
'ParsingRules', 'ModelingRules', 'CorrelationRules',
'XSIAMDashboards', 'XSIAMReports', 'Triggers', 'Wizards'
}
assert keys == expected_keys, f'missing keys: {expected_keys.difference(keys)}\n' \
f' unexpected keys: {keys.difference(expected_keys)}'
def update_id_set(repo):
with ChangeCWD(repo.path):
id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False)
id_set_creator.create_id_set()
def generate_script_doc(input_path,
examples,
output: str = None,
permissions: str = None,
limitations: str = None,
insecure: bool = False,
verbose: bool = False):
try:
doc: list = []
errors: list = []
example_section: list = []
if not output: # default output dir will be the dir of the input file
output = os.path.dirname(os.path.realpath(input_path))
if examples:
if os.path.isfile(examples):
with open(examples, 'r') as examples_file:
examples = examples_file.read().splitlines()
else:
examples = examples.split(',')
for i, example in enumerate(examples):
if not example.startswith('!'):
examples[i] = f'!{examples}'
example_dict, build_errors = build_example_dict(examples, insecure)
script_name = list(
example_dict.keys())[0] if example_dict else None
example_section, example_errors = generate_script_example(
script_name, example_dict)
errors.extend(build_errors)
errors.extend(example_errors)
else:
errors.append(
'Note: Script example was not provided. For a more complete documentation,run with the -e '
'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".'
)
script = get_yaml(input_path)
# get script data
script_info = get_script_info(input_path)
script_id = script.get('commonfields')['id']
# get script dependencies
dependencies, _ = get_depends_on(script)
# get the script usages by the id set
if not os.path.isfile(DEFAULT_ID_SET_PATH):
id_set_creator = IDSetCreator(output='', print_logs=False)
id_set, _, _ = id_set_creator.create_id_set()
else:
id_set = open_id_set_file(DEFAULT_ID_SET_PATH)
used_in = get_used_in(id_set, script_id)
description = script.get('comment', '')
# get inputs/outputs
inputs, inputs_errors = get_inputs(script)
outputs, outputs_errors = get_outputs(script)
errors.extend(inputs_errors)
errors.extend(outputs_errors)
if not description:
errors.append(
'Error! You are missing a description for the Script')
doc.append(description + '\n')
doc.extend(generate_table_section(script_info, 'Script Data'))
if dependencies:
doc.extend(
generate_list_section(
'Dependencies',
dependencies,
True,
text='This script uses the following commands and scripts.'
))
# Script global permissions
if permissions == 'general':
doc.extend(generate_section('Permissions', ''))
if used_in:
if len(used_in) <= 10:
doc.extend(
generate_list_section(
'Used In',
used_in,
True,
text=
'This script is used in the following playbooks and scripts.'
))
else: # if we have more than 10 use a sample
print_warning(
f'"Used In" section found too many scripts/playbooks ({len(used_in)}). Will use a sample of 10.'
' Full list is available as a comment in the README file.')
sample_used_in = random.sample(used_in, 10)
doc.extend(
generate_list_section(
'Used In',
sorted(sample_used_in),
True,
text=
'Sample usage of this script can be found in the following playbooks and scripts.'
))
used_in_str = '\n'.join(used_in)
doc.append(
f"<!--\nUsed In: list was truncated. Full list commented out for reference:\n\n{used_in_str}\n -->\n"
)
doc.extend(
generate_table_section(inputs, 'Inputs',
'There are no inputs for this script.'))
doc.extend(
generate_table_section(outputs, 'Outputs',
'There are no outputs for this script.'))
if example_section:
doc.extend(example_section)
# Known limitations
if limitations:
doc.extend(
generate_numbered_section('Known Limitations', limitations))
doc_text = '\n'.join(doc)
save_output(output, 'README.md', doc_text)
if errors:
print_warning('Possible Errors:')
for error in errors:
print_warning(error)
except Exception as ex:
if verbose:
raise
else:
print_error(f'Error: {str(ex)}')
return
def test_create_id_set_output(self):
id_set_creator = IDSetCreator(self.file_path)
id_set_creator.create_id_set()
assert os.path.exists(self.file_path)
def generate_script_doc(input,
examples,
output: str = None,
permissions: str = None,
limitations: str = None,
insecure: bool = False,
verbose: bool = False):
try:
doc = []
errors = []
used_in = []
example_section = []
if not output: # default output dir will be the dir of the input file
output = os.path.dirname(os.path.realpath(input))
if examples:
if not examples.startswith('!'):
examples = f'!{examples}'
example_dict, build_errors = build_example_dict([examples],
insecure)
script_name = examples.split(' ')[0][1:]
example_section, example_errors = generate_script_example(
script_name, example_dict)
errors.extend(build_errors)
errors.extend(example_errors)
else:
errors.append(
f'Note: Script example was not provided. For a more complete documentation,run with the -e '
f'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".'
)
script = get_yaml(input)
# get script data
secript_info = get_script_info(input)
script_id = script.get('commonfields')['id']
# get script dependencies
dependencies, _ = get_depends_on(script)
# get the script usages by the id set
id_set_creator = IDSetCreator()
id_set = id_set_creator.create_id_set()
used_in = get_used_in(id_set, script_id)
description = script.get('comment', '')
deprecated = script.get('deprecated', False)
# get inputs/outputs
inputs, inputs_errors = get_inputs(script)
outputs, outputs_errors = get_outputs(script)
errors.extend(inputs_errors)
errors.extend(outputs_errors)
if not description:
errors.append(
'Error! You are missing description for the playbook')
if deprecated:
doc.append('`Deprecated`')
doc.append(description)
doc.extend(generate_table_section(secript_info, 'Script Data'))
if dependencies:
doc.extend(
generate_list_section(
'Dependencies',
dependencies,
True,
text='This script uses the following commands and scripts.'
))
# Script global permissions
if permissions == 'general':
doc.extend(generate_section('Permissions', ''))
if used_in:
doc.extend(
generate_list_section(
'Used In',
used_in,
True,
text=
'This script is used in the following playbooks and scripts.'
))
doc.extend(
generate_table_section(inputs, 'Inputs',
'There are no inputs for this script.'))
doc.extend(
generate_table_section(outputs, 'Outputs',
'There are no outputs for this script.'))
if example_section:
doc.extend(example_section)
# Known limitations
if limitations:
doc.extend(
generate_numbered_section('Known Limitations', limitations))
doc_text = '\n'.join(doc)
save_output(output, 'README.md', doc_text)
if errors:
print_warning('Possible Errors:')
for error in errors:
print_warning(error)
except Exception as ex:
if verbose:
raise
else:
print_error(f'Error: {str(ex)}')
return
def id_set_command(**kwargs):
id_set_creator = IDSetCreator(**kwargs)
id_set_creator.create_id_set()