def setup_id_set(): integration1 = { 'Integration1': OrderedDict([('name', 'Integration1'), ('commands', ['test-command_1', 'test-command'])]) } integration2 = { 'Integration2': OrderedDict([('name', 'Integration2'), ('commands', ['test-command', 'test-command_2'])]) } playbook1 = { 'Playbook1': OrderedDict([('name', 'Playbook1'), ('command_to_integration', { 'test-command': "", 'test-command_1': "" })]) } playbook2 = { 'Playbook2': OrderedDict([('name', 'Playbook2'), ('command_to_integration', { 'test-command': "", 'test-command_2': "" })]) } id_set_creator = IDSetCreator(print_logs=False) id_set_creator.id_set["integrations"] = [integration1, integration2] id_set_creator.id_set["playbooks"] = [playbook1, playbook2] return id_set_creator
def test_create_id_set_on_specific_empty_pack(self, repo, mocker): """ Given - an empty pack to create from it ID set When - create ID set on this pack Then - ensure that an ID set is created and no error is returned - ensure output id_set is empty """ pack = repo.create_pack() repo.add_pack_metadata_file(pack.path, json.dumps(METADATA)) import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'should_skip_item_by_mp', return_value=False) id_set_creator = IDSetCreator(self.file_path, pack.path) id_set_creator.create_id_set() with open(self.file_path, 'r') as id_set_file: private_id_set = json.load(id_set_file) for content_entity, content_entity_value_list in private_id_set.items( ): if content_entity != 'Packs': assert len(content_entity_value_list) == 0 else: assert len(content_entity_value_list) == 1
def test_create_id_set_on_specific_pack(self, repo): """ Given - two packs with integrations to create an ID set from When - create ID set on one of the packs Then - ensure there is only one integration in the ID set integrations list - ensure output id_set contains only the pack on which created the ID set on - ensure output id_set does not contain the second pack """ packs = repo.packs pack_to_create_id_set_on = repo.create_pack('pack_to_create_id_set_on') pack_to_create_id_set_on.create_integration(yml={ 'commonfields': { 'id': 'id1' }, 'category': '', 'name': 'integration to create id set', 'script': { 'type': 'python' } }, name='integration1') packs.append(pack_to_create_id_set_on) pack_to_not_create_id_set_on = repo.create_pack( 'pack_to_not_create_id_set_on') pack_to_not_create_id_set_on.create_integration(yml={ 'commonfields': { 'id2': 'id' }, 'category': '', 'name': 'integration to not create id set' }, name='integration2') packs.append(pack_to_not_create_id_set_on) id_set_creator = IDSetCreator(self.file_path, pack_to_create_id_set_on.path) id_set_creator.create_id_set() with open(self.file_path, 'r') as id_set_file: private_id_set = json.load(id_set_file) assert len(private_id_set['integrations']) == 1 assert private_id_set['integrations'][0].get('id1', {}).get( 'name', '') == 'integration to create id set' assert private_id_set['integrations'][0].get('id2', {}).get('name', '') == '' assert private_id_set['Packs']['pack_to_create_id_set_on'][ 'ContentItems']['integrations'] == ['id1']
def test_create_id_set_on_specific_pack_output(self): """ Given - input - specific pack to create from it ID set - output - path to return the created ID set When - create ID set on this pack Then - ensure that the created ID set is in the path of the output """ id_set_creator = IDSetCreator(self.file_path, input='Packs/AMP') id_set_creator.create_id_set() assert os.path.exists(self.file_path)
def test_create_id_set_no_output(self): id_set_creator = IDSetCreator() id_set = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None assert 'scripts' in id_set.keys() assert 'integrations' in id_set.keys() assert 'playbooks' in id_set.keys() assert 'TestPlaybooks' in id_set.keys() assert 'Classifiers' in id_set.keys() assert 'Dashboards' in id_set.keys() assert 'IncidentFields' in id_set.keys() assert 'IncidentTypes' in id_set.keys() assert 'IndicatorFields' in id_set.keys() assert 'Layouts' in id_set.keys() assert 'Reports' in id_set.keys() assert 'Widgets' in id_set.keys()
def test_create_id_set_flow(repo, mocker): # Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test mocker.patch.dict(os.environ, {'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'}) number_of_packs_to_create = 10 repo.setup_content_repo(number_of_packs_to_create) with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set() id_set_content = repo.id_set.read_json_as_dict() assert not IsEqualFunctions.is_dicts_equal(id_set_content, {}) assert set(id_set_content.keys()) == set(ID_SET_ENTITIES + ['Packs']) for id_set_entity in ID_SET_ENTITIES: if id_set_entity in [ 'ParsingRules', 'ModelingRules', 'CorrelationRules', 'XSIAMDashboards', 'XSIAMReports', 'Triggers' ]: continue entity_content_in_id_set = id_set_content.get(id_set_entity) assert entity_content_in_id_set, f'ID set for {id_set_entity} is empty' factor = 1 if id_set_entity in {'Layouts', 'TestPlaybooks', 'Jobs'}: ''' Layouts: The folder contains both layouts and layoutcontainers TestPlaybooks: each integration and script has a test playbook Jobs: The default test suite pack has two jobs (is_feed=true, is_feed=false), and a playbook ''' factor = 2 elif id_set_entity == 'playbooks': ''' One playbook is generated for every pack, And one more is created for each of the 2 Job objects that are automatically created in every pack. ''' factor = 3 assert len( entity_content_in_id_set) == factor * number_of_packs_to_create
def test_create_id_set_no_output(self, mocker): import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'cpu_count', return_value=1) id_set_creator = IDSetCreator(output=None) id_set = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None assert 'scripts' in id_set.keys() assert 'integrations' in id_set.keys() assert 'playbooks' in id_set.keys() assert 'TestPlaybooks' in id_set.keys() assert 'Classifiers' in id_set.keys() assert 'Dashboards' in id_set.keys() assert 'IncidentFields' in id_set.keys() assert 'IncidentTypes' in id_set.keys() assert 'IndicatorFields' in id_set.keys() assert 'IndicatorTypes' in id_set.keys() assert 'Layouts' in id_set.keys() assert 'Reports' in id_set.keys() assert 'Widgets' in id_set.keys() assert 'Mappers' in id_set.keys()
def test_create_id_set_no_output(self, mocker): import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'cpu_count', return_value=1) id_set_creator = IDSetCreator(output=None) id_set, _, _ = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None keys = set(id_set.keys()) expected_keys = { 'scripts', 'playbooks', 'integrations', 'TestPlaybooks', 'Classifiers', 'Dashboards', 'IncidentFields', 'IncidentTypes', 'IndicatorFields', 'IndicatorTypes', 'Layouts', 'Reports', 'Widgets', 'Mappers', 'Packs', 'GenericTypes', 'GenericFields', 'GenericModules', 'GenericDefinitions', 'Lists', 'Jobs', 'ParsingRules', 'ModelingRules', 'CorrelationRules', 'XSIAMDashboards', 'XSIAMReports', 'Triggers', 'Wizards' } assert keys == expected_keys, f'missing keys: {expected_keys.difference(keys)}\n' \ f' unexpected keys: {keys.difference(expected_keys)}'
def test_create_id_set_flow(repo): number_of_packs_to_create = 10 repo.setup_content_repo(number_of_packs_to_create) with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set() id_set_content = repo.id_set.read_json_as_dict() assert not IsEqualFunctions.is_dicts_equal(id_set_content, {}) assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()), ID_SET_ENTITIES) for id_set_entity in ID_SET_ENTITIES: entity_content_in_id_set = id_set_content.get(id_set_entity) assert entity_content_in_id_set # Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects if id_set_entity != 'Layouts': assert len(entity_content_in_id_set) == number_of_packs_to_create else: assert len( entity_content_in_id_set) == number_of_packs_to_create * 2
def find_dependencies(pack_name, id_set_path='', exclude_ignored_dependencies=True, update_pack_metadata=True, silent_mode=False, debug_file_path=''): """ Main function for dependencies search and pack metadata update. Args: pack_name (str): pack id, currently pack folder name is in use. id_set_path (str): id set json. debug_file_path (str): path to dependency explanations file. silent_mode (bool): Determines whether to echo the dependencies or not. update_pack_metadata (bool): Determines whether to update to pack metadata or not. exclude_ignored_dependencies (bool): Determines whether to include unsupported dependencies or not. Returns: Dict: first level dependencies of a given pack. """ if not id_set_path or not os.path.isfile(id_set_path): id_set = IDSetCreator(print_logs=False).create_id_set() else: with open(id_set_path, 'r') as id_set_file: id_set = json.load(id_set_file) with VerboseFile(debug_file_path) as verbose_file: dependency_graph = PackDependencies.build_dependency_graph( pack_id=pack_name, id_set=id_set, verbose_file=verbose_file, exclude_ignored_dependencies=exclude_ignored_dependencies) first_level_dependencies, _ = parse_for_pack_metadata( dependency_graph, pack_name) if update_pack_metadata: update_pack_metadata_with_dependencies(pack_name, first_level_dependencies) if not silent_mode: # print the found pack dependency results click.echo( click.style(f"Found dependencies result for {pack_name} pack:", bold=True)) dependency_result = json.dumps(first_level_dependencies, indent=4) click.echo(click.style(dependency_result, bold=True)) return first_level_dependencies
def test_generic_command_that_uses_a_specific_brand(repo): """ Given - A playbook with a generic command (send-notification, etc.) that uses a specific brand When - Updating the commands_to_integrations fields in playbooks Then - Do not update 'command_to_integration' fields with any other brands """ integrations = [ { 'Slack': OrderedDict([ ('name', 'Slack'), ('commands', ['send-notification']), ]) }, { 'Syslog': OrderedDict([ ('name', 'Syslog'), ('commands', ['send-notification']), ]) }, ] playbooks = [{ 'Playbook': OrderedDict([ ('name', 'Playbook'), ('command_to_integration', { 'send-notification': 'Slack', }), ]), }] id_set_creator = IDSetCreator(print_logs=False) id_set_creator.id_set["integrations"] = integrations id_set_creator.id_set["playbooks"] = playbooks id_set_creator.add_command_to_implementing_integrations_mapping() playbook = id_set_creator.id_set['playbooks'][0]['Playbook'] assert playbook['command_to_integration'][ 'send-notification'] == 'Slack'
def generate_script_doc(input, examples, output: str = None, permissions: str = None, limitations: str = None, insecure: bool = False, verbose: bool = False): try: doc = [] errors = [] used_in = [] example_section = [] if not output: # default output dir will be the dir of the input file output = os.path.dirname(os.path.realpath(input)) if examples: if not examples.startswith('!'): examples = f'!{examples}' example_dict, build_errors = build_example_dict([examples], insecure) script_name = examples.split(' ')[0][1:] example_section, example_errors = generate_script_example( script_name, example_dict) errors.extend(build_errors) errors.extend(example_errors) else: errors.append( f'Note: Script example was not provided. For a more complete documentation,run with the -e ' f'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".' ) script = get_yaml(input) # get script data secript_info = get_script_info(input) script_id = script.get('commonfields')['id'] # get script dependencies dependencies, _ = get_depends_on(script) # get the script usages by the id set id_set_creator = IDSetCreator() id_set = id_set_creator.create_id_set() used_in = get_used_in(id_set, script_id) description = script.get('comment', '') deprecated = script.get('deprecated', False) # get inputs/outputs inputs, inputs_errors = get_inputs(script) outputs, outputs_errors = get_outputs(script) errors.extend(inputs_errors) errors.extend(outputs_errors) if not description: errors.append( 'Error! You are missing description for the playbook') if deprecated: doc.append('`Deprecated`') doc.append(description) doc.extend(generate_table_section(secript_info, 'Script Data')) if dependencies: doc.extend( generate_list_section( 'Dependencies', dependencies, True, text='This script uses the following commands and scripts.' )) # Script global permissions if permissions == 'general': doc.extend(generate_section('Permissions', '')) if used_in: doc.extend( generate_list_section( 'Used In', used_in, True, text= 'This script is used in the following playbooks and scripts.' )) doc.extend( generate_table_section(inputs, 'Inputs', 'There are no inputs for this script.')) doc.extend( generate_table_section(outputs, 'Outputs', 'There are no outputs for this script.')) if example_section: doc.extend(example_section) # Known limitations if limitations: doc.extend( generate_numbered_section('Known Limitations', limitations)) doc_text = '\n'.join(doc) save_output(output, 'README.md', doc_text) if errors: print_warning('Possible Errors:') for error in errors: print_warning(error) except Exception as ex: if verbose: raise else: print_error(f'Error: {str(ex)}') return
def id_set_command(**kwargs): id_set_creator = IDSetCreator(**kwargs) id_set_creator.create_id_set()
def test_do_not_modify_specific_brand(repo): """ Given - playbook with a command using a specific brand - playbook with a command using a specific brand When - updating the commands_to_integrations fields in playbooks Then - only update commands that don't have a specific brand """ integrations = [ { 'MainInteg': OrderedDict([ ('name', 'MainInteg'), ('commands', ['generic-command']), ]) }, { 'SecondaryInteg': OrderedDict([ ('name', 'SecondaryInteg'), ('commands', ['generic-command', 'specific-command']), ]) }, ] playbooks = [ { 'Playbook1': OrderedDict([ ('name', 'Playbook1'), ('command_to_integration', { 'specific-command': "", 'generic-command': "", }), ]), }, { 'Playbook2': OrderedDict([ ('name', 'Playbook2'), ('command_to_integration', { 'generic-command': 'MainInteg', 'no-integration': '', }), ]), }, ] id_set_creator = IDSetCreator(print_logs=False) id_set_creator.id_set["integrations"] = integrations id_set_creator.id_set["playbooks"] = playbooks id_set_creator.add_command_to_implementing_integrations_mapping() playbook1 = id_set_creator.id_set['playbooks'][0]['Playbook1'] playbook2 = id_set_creator.id_set['playbooks'][1]['Playbook2'] assert playbook1['command_to_integration']['specific-command'] == [ 'SecondaryInteg' ] assert playbook1['command_to_integration']['generic-command'] == [ 'MainInteg', 'SecondaryInteg' ] assert playbook2['command_to_integration'][ 'generic-command'] == 'MainInteg' assert playbook2['command_to_integration']['no-integration'] == ''