def test_create_id_set_flow(repo, mocker): # Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test mocker.patch.dict(os.environ, {'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'}) number_of_packs_to_create = 10 repo.setup_content_repo(number_of_packs_to_create) with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set() id_set_content = repo.id_set.read_json_as_dict() assert not IsEqualFunctions.is_dicts_equal(id_set_content, {}) assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()), ID_SET_ENTITIES) for id_set_entity in ID_SET_ENTITIES: entity_content_in_id_set = id_set_content.get(id_set_entity) assert entity_content_in_id_set # Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects if id_set_entity != 'Layouts': assert len(entity_content_in_id_set) == number_of_packs_to_create else: assert len( entity_content_in_id_set) == number_of_packs_to_create * 2
def test_create_id_set_on_specific_empty_pack(self, repo, mocker): """ Given - an empty pack to create from it ID set When - create ID set on this pack Then - ensure that an ID set is created and no error is returned - ensure output id_set is empty """ pack = repo.create_pack() repo.add_pack_metadata_file(pack.path, json.dumps(METADATA)) import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'should_skip_item_by_mp', return_value=False) id_set_creator = IDSetCreator(self.file_path, pack.path) id_set_creator.create_id_set() with open(self.file_path, 'r') as id_set_file: private_id_set = json.load(id_set_file) for content_entity, content_entity_value_list in private_id_set.items( ): if content_entity != 'Packs': assert len(content_entity_value_list) == 0 else: assert len(content_entity_value_list) == 1
def test_create_id_set_on_specific_pack(self, repo): """ Given - two packs with integrations to create an ID set from When - create ID set on one of the packs Then - ensure there is only one integration in the ID set integrations list - ensure output id_set contains only the pack on which created the ID set on - ensure output id_set does not contain the second pack """ packs = repo.packs pack_to_create_id_set_on = repo.create_pack('pack_to_create_id_set_on') pack_to_create_id_set_on.create_integration(yml={ 'commonfields': { 'id': 'id1' }, 'category': '', 'name': 'integration to create id set', 'script': { 'type': 'python' } }, name='integration1') packs.append(pack_to_create_id_set_on) pack_to_not_create_id_set_on = repo.create_pack( 'pack_to_not_create_id_set_on') pack_to_not_create_id_set_on.create_integration(yml={ 'commonfields': { 'id2': 'id' }, 'category': '', 'name': 'integration to not create id set' }, name='integration2') packs.append(pack_to_not_create_id_set_on) id_set_creator = IDSetCreator(self.file_path, pack_to_create_id_set_on.path) id_set_creator.create_id_set() with open(self.file_path, 'r') as id_set_file: private_id_set = json.load(id_set_file) assert len(private_id_set['integrations']) == 1 assert private_id_set['integrations'][0].get('id1', {}).get( 'name', '') == 'integration to create id set' assert private_id_set['integrations'][0].get('id2', {}).get('name', '') == '' assert private_id_set['Packs']['pack_to_create_id_set_on'][ 'ContentItems']['integrations'] == ['id1']
def test_create_id_set_on_specific_pack_output(self): """ Given - input - specific pack to create from it ID set - output - path to return the created ID set When - create ID set on this pack Then - ensure that the created ID set is in the path of the output """ id_set_creator = IDSetCreator(self.file_path, input='Packs/AMP') id_set_creator.create_id_set() assert os.path.exists(self.file_path)
def test_create_id_set_flow(repo, mocker): # Note: if DEMISTO_SDK_ID_SET_REFRESH_INTERVAL is set it can fail the test mocker.patch.dict(os.environ, {'DEMISTO_SDK_ID_SET_REFRESH_INTERVAL': '-1'}) number_of_packs_to_create = 10 repo.setup_content_repo(number_of_packs_to_create) with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set() id_set_content = repo.id_set.read_json_as_dict() assert not IsEqualFunctions.is_dicts_equal(id_set_content, {}) assert set(id_set_content.keys()) == set(ID_SET_ENTITIES + ['Packs']) for id_set_entity in ID_SET_ENTITIES: if id_set_entity in [ 'ParsingRules', 'ModelingRules', 'CorrelationRules', 'XSIAMDashboards', 'XSIAMReports', 'Triggers' ]: continue entity_content_in_id_set = id_set_content.get(id_set_entity) assert entity_content_in_id_set, f'ID set for {id_set_entity} is empty' factor = 1 if id_set_entity in {'Layouts', 'TestPlaybooks', 'Jobs'}: ''' Layouts: The folder contains both layouts and layoutcontainers TestPlaybooks: each integration and script has a test playbook Jobs: The default test suite pack has two jobs (is_feed=true, is_feed=false), and a playbook ''' factor = 2 elif id_set_entity == 'playbooks': ''' One playbook is generated for every pack, And one more is created for each of the 2 Job objects that are automatically created in every pack. ''' factor = 3 assert len( entity_content_in_id_set) == factor * number_of_packs_to_create
def test_create_id_set_flow(repo): number_of_packs_to_create = 10 repo.setup_content_repo(number_of_packs_to_create) with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set() id_set_content = repo.id_set.read_json_as_dict() assert not IsEqualFunctions.is_dicts_equal(id_set_content, {}) assert IsEqualFunctions.is_lists_equal(list(id_set_content.keys()), ID_SET_ENTITIES) for id_set_entity in ID_SET_ENTITIES: entity_content_in_id_set = id_set_content.get(id_set_entity) assert entity_content_in_id_set # Since Layouts folder contains both layouts and layoutcontainers then this folder has 2 * amount objects if id_set_entity != 'Layouts': assert len(entity_content_in_id_set) == number_of_packs_to_create else: assert len( entity_content_in_id_set) == number_of_packs_to_create * 2
def test_create_id_set_no_output(self): id_set_creator = IDSetCreator() id_set = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None assert 'scripts' in id_set.keys() assert 'integrations' in id_set.keys() assert 'playbooks' in id_set.keys() assert 'TestPlaybooks' in id_set.keys() assert 'Classifiers' in id_set.keys() assert 'Dashboards' in id_set.keys() assert 'IncidentFields' in id_set.keys() assert 'IncidentTypes' in id_set.keys() assert 'IndicatorFields' in id_set.keys() assert 'Layouts' in id_set.keys() assert 'Reports' in id_set.keys() assert 'Widgets' in id_set.keys()
def test_create_id_set_no_output(self, mocker): import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'cpu_count', return_value=1) id_set_creator = IDSetCreator(output=None) id_set = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None assert 'scripts' in id_set.keys() assert 'integrations' in id_set.keys() assert 'playbooks' in id_set.keys() assert 'TestPlaybooks' in id_set.keys() assert 'Classifiers' in id_set.keys() assert 'Dashboards' in id_set.keys() assert 'IncidentFields' in id_set.keys() assert 'IncidentTypes' in id_set.keys() assert 'IndicatorFields' in id_set.keys() assert 'IndicatorTypes' in id_set.keys() assert 'Layouts' in id_set.keys() assert 'Reports' in id_set.keys() assert 'Widgets' in id_set.keys() assert 'Mappers' in id_set.keys()
def test_create_id_set_no_output(self, mocker): import demisto_sdk.commands.common.update_id_set as uis mocker.patch.object(uis, 'cpu_count', return_value=1) id_set_creator = IDSetCreator(output=None) id_set, _, _ = id_set_creator.create_id_set() assert not os.path.exists(self.file_path) assert id_set is not None keys = set(id_set.keys()) expected_keys = { 'scripts', 'playbooks', 'integrations', 'TestPlaybooks', 'Classifiers', 'Dashboards', 'IncidentFields', 'IncidentTypes', 'IndicatorFields', 'IndicatorTypes', 'Layouts', 'Reports', 'Widgets', 'Mappers', 'Packs', 'GenericTypes', 'GenericFields', 'GenericModules', 'GenericDefinitions', 'Lists', 'Jobs', 'ParsingRules', 'ModelingRules', 'CorrelationRules', 'XSIAMDashboards', 'XSIAMReports', 'Triggers', 'Wizards' } assert keys == expected_keys, f'missing keys: {expected_keys.difference(keys)}\n' \ f' unexpected keys: {keys.difference(expected_keys)}'
def update_id_set(repo): with ChangeCWD(repo.path): id_set_creator = IDSetCreator(repo.id_set.path, print_logs=False) id_set_creator.create_id_set()
def generate_script_doc(input_path, examples, output: str = None, permissions: str = None, limitations: str = None, insecure: bool = False, verbose: bool = False): try: doc: list = [] errors: list = [] example_section: list = [] if not output: # default output dir will be the dir of the input file output = os.path.dirname(os.path.realpath(input_path)) if examples: if os.path.isfile(examples): with open(examples, 'r') as examples_file: examples = examples_file.read().splitlines() else: examples = examples.split(',') for i, example in enumerate(examples): if not example.startswith('!'): examples[i] = f'!{examples}' example_dict, build_errors = build_example_dict(examples, insecure) script_name = list( example_dict.keys())[0] if example_dict else None example_section, example_errors = generate_script_example( script_name, example_dict) errors.extend(build_errors) errors.extend(example_errors) else: errors.append( 'Note: Script example was not provided. For a more complete documentation,run with the -e ' 'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".' ) script = get_yaml(input_path) # get script data script_info = get_script_info(input_path) script_id = script.get('commonfields')['id'] # get script dependencies dependencies, _ = get_depends_on(script) # get the script usages by the id set if not os.path.isfile(DEFAULT_ID_SET_PATH): id_set_creator = IDSetCreator(output='', print_logs=False) id_set, _, _ = id_set_creator.create_id_set() else: id_set = open_id_set_file(DEFAULT_ID_SET_PATH) used_in = get_used_in(id_set, script_id) description = script.get('comment', '') # get inputs/outputs inputs, inputs_errors = get_inputs(script) outputs, outputs_errors = get_outputs(script) errors.extend(inputs_errors) errors.extend(outputs_errors) if not description: errors.append( 'Error! You are missing a description for the Script') doc.append(description + '\n') doc.extend(generate_table_section(script_info, 'Script Data')) if dependencies: doc.extend( generate_list_section( 'Dependencies', dependencies, True, text='This script uses the following commands and scripts.' )) # Script global permissions if permissions == 'general': doc.extend(generate_section('Permissions', '')) if used_in: if len(used_in) <= 10: doc.extend( generate_list_section( 'Used In', used_in, True, text= 'This script is used in the following playbooks and scripts.' )) else: # if we have more than 10 use a sample print_warning( f'"Used In" section found too many scripts/playbooks ({len(used_in)}). Will use a sample of 10.' ' Full list is available as a comment in the README file.') sample_used_in = random.sample(used_in, 10) doc.extend( generate_list_section( 'Used In', sorted(sample_used_in), True, text= 'Sample usage of this script can be found in the following playbooks and scripts.' )) used_in_str = '\n'.join(used_in) doc.append( f"<!--\nUsed In: list was truncated. Full list commented out for reference:\n\n{used_in_str}\n -->\n" ) doc.extend( generate_table_section(inputs, 'Inputs', 'There are no inputs for this script.')) doc.extend( generate_table_section(outputs, 'Outputs', 'There are no outputs for this script.')) if example_section: doc.extend(example_section) # Known limitations if limitations: doc.extend( generate_numbered_section('Known Limitations', limitations)) doc_text = '\n'.join(doc) save_output(output, 'README.md', doc_text) if errors: print_warning('Possible Errors:') for error in errors: print_warning(error) except Exception as ex: if verbose: raise else: print_error(f'Error: {str(ex)}') return
def test_create_id_set_output(self): id_set_creator = IDSetCreator(self.file_path) id_set_creator.create_id_set() assert os.path.exists(self.file_path)
def generate_script_doc(input, examples, output: str = None, permissions: str = None, limitations: str = None, insecure: bool = False, verbose: bool = False): try: doc = [] errors = [] used_in = [] example_section = [] if not output: # default output dir will be the dir of the input file output = os.path.dirname(os.path.realpath(input)) if examples: if not examples.startswith('!'): examples = f'!{examples}' example_dict, build_errors = build_example_dict([examples], insecure) script_name = examples.split(' ')[0][1:] example_section, example_errors = generate_script_example( script_name, example_dict) errors.extend(build_errors) errors.extend(example_errors) else: errors.append( f'Note: Script example was not provided. For a more complete documentation,run with the -e ' f'option with an example command. For example: -e "!ConvertFile entry_id=<entry_id>".' ) script = get_yaml(input) # get script data secript_info = get_script_info(input) script_id = script.get('commonfields')['id'] # get script dependencies dependencies, _ = get_depends_on(script) # get the script usages by the id set id_set_creator = IDSetCreator() id_set = id_set_creator.create_id_set() used_in = get_used_in(id_set, script_id) description = script.get('comment', '') deprecated = script.get('deprecated', False) # get inputs/outputs inputs, inputs_errors = get_inputs(script) outputs, outputs_errors = get_outputs(script) errors.extend(inputs_errors) errors.extend(outputs_errors) if not description: errors.append( 'Error! You are missing description for the playbook') if deprecated: doc.append('`Deprecated`') doc.append(description) doc.extend(generate_table_section(secript_info, 'Script Data')) if dependencies: doc.extend( generate_list_section( 'Dependencies', dependencies, True, text='This script uses the following commands and scripts.' )) # Script global permissions if permissions == 'general': doc.extend(generate_section('Permissions', '')) if used_in: doc.extend( generate_list_section( 'Used In', used_in, True, text= 'This script is used in the following playbooks and scripts.' )) doc.extend( generate_table_section(inputs, 'Inputs', 'There are no inputs for this script.')) doc.extend( generate_table_section(outputs, 'Outputs', 'There are no outputs for this script.')) if example_section: doc.extend(example_section) # Known limitations if limitations: doc.extend( generate_numbered_section('Known Limitations', limitations)) doc_text = '\n'.join(doc) save_output(output, 'README.md', doc_text) if errors: print_warning('Possible Errors:') for error in errors: print_warning(error) except Exception as ex: if verbose: raise else: print_error(f'Error: {str(ex)}') return
def id_set_command(**kwargs): id_set_creator = IDSetCreator(**kwargs) id_set_creator.create_id_set()