def test_integration_has_no_test_playbook_should_fail_on_validation(mocker): """ Given - integration_a was modified - no tests provided for integration_a When - filtering tests to run Then - ensure the validation is failing """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag try: # Given # - integration_a was modified # - no tests provided for integration_a fake_integration = TestUtils.create_integration(name='integration_a', with_commands=['a-command']) # mark as modified TestUtils.mock_get_modified_files(mocker, modified_files_list=[ fake_integration['path'] ]) # - both in conf.json fake_conf = TestUtils.create_tests_conf() fake_id_set = TestUtils.create_id_set( with_integration=fake_integration['id_set'] ) # When # - filtering tests to run get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', two_before_ga_ver=TWO_BEFORE_GA_VERSION, conf=fake_conf, id_set=fake_id_set ) # Then # - ensure the validation is failing assert collect_tests_and_content_packs._FAILED finally: # delete the mocked files TestUtils.delete_files([ fake_integration['path'] ]) # reset _FAILED flag collect_tests_and_content_packs._FAILED = False
def test_conf_has_modified(mocker): """ Given - Tests/conf.json has been modified When - filtering tests to run Then - ensure the validation not failing """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag try: # Given # - Tests/conf.json has been modified TestUtils.mock_get_modified_files(mocker, modified_files_list=[], is_conf_json=True) TestUtils.mock_run_command( mocker, on_command='git diff origin/master...dummy_branch Tests/conf.json', return_value='something' ) # - both in conf.json fake_conf = TestUtils.create_tests_conf() fake_id_set = TestUtils.create_id_set() # When # - filtering tests to run get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', two_before_ga_ver=TWO_BEFORE_GA_VERSION, conf=fake_conf, id_set=fake_id_set ) # Then # - ensure the validation not failing assert not collect_tests_and_content_packs._FAILED finally: # reset _FAILED flag collect_tests_and_content_packs._FAILED = False
def test_modified_integration_content_pack_is_collected(mocker): """ Given - Modified integration named GreatIntegration which is in pack named GreatPack. When - Collecting content packs to install. Then - Ensure the content pack GreatPack is collected. - Ensure the collection runs successfully. """ mocker.patch.object(Tests.scripts.collect_tests_and_content_packs, 'should_test_content_pack', return_value=(True, '')) mocker.patch.object(content_packs_util, 'is_pack_deprecated', return_value=False) from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag pack_name = "GreatPack" integration_name = "GreatIntegration" test_name = "GreatTest" fake_integration = TestUtils.create_integration( name=integration_name, with_commands=["great-command"], pack=pack_name ) fake_test_playbook = TestUtils.create_test_playbook(name=test_name, with_scripts=["FetchFromInstance"]) try: TestUtils.mock_get_modified_files(mocker, modified_files_list=[fake_integration['path']]) fake_id_set = TestUtils.create_id_set( with_integration=fake_integration["id_set"], with_test_playbook=fake_test_playbook["id_set"] ) fake_conf = TestUtils.create_tests_conf( with_test_configuration={ "integrations": integration_name, "playbookID": test_name } ) filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string="", branch_name="dummy-branch", conf=fake_conf, id_set=fake_id_set ) assert content_packs == {"Base", "DeveloperTools", pack_name} assert filtered_tests == {test_name} assert not collect_tests_and_content_packs._FAILED finally: TestUtils.delete_files([ fake_integration["path"], fake_test_playbook["path"] ]) collect_tests_and_content_packs._FAILED = False
def get_mock_test_list(two_before_ga=TWO_BEFORE_GA_VERSION, get_modified_files_ret=None, mocker=None, git_diff_ret=''): branch_name = 'BranchA' if get_modified_files_ret is not None: mocker.patch( 'Tests.scripts.collect_tests_and_content_packs.get_modified_files_for_testing', return_value=get_modified_files_ret ) tests, content_packs = get_test_list_and_content_packs_to_install( git_diff_ret, branch_name, two_before_ga, id_set=MOCK_ID_SET, conf=TestConf(MOCK_CONF) ) return tests, content_packs
def test_modified_integration_content_pack_is_collected(mocker): """ Given - Modified integration named GreatIntegration which is in pack named GreatPack. When - Collecting content packs to install. Then - Ensure the content pack GreatPack is collected. - Ensure the collection runs successfully. """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag pack_name = "GreatPack" integration_name = "GreatIntegration" test_name = "GreatTest" fake_integration = TestUtils.create_integration( name=integration_name, with_commands=["great-command"], pack=pack_name ) fake_test_playbook = TestUtils.create_test_playbook(name=test_name, with_scripts=["FetchFromInstance"]) try: TestUtils.mock_get_modified_files(mocker, modified_files_list=[fake_integration['path']]) fake_id_set = TestUtils.create_id_set( with_integration=fake_integration["id_set"], with_test_playbook=fake_test_playbook["id_set"] ) fake_conf = TestUtils.create_tests_conf( with_test_configuration={ "integrations": integration_name, "playbookID": test_name } ) filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string="", branch_name="dummy-branch", two_before_ga_ver=TWO_BEFORE_GA_VERSION, conf=fake_conf, id_set=fake_id_set ) assert content_packs == {pack_name} assert not collect_tests_and_content_packs._FAILED finally: TestUtils.delete_files([ fake_integration["path"] ]) collect_tests_and_content_packs._FAILED = False
def test_skipped_integration_should_not_be_tested(mocker): """ Given - conf.json file with IntegrationA is skipped - no tests provided for IntegrationA When - filtering tests to run Then - ensure IntegrationA is skipped - ensure the validation not failing """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag # Given # - conf.json file with IntegrationA is skipped # - no tests provided for IntegrationA fake_integration = TestUtils.create_integration(name='integration_a', with_commands=['a-command']) # mark as modified TestUtils.mock_get_modified_files(mocker, modified_files_list=[ fake_integration['path'] ]) mock_conf_dict = copy.deepcopy(MOCK_CONF) mock_conf_dict['skipped_integrations']['integration_a'] = 'comment' fake_id_set = TestUtils.create_id_set() # When # - filtering tests to run filtered_tests = get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', two_before_ga_ver=TWO_BEFORE_GA_VERSION, conf=TestConf(mock_conf_dict), id_set=fake_id_set ) # Then # - ensure IntegrationA is skipped assert 'integration_a' not in filtered_tests # - ensure the validation not failing assert not collect_tests_and_content_packs._FAILED
def test_pack_ignore_test_is_skipped(mocker): """ Given - Modified integration named GreatIntegration which is in pack named GreatPack. - Test playbook that is skipped in .pack-ignore When - Collecting content packs to install and tests. Then - Ensure test playbook is not collected - Ensure the content pack GreatPack is collected. - Ensure the collection runs successfully. """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag pack_name = "GreatPack" integration_name = "GreatIntegration" test_name = "GreatTest" fake_integration = TestUtils.create_integration( name=integration_name, with_commands=["great-command"], pack=pack_name) fake_test_playbook = TestUtils.create_test_playbook( name=test_name, with_scripts=["FetchFromInstance"], with_pack='GreatPack') pack_ignore_mgr = TestUtils.PackIgnoreManager( os.path.basename(fake_test_playbook['path'])) try: mocker.patch.object(os.path, 'join', return_value=fake_test_playbook['path']) mocker.patch.object(Tests.scripts.utils.content_packs_util, 'get_pack_metadata', return_value={PACK_METADATA_SUPPORT: 'xsoar'}) mocker.patch.object(demisto_sdk_tools, 'get_pack_ignore_file_path', return_value=pack_ignore_mgr.pack_ignore_path) TestUtils.mock_get_modified_files( mocker, modified_files_list=[fake_integration['path']]) fake_id_set = TestUtils.create_id_set( with_integration=fake_integration["id_set"], with_test_playbook=fake_test_playbook["id_set"]) fake_conf = TestUtils.create_tests_conf(with_test_configuration={ "integrations": integration_name, "playbookID": test_name }) # add test playbook to ignore list and clean afterward with pack_ignore_mgr: filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string="", branch_name="dummy-branch", conf=fake_conf, id_set=fake_id_set) assert content_packs == { "HelloWorld", "Gmail", "Base", "DeveloperTools", pack_name } assert filtered_tests == { "Sanity Test - Playbook with no integration", "Sanity Test - Playbook with integration", "Sanity Test - Playbook with mocked integration", "Sanity Test - Playbook with Unmockable Integration" } assert not collect_tests_and_content_packs._FAILED finally: TestUtils.delete_files( [fake_integration["path"], fake_test_playbook["path"]]) collect_tests_and_content_packs._FAILED = False
def test_mismatching_script_id(self, mocker, tmp_path): """ Given - script_a was modified - tests were provided for script_a with mismatching id When - filtering tests to run Then - ensure test_playbook_a will run/returned """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag # Given # - script_a exists script_name = 'script_a' fake_script = TestUtils.create_script(name=script_name) # - tests were provided for script_a with mismatching id id_set_obj = fake_script['id_set'][script_name] fake_script['id_set'] = {'wrong_id': id_set_obj} # Assuming pack is XSOAR supported mocker.patch.object(Tests.scripts.utils.content_packs_util, 'get_pack_metadata', return_value={PACK_METADATA_SUPPORT: 'xsoar'}) create_temp_dir_with_metadata(tmp_path, 'pack_a', {PACK_METADATA_SUPPORT: 'xsoar'}) mocker.patch.object(Tests.scripts.utils.content_packs_util, 'PACKS_DIR', tmp_path / PACKS_DIR) # mark as modified TestUtils.mock_get_modified_files( mocker, modified_files_list=[fake_script['path']]) # - test_playbook_a exists that should test script_a fake_test_playbook = TestUtils.create_test_playbook( name='test_playbook_a', with_scripts=[script_name], with_pack='pack_a') try: # - both in conf.json fake_conf = TestUtils.create_tests_conf( with_test_configuration={'playbookID': 'test_playbook_a'}) fake_id_set = TestUtils.create_id_set( with_scripts=fake_script['id_set'], with_test_playbook=fake_test_playbook['id_set']) # When # - filtering tests to run filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', conf=fake_conf, id_set=fake_id_set) # Then # - ensure test_playbook_a will run/returned assert 'test_playbook_a' in filtered_tests assert content_packs == {"Base", "DeveloperTools", "pack_a"} # - ensure the validation not failing assert not collect_tests_and_content_packs._FAILED finally: # delete the mocked files TestUtils.delete_files( [fake_script['path'], fake_test_playbook['path']]) # reset _FAILED flag collect_tests_and_content_packs._FAILED = False
def test_dont_fail_integration_on_no_tests_if_it_has_test_playbook_in_conf( mocker): """ If there is an integration in conf.json configured with test playbook Ensure that this integration don't fails on validation. Given - integration_a that fetches incidents - test_playbook_a exists that should test FetchFromInstance of integration_a - both in conf.json When - filtering tests to run Then - ensure test_playbook_a will run/returned - ensure the validation not failing """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag # Given # - integration_a exists fake_integration = TestUtils.create_integration( name='integration_a', with_commands=['a-command']) # mark as modified TestUtils.mock_get_modified_files( mocker, modified_files_list=[fake_integration['path']]) # - test_playbook_a exists that should test FetchFromInstance of integration_a fake_test_playbook = TestUtils.create_test_playbook( name='test_playbook_a', with_scripts=['FetchFromInstance']) try: # - both in conf.json fake_conf = TestUtils.create_tests_conf(with_test_configuration={ 'integrations': 'integration_a', 'playbookID': 'test_playbook_a' }) fake_id_set = TestUtils.create_id_set( with_integration=fake_integration['id_set'], with_test_playbook=fake_test_playbook['id_set']) # When # - filtering tests to run filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', conf=fake_conf, id_set=fake_id_set) # Then # - ensure test_playbook_a will run/returned assert 'test_playbook_a' in filtered_tests # - ensure the validation not failing assert not collect_tests_and_content_packs._FAILED finally: # delete the mocked files TestUtils.delete_files( [fake_integration['path'], fake_test_playbook['path']]) # reset _FAILED flag collect_tests_and_content_packs._FAILED = False
def test_mismatching_script_id(self, mocker): """ Given - integration_a was modified - tests were provided for integration_a with mismatching id When - filtering tests to run Then - ensure test_playbook_a will run/returned """ from Tests.scripts import collect_tests_and_content_packs collect_tests_and_content_packs._FAILED = False # reset the FAILED flag # Given # - integration_a exists script_name = 'script_a' fake_script = TestUtils.create_script(name=script_name) # - tests were provided for integration_a with mismatching id id_set_obj = fake_script['id_set'][script_name] fake_script['id_set'] = {'wrong_id': id_set_obj} # mark as modified TestUtils.mock_get_modified_files(mocker, modified_files_list=[ fake_script['path'] ]) # - test_playbook_a exists that should test script_a fake_test_playbook = TestUtils.create_test_playbook(name='test_playbook_a', with_scripts=[script_name]) try: # - both in conf.json fake_conf = TestUtils.create_tests_conf( with_test_configuration={ 'playbookID': 'test_playbook_a' } ) fake_id_set = TestUtils.create_id_set( with_scripts=fake_script['id_set'], with_test_playbook=fake_test_playbook['id_set'] ) # When # - filtering tests to run filtered_tests, content_packs = get_test_list_and_content_packs_to_install( files_string='', branch_name='dummy_branch', two_before_ga_ver=TWO_BEFORE_GA_VERSION, conf=fake_conf, id_set=fake_id_set ) # Then # - ensure test_playbook_a will run/returned assert 'test_playbook_a' in filtered_tests assert content_packs == {"Base", "DeveloperTools"} # - ensure the validation not failing assert not collect_tests_and_content_packs._FAILED finally: # delete the mocked files TestUtils.delete_files([ fake_script['path'], fake_test_playbook['path'] ]) # reset _FAILED flag collect_tests_and_content_packs._FAILED = False