def test_integration_has_no_test_playbook_should_fail_on_validation(mocker):
"""
Given
- integration_a was modified
- no tests provided for integration_a
When
- filtering tests to run
Then
- ensure the validation is failing
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
try:
# Given
# - integration_a was modified
# - no tests provided for integration_a
fake_integration = TestUtils.create_integration(name='integration_a', with_commands=['a-command'])
# mark as modified
TestUtils.mock_get_modified_files(mocker,
modified_files_list=[
fake_integration['path']
])
# - both in conf.json
fake_conf = TestUtils.create_tests_conf()
fake_id_set = TestUtils.create_id_set(
with_integration=fake_integration['id_set']
)
# When
# - filtering tests to run
get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
two_before_ga_ver=TWO_BEFORE_GA_VERSION,
conf=fake_conf,
id_set=fake_id_set
)
# Then
# - ensure the validation is failing
assert collect_tests_and_content_packs._FAILED
finally:
# delete the mocked files
TestUtils.delete_files([
fake_integration['path']
])
# reset _FAILED flag
collect_tests_and_content_packs._FAILED = False
def test_conf_has_modified(mocker):
"""
Given
- Tests/conf.json has been modified
When
- filtering tests to run
Then
- ensure the validation not failing
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
try:
# Given
# - Tests/conf.json has been modified
TestUtils.mock_get_modified_files(mocker,
modified_files_list=[],
is_conf_json=True)
TestUtils.mock_run_command(
mocker,
on_command='git diff origin/master...dummy_branch Tests/conf.json',
return_value='something'
)
# - both in conf.json
fake_conf = TestUtils.create_tests_conf()
fake_id_set = TestUtils.create_id_set()
# When
# - filtering tests to run
get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
two_before_ga_ver=TWO_BEFORE_GA_VERSION,
conf=fake_conf,
id_set=fake_id_set
)
# Then
# - ensure the validation not failing
assert not collect_tests_and_content_packs._FAILED
finally:
# reset _FAILED flag
collect_tests_and_content_packs._FAILED = False
def test_modified_integration_content_pack_is_collected(mocker):
"""
Given
- Modified integration named GreatIntegration which is in pack named GreatPack.
When
- Collecting content packs to install.
Then
- Ensure the content pack GreatPack is collected.
- Ensure the collection runs successfully.
"""
mocker.patch.object(Tests.scripts.collect_tests_and_content_packs, 'should_test_content_pack',
return_value=(True, ''))
mocker.patch.object(content_packs_util, 'is_pack_deprecated', return_value=False)
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
pack_name = "GreatPack"
integration_name = "GreatIntegration"
test_name = "GreatTest"
fake_integration = TestUtils.create_integration(
name=integration_name, with_commands=["great-command"], pack=pack_name
)
fake_test_playbook = TestUtils.create_test_playbook(name=test_name, with_scripts=["FetchFromInstance"])
try:
TestUtils.mock_get_modified_files(mocker, modified_files_list=[fake_integration['path']])
fake_id_set = TestUtils.create_id_set(
with_integration=fake_integration["id_set"],
with_test_playbook=fake_test_playbook["id_set"]
)
fake_conf = TestUtils.create_tests_conf(
with_test_configuration={
"integrations": integration_name,
"playbookID": test_name
}
)
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string="",
branch_name="dummy-branch",
conf=fake_conf,
id_set=fake_id_set
)
assert content_packs == {"Base", "DeveloperTools", pack_name}
assert filtered_tests == {test_name}
assert not collect_tests_and_content_packs._FAILED
finally:
TestUtils.delete_files([
fake_integration["path"],
fake_test_playbook["path"]
])
collect_tests_and_content_packs._FAILED = False
def get_mock_test_list(two_before_ga=TWO_BEFORE_GA_VERSION, get_modified_files_ret=None, mocker=None, git_diff_ret=''):
branch_name = 'BranchA'
if get_modified_files_ret is not None:
mocker.patch(
'Tests.scripts.collect_tests_and_content_packs.get_modified_files_for_testing',
return_value=get_modified_files_ret
)
tests, content_packs = get_test_list_and_content_packs_to_install(
git_diff_ret, branch_name, two_before_ga, id_set=MOCK_ID_SET, conf=TestConf(MOCK_CONF)
)
return tests, content_packs
def test_modified_integration_content_pack_is_collected(mocker):
"""
Given
- Modified integration named GreatIntegration which is in pack named GreatPack.
When
- Collecting content packs to install.
Then
- Ensure the content pack GreatPack is collected.
- Ensure the collection runs successfully.
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
pack_name = "GreatPack"
integration_name = "GreatIntegration"
test_name = "GreatTest"
fake_integration = TestUtils.create_integration(
name=integration_name, with_commands=["great-command"], pack=pack_name
)
fake_test_playbook = TestUtils.create_test_playbook(name=test_name, with_scripts=["FetchFromInstance"])
try:
TestUtils.mock_get_modified_files(mocker, modified_files_list=[fake_integration['path']])
fake_id_set = TestUtils.create_id_set(
with_integration=fake_integration["id_set"],
with_test_playbook=fake_test_playbook["id_set"]
)
fake_conf = TestUtils.create_tests_conf(
with_test_configuration={
"integrations": integration_name,
"playbookID": test_name
}
)
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string="",
branch_name="dummy-branch",
two_before_ga_ver=TWO_BEFORE_GA_VERSION,
conf=fake_conf,
id_set=fake_id_set
)
assert content_packs == {pack_name}
assert not collect_tests_and_content_packs._FAILED
finally:
TestUtils.delete_files([
fake_integration["path"]
])
collect_tests_and_content_packs._FAILED = False
def test_skipped_integration_should_not_be_tested(mocker):
"""
Given
- conf.json file with IntegrationA is skipped
- no tests provided for IntegrationA
When
- filtering tests to run
Then
- ensure IntegrationA is skipped
- ensure the validation not failing
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
# Given
# - conf.json file with IntegrationA is skipped
# - no tests provided for IntegrationA
fake_integration = TestUtils.create_integration(name='integration_a', with_commands=['a-command'])
# mark as modified
TestUtils.mock_get_modified_files(mocker,
modified_files_list=[
fake_integration['path']
])
mock_conf_dict = copy.deepcopy(MOCK_CONF)
mock_conf_dict['skipped_integrations']['integration_a'] = 'comment'
fake_id_set = TestUtils.create_id_set()
# When
# - filtering tests to run
filtered_tests = get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
two_before_ga_ver=TWO_BEFORE_GA_VERSION,
conf=TestConf(mock_conf_dict),
id_set=fake_id_set
)
# Then
# - ensure IntegrationA is skipped
assert 'integration_a' not in filtered_tests
# - ensure the validation not failing
assert not collect_tests_and_content_packs._FAILED
def test_pack_ignore_test_is_skipped(mocker):
"""
Given
- Modified integration named GreatIntegration which is in pack named GreatPack.
- Test playbook that is skipped in .pack-ignore
When
- Collecting content packs to install and tests.
Then
- Ensure test playbook is not collected
- Ensure the content pack GreatPack is collected.
- Ensure the collection runs successfully.
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
pack_name = "GreatPack"
integration_name = "GreatIntegration"
test_name = "GreatTest"
fake_integration = TestUtils.create_integration(
name=integration_name, with_commands=["great-command"], pack=pack_name)
fake_test_playbook = TestUtils.create_test_playbook(
name=test_name,
with_scripts=["FetchFromInstance"],
with_pack='GreatPack')
pack_ignore_mgr = TestUtils.PackIgnoreManager(
os.path.basename(fake_test_playbook['path']))
try:
mocker.patch.object(os.path,
'join',
return_value=fake_test_playbook['path'])
mocker.patch.object(Tests.scripts.utils.content_packs_util,
'get_pack_metadata',
return_value={PACK_METADATA_SUPPORT: 'xsoar'})
mocker.patch.object(demisto_sdk_tools,
'get_pack_ignore_file_path',
return_value=pack_ignore_mgr.pack_ignore_path)
TestUtils.mock_get_modified_files(
mocker, modified_files_list=[fake_integration['path']])
fake_id_set = TestUtils.create_id_set(
with_integration=fake_integration["id_set"],
with_test_playbook=fake_test_playbook["id_set"])
fake_conf = TestUtils.create_tests_conf(with_test_configuration={
"integrations": integration_name,
"playbookID": test_name
})
# add test playbook to ignore list and clean afterward
with pack_ignore_mgr:
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string="",
branch_name="dummy-branch",
conf=fake_conf,
id_set=fake_id_set)
assert content_packs == {
"HelloWorld", "Gmail", "Base", "DeveloperTools", pack_name
}
assert filtered_tests == {
"Sanity Test - Playbook with no integration",
"Sanity Test - Playbook with integration",
"Sanity Test - Playbook with mocked integration",
"Sanity Test - Playbook with Unmockable Integration"
}
assert not collect_tests_and_content_packs._FAILED
finally:
TestUtils.delete_files(
[fake_integration["path"], fake_test_playbook["path"]])
collect_tests_and_content_packs._FAILED = False
def test_mismatching_script_id(self, mocker, tmp_path):
"""
Given
- script_a was modified
- tests were provided for script_a with mismatching id
When
- filtering tests to run
Then
- ensure test_playbook_a will run/returned
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
# Given
# - script_a exists
script_name = 'script_a'
fake_script = TestUtils.create_script(name=script_name)
# - tests were provided for script_a with mismatching id
id_set_obj = fake_script['id_set'][script_name]
fake_script['id_set'] = {'wrong_id': id_set_obj}
# Assuming pack is XSOAR supported
mocker.patch.object(Tests.scripts.utils.content_packs_util,
'get_pack_metadata',
return_value={PACK_METADATA_SUPPORT: 'xsoar'})
create_temp_dir_with_metadata(tmp_path, 'pack_a',
{PACK_METADATA_SUPPORT: 'xsoar'})
mocker.patch.object(Tests.scripts.utils.content_packs_util,
'PACKS_DIR', tmp_path / PACKS_DIR)
# mark as modified
TestUtils.mock_get_modified_files(
mocker, modified_files_list=[fake_script['path']])
# - test_playbook_a exists that should test script_a
fake_test_playbook = TestUtils.create_test_playbook(
name='test_playbook_a',
with_scripts=[script_name],
with_pack='pack_a')
try:
# - both in conf.json
fake_conf = TestUtils.create_tests_conf(
with_test_configuration={'playbookID': 'test_playbook_a'})
fake_id_set = TestUtils.create_id_set(
with_scripts=fake_script['id_set'],
with_test_playbook=fake_test_playbook['id_set'])
# When
# - filtering tests to run
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
conf=fake_conf,
id_set=fake_id_set)
# Then
# - ensure test_playbook_a will run/returned
assert 'test_playbook_a' in filtered_tests
assert content_packs == {"Base", "DeveloperTools", "pack_a"}
# - ensure the validation not failing
assert not collect_tests_and_content_packs._FAILED
finally:
# delete the mocked files
TestUtils.delete_files(
[fake_script['path'], fake_test_playbook['path']])
# reset _FAILED flag
collect_tests_and_content_packs._FAILED = False
def test_dont_fail_integration_on_no_tests_if_it_has_test_playbook_in_conf(
mocker):
"""
If there is an integration in conf.json configured with test playbook
Ensure that this integration don't fails on validation.
Given
- integration_a that fetches incidents
- test_playbook_a exists that should test FetchFromInstance of integration_a
- both in conf.json
When
- filtering tests to run
Then
- ensure test_playbook_a will run/returned
- ensure the validation not failing
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
# Given
# - integration_a exists
fake_integration = TestUtils.create_integration(
name='integration_a', with_commands=['a-command'])
# mark as modified
TestUtils.mock_get_modified_files(
mocker, modified_files_list=[fake_integration['path']])
# - test_playbook_a exists that should test FetchFromInstance of integration_a
fake_test_playbook = TestUtils.create_test_playbook(
name='test_playbook_a', with_scripts=['FetchFromInstance'])
try:
# - both in conf.json
fake_conf = TestUtils.create_tests_conf(with_test_configuration={
'integrations': 'integration_a',
'playbookID': 'test_playbook_a'
})
fake_id_set = TestUtils.create_id_set(
with_integration=fake_integration['id_set'],
with_test_playbook=fake_test_playbook['id_set'])
# When
# - filtering tests to run
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
conf=fake_conf,
id_set=fake_id_set)
# Then
# - ensure test_playbook_a will run/returned
assert 'test_playbook_a' in filtered_tests
# - ensure the validation not failing
assert not collect_tests_and_content_packs._FAILED
finally:
# delete the mocked files
TestUtils.delete_files(
[fake_integration['path'], fake_test_playbook['path']])
# reset _FAILED flag
collect_tests_and_content_packs._FAILED = False
def test_mismatching_script_id(self, mocker):
"""
Given
- integration_a was modified
- tests were provided for integration_a with mismatching id
When
- filtering tests to run
Then
- ensure test_playbook_a will run/returned
"""
from Tests.scripts import collect_tests_and_content_packs
collect_tests_and_content_packs._FAILED = False # reset the FAILED flag
# Given
# - integration_a exists
script_name = 'script_a'
fake_script = TestUtils.create_script(name=script_name)
# - tests were provided for integration_a with mismatching id
id_set_obj = fake_script['id_set'][script_name]
fake_script['id_set'] = {'wrong_id': id_set_obj}
# mark as modified
TestUtils.mock_get_modified_files(mocker,
modified_files_list=[
fake_script['path']
])
# - test_playbook_a exists that should test script_a
fake_test_playbook = TestUtils.create_test_playbook(name='test_playbook_a',
with_scripts=[script_name])
try:
# - both in conf.json
fake_conf = TestUtils.create_tests_conf(
with_test_configuration={
'playbookID': 'test_playbook_a'
}
)
fake_id_set = TestUtils.create_id_set(
with_scripts=fake_script['id_set'],
with_test_playbook=fake_test_playbook['id_set']
)
# When
# - filtering tests to run
filtered_tests, content_packs = get_test_list_and_content_packs_to_install(
files_string='',
branch_name='dummy_branch',
two_before_ga_ver=TWO_BEFORE_GA_VERSION,
conf=fake_conf,
id_set=fake_id_set
)
# Then
# - ensure test_playbook_a will run/returned
assert 'test_playbook_a' in filtered_tests
assert content_packs == {"Base", "DeveloperTools"}
# - ensure the validation not failing
assert not collect_tests_and_content_packs._FAILED
finally:
# delete the mocked files
TestUtils.delete_files([
fake_script['path'],
fake_test_playbook['path']
])
# reset _FAILED flag
collect_tests_and_content_packs._FAILED = False
