def execute_testing(server,
server_ip,
server_version,
server_numeric_version,
is_ami=True):
print("Executing tests with the server {} - and the server ip {}".format(
server, server_ip))
options = options_handler()
username = options.user
password = options.password
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
slack = options.slack
circle_ci = options.circleci
build_number = options.buildNumber
build_name = options.buildName
if not (username and password and server):
print_error('You must provide server user & password arguments')
sys.exit(1)
c = demisto.DemistoClient(None, server, username, password)
res = c.Login()
if res.status_code != 200:
print_error("Login has failed with status code " +
str(res.status_code))
sys.exit(1)
demisto_api_key = generate_demisto_api_key(c)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nigthly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests(
)
if is_filter_configured and not run_all_tests:
is_nightly = True
if not tests or len(tests) == 0:
print('no integrations are configured for test')
return
proxy = None
if is_ami:
ami = AMIConnection(server_ip)
ami.clone_mock_data()
proxy = MITMProxy(c, server_ip)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
disable_all_integrations(c)
if is_ami:
# move all mock tests to the top of the list
mock_tests, mockless_tests = organize_tests(tests,
unmockable_integrations,
skipped_integrations_conf,
nightly_integrations)
else: # In case of a non AMI run we don't want to use the mocking mechanism
mockless_tests = tests
# first run the mock tests to avoid mockless side effects in container
if is_ami and mock_tests:
proxy.configure_proxy_in_demisto(proxy.ami.docker_ip + ':' +
proxy.PROXY_PORT)
for t in mock_tests:
run_test_scenario(t, c, proxy, default_test_timeout,
skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration,
is_nightly, run_all_tests, is_filter_configured,
filtered_tests, skipped_tests, demisto_api_key,
secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks,
slack, circle_ci, build_number, server,
build_name, server_numeric_version)
print("\nRunning mock-disabled tests")
proxy.configure_proxy_in_demisto('')
print("Restarting demisto service")
restart_demisto_service(ami, c)
print("Demisto service restarted\n")
for t in mockless_tests:
run_test_scenario(t, c, proxy, default_test_timeout,
skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration,
is_nightly, run_all_tests, is_filter_configured,
filtered_tests, skipped_tests, demisto_api_key,
secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks, slack,
circle_ci, build_number, server, build_name,
server_numeric_version, is_ami)
print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests,
skipped_integration, unmockable_integrations, proxy,
is_ami)
create_result_files(failed_playbooks, skipped_integration, skipped_tests)
if is_ami and build_name == 'master':
print("Pushing new/updated mock files to mock git repo.")
ami.upload_mock_files(build_name, build_number)
if len(failed_playbooks):
file_path = "./Tests/is_build_failed_{}.txt".format(
server_version.replace(' ', ''))
with open(file_path, "w") as is_build_failed_file:
is_build_failed_file.write('Build failed')
sys.exit(1)
def execute_testing(server, server_ip, server_version, server_numeric_version, is_ami=True):
print("Executing tests with the server {} - and the server ip {}".format(server, server_ip))
options = options_handler()
conf_path = options.conf
secret_conf_path = options.secret
is_nightly = options.nightly
is_memory_check = options.memCheck
slack = options.slack
circle_ci = options.circleci
build_number = options.buildNumber
build_name = options.buildName
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
demisto_api_key = secret_conf.get('temp_apikey')
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests()
if is_filter_configured and not run_all_tests:
is_nightly = True
if not tests or len(tests) == 0:
print('no integrations are configured for test')
return
proxy = None
if is_ami:
ami = AMIConnection(server_ip)
ami.clone_mock_data()
proxy = MITMProxy(server_ip)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
disable_all_integrations(demisto_api_key, server)
if is_ami:
# move all mock tests to the top of the list
mock_tests, mockless_tests = organize_tests(tests, unmockable_integrations, skipped_integrations_conf,
nightly_integrations)
else: # In case of a non AMI run we don't want to use the mocking mechanism
mockless_tests = tests
if is_nightly and is_memory_check:
mem_lim, err = get_docker_limit()
send_slack_message(slack, SLACK_MEM_CHANNEL_ID,
'Build Number: {0}\n Server Address: {1}\nMemory Limit: {2}'.format(build_number, server,
mem_lim),
'Content CircleCI', 'False')
# first run the mock tests to avoid mockless side effects in container
if is_ami and mock_tests:
proxy.configure_proxy_in_demisto(demisto_api_key, server, proxy.ami.docker_ip + ':' + proxy.PROXY_PORT)
for t in mock_tests:
run_test_scenario(t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests,
is_filter_configured,
filtered_tests, skipped_tests, secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server,
build_name, server_numeric_version, demisto_api_key)
print("\nRunning mock-disabled tests")
proxy.configure_proxy_in_demisto(demisto_api_key, server, '')
print("Restarting demisto service")
restart_demisto_service(ami, demisto_api_key, server)
print("Demisto service restarted\n")
for t in mockless_tests:
run_test_scenario(t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests,
is_filter_configured,
filtered_tests, skipped_tests, secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server,
build_name, server_numeric_version, demisto_api_key, is_ami)
print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration, unmockable_integrations,
proxy, is_ami)
create_result_files(failed_playbooks, skipped_integration, skipped_tests)
if is_ami and build_name == 'master':
print("Pushing new/updated mock files to mock git repo.")
ami.upload_mock_files(build_name, build_number)
if len(failed_playbooks):
print("Some tests have failed. Not destroying instances.")
sys.exit(1)
else:
file_path = "./Tests/is_build_passed_{}.txt".format(server_version.replace(' ', ''))
with open(file_path, "w") as is_build_passed_file:
is_build_passed_file.write('Build passed')
def execute_testing(tests_settings: SettingsTester,
server_ip: str,
all_tests: set,
tests_data_keeper: DataKeeperTester,
prints_manager: ParallelPrintsManager,
thread_index: int = 0):
"""
Main function used to handle the testing process. Starts by turning off telemetry and disabling
any left over tests. Afterwards it will create a test queue object which then is used to run the
specific test scenario.
:param tests_settings: SettingsTester object which contains the test variables
:param server_ip: IP address of the server. Will be formatted before use.
:param all_tests: All tests currently in the test conf.
:param tests_data_keeper: Object containing all the test results. Used by report tests function.
:param prints_manager: PrintsManager object used in reporting. Will be deprecated.
:param thread_index: Integer indicating what thread the test is running on.
:return: No object is returned, just updates the tests_data_keep object.
"""
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion
start_message = "Executing tests with the server {} - and the server ip {}".format(
server, server_ip)
prints_manager.add_print_job(start_message, print, thread_index)
slack = tests_settings.slack
circle_ci = tests_settings.circleci
build_number = tests_settings.buildNumber
build_name = tests_settings.buildName
conf, secret_conf = load_conf_files(tests_settings.conf_path,
tests_settings.secret_conf_path)
demisto_api_key = tests_settings.api_key
demisto_user = secret_conf['username']
demisto_pass = secret_conf['userPassword']
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests = extract_filtered_tests()
if not tests or len(tests) == 0:
prints_manager.add_print_job('no integrations are configured for test',
print, thread_index)
prints_manager.execute_thread_prints(thread_index)
return
xsoar_client = demisto_client.configure(base_url=server,
username=demisto_user,
password=demisto_pass,
verify_ssl=False)
# turn off telemetry
turn_off_telemetry(xsoar_client)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
playbook_skipped_integration = set([])
disable_all_integrations(xsoar_client,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
# Private builds do not use mocking. Here we copy the mocked test list to the unmockable list.
private_tests = get_test_records_of_given_test_names(
tests_settings, all_tests)
try:
# first run the mock tests to avoid mockless side effects in container
prints_manager.add_print_job("\nRunning private tests", print,
thread_index)
executed_in_current_round, private_tests_queue = initialize_queue_and_executed_tests_set(
private_tests)
while not private_tests_queue.empty():
t = private_tests_queue.get()
executed_in_current_round = update_round_set_and_sleep_if_round_completed(
executed_in_current_round, prints_manager, t, thread_index)
run_private_test_scenario(tests_settings,
t,
default_test_timeout,
skipped_tests_conf,
nightly_integrations,
skipped_integrations_conf,
skipped_integration,
filtered_tests,
skipped_tests,
secret_params,
failed_playbooks,
playbook_skipped_integration,
succeed_playbooks,
slack,
circle_ci,
build_number,
server,
build_name,
server_numeric_version,
demisto_user,
demisto_pass,
demisto_api_key,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
except Exception as exc:
if exc.__class__ == ApiException:
error_message = exc.body
else:
error_message = f'~~ Thread {thread_index + 1} failed ~~\n{str(exc)}\n{traceback.format_exc()}'
prints_manager.add_print_job(error_message, print_error, thread_index)
prints_manager.execute_thread_prints(thread_index)
failed_playbooks.append(f'~~ Thread {thread_index + 1} failed ~~')
raise
finally:
tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks,
skipped_tests, skipped_integration,
unmockable_integrations)
def execute_testing(tests_settings, server_ip, mockable_tests_names, unmockable_tests_names,
tests_data_keeper, prints_manager, thread_index=0, is_ami=True):
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion
start_message = "Executing tests with the server {} - and the server ip {}".format(server, server_ip)
prints_manager.add_print_job(start_message, print, thread_index)
is_nightly = tests_settings.nightly
is_memory_check = tests_settings.memCheck
slack = tests_settings.slack
circle_ci = tests_settings.circleci
build_number = tests_settings.buildNumber
build_name = tests_settings.buildName
conf, secret_conf = load_conf_files(tests_settings.conf_path, tests_settings.secret_conf_path)
demisto_api_key = tests_settings.api_key
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests(tests_settings.nightly)
if is_filter_configured and not run_all_tests:
is_nightly = True
if not tests or len(tests) == 0:
prints_manager.add_print_job('no integrations are configured for test', print, thread_index)
prints_manager.execute_thread_prints(thread_index)
return
proxy = None
if is_ami:
ami = AMIConnection(server_ip)
ami.clone_mock_data()
proxy = MITMProxy(server_ip)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
disable_all_integrations(demisto_api_key, server, prints_manager, thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
mockable_tests = get_test_records_of_given_test_names(tests_settings, mockable_tests_names)
unmockable_tests = get_test_records_of_given_test_names(tests_settings, unmockable_tests_names)
if is_nightly and is_memory_check:
mem_lim, err = get_docker_limit()
send_slack_message(slack, SLACK_MEM_CHANNEL_ID,
'Build Number: {0}\n Server Address: {1}\nMemory Limit: {2}'.format(build_number, server,
mem_lim),
'Content CircleCI', 'False')
# first run the mock tests to avoid mockless side effects in container
if is_ami and mockable_tests:
proxy.configure_proxy_in_demisto(demisto_api_key, server, proxy.ami.docker_ip + ':' + proxy.PROXY_PORT)
for t in mockable_tests:
run_test_scenario(tests_settings, t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests,
is_filter_configured,
filtered_tests, skipped_tests, secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server,
build_name, server_numeric_version, demisto_api_key, prints_manager,
thread_index=thread_index)
prints_manager.add_print_job("\nRunning mock-disabled tests", print, thread_index)
proxy.configure_proxy_in_demisto(demisto_api_key, server, '')
prints_manager.add_print_job('Resetting containers', print, thread_index)
client = demisto_client.configure(base_url=server, api_key=demisto_api_key, verify_ssl=False)
body, status_code, headers = demisto_client.generic_request_func(self=client, method='POST',
path='/containers/reset')
if status_code != 200:
error_msg = 'Request to reset containers failed with status code "{}"\n{}'.format(status_code, body)
prints_manager.add_print_job(error_msg, print_error, thread_index)
prints_manager.execute_thread_prints(thread_index)
sys.exit(1)
sleep(10)
for t in unmockable_tests:
run_test_scenario(tests_settings, t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations,
skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests,
is_filter_configured,
filtered_tests, skipped_tests, secret_params, failed_playbooks,
unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server,
build_name, server_numeric_version, demisto_api_key,
prints_manager, thread_index, is_ami)
prints_manager.execute_thread_prints(thread_index)
tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks, skipped_tests,
skipped_integration, unmockable_integrations)
if not tests_settings.is_local_run:
tests_data_keeper.add_proxy_related_test_data(proxy)
if is_ami and build_name == 'master':
updating_mocks_msg = "Pushing new/updated mock files to mock git repo."
prints_manager.add_print_job(updating_mocks_msg, print, thread_index)
ami.upload_mock_files(build_name, build_number)