def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
servers = determine_servers_urls(ami_env)
conf_path = options.conf
secret_conf_path = options.secret
prints_manager = ParallelPrintsManager(1)
server_numeric_version = get_server_numeric_version(ami_env)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
clients = []
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
clients.append(client)
testing_client = clients[0]
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(
is_nightly=False)
tests_for_iteration = tests
if run_all_tests:
# Use all tests for testing, leave 'tests_for_iteration' as is
pass
elif filter_configured and filtered_tests:
tests_for_iteration = [
test for test in tests
if test.get('playbookID', '') in filtered_tests
]
tests_for_iteration = filter_tests_with_incompatible_version(
tests_for_iteration, server_numeric_version, prints_manager)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_names, modified_integrations_names = get_new_and_modified_integrations(
git_sha1)
if new_integrations_names:
print_warning('New Integrations Since Last Release:\n{}\n'.format(
'\n'.join(new_integrations_names)))
if modified_integrations_names:
print_warning('Updated Integrations Since Last Release:\n{}\n'.format(
'\n'.join(modified_integrations_names)))
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
integrations = get_integrations_for_test(test,
skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
print_warning('All Integrations for test "{}":'.format(
test.get('playbookID')))
print_warning(integrations_names)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names,
modified_integrations_names)
integrations_msg = '\n'.join([
'"{}" - {}'.format(key, val)
for key, val in integration_to_status.items()
])
print_warning('{}\n'.format(integrations_msg))
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
new_ints_params_set = set_integration_params(new_integrations,
secret_params,
instance_names_conf)
ints_to_configure_params_set = set_integration_params(
integrations_to_configure, secret_params, instance_names_conf)
if not new_ints_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(new_integrations)))
if not ints_to_configure_params_set:
print_error(
'failed setting parameters for integrations "{}"'.format(
'\n'.join(integrations_to_configure)))
if not (new_ints_params_set and ints_to_configure_params_set):
continue
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
print_warning(
'Start of Instance Testing ("Test" button) prior to Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Pre-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(clients))
# For each server url we install content
for thread_index, (client, server_url) in enumerate(zip(clients, servers)):
t = Thread(target=update_content_on_demisto_instance,
kwargs={
'client': client,
'server': server_url,
'prints_manager': threads_prints_manager,
'thread_index': thread_index
})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
new_integration_module_instances.append(
configure_integration_instance(integration, testing_client,
prints_manager))
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
print_warning(
'Start of Instance Testing ("Test" button) after the Content Update:'
)
else:
print_warning(
'No integrations to configure for the chosen tests. (Post-update)')
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(
instance_name, integration_of_instance)
print(msg)
# If there is a failure, __test_integration_instance will print it
success = __test_integration_instance(testing_client, instance,
prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url,
username=username,
password=password,
verify_ssl=False)
__disable_integrations_instances(client, all_module_instances,
prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails,
new_integrations_names)
if not success:
sys.exit(1)
def execute_testing(tests_settings: SettingsTester,
server_ip: str,
all_tests: set,
tests_data_keeper: DataKeeperTester,
prints_manager: ParallelPrintsManager,
thread_index: int = 0):
"""
Main function used to handle the testing process. Starts by turning off telemetry and disabling
any left over tests. Afterwards it will create a test queue object which then is used to run the
specific test scenario.
:param tests_settings: SettingsTester object which contains the test variables
:param server_ip: IP address of the server. Will be formatted before use.
:param all_tests: All tests currently in the test conf.
:param tests_data_keeper: Object containing all the test results. Used by report tests function.
:param prints_manager: PrintsManager object used in reporting. Will be deprecated.
:param thread_index: Integer indicating what thread the test is running on.
:return: No object is returned, just updates the tests_data_keep object.
"""
server = SERVER_URL.format(server_ip)
server_numeric_version = tests_settings.serverNumericVersion
start_message = "Executing tests with the server {} - and the server ip {}".format(
server, server_ip)
prints_manager.add_print_job(start_message, print, thread_index)
slack = tests_settings.slack
circle_ci = tests_settings.circleci
build_number = tests_settings.buildNumber
build_name = tests_settings.buildName
conf, secret_conf = load_conf_files(tests_settings.conf_path,
tests_settings.secret_conf_path)
demisto_api_key = tests_settings.api_key
demisto_user = secret_conf['username']
demisto_pass = secret_conf['userPassword']
default_test_timeout = conf.get('testTimeout', 30)
tests = conf['tests']
skipped_tests_conf = conf['skipped_tests']
nightly_integrations = conf['nightly_integrations']
skipped_integrations_conf = conf['skipped_integrations']
unmockable_integrations = conf['unmockable_integrations']
secret_params = secret_conf['integrations'] if secret_conf else []
filtered_tests = extract_filtered_tests()
if not tests or len(tests) == 0:
prints_manager.add_print_job('no integrations are configured for test',
print, thread_index)
prints_manager.execute_thread_prints(thread_index)
return
xsoar_client = demisto_client.configure(base_url=server,
username=demisto_user,
password=demisto_pass,
verify_ssl=False)
# turn off telemetry
turn_off_telemetry(xsoar_client)
failed_playbooks = []
succeed_playbooks = []
skipped_tests = set([])
skipped_integration = set([])
playbook_skipped_integration = set([])
disable_all_integrations(xsoar_client,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
# Private builds do not use mocking. Here we copy the mocked test list to the unmockable list.
private_tests = get_test_records_of_given_test_names(
tests_settings, all_tests)
try:
# first run the mock tests to avoid mockless side effects in container
prints_manager.add_print_job("\nRunning private tests", print,
thread_index)
executed_in_current_round, private_tests_queue = initialize_queue_and_executed_tests_set(
private_tests)
while not private_tests_queue.empty():
t = private_tests_queue.get()
executed_in_current_round = update_round_set_and_sleep_if_round_completed(
executed_in_current_round, prints_manager, t, thread_index)
run_private_test_scenario(tests_settings,
t,
default_test_timeout,
skipped_tests_conf,
nightly_integrations,
skipped_integrations_conf,
skipped_integration,
filtered_tests,
skipped_tests,
secret_params,
failed_playbooks,
playbook_skipped_integration,
succeed_playbooks,
slack,
circle_ci,
build_number,
server,
build_name,
server_numeric_version,
demisto_user,
demisto_pass,
demisto_api_key,
prints_manager,
thread_index=thread_index)
prints_manager.execute_thread_prints(thread_index)
except Exception as exc:
if exc.__class__ == ApiException:
error_message = exc.body
else:
error_message = f'~~ Thread {thread_index + 1} failed ~~\n{str(exc)}\n{traceback.format_exc()}'
prints_manager.add_print_job(error_message, print_error, thread_index)
prints_manager.execute_thread_prints(thread_index)
failed_playbooks.append(f'~~ Thread {thread_index + 1} failed ~~')
raise
finally:
tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks,
skipped_tests, skipped_integration,
unmockable_integrations)
def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
conf_path = options.conf
secret_conf_path = options.secret
branch_name = options.branch
ci_build_number = options.build_number
servers = determine_servers_urls(ami_env)
server_numeric_version = get_server_numeric_version(ami_env)
prints_manager = ParallelPrintsManager(1)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
for server in servers:
client = demisto_client.configure(base_url=server, username=username, password=password,
verify_ssl=False)
set_marketplace_gcp_bucket_for_build(client, prints_manager, branch_name, ci_build_number)
print('Restarting servers to apply GCS server config ...')
ssh_string = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {}@{} ' \
'"sudo systemctl restart demisto"'
try:
subprocess.check_output(
ssh_string.format('ec2-user', server.replace('https://', '')), shell=True)
except subprocess.CalledProcessError as exc:
print(exc.output)
print('Done restarting servers.')
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(is_nightly=options.is_nightly)
tests_for_iteration = tests
if run_all_tests:
# skip test button testing
skipped_instance_test_message = 'Not running instance tests when {} is turned on'.format(RUN_ALL_TESTS_FORMAT)
prints_manager.add_print_job(skipped_instance_test_message, print_warning, 0)
tests_for_iteration = []
elif filter_configured and filtered_tests:
tests_for_iteration = [test for test in tests if test.get('playbookID', '') in filtered_tests]
tests_for_iteration = filter_tests_with_incompatible_version(tests_for_iteration, server_numeric_version,
prints_manager)
prints_manager.execute_thread_prints(0)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_files, modified_integrations_files = get_new_and_modified_integration_files(git_sha1)
new_integrations_names, modified_integrations_names = [], []
installed_content_packs_successfully = True
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
# sleep for one minute before starting to search and install packs to ensure bucket is ready
prints_manager.add_print_job('Sleeping for 1 minute...', print_warning, 0)
prints_manager.execute_thread_prints(0)
sleep(60)
pack_ids = get_pack_ids_to_install()
# install content packs in every server
for server_url in servers:
try:
client = demisto_client.configure(base_url=server_url, username=username, password=password,
verify_ssl=False)
search_and_install_packs_and_their_dependencies(pack_ids, client, prints_manager, options.is_nightly)
except Exception as exc:
prints_manager.add_print_job(str(exc), print_error, 0)
prints_manager.execute_thread_prints(0)
installed_content_packs_successfully = False
if new_integrations_files:
new_integrations_names = get_integration_names_from_files(new_integrations_files)
new_integrations_names_message = \
'New Integrations Since Last Release:\n{}\n'.format('\n'.join(new_integrations_names))
prints_manager.add_print_job(new_integrations_names_message, print_warning, 0)
if modified_integrations_files:
modified_integrations_names = get_integration_names_from_files(modified_integrations_files)
modified_integrations_names_message = \
'Updated Integrations Since Last Release:\n{}\n'.format('\n'.join(modified_integrations_names))
prints_manager.add_print_job(modified_integrations_names_message, print_warning, 0)
prints_manager.execute_thread_prints(0)
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integrations = get_integrations_for_test(test, skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
prints_manager.add_print_job('All Integrations for test "{}":'.format(test.get('playbookID')), print_warning, 0)
prints_manager.add_print_job(integrations_names, print_warning, 0)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names
)
integrations_msg = '\n'.join(['"{}" - {}'.format(key, val) for key, val in integration_to_status.items()])
prints_manager.add_print_job('{}\n'.format(integrations_msg), print_warning, 0)
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf,
placeholders_map)
ints_to_configure_params_set = set_integration_params(integrations_to_configure, secret_params,
instance_names_conf, placeholders_map)
if not new_ints_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(new_integrations)), print_error, 0)
if not ints_to_configure_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(integrations_to_configure)),
print_error, 0)
if not (new_ints_params_set and ints_to_configure_params_set):
continue
prints_manager.execute_thread_prints(0)
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if module_instance:
module_instances.append(module_instance)
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
preupdate_success = set()
postupdate_success = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) prior to Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Pre-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
else:
preupdate_success.add((instance_name, integration_of_instance))
if LooseVersion(server_numeric_version) < LooseVersion('6.0.0'):
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(servers))
# For each server url we install content
for thread_index, server_url in enumerate(servers):
client = demisto_client.configure(base_url=server_url, username=username,
password=password, verify_ssl=False)
t = Thread(target=update_content_on_demisto_instance,
kwargs={'client': client, 'server': server_url, 'ami_name': ami_env,
'prints_manager': threads_prints_manager,
'thread_index': thread_index})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_integration_module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if new_integration_module_instance:
new_integration_module_instances.append(new_integration_module_instance)
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) after the Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Post-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
else:
postupdate_success.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False)
__disable_integrations_instances(client, all_module_instances, prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails, preupdate_success, postupdate_success,
new_integrations_names, prints_manager)
prints_manager.execute_thread_prints(0)
if not success or not installed_content_packs_successfully:
sys.exit(2)
