def manage_tests(tests_settings: SettingsTester):
"""
This function manages the execution of Demisto's tests.
Args:
tests_settings (SettingsTester): An object containing all the relevant data regarding how the
tests should be ran.
"""
tests_settings.serverNumericVersion = get_server_numeric_version(tests_settings.serverVersion,
tests_settings.is_local_run)
instances_ips = get_instances_ips_and_names(tests_settings)
number_of_instances = len(instances_ips)
prints_manager = ParallelPrintsManager(number_of_instances)
tests_data_keeper = DataKeeperTester()
for ami_instance_name, ami_instance_ip in instances_ips:
if ami_instance_name == tests_settings.serverVersion:
print_color("Starting private testing for {}".format(ami_instance_name), LOG_COLORS.GREEN)
print("Starts tests with server url - https://{}".format(ami_instance_ip))
all_tests = get_all_tests(tests_settings)
execute_testing(tests_settings, ami_instance_ip, all_tests, tests_data_keeper,
prints_manager, thread_index=0)
sleep(8)
print_test_summary(tests_data_keeper, tests_settings.isAMI)
create_result_files(tests_data_keeper)
if tests_data_keeper.failed_playbooks:
tests_failed_msg = "Some tests have failed. Not destroying instances."
print(tests_failed_msg)
sys.exit(1)
def manage_tests(tests_settings: SettingsTester):
"""
This function manages the execution of Demisto's tests.
Args:
tests_settings (SettingsTester): An object containing all the relevant data regarding how the
tests should be ran.
"""
tests_settings.serverNumericVersion = get_server_numeric_version(
tests_settings.serverVersion, tests_settings.is_local_run)
instances_ips = get_instances_ips_and_names(tests_settings)
tests_data_keeper = DataKeeperTester()
for ami_instance_name, ami_instance_ip in instances_ips:
if ami_instance_name == tests_settings.serverVersion:
logging.info(f"Starting private testing for {ami_instance_name}")
logging.info(
f"Starts tests with server url - https://{ami_instance_ip}")
all_tests = get_all_tests(tests_settings)
execute_testing(tests_settings, ami_instance_ip, all_tests,
tests_data_keeper)
sleep(8)
print_test_summary(tests_data_keeper, tests_settings.isAMI)
create_result_files(tests_data_keeper)
if tests_data_keeper.failed_playbooks:
tests_failed_msg = "Some tests have failed. Not destroying instances."
print(tests_failed_msg)
sys.exit(1)
def main():
options = options_handler()
username = options.user
password = options.password
ami_env = options.ami_env
git_sha1 = options.git_sha1
conf_path = options.conf
secret_conf_path = options.secret
branch_name = options.branch
ci_build_number = options.build_number
servers = determine_servers_urls(ami_env)
server_numeric_version = get_server_numeric_version(ami_env)
prints_manager = ParallelPrintsManager(1)
conf, secret_conf = load_conf_files(conf_path, secret_conf_path)
secret_params = secret_conf.get('integrations', []) if secret_conf else []
username = secret_conf.get('username') if not username else username
password = secret_conf.get('userPassword') if not password else password
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
for server in servers:
client = demisto_client.configure(base_url=server, username=username, password=password,
verify_ssl=False)
set_marketplace_gcp_bucket_for_build(client, prints_manager, branch_name, ci_build_number)
print('Restarting servers to apply GCS server config ...')
ssh_string = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {}@{} ' \
'"sudo systemctl restart demisto"'
try:
subprocess.check_output(
ssh_string.format('ec2-user', server.replace('https://', '')), shell=True)
except subprocess.CalledProcessError as exc:
print(exc.output)
print('Done restarting servers.')
tests = conf['tests']
skipped_integrations_conf = conf['skipped_integrations']
all_module_instances = []
filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(is_nightly=options.is_nightly)
tests_for_iteration = tests
if run_all_tests:
# skip test button testing
skipped_instance_test_message = 'Not running instance tests when {} is turned on'.format(RUN_ALL_TESTS_FORMAT)
prints_manager.add_print_job(skipped_instance_test_message, print_warning, 0)
tests_for_iteration = []
elif filter_configured and filtered_tests:
tests_for_iteration = [test for test in tests if test.get('playbookID', '') in filtered_tests]
tests_for_iteration = filter_tests_with_incompatible_version(tests_for_iteration, server_numeric_version,
prints_manager)
prints_manager.execute_thread_prints(0)
# get a list of brand new integrations that way we filter them out to only configure instances
# after updating content
new_integrations_files, modified_integrations_files = get_new_and_modified_integration_files(git_sha1)
new_integrations_names, modified_integrations_names = [], []
installed_content_packs_successfully = True
if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'):
# sleep for one minute before starting to search and install packs to ensure bucket is ready
prints_manager.add_print_job('Sleeping for 1 minute...', print_warning, 0)
prints_manager.execute_thread_prints(0)
sleep(60)
pack_ids = get_pack_ids_to_install()
# install content packs in every server
for server_url in servers:
try:
client = demisto_client.configure(base_url=server_url, username=username, password=password,
verify_ssl=False)
search_and_install_packs_and_their_dependencies(pack_ids, client, prints_manager, options.is_nightly)
except Exception as exc:
prints_manager.add_print_job(str(exc), print_error, 0)
prints_manager.execute_thread_prints(0)
installed_content_packs_successfully = False
if new_integrations_files:
new_integrations_names = get_integration_names_from_files(new_integrations_files)
new_integrations_names_message = \
'New Integrations Since Last Release:\n{}\n'.format('\n'.join(new_integrations_names))
prints_manager.add_print_job(new_integrations_names_message, print_warning, 0)
if modified_integrations_files:
modified_integrations_names = get_integration_names_from_files(modified_integrations_files)
modified_integrations_names_message = \
'Updated Integrations Since Last Release:\n{}\n'.format('\n'.join(modified_integrations_names))
prints_manager.add_print_job(modified_integrations_names_message, print_warning, 0)
prints_manager.execute_thread_prints(0)
# Each test is a dictionary from Tests/conf.json which may contain the following fields
# "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion"
# Note that only the "playbookID" field is required with all of the others being optional.
# Most tests have an "integrations" field listing the integration used for that playbook
# and sometimes an "instance_names" field which is used when there are multiple instances
# of an integration that we want to configure with different configuration values. Look at
# [conf.json](../conf.json) for examples
brand_new_integrations = []
for test in tests_for_iteration:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integrations = get_integrations_for_test(test, skipped_integrations_conf)
instance_names_conf = test.get('instance_names', [])
if not isinstance(instance_names_conf, list):
instance_names_conf = [instance_names_conf]
integrations_names = [i.get('name') for i in integrations]
prints_manager.add_print_job('All Integrations for test "{}":'.format(test.get('playbookID')), print_warning, 0)
prints_manager.add_print_job(integrations_names, print_warning, 0)
new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations(
integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names
)
integrations_msg = '\n'.join(['"{}" - {}'.format(key, val) for key, val in integration_to_status.items()])
prints_manager.add_print_job('{}\n'.format(integrations_msg), print_warning, 0)
integrations_to_configure = modified_integrations[:]
integrations_to_configure.extend(unchanged_integrations)
# set params for new integrations and [modified + unchanged] integrations, then add the new ones
# to brand_new_integrations list for later use
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf,
placeholders_map)
ints_to_configure_params_set = set_integration_params(integrations_to_configure, secret_params,
instance_names_conf, placeholders_map)
if not new_ints_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(new_integrations)), print_error, 0)
if not ints_to_configure_params_set:
prints_manager.add_print_job(
'failed setting parameters for integrations "{}"'.format('\n'.join(integrations_to_configure)),
print_error, 0)
if not (new_ints_params_set and ints_to_configure_params_set):
continue
prints_manager.execute_thread_prints(0)
brand_new_integrations.extend(new_integrations)
module_instances = []
for integration in integrations_to_configure:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if module_instance:
module_instances.append(module_instance)
all_module_instances.extend(module_instances)
preupdate_fails = set()
postupdate_fails = set()
preupdate_success = set()
postupdate_success = set()
# Test all module instances (of modified + unchanged integrations) pre-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) prior to Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Pre-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password,
verify_ssl=False)
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
preupdate_fails.add((instance_name, integration_of_instance))
else:
preupdate_success.add((instance_name, integration_of_instance))
if LooseVersion(server_numeric_version) < LooseVersion('6.0.0'):
threads_list = []
threads_prints_manager = ParallelPrintsManager(len(servers))
# For each server url we install content
for thread_index, server_url in enumerate(servers):
client = demisto_client.configure(base_url=server_url, username=username,
password=password, verify_ssl=False)
t = Thread(target=update_content_on_demisto_instance,
kwargs={'client': client, 'server': server_url, 'ami_name': ami_env,
'prints_manager': threads_prints_manager,
'thread_index': thread_index})
threads_list.append(t)
run_threads_list(threads_list)
# configure instances for new integrations
new_integration_module_instances = []
for integration in brand_new_integrations:
placeholders_map = {'%%SERVER_HOST%%': servers[0]}
new_integration_module_instance = configure_integration_instance(integration, testing_client, prints_manager,
placeholders_map)
if new_integration_module_instance:
new_integration_module_instances.append(new_integration_module_instance)
all_module_instances.extend(new_integration_module_instances)
# After content upload has completed - test ("Test" button) integration instances
# Test all module instances (of pre-existing AND new integrations) post-updating content
if all_module_instances:
# only print start message if there are instances to configure
prints_manager.add_print_job('Start of Instance Testing ("Test" button) after the Content Update:',
print_warning, 0)
else:
prints_manager.add_print_job('No integrations to configure for the chosen tests. (Post-update)',
print_warning, 0)
prints_manager.execute_thread_prints(0)
for instance in all_module_instances:
integration_of_instance = instance.get('brand', '')
instance_name = instance.get('name', '')
msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(instance_name,
integration_of_instance)
prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN)
prints_manager.execute_thread_prints(0)
# If there is a failure, __test_integration_instance will print it
success, _ = __test_integration_instance(testing_client, instance, prints_manager)
prints_manager.execute_thread_prints(0)
if not success:
postupdate_fails.add((instance_name, integration_of_instance))
else:
postupdate_success.add((instance_name, integration_of_instance))
# reinitialize all clients since their authorization has probably expired by now
for server_url in servers:
client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False)
__disable_integrations_instances(client, all_module_instances, prints_manager)
prints_manager.execute_thread_prints(0)
success = report_tests_status(preupdate_fails, postupdate_fails, preupdate_success, postupdate_success,
new_integrations_names, prints_manager)
prints_manager.execute_thread_prints(0)
if not success or not installed_content_packs_successfully:
sys.exit(2)