def main(): options = options_handler() username = options.user password = options.password ami_env = options.ami_env git_sha1 = options.git_sha1 servers = determine_servers_urls(ami_env) conf_path = options.conf secret_conf_path = options.secret prints_manager = ParallelPrintsManager(1) server_numeric_version = get_server_numeric_version(ami_env) conf, secret_conf = load_conf_files(conf_path, secret_conf_path) secret_params = secret_conf.get('integrations', []) if secret_conf else [] username = secret_conf.get('username') if not username else username password = secret_conf.get('userPassword') if not password else password clients = [] for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) clients.append(client) testing_client = clients[0] tests = conf['tests'] skipped_integrations_conf = conf['skipped_integrations'] all_module_instances = [] filtered_tests, filter_configured, run_all_tests = extract_filtered_tests( is_nightly=False) tests_for_iteration = tests if run_all_tests: # Use all tests for testing, leave 'tests_for_iteration' as is pass elif filter_configured and filtered_tests: tests_for_iteration = [ test for test in tests if test.get('playbookID', '') in filtered_tests ] tests_for_iteration = filter_tests_with_incompatible_version( tests_for_iteration, server_numeric_version, prints_manager) # get a list of brand new integrations that way we filter them out to only configure instances # after updating content new_integrations_names, modified_integrations_names = get_new_and_modified_integrations( git_sha1) if new_integrations_names: print_warning('New Integrations Since Last Release:\n{}\n'.format( '\n'.join(new_integrations_names))) if modified_integrations_names: print_warning('Updated Integrations Since Last Release:\n{}\n'.format( '\n'.join(modified_integrations_names))) # Each test is a dictionary from Tests/conf.json which may contain the following fields # "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion" # Note that only the "playbookID" field is required with all of the others being optional. # Most tests have an "integrations" field listing the integration used for that playbook # and sometimes an "instance_names" field which is used when there are multiple instances # of an integration that we want to configure with different configuration values. Look at # [conf.json](../conf.json) for examples brand_new_integrations = [] for test in tests_for_iteration: integrations = get_integrations_for_test(test, skipped_integrations_conf) instance_names_conf = test.get('instance_names', []) if not isinstance(instance_names_conf, list): instance_names_conf = [instance_names_conf] integrations_names = [i.get('name') for i in integrations] print_warning('All Integrations for test "{}":'.format( test.get('playbookID'))) print_warning(integrations_names) new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations( integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names) integrations_msg = '\n'.join([ '"{}" - {}'.format(key, val) for key, val in integration_to_status.items() ]) print_warning('{}\n'.format(integrations_msg)) integrations_to_configure = modified_integrations[:] integrations_to_configure.extend(unchanged_integrations) # set params for new integrations and [modified + unchanged] integrations, then add the new ones # to brand_new_integrations list for later use new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf) ints_to_configure_params_set = set_integration_params( integrations_to_configure, secret_params, instance_names_conf) if not new_ints_params_set: print_error( 'failed setting parameters for integrations "{}"'.format( '\n'.join(new_integrations))) if not ints_to_configure_params_set: print_error( 'failed setting parameters for integrations "{}"'.format( '\n'.join(integrations_to_configure))) if not (new_ints_params_set and ints_to_configure_params_set): continue brand_new_integrations.extend(new_integrations) module_instances = [] for integration in integrations_to_configure: module_instances.append( configure_integration_instance(integration, testing_client, prints_manager)) all_module_instances.extend(module_instances) preupdate_fails = set() postupdate_fails = set() # Test all module instances (of modified + unchanged integrations) pre-updating content if all_module_instances: # only print start message if there are instances to configure print_warning( 'Start of Instance Testing ("Test" button) prior to Content Update:' ) else: print_warning( 'No integrations to configure for the chosen tests. (Pre-update)') for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format( instance_name, integration_of_instance) print(msg) # If there is a failure, __test_integration_instance will print it success = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: preupdate_fails.add((instance_name, integration_of_instance)) threads_list = [] threads_prints_manager = ParallelPrintsManager(len(clients)) # For each server url we install content for thread_index, (client, server_url) in enumerate(zip(clients, servers)): t = Thread(target=update_content_on_demisto_instance, kwargs={ 'client': client, 'server': server_url, 'prints_manager': threads_prints_manager, 'thread_index': thread_index }) threads_list.append(t) run_threads_list(threads_list) # configure instances for new integrations new_integration_module_instances = [] for integration in brand_new_integrations: new_integration_module_instances.append( configure_integration_instance(integration, testing_client, prints_manager)) all_module_instances.extend(new_integration_module_instances) # After content upload has completed - test ("Test" button) integration instances # Test all module instances (of pre-existing AND new integrations) post-updating content if all_module_instances: print_warning( 'Start of Instance Testing ("Test" button) after the Content Update:' ) else: print_warning( 'No integrations to configure for the chosen tests. (Post-update)') for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format( instance_name, integration_of_instance) print(msg) # If there is a failure, __test_integration_instance will print it success = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: postupdate_fails.add((instance_name, integration_of_instance)) # reinitialize all clients since their authorization has probably expired by now for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) __disable_integrations_instances(client, all_module_instances, prints_manager) prints_manager.execute_thread_prints(0) success = report_tests_status(preupdate_fails, postupdate_fails, new_integrations_names) if not success: sys.exit(1)
def execute_testing(tests_settings: SettingsTester, server_ip: str, all_tests: set, tests_data_keeper: DataKeeperTester, prints_manager: ParallelPrintsManager, thread_index: int = 0): """ Main function used to handle the testing process. Starts by turning off telemetry and disabling any left over tests. Afterwards it will create a test queue object which then is used to run the specific test scenario. :param tests_settings: SettingsTester object which contains the test variables :param server_ip: IP address of the server. Will be formatted before use. :param all_tests: All tests currently in the test conf. :param tests_data_keeper: Object containing all the test results. Used by report tests function. :param prints_manager: PrintsManager object used in reporting. Will be deprecated. :param thread_index: Integer indicating what thread the test is running on. :return: No object is returned, just updates the tests_data_keep object. """ server = SERVER_URL.format(server_ip) server_numeric_version = tests_settings.serverNumericVersion start_message = "Executing tests with the server {} - and the server ip {}".format( server, server_ip) prints_manager.add_print_job(start_message, print, thread_index) slack = tests_settings.slack circle_ci = tests_settings.circleci build_number = tests_settings.buildNumber build_name = tests_settings.buildName conf, secret_conf = load_conf_files(tests_settings.conf_path, tests_settings.secret_conf_path) demisto_api_key = tests_settings.api_key demisto_user = secret_conf['username'] demisto_pass = secret_conf['userPassword'] default_test_timeout = conf.get('testTimeout', 30) tests = conf['tests'] skipped_tests_conf = conf['skipped_tests'] nightly_integrations = conf['nightly_integrations'] skipped_integrations_conf = conf['skipped_integrations'] unmockable_integrations = conf['unmockable_integrations'] secret_params = secret_conf['integrations'] if secret_conf else [] filtered_tests = extract_filtered_tests() if not tests or len(tests) == 0: prints_manager.add_print_job('no integrations are configured for test', print, thread_index) prints_manager.execute_thread_prints(thread_index) return xsoar_client = demisto_client.configure(base_url=server, username=demisto_user, password=demisto_pass, verify_ssl=False) # turn off telemetry turn_off_telemetry(xsoar_client) failed_playbooks = [] succeed_playbooks = [] skipped_tests = set([]) skipped_integration = set([]) playbook_skipped_integration = set([]) disable_all_integrations(xsoar_client, prints_manager, thread_index=thread_index) prints_manager.execute_thread_prints(thread_index) # Private builds do not use mocking. Here we copy the mocked test list to the unmockable list. private_tests = get_test_records_of_given_test_names( tests_settings, all_tests) try: # first run the mock tests to avoid mockless side effects in container prints_manager.add_print_job("\nRunning private tests", print, thread_index) executed_in_current_round, private_tests_queue = initialize_queue_and_executed_tests_set( private_tests) while not private_tests_queue.empty(): t = private_tests_queue.get() executed_in_current_round = update_round_set_and_sleep_if_round_completed( executed_in_current_round, prints_manager, t, thread_index) run_private_test_scenario(tests_settings, t, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, filtered_tests, skipped_tests, secret_params, failed_playbooks, playbook_skipped_integration, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, demisto_user, demisto_pass, demisto_api_key, prints_manager, thread_index=thread_index) prints_manager.execute_thread_prints(thread_index) except Exception as exc: if exc.__class__ == ApiException: error_message = exc.body else: error_message = f'~~ Thread {thread_index + 1} failed ~~\n{str(exc)}\n{traceback.format_exc()}' prints_manager.add_print_job(error_message, print_error, thread_index) prints_manager.execute_thread_prints(thread_index) failed_playbooks.append(f'~~ Thread {thread_index + 1} failed ~~') raise finally: tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration, unmockable_integrations)
def main(): options = options_handler() username = options.user password = options.password ami_env = options.ami_env git_sha1 = options.git_sha1 conf_path = options.conf secret_conf_path = options.secret branch_name = options.branch ci_build_number = options.build_number servers = determine_servers_urls(ami_env) server_numeric_version = get_server_numeric_version(ami_env) prints_manager = ParallelPrintsManager(1) conf, secret_conf = load_conf_files(conf_path, secret_conf_path) secret_params = secret_conf.get('integrations', []) if secret_conf else [] username = secret_conf.get('username') if not username else username password = secret_conf.get('userPassword') if not password else password if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'): for server in servers: client = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) set_marketplace_gcp_bucket_for_build(client, prints_manager, branch_name, ci_build_number) print('Restarting servers to apply GCS server config ...') ssh_string = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {}@{} ' \ '"sudo systemctl restart demisto"' try: subprocess.check_output( ssh_string.format('ec2-user', server.replace('https://', '')), shell=True) except subprocess.CalledProcessError as exc: print(exc.output) print('Done restarting servers.') tests = conf['tests'] skipped_integrations_conf = conf['skipped_integrations'] all_module_instances = [] filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(is_nightly=options.is_nightly) tests_for_iteration = tests if run_all_tests: # skip test button testing skipped_instance_test_message = 'Not running instance tests when {} is turned on'.format(RUN_ALL_TESTS_FORMAT) prints_manager.add_print_job(skipped_instance_test_message, print_warning, 0) tests_for_iteration = [] elif filter_configured and filtered_tests: tests_for_iteration = [test for test in tests if test.get('playbookID', '') in filtered_tests] tests_for_iteration = filter_tests_with_incompatible_version(tests_for_iteration, server_numeric_version, prints_manager) prints_manager.execute_thread_prints(0) # get a list of brand new integrations that way we filter them out to only configure instances # after updating content new_integrations_files, modified_integrations_files = get_new_and_modified_integration_files(git_sha1) new_integrations_names, modified_integrations_names = [], [] installed_content_packs_successfully = True if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'): # sleep for one minute before starting to search and install packs to ensure bucket is ready prints_manager.add_print_job('Sleeping for 1 minute...', print_warning, 0) prints_manager.execute_thread_prints(0) sleep(60) pack_ids = get_pack_ids_to_install() # install content packs in every server for server_url in servers: try: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) search_and_install_packs_and_their_dependencies(pack_ids, client, prints_manager, options.is_nightly) except Exception as exc: prints_manager.add_print_job(str(exc), print_error, 0) prints_manager.execute_thread_prints(0) installed_content_packs_successfully = False if new_integrations_files: new_integrations_names = get_integration_names_from_files(new_integrations_files) new_integrations_names_message = \ 'New Integrations Since Last Release:\n{}\n'.format('\n'.join(new_integrations_names)) prints_manager.add_print_job(new_integrations_names_message, print_warning, 0) if modified_integrations_files: modified_integrations_names = get_integration_names_from_files(modified_integrations_files) modified_integrations_names_message = \ 'Updated Integrations Since Last Release:\n{}\n'.format('\n'.join(modified_integrations_names)) prints_manager.add_print_job(modified_integrations_names_message, print_warning, 0) prints_manager.execute_thread_prints(0) # Each test is a dictionary from Tests/conf.json which may contain the following fields # "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion" # Note that only the "playbookID" field is required with all of the others being optional. # Most tests have an "integrations" field listing the integration used for that playbook # and sometimes an "instance_names" field which is used when there are multiple instances # of an integration that we want to configure with different configuration values. Look at # [conf.json](../conf.json) for examples brand_new_integrations = [] for test in tests_for_iteration: testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password, verify_ssl=False) integrations = get_integrations_for_test(test, skipped_integrations_conf) instance_names_conf = test.get('instance_names', []) if not isinstance(instance_names_conf, list): instance_names_conf = [instance_names_conf] integrations_names = [i.get('name') for i in integrations] prints_manager.add_print_job('All Integrations for test "{}":'.format(test.get('playbookID')), print_warning, 0) prints_manager.add_print_job(integrations_names, print_warning, 0) new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations( integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names ) integrations_msg = '\n'.join(['"{}" - {}'.format(key, val) for key, val in integration_to_status.items()]) prints_manager.add_print_job('{}\n'.format(integrations_msg), print_warning, 0) integrations_to_configure = modified_integrations[:] integrations_to_configure.extend(unchanged_integrations) # set params for new integrations and [modified + unchanged] integrations, then add the new ones # to brand_new_integrations list for later use placeholders_map = {'%%SERVER_HOST%%': servers[0]} new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf, placeholders_map) ints_to_configure_params_set = set_integration_params(integrations_to_configure, secret_params, instance_names_conf, placeholders_map) if not new_ints_params_set: prints_manager.add_print_job( 'failed setting parameters for integrations "{}"'.format('\n'.join(new_integrations)), print_error, 0) if not ints_to_configure_params_set: prints_manager.add_print_job( 'failed setting parameters for integrations "{}"'.format('\n'.join(integrations_to_configure)), print_error, 0) if not (new_ints_params_set and ints_to_configure_params_set): continue prints_manager.execute_thread_prints(0) brand_new_integrations.extend(new_integrations) module_instances = [] for integration in integrations_to_configure: placeholders_map = {'%%SERVER_HOST%%': servers[0]} module_instance = configure_integration_instance(integration, testing_client, prints_manager, placeholders_map) if module_instance: module_instances.append(module_instance) all_module_instances.extend(module_instances) preupdate_fails = set() postupdate_fails = set() preupdate_success = set() postupdate_success = set() # Test all module instances (of modified + unchanged integrations) pre-updating content if all_module_instances: # only print start message if there are instances to configure prints_manager.add_print_job('Start of Instance Testing ("Test" button) prior to Content Update:', print_warning, 0) else: prints_manager.add_print_job('No integrations to configure for the chosen tests. (Pre-update)', print_warning, 0) prints_manager.execute_thread_prints(0) for instance in all_module_instances: testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password, verify_ssl=False) integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(instance_name, integration_of_instance) prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN) prints_manager.execute_thread_prints(0) # If there is a failure, __test_integration_instance will print it success, _ = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: preupdate_fails.add((instance_name, integration_of_instance)) else: preupdate_success.add((instance_name, integration_of_instance)) if LooseVersion(server_numeric_version) < LooseVersion('6.0.0'): threads_list = [] threads_prints_manager = ParallelPrintsManager(len(servers)) # For each server url we install content for thread_index, server_url in enumerate(servers): client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) t = Thread(target=update_content_on_demisto_instance, kwargs={'client': client, 'server': server_url, 'ami_name': ami_env, 'prints_manager': threads_prints_manager, 'thread_index': thread_index}) threads_list.append(t) run_threads_list(threads_list) # configure instances for new integrations new_integration_module_instances = [] for integration in brand_new_integrations: placeholders_map = {'%%SERVER_HOST%%': servers[0]} new_integration_module_instance = configure_integration_instance(integration, testing_client, prints_manager, placeholders_map) if new_integration_module_instance: new_integration_module_instances.append(new_integration_module_instance) all_module_instances.extend(new_integration_module_instances) # After content upload has completed - test ("Test" button) integration instances # Test all module instances (of pre-existing AND new integrations) post-updating content if all_module_instances: # only print start message if there are instances to configure prints_manager.add_print_job('Start of Instance Testing ("Test" button) after the Content Update:', print_warning, 0) else: prints_manager.add_print_job('No integrations to configure for the chosen tests. (Post-update)', print_warning, 0) prints_manager.execute_thread_prints(0) for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(instance_name, integration_of_instance) prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN) prints_manager.execute_thread_prints(0) # If there is a failure, __test_integration_instance will print it success, _ = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: postupdate_fails.add((instance_name, integration_of_instance)) else: postupdate_success.add((instance_name, integration_of_instance)) # reinitialize all clients since their authorization has probably expired by now for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) __disable_integrations_instances(client, all_module_instances, prints_manager) prints_manager.execute_thread_prints(0) success = report_tests_status(preupdate_fails, postupdate_fails, preupdate_success, postupdate_success, new_integrations_names, prints_manager) prints_manager.execute_thread_prints(0) if not success or not installed_content_packs_successfully: sys.exit(2)