def main(): options = options_handler() username = options.user password = options.password ami_env = options.ami_env git_sha1 = options.git_sha1 servers = determine_servers_urls(ami_env) conf_path = options.conf secret_conf_path = options.secret prints_manager = ParallelPrintsManager(1) server_numeric_version = get_server_numeric_version(ami_env) conf, secret_conf = load_conf_files(conf_path, secret_conf_path) secret_params = secret_conf.get('integrations', []) if secret_conf else [] username = secret_conf.get('username') if not username else username password = secret_conf.get('userPassword') if not password else password clients = [] for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) clients.append(client) testing_client = clients[0] tests = conf['tests'] skipped_integrations_conf = conf['skipped_integrations'] all_module_instances = [] filtered_tests, filter_configured, run_all_tests = extract_filtered_tests( is_nightly=False) tests_for_iteration = tests if run_all_tests: # Use all tests for testing, leave 'tests_for_iteration' as is pass elif filter_configured and filtered_tests: tests_for_iteration = [ test for test in tests if test.get('playbookID', '') in filtered_tests ] tests_for_iteration = filter_tests_with_incompatible_version( tests_for_iteration, server_numeric_version, prints_manager) # get a list of brand new integrations that way we filter them out to only configure instances # after updating content new_integrations_names, modified_integrations_names = get_new_and_modified_integrations( git_sha1) if new_integrations_names: print_warning('New Integrations Since Last Release:\n{}\n'.format( '\n'.join(new_integrations_names))) if modified_integrations_names: print_warning('Updated Integrations Since Last Release:\n{}\n'.format( '\n'.join(modified_integrations_names))) # Each test is a dictionary from Tests/conf.json which may contain the following fields # "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion" # Note that only the "playbookID" field is required with all of the others being optional. # Most tests have an "integrations" field listing the integration used for that playbook # and sometimes an "instance_names" field which is used when there are multiple instances # of an integration that we want to configure with different configuration values. Look at # [conf.json](../conf.json) for examples brand_new_integrations = [] for test in tests_for_iteration: integrations = get_integrations_for_test(test, skipped_integrations_conf) instance_names_conf = test.get('instance_names', []) if not isinstance(instance_names_conf, list): instance_names_conf = [instance_names_conf] integrations_names = [i.get('name') for i in integrations] print_warning('All Integrations for test "{}":'.format( test.get('playbookID'))) print_warning(integrations_names) new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations( integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names) integrations_msg = '\n'.join([ '"{}" - {}'.format(key, val) for key, val in integration_to_status.items() ]) print_warning('{}\n'.format(integrations_msg)) integrations_to_configure = modified_integrations[:] integrations_to_configure.extend(unchanged_integrations) # set params for new integrations and [modified + unchanged] integrations, then add the new ones # to brand_new_integrations list for later use new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf) ints_to_configure_params_set = set_integration_params( integrations_to_configure, secret_params, instance_names_conf) if not new_ints_params_set: print_error( 'failed setting parameters for integrations "{}"'.format( '\n'.join(new_integrations))) if not ints_to_configure_params_set: print_error( 'failed setting parameters for integrations "{}"'.format( '\n'.join(integrations_to_configure))) if not (new_ints_params_set and ints_to_configure_params_set): continue brand_new_integrations.extend(new_integrations) module_instances = [] for integration in integrations_to_configure: module_instances.append( configure_integration_instance(integration, testing_client, prints_manager)) all_module_instances.extend(module_instances) preupdate_fails = set() postupdate_fails = set() # Test all module instances (of modified + unchanged integrations) pre-updating content if all_module_instances: # only print start message if there are instances to configure print_warning( 'Start of Instance Testing ("Test" button) prior to Content Update:' ) else: print_warning( 'No integrations to configure for the chosen tests. (Pre-update)') for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format( instance_name, integration_of_instance) print(msg) # If there is a failure, __test_integration_instance will print it success = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: preupdate_fails.add((instance_name, integration_of_instance)) threads_list = [] threads_prints_manager = ParallelPrintsManager(len(clients)) # For each server url we install content for thread_index, (client, server_url) in enumerate(zip(clients, servers)): t = Thread(target=update_content_on_demisto_instance, kwargs={ 'client': client, 'server': server_url, 'prints_manager': threads_prints_manager, 'thread_index': thread_index }) threads_list.append(t) run_threads_list(threads_list) # configure instances for new integrations new_integration_module_instances = [] for integration in brand_new_integrations: new_integration_module_instances.append( configure_integration_instance(integration, testing_client, prints_manager)) all_module_instances.extend(new_integration_module_instances) # After content upload has completed - test ("Test" button) integration instances # Test all module instances (of pre-existing AND new integrations) post-updating content if all_module_instances: print_warning( 'Start of Instance Testing ("Test" button) after the Content Update:' ) else: print_warning( 'No integrations to configure for the chosen tests. (Post-update)') for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format( instance_name, integration_of_instance) print(msg) # If there is a failure, __test_integration_instance will print it success = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: postupdate_fails.add((instance_name, integration_of_instance)) # reinitialize all clients since their authorization has probably expired by now for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) __disable_integrations_instances(client, all_module_instances, prints_manager) prints_manager.execute_thread_prints(0) success = report_tests_status(preupdate_fails, postupdate_fails, new_integrations_names) if not success: sys.exit(1)
def main(): options = options_handler() username = options.user password = options.password ami_env = options.ami_env git_sha1 = options.git_sha1 conf_path = options.conf secret_conf_path = options.secret branch_name = options.branch ci_build_number = options.build_number servers = determine_servers_urls(ami_env) server_numeric_version = get_server_numeric_version(ami_env) prints_manager = ParallelPrintsManager(1) conf, secret_conf = load_conf_files(conf_path, secret_conf_path) secret_params = secret_conf.get('integrations', []) if secret_conf else [] username = secret_conf.get('username') if not username else username password = secret_conf.get('userPassword') if not password else password if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'): for server in servers: client = demisto_client.configure(base_url=server, username=username, password=password, verify_ssl=False) set_marketplace_gcp_bucket_for_build(client, prints_manager, branch_name, ci_build_number) print('Restarting servers to apply GCS server config ...') ssh_string = 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {}@{} ' \ '"sudo systemctl restart demisto"' try: subprocess.check_output( ssh_string.format('ec2-user', server.replace('https://', '')), shell=True) except subprocess.CalledProcessError as exc: print(exc.output) print('Done restarting servers.') tests = conf['tests'] skipped_integrations_conf = conf['skipped_integrations'] all_module_instances = [] filtered_tests, filter_configured, run_all_tests = extract_filtered_tests(is_nightly=options.is_nightly) tests_for_iteration = tests if run_all_tests: # skip test button testing skipped_instance_test_message = 'Not running instance tests when {} is turned on'.format(RUN_ALL_TESTS_FORMAT) prints_manager.add_print_job(skipped_instance_test_message, print_warning, 0) tests_for_iteration = [] elif filter_configured and filtered_tests: tests_for_iteration = [test for test in tests if test.get('playbookID', '') in filtered_tests] tests_for_iteration = filter_tests_with_incompatible_version(tests_for_iteration, server_numeric_version, prints_manager) prints_manager.execute_thread_prints(0) # get a list of brand new integrations that way we filter them out to only configure instances # after updating content new_integrations_files, modified_integrations_files = get_new_and_modified_integration_files(git_sha1) new_integrations_names, modified_integrations_names = [], [] installed_content_packs_successfully = True if LooseVersion(server_numeric_version) >= LooseVersion('6.0.0'): # sleep for one minute before starting to search and install packs to ensure bucket is ready prints_manager.add_print_job('Sleeping for 1 minute...', print_warning, 0) prints_manager.execute_thread_prints(0) sleep(60) pack_ids = get_pack_ids_to_install() # install content packs in every server for server_url in servers: try: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) search_and_install_packs_and_their_dependencies(pack_ids, client, prints_manager, options.is_nightly) except Exception as exc: prints_manager.add_print_job(str(exc), print_error, 0) prints_manager.execute_thread_prints(0) installed_content_packs_successfully = False if new_integrations_files: new_integrations_names = get_integration_names_from_files(new_integrations_files) new_integrations_names_message = \ 'New Integrations Since Last Release:\n{}\n'.format('\n'.join(new_integrations_names)) prints_manager.add_print_job(new_integrations_names_message, print_warning, 0) if modified_integrations_files: modified_integrations_names = get_integration_names_from_files(modified_integrations_files) modified_integrations_names_message = \ 'Updated Integrations Since Last Release:\n{}\n'.format('\n'.join(modified_integrations_names)) prints_manager.add_print_job(modified_integrations_names_message, print_warning, 0) prints_manager.execute_thread_prints(0) # Each test is a dictionary from Tests/conf.json which may contain the following fields # "playbookID", "integrations", "instance_names", "timeout", "nightly", "fromversion", "toversion" # Note that only the "playbookID" field is required with all of the others being optional. # Most tests have an "integrations" field listing the integration used for that playbook # and sometimes an "instance_names" field which is used when there are multiple instances # of an integration that we want to configure with different configuration values. Look at # [conf.json](../conf.json) for examples brand_new_integrations = [] for test in tests_for_iteration: testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password, verify_ssl=False) integrations = get_integrations_for_test(test, skipped_integrations_conf) instance_names_conf = test.get('instance_names', []) if not isinstance(instance_names_conf, list): instance_names_conf = [instance_names_conf] integrations_names = [i.get('name') for i in integrations] prints_manager.add_print_job('All Integrations for test "{}":'.format(test.get('playbookID')), print_warning, 0) prints_manager.add_print_job(integrations_names, print_warning, 0) new_integrations, modified_integrations, unchanged_integrations, integration_to_status = group_integrations( integrations, skipped_integrations_conf, new_integrations_names, modified_integrations_names ) integrations_msg = '\n'.join(['"{}" - {}'.format(key, val) for key, val in integration_to_status.items()]) prints_manager.add_print_job('{}\n'.format(integrations_msg), print_warning, 0) integrations_to_configure = modified_integrations[:] integrations_to_configure.extend(unchanged_integrations) # set params for new integrations and [modified + unchanged] integrations, then add the new ones # to brand_new_integrations list for later use placeholders_map = {'%%SERVER_HOST%%': servers[0]} new_ints_params_set = set_integration_params(new_integrations, secret_params, instance_names_conf, placeholders_map) ints_to_configure_params_set = set_integration_params(integrations_to_configure, secret_params, instance_names_conf, placeholders_map) if not new_ints_params_set: prints_manager.add_print_job( 'failed setting parameters for integrations "{}"'.format('\n'.join(new_integrations)), print_error, 0) if not ints_to_configure_params_set: prints_manager.add_print_job( 'failed setting parameters for integrations "{}"'.format('\n'.join(integrations_to_configure)), print_error, 0) if not (new_ints_params_set and ints_to_configure_params_set): continue prints_manager.execute_thread_prints(0) brand_new_integrations.extend(new_integrations) module_instances = [] for integration in integrations_to_configure: placeholders_map = {'%%SERVER_HOST%%': servers[0]} module_instance = configure_integration_instance(integration, testing_client, prints_manager, placeholders_map) if module_instance: module_instances.append(module_instance) all_module_instances.extend(module_instances) preupdate_fails = set() postupdate_fails = set() preupdate_success = set() postupdate_success = set() # Test all module instances (of modified + unchanged integrations) pre-updating content if all_module_instances: # only print start message if there are instances to configure prints_manager.add_print_job('Start of Instance Testing ("Test" button) prior to Content Update:', print_warning, 0) else: prints_manager.add_print_job('No integrations to configure for the chosen tests. (Pre-update)', print_warning, 0) prints_manager.execute_thread_prints(0) for instance in all_module_instances: testing_client = demisto_client.configure(base_url=servers[0], username=username, password=password, verify_ssl=False) integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}".'.format(instance_name, integration_of_instance) prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN) prints_manager.execute_thread_prints(0) # If there is a failure, __test_integration_instance will print it success, _ = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: preupdate_fails.add((instance_name, integration_of_instance)) else: preupdate_success.add((instance_name, integration_of_instance)) if LooseVersion(server_numeric_version) < LooseVersion('6.0.0'): threads_list = [] threads_prints_manager = ParallelPrintsManager(len(servers)) # For each server url we install content for thread_index, server_url in enumerate(servers): client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) t = Thread(target=update_content_on_demisto_instance, kwargs={'client': client, 'server': server_url, 'ami_name': ami_env, 'prints_manager': threads_prints_manager, 'thread_index': thread_index}) threads_list.append(t) run_threads_list(threads_list) # configure instances for new integrations new_integration_module_instances = [] for integration in brand_new_integrations: placeholders_map = {'%%SERVER_HOST%%': servers[0]} new_integration_module_instance = configure_integration_instance(integration, testing_client, prints_manager, placeholders_map) if new_integration_module_instance: new_integration_module_instances.append(new_integration_module_instance) all_module_instances.extend(new_integration_module_instances) # After content upload has completed - test ("Test" button) integration instances # Test all module instances (of pre-existing AND new integrations) post-updating content if all_module_instances: # only print start message if there are instances to configure prints_manager.add_print_job('Start of Instance Testing ("Test" button) after the Content Update:', print_warning, 0) else: prints_manager.add_print_job('No integrations to configure for the chosen tests. (Post-update)', print_warning, 0) prints_manager.execute_thread_prints(0) for instance in all_module_instances: integration_of_instance = instance.get('brand', '') instance_name = instance.get('name', '') msg = 'Testing ("Test" button) for instance "{}" of integration "{}" .'.format(instance_name, integration_of_instance) prints_manager.add_print_job(msg, print_color, 0, LOG_COLORS.GREEN) prints_manager.execute_thread_prints(0) # If there is a failure, __test_integration_instance will print it success, _ = __test_integration_instance(testing_client, instance, prints_manager) prints_manager.execute_thread_prints(0) if not success: postupdate_fails.add((instance_name, integration_of_instance)) else: postupdate_success.add((instance_name, integration_of_instance)) # reinitialize all clients since their authorization has probably expired by now for server_url in servers: client = demisto_client.configure(base_url=server_url, username=username, password=password, verify_ssl=False) __disable_integrations_instances(client, all_module_instances, prints_manager) prints_manager.execute_thread_prints(0) success = report_tests_status(preupdate_fails, postupdate_fails, preupdate_success, postupdate_success, new_integrations_names, prints_manager) prints_manager.execute_thread_prints(0) if not success or not installed_content_packs_successfully: sys.exit(2)