def execute_testing(server, server_ip, server_version, server_numeric_version, is_ami=True): print("Executing tests with the server {} - and the server ip {}".format( server, server_ip)) options = options_handler() username = options.user password = options.password conf_path = options.conf secret_conf_path = options.secret is_nightly = options.nightly slack = options.slack circle_ci = options.circleci build_number = options.buildNumber build_name = options.buildName if not (username and password and server): print_error('You must provide server user & password arguments') sys.exit(1) c = demisto.DemistoClient(None, server, username, password) res = c.Login() if res.status_code != 200: print_error("Login has failed with status code " + str(res.status_code)) sys.exit(1) demisto_api_key = generate_demisto_api_key(c) conf, secret_conf = load_conf_files(conf_path, secret_conf_path) default_test_timeout = conf.get('testTimeout', 30) tests = conf['tests'] skipped_tests_conf = conf['skipped_tests'] nightly_integrations = conf['nigthly_integrations'] skipped_integrations_conf = conf['skipped_integrations'] unmockable_integrations = conf['unmockable_integrations'] secret_params = secret_conf['integrations'] if secret_conf else [] filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests( ) if is_filter_configured and not run_all_tests: is_nightly = True if not tests or len(tests) == 0: print('no integrations are configured for test') return proxy = None if is_ami: ami = AMIConnection(server_ip) ami.clone_mock_data() proxy = MITMProxy(c, server_ip) failed_playbooks = [] succeed_playbooks = [] skipped_tests = set([]) skipped_integration = set([]) disable_all_integrations(c) if is_ami: # move all mock tests to the top of the list mock_tests, mockless_tests = organize_tests(tests, unmockable_integrations, skipped_integrations_conf, nightly_integrations) else: # In case of a non AMI run we don't want to use the mocking mechanism mockless_tests = tests # first run the mock tests to avoid mockless side effects in container if is_ami and mock_tests: proxy.configure_proxy_in_demisto(proxy.ami.docker_ip + ':' + proxy.PROXY_PORT) for t in mock_tests: run_test_scenario(t, c, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, demisto_api_key, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version) print("\nRunning mock-disabled tests") proxy.configure_proxy_in_demisto('') print("Restarting demisto service") restart_demisto_service(ami, c) print("Demisto service restarted\n") for t in mockless_tests: run_test_scenario(t, c, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, demisto_api_key, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, is_ami) print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration, unmockable_integrations, proxy, is_ami) create_result_files(failed_playbooks, skipped_integration, skipped_tests) if is_ami and build_name == 'master': print("Pushing new/updated mock files to mock git repo.") ami.upload_mock_files(build_name, build_number) if len(failed_playbooks): file_path = "./Tests/is_build_failed_{}.txt".format( server_version.replace(' ', '')) with open(file_path, "w") as is_build_failed_file: is_build_failed_file.write('Build failed') sys.exit(1)
def execute_testing(server, server_ip, server_version, server_numeric_version, is_ami=True): print("Executing tests with the server {} - and the server ip {}".format(server, server_ip)) options = options_handler() conf_path = options.conf secret_conf_path = options.secret is_nightly = options.nightly is_memory_check = options.memCheck slack = options.slack circle_ci = options.circleci build_number = options.buildNumber build_name = options.buildName conf, secret_conf = load_conf_files(conf_path, secret_conf_path) demisto_api_key = secret_conf.get('temp_apikey') default_test_timeout = conf.get('testTimeout', 30) tests = conf['tests'] skipped_tests_conf = conf['skipped_tests'] nightly_integrations = conf['nightly_integrations'] skipped_integrations_conf = conf['skipped_integrations'] unmockable_integrations = conf['unmockable_integrations'] secret_params = secret_conf['integrations'] if secret_conf else [] filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests() if is_filter_configured and not run_all_tests: is_nightly = True if not tests or len(tests) == 0: print('no integrations are configured for test') return proxy = None if is_ami: ami = AMIConnection(server_ip) ami.clone_mock_data() proxy = MITMProxy(server_ip) failed_playbooks = [] succeed_playbooks = [] skipped_tests = set([]) skipped_integration = set([]) disable_all_integrations(demisto_api_key, server) if is_ami: # move all mock tests to the top of the list mock_tests, mockless_tests = organize_tests(tests, unmockable_integrations, skipped_integrations_conf, nightly_integrations) else: # In case of a non AMI run we don't want to use the mocking mechanism mockless_tests = tests if is_nightly and is_memory_check: mem_lim, err = get_docker_limit() send_slack_message(slack, SLACK_MEM_CHANNEL_ID, 'Build Number: {0}\n Server Address: {1}\nMemory Limit: {2}'.format(build_number, server, mem_lim), 'Content CircleCI', 'False') # first run the mock tests to avoid mockless side effects in container if is_ami and mock_tests: proxy.configure_proxy_in_demisto(demisto_api_key, server, proxy.ami.docker_ip + ':' + proxy.PROXY_PORT) for t in mock_tests: run_test_scenario(t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, demisto_api_key) print("\nRunning mock-disabled tests") proxy.configure_proxy_in_demisto(demisto_api_key, server, '') print("Restarting demisto service") restart_demisto_service(ami, demisto_api_key, server) print("Demisto service restarted\n") for t in mockless_tests: run_test_scenario(t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, demisto_api_key, is_ami) print_test_summary(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration, unmockable_integrations, proxy, is_ami) create_result_files(failed_playbooks, skipped_integration, skipped_tests) if is_ami and build_name == 'master': print("Pushing new/updated mock files to mock git repo.") ami.upload_mock_files(build_name, build_number) if len(failed_playbooks): print("Some tests have failed. Not destroying instances.") sys.exit(1) else: file_path = "./Tests/is_build_passed_{}.txt".format(server_version.replace(' ', '')) with open(file_path, "w") as is_build_passed_file: is_build_passed_file.write('Build passed')
assert clean_filename(u'n&%ew $r#eplac@es', replace='&%$#@') == 'n__ew _r_eplac_es' def test_silence_output(): # TODO: How to check output is redirected to /dev/null ? pass def test_get_paths(): test_playbook_id = u'test_playbook' assert get_mock_file_path( test_playbook_id) == 'test_playbook/test_playbook.mock' assert get_log_file_path( test_playbook_id) == 'test_playbook/test_playbook_playback.log' assert get_log_file_path( test_playbook_id, record=True) == 'test_playbook/test_playbook_record.log' assert get_folder_path(test_playbook_id) == 'test_playbook/' # TODO: Maybe mock subprocess functions?? with patch('Tests.mock_server.AMIConnection._get_docker_ip') as mock: mock.return_value = "2.2.2.2" ami = AMIConnection('1.1.1.1') def test_ami(): assert ami.public_ip == '1.1.1.1' assert ami.docker_ip == '2.2.2.2'
def execute_testing(tests_settings, server_ip, mockable_tests_names, unmockable_tests_names, tests_data_keeper, prints_manager, thread_index=0, is_ami=True): server = SERVER_URL.format(server_ip) server_numeric_version = tests_settings.serverNumericVersion start_message = "Executing tests with the server {} - and the server ip {}".format(server, server_ip) prints_manager.add_print_job(start_message, print, thread_index) is_nightly = tests_settings.nightly is_memory_check = tests_settings.memCheck slack = tests_settings.slack circle_ci = tests_settings.circleci build_number = tests_settings.buildNumber build_name = tests_settings.buildName conf, secret_conf = load_conf_files(tests_settings.conf_path, tests_settings.secret_conf_path) demisto_api_key = tests_settings.api_key default_test_timeout = conf.get('testTimeout', 30) tests = conf['tests'] skipped_tests_conf = conf['skipped_tests'] nightly_integrations = conf['nightly_integrations'] skipped_integrations_conf = conf['skipped_integrations'] unmockable_integrations = conf['unmockable_integrations'] secret_params = secret_conf['integrations'] if secret_conf else [] filtered_tests, is_filter_configured, run_all_tests = extract_filtered_tests(tests_settings.nightly) if is_filter_configured and not run_all_tests: is_nightly = True if not tests or len(tests) == 0: prints_manager.add_print_job('no integrations are configured for test', print, thread_index) prints_manager.execute_thread_prints(thread_index) return proxy = None if is_ami: ami = AMIConnection(server_ip) ami.clone_mock_data() proxy = MITMProxy(server_ip) failed_playbooks = [] succeed_playbooks = [] skipped_tests = set([]) skipped_integration = set([]) disable_all_integrations(demisto_api_key, server, prints_manager, thread_index=thread_index) prints_manager.execute_thread_prints(thread_index) mockable_tests = get_test_records_of_given_test_names(tests_settings, mockable_tests_names) unmockable_tests = get_test_records_of_given_test_names(tests_settings, unmockable_tests_names) if is_nightly and is_memory_check: mem_lim, err = get_docker_limit() send_slack_message(slack, SLACK_MEM_CHANNEL_ID, 'Build Number: {0}\n Server Address: {1}\nMemory Limit: {2}'.format(build_number, server, mem_lim), 'Content CircleCI', 'False') # first run the mock tests to avoid mockless side effects in container if is_ami and mockable_tests: proxy.configure_proxy_in_demisto(demisto_api_key, server, proxy.ami.docker_ip + ':' + proxy.PROXY_PORT) for t in mockable_tests: run_test_scenario(tests_settings, t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, demisto_api_key, prints_manager, thread_index=thread_index) prints_manager.add_print_job("\nRunning mock-disabled tests", print, thread_index) proxy.configure_proxy_in_demisto(demisto_api_key, server, '') prints_manager.add_print_job('Resetting containers', print, thread_index) client = demisto_client.configure(base_url=server, api_key=demisto_api_key, verify_ssl=False) body, status_code, headers = demisto_client.generic_request_func(self=client, method='POST', path='/containers/reset') if status_code != 200: error_msg = 'Request to reset containers failed with status code "{}"\n{}'.format(status_code, body) prints_manager.add_print_job(error_msg, print_error, thread_index) prints_manager.execute_thread_prints(thread_index) sys.exit(1) sleep(10) for t in unmockable_tests: run_test_scenario(tests_settings, t, proxy, default_test_timeout, skipped_tests_conf, nightly_integrations, skipped_integrations_conf, skipped_integration, is_nightly, run_all_tests, is_filter_configured, filtered_tests, skipped_tests, secret_params, failed_playbooks, unmockable_integrations, succeed_playbooks, slack, circle_ci, build_number, server, build_name, server_numeric_version, demisto_api_key, prints_manager, thread_index, is_ami) prints_manager.execute_thread_prints(thread_index) tests_data_keeper.add_tests_data(succeed_playbooks, failed_playbooks, skipped_tests, skipped_integration, unmockable_integrations) if not tests_settings.is_local_run: tests_data_keeper.add_proxy_related_test_data(proxy) if is_ami and build_name == 'master': updating_mocks_msg = "Pushing new/updated mock files to mock git repo." prints_manager.add_print_job(updating_mocks_msg, print, thread_index) ami.upload_mock_files(build_name, build_number)