def run_cleanup(self, run_properties: RunProperties, versions: list = ["3.3.3"]): """main""" installer_set = create_config_installer_set(versions, self.base_config, "all", run_properties) inst = installer_set[0][1] if inst.calc_config_file_name().is_file(): inst.load_config() inst.cfg.interactive = False inst.stop_service() installer_set[0][0].set_directories(inst.cfg) kill_all_processes() kill_all_processes() starter_mode = [ RunnerType.LEADER_FOLLOWER, RunnerType.ACTIVE_FAILOVER, RunnerType.CLUSTER, RunnerType.DC2DC, ] for runner_type in starter_mode: assert runner_type runner = make_runner(runner_type, False, "none", [], installer_set, run_properties) runner.cleanup() if inst.calc_config_file_name().is_file(): try: inst.un_install_debug_package() except Exception: print("nothing to uninstall") try: inst.un_install_client_package() except Exception: print("nothing to uninstall") inst.un_install_server_package() else: print("Cannot uninstall package without config.yml!") inst.cleanup_system()
def teardown_suite(self): """Teardown suite environment: Debug symbols test suite""" kill_all_processes() # If there are failed test cases, save the contents of the installation directories. # This must be done not more than once per suite run to save space, # therefore it shouldn't be done in a @collect_crash_data method. if self.there_are_failed_tests(): if self.installer.cfg.debug_install_prefix.exists(): archive = shutil.make_archive( "debug_package_installation_dir", "gztar", self.installer.cfg.debug_install_prefix, self.installer.cfg.debug_install_prefix, ) attach.file(archive, "Debug package installation directory", "application/x-tar", "tgz") if self.installer.cfg.install_prefix.exists(): archive = shutil.make_archive( "server_package_installation_dir", "gztar", self.installer.cfg.install_prefix, self.installer.cfg.install_prefix, ) attach.file(archive, "Server package installation directory", "application/x-tar", "tgz")
def run_test(zip_package): """ main """ logging.getLogger().setLevel(logging.DEBUG) install_config = InstallerConfig('3.3.3', True, False, False, zip_package, Path("/tmp/"), Path("/"), "127.0.0.1", "", False, False) inst = make_installer(install_config) if inst.calc_config_file_name().is_file(): inst.load_config() inst.cfg.interactive = False inst.stop_service() kill_all_processes() kill_all_processes() starter_mode = [ RunnerType.LEADER_FOLLOWER, RunnerType.ACTIVE_FAILOVER, RunnerType.CLUSTER ] # , # RunnerType.DC2DC] here __init__ will create stuff, TODO. for runner_type in starter_mode: assert runner_type runner = make_runner(runner_type, 'none', [], inst.cfg, inst, None) runner.cleanup() if inst.calc_config_file_name().is_file(): try: inst.un_install_debug_package() except: print('nothing to uninstall') inst.un_install_package() else: print('Cannot uninstall package without config.yml!') inst.cleanup_system()
def main(**kwargs): """ main """ kwargs['stress_upgrade'] = False kwargs['package_dir'] = Path(kwargs['package_dir']) kwargs['test_data_dir'] = Path(kwargs['test_data_dir']) kwargs['alluredir'] = Path(kwargs['alluredir']) kwargs['hb_cli_cfg'] = HotBackupCliCfg("disabled", "", "", "", "", "", "") kwargs['base_config'] = InstallerBaseConfig.from_dict(**kwargs) do_install = kwargs['mode'] in ["all", "install"] do_uninstall = kwargs['mode'] in ["all", "uninstall"] launch_dir = Path.cwd() if "WORKSPACE" in os.environ: launch_dir = Path(os.environ["WORKSPACE"]) if not kwargs['test_data_dir'].is_absolute(): kwargs['test_data_dir'] = launch_dir / kwargs['test_data_dir'] if not kwargs['test_data_dir'].exists(): kwargs['test_data_dir'].mkdir(parents=True, exist_ok=True) os.chdir(kwargs['test_data_dir']) if not kwargs['package_dir'].is_absolute(): kwargs['package_dir'] = (launch_dir / kwargs['package_dir']).resolve() if not kwargs['package_dir'].exists(): kwargs['package_dir'].mkdir(parents=True, exist_ok=True) init_allure(results_dir=kwargs['alluredir'], clean=kwargs['clean_alluredir'], zip_package=kwargs['zip_package']) lh.section("startup") install_config = InstallerConfig( kwargs['new_version'], kwargs['verbose'], kwargs['enterprise'], kwargs['encryption_at_rest'], kwargs['zip_package'], kwargs['src_testing'], kwargs['hb_cli_cfg'], kwargs['package_dir'], kwargs['test_data_dir'], kwargs['mode'], kwargs['publicip'], kwargs['interactive'], False, False) install_config.source = True inst = make_installer(install_config) inst.cfg.scenario = Path(kwargs['scenario']) runner = Testing(RunnerType.TESTING, False, inst, kwargs['selenium'], kwargs['selenium_driver_args'], "bla", False, False) runner.do_install = do_install runner.do_uninstall = do_uninstall failed = False if not runner.starter_prepare_env_impl(): failed = True if len(kwargs['frontends']) == 0: kill_all_processes() return 0 if not failed else 1
def run_perf_test(self, deployment_mode, versions: list, frontends, scenario, run_props: RunProperties): # fmt: on """ main """ do_install = deployment_mode in ["all", "install"] do_uninstall = deployment_mode in ["all", "uninstall"] installers = create_config_installer_set( versions, self.base_config, deployment_mode, run_props ) lh.section("configuration") print( """ mode: {mode} {cfg_repr} """.format( **{"mode": str(deployment_mode), "cfg_repr": repr(installers[0][0])} ) ) inst = installers[0][1] split_host = re.compile(r"([a-z]*)://([0-9.:]*):(\d*)") if len(frontends) > 0: for frontend in frontends: print("remote") host_parts = re.split(split_host, frontend) inst.cfg.add_frontend(host_parts[1], host_parts[2], host_parts[3]) inst.cfg.scenario = Path(scenario) runner = ClusterPerf( RunnerType.CLUSTER, self.abort_on_error, installers, self.selenium, self.selenium_driver_args, "perf", run_props, use_auto_certs=self.use_auto_certs ) runner.do_install = do_install runner.do_uninstall = do_uninstall failed = False if not runner.run(): failed = True if len(frontends) == 0: kill_all_processes() return failed
def save_data_dir(self): """save data dir and logs in case a test failed""" kill_all_processes() if self.starter.basedir.exists(): archive = shutil.make_archive( f"LicenseManagerSingleServerTestSuite(v. {self.base_cfg.version})", "bztar", self.starter.basedir) attach.file(archive, "test dir archive", "application/x-bzip2", "tar.bz2") else: print("test basedir doesn't exist, won't create report tar")
def install(self, inst): """install the package to the system""" self.progress(True, "{0} - install package".format(str(self.name))) kill_all_processes(False) if self.do_install: lh.subsubsection("installing server package") inst.install_server_package() self.cfg.set_directories(inst.cfg) lh.subsubsection("checking files") inst.check_installed_files() lh.subsubsection("saving config") inst.save_config() lh.subsubsection("checking if service is up") if inst.check_service_up(): lh.subsubsection("stopping service") inst.stop_service() inst.broadcast_bind() lh.subsubsection("outputting version") inst.output_arangod_version() inst.get_starter_version() inst.get_sync_version() lh.subsubsection("starting service") inst.start_service() inst.check_installed_paths() inst.check_engine_file() if not self.new_installer: # only install debug package for new package. self.progress(True, "installing debug package:") # start / stop if inst.check_service_up(): inst.stop_service() inst.start_service() print(inst.cfg.semver) sys_arangosh = ArangoshExecutor(inst.cfg, inst.instance) logging.debug("self test after installation") if inst.cfg.have_system_service: sys_arangosh.self_test() if self.do_system_test: sys_arangosh.js_version_check() # TODO: here we should invoke Makedata for the system installation. logging.debug("stop system service to make ports available for starter") inst.stop_service()
def create_arangod_dump(installer, starter_dir: str, dump_file_dir: str): """create arangod memory dump file""" starter = StarterManager( basecfg=installer.cfg, install_prefix=Path(starter_dir), instance_prefix="single", expect_instances=[InstanceType.SINGLE], mode="single", jwt_str="single", ) dump_filename = None try: with step("Start a single server deployment"): starter.run_starter() starter.detect_instances() starter.detect_instance_pids() starter.set_passvoid("") pid = starter.all_instances[0].pid with step("Create a dump of arangod process"): cmd = ["procdump", "-ma", str(pid), dump_file_dir] lh.log_cmd(cmd) with psutil.Popen(cmd, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: (procdump_out, procdump_err) = proc.communicate() procdump_str = str(procdump_out, "UTF-8") attach(procdump_str, "procdump sdtout") attach(str(procdump_err), "procdump stderr") success_string = "Dump 1 complete" filename_regex = re.compile( r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$", re.MULTILINE) match = re.search(filename_regex, procdump_str) if procdump_str.find(success_string) < 0 or not match: raise Exception( "procdump wasn't able to create a dump file: " + procdump_str) dump_filename = match.group("filename") finally: starter.terminate_instance() kill_all_processes() return dump_filename
def test_debug_symbols_attach_to_process_windows(self): """Debug arangod executable by attaching debugger to a running process (Windows)""" starter = StarterManager( basecfg=self.installer.cfg, install_prefix=Path(DebuggerTestSuite.STARTER_DIR), instance_prefix="single", expect_instances=[InstanceType.SINGLE], mode="single", jwt_str="single", ) try: with step("Start a single server deployment"): starter.run_starter() starter.detect_instances() starter.detect_instance_pids() starter.set_passvoid("") pid = starter.all_instances[0].pid pdb_dir = str(self.installer.cfg.debug_install_prefix) with step( "Check that stack trace with function names and line numbers can be acquired from cdb" ): cmd = " ".join([ "cdb", "-pv", "-p", str(pid), "-y", pdb_dir, "-lines", "-n" ]) attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT) cdb = wexpect.spawn(cmd) cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300) cdb.sendline("k") cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300) stack = cdb.before cdb.sendline("q") attach(stack, "Stacktrace from cdb output", attachment_type=AttachmentType.TEXT) assert "arangod!main" in stack, "Stack must contain real function names." assert "arangod.cpp" in stack, "Stack must contain real source file names." finally: starter.terminate_instance() kill_all_processes()
def teardown_suite(self): """Teardown suite environment: single server""" self.starter.terminate_instance() kill_all_processes() self.cleanup()
def save_data_dir(self): """save data dir and logs in case a test failed""" kill_all_processes() self.runner.zip_test_dir()
def run_test(old_version, new_version, verbose, package_dir, test_data_dir, enterprise, encryption_at_rest, zip_package, interactive, mode, starter_mode, publicip, selenium, selenium_driver_args): """ main """ lh.section("configuration") print("version: " + str(new_version)) print("using enterpise: " + str(enterprise)) print("using encryption at rest: " + str(encryption_at_rest)) print("using zip: " + str(zip_package)) print("package directory: " + str(package_dir)) print("mode: " + str(mode)) print("starter mode: " + str(starter_mode)) print("public ip: " + str(publicip)) print("interactive: " + str(interactive)) print("verbose: " + str(verbose)) if mode not in ['all', 'install', 'system', 'tests', 'uninstall']: raise Exception("unsupported mode %s!" % mode) do_install = mode in ["all", "install"] do_uninstall = mode in ["all", "uninstall"] lh.section("startup") if verbose: logging.info("setting debug level to debug (verbose)") logging.getLogger().setLevel(logging.DEBUG) install_config = InstallerConfig(new_version, verbose, enterprise, encryption_at_rest, zip_package, Path(package_dir), Path(test_data_dir), mode, publicip, interactive, False) inst = make_installer(install_config) if starter_mode == 'all': starter_mode = [ RunnerType.LEADER_FOLLOWER, RunnerType.ACTIVE_FAILOVER, RunnerType.CLUSTER ] if enterprise: starter_mode.append(RunnerType.DC2DC) elif starter_mode == 'LF': starter_mode = [RunnerType.LEADER_FOLLOWER] elif starter_mode == 'AFO': starter_mode = [RunnerType.ACTIVE_FAILOVER] elif starter_mode == 'CL': starter_mode = [RunnerType.CLUSTER] elif starter_mode == 'DC': starter_mode = [RunnerType.DC2DC] elif starter_mode == 'DCendurance': starter_mode = [RunnerType.DC2DCENDURANCE] elif starter_mode == 'none': starter_mode = [RunnerType.NONE] else: raise Exception("invalid starter mode: " + starter_mode) count = 1 for runner_type in starter_mode: assert runner_type runner = make_runner(runner_type, selenium, selenium_driver_args, inst.cfg, inst, None) # install on first run: runner.do_install = (count == 1) and do_install # only uninstall after the last test: runner.do_uninstall = (count == len(starter_mode)) and do_uninstall failed = False if not runner.run(): failed = True kill_all_processes() count += 1 return 0 if not failed else 1
def run_test(old_version, new_version, verbose, package_dir, test_data_dir, enterprise, encryption_at_rest, zip_package, interactive, mode, starter_mode, publicip, scenario, frontends, selenium, selenium_driver_args): """ main """ lh.section("configuration") print("version: " + str(new_version)) print("using enterpise: " + str(enterprise)) print("using zip: " + str(zip_package)) print("package directory: " + str(package_dir)) print("mode: " + str(mode)) print("starter mode: " + str(starter_mode)) print("public ip: " + str(publicip)) print("interactive: " + str(interactive)) print("scenario: " + str(scenario)) print("verbose: " + str(verbose)) if mode not in ['all', 'install', 'system', 'tests', 'uninstall']: raise Exception("unsupported mode %s!" % mode) do_install = mode in ["all", "install"] do_uninstall = mode in ["all", "uninstall"] lh.section("startup") if verbose: logging.info("setting debug level to debug (verbose)") logging.getLogger().setLevel(logging.DEBUG) install_config = InstallerConfig(new_version, verbose, enterprise, encryption_at_rest, zip_package, Path(package_dir), Path(test_data_dir), mode, publicip, interactive, False) split_host = re.compile(r'([a-z]*)://([0-9.:]*):(\d*)') inst = make_installer(install_config) if len(frontends) > 0: for frontend in frontends: print('remote') host_parts = re.split(split_host, frontend) inst.cfg.add_frontend(host_parts[1], host_parts[2], host_parts[3]) inst.cfg.scenario = Path(scenario) runner = ClusterPerf(RunnerType.CLUSTER, inst.cfg, inst, None, None, selenium, selenium_driver_args) runner.do_install = do_install runner.do_uninstall = do_uninstall failed = False if not runner.run(): failed = True if len(frontends) == 0: kill_all_processes() return 0 if not failed else 1
def run_test(self, deployment_mode, versions: list, run_props: RunProperties): # fmt: on """ main """ results = [] do_install = deployment_mode in ["all", "install"] do_uninstall = deployment_mode in ["all", "uninstall"] installers = create_config_installer_set( versions, self.base_config, deployment_mode, run_props ) lh.section("configuration") print( """ mode: {mode} {cfg_repr} """.format( **{"mode": str(deployment_mode), "cfg_repr": repr(installers[0][0])} ) ) count = 1 for runner_type in STARTER_MODES[self.base_config.starter_mode]: with AllureTestSuiteContext(properties=run_props, versions=versions, parent_test_suite_name=None, auto_generate_parent_test_suite_name=True, suite_name=runner_strings[runner_type], runner_type=None, installer_type=installers[0][1].installer_type): with RtaTestcase(runner_strings[runner_type] + " main flow") as testcase: if not run_props.supports_dc2dc() and runner_type == RunnerType.DC2DC: testcase.context.status = Status.SKIPPED testcase.context.statusDetails = StatusDetails( message="DC2DC is not applicable to Community packages.\nDC2DC is not supported on Windows.") continue one_result = { "testrun name": run_props.testrun_name, "testscenario": runner_strings[runner_type], "success": True, "messages": [], "progress": "", } runner = make_runner( runner_type, self.abort_on_error, self.selenium, self.selenium_driver_args, installers, run_props, use_auto_certs=self.use_auto_certs, ) # install on first run: runner.do_install = (count == 1) and do_install # only uninstall after the last test: runner.do_uninstall = (count == len( STARTER_MODES[deployment_mode])) and do_uninstall try: runner.run() runner.cleanup() testcase.context.status = Status.PASSED # pylint: disable=broad-except except Exception as ex: one_result["success"] = False one_result["messages"].append(str(ex)) one_result["progress"] += runner.get_progress() runner.take_screenshot() runner.agency_acquire_dump() runner.search_for_warnings() runner.quit_selenium() kill_all_processes() runner.zip_test_dir() self.copy_packages_to_result(installers) testcase.context.status = Status.FAILED testcase.context.statusDetails = StatusDetails(message=str(ex), trace="".join( traceback.TracebackException.from_exception( ex).format())) lh.section("uninstall on error") installers[0][1].un_install_debug_package() installers[0][1].un_install_server_package() installers[0][1].cleanup_system() if self.abort_on_error: raise ex traceback.print_exc() lh.section("uninstall on error") try: runner.cleanup() finally: pass results.append(one_result) continue if runner.ui_tests_failed: failed_test_names = [f'"{row["Name"]}"' for row in runner.ui_test_results_table if not row["Result"] == "PASSED"] one_result["success"] = False # pylint: disable=line-too-long one_result[ "messages"].append( f'The following UI tests failed: {", ".join(failed_test_names)}. See allure report for details.') kill_all_processes() count += 1 return results
def run_upgrade(self, versions: list, run_props: RunProperties): """execute upgrade tests""" lh.section("startup") results = [] for runner_type in STARTER_MODES[self.base_config.starter_mode]: installers = create_config_installer_set( versions, self.base_config, "all", run_props, ) old_inst = installers[0][1] new_inst = installers[1][1] with AllureTestSuiteContext( properties=run_props, versions=versions, parent_test_suite_name=None, auto_generate_parent_test_suite_name=True, suite_name=runner_strings[runner_type], runner_type=None, installer_type=new_inst.installer_type, ): with RtaTestcase(runner_strings[runner_type] + " main flow") as testcase: if not run_props.supports_dc2dc() and runner_type == RunnerType.DC2DC: testcase.context.status = Status.SKIPPED testcase.context.statusDetails = StatusDetails( message="DC2DC is not applicable to Community packages.\nDC2DC is not supported on Windows." ) continue one_result = { "testrun name": run_props.testrun_name, "testscenario": runner_strings[runner_type], "success": True, "messages": [], "progress": "", } try: kill_all_processes() runner = None lh.section("configuration") print( """ starter mode: {starter_mode} old version: {old_version} {cfg_repr} """.format( **{ "starter_mode": str(self.base_config.starter_mode), "old_version": str(versions[0]), "cfg_repr": repr(installers[1][0]), } ) ) if runner_type: runner = make_runner( runner_type, self.abort_on_error, self.selenium, self.selenium_driver_args, installers, run_props, use_auto_certs=self.use_auto_certs, ) if runner: try: runner.run() runner.cleanup() testcase.context.status = Status.PASSED except Exception as ex: one_result["success"] = False one_result["messages"].append(str(ex)) one_result["progress"] += runner.get_progress() runner.take_screenshot() runner.agency_acquire_dump() runner.search_for_warnings() runner.quit_selenium() kill_all_processes() runner.zip_test_dir() self.copy_packages_to_result(installers) testcase.context.status = Status.FAILED testcase.context.statusDetails = StatusDetails( message=str(ex), trace="".join(traceback.TracebackException.from_exception(ex).format()), ) if self.abort_on_error: raise ex one_result["progress"] += str(ex) + "".join( traceback.TracebackException.from_exception(ex).format() ) traceback.print_exc() lh.section("uninstall on error") old_inst.un_install_debug_package() old_inst.un_install_server_package() old_inst.cleanup_system() try: runner.cleanup() finally: pass results.append(one_result) continue if runner.ui_tests_failed: failed_test_names = [ f'"{row["Name"]}"' for row in runner.ui_test_results_table if not row["Result"] == "PASSED" ] one_result["success"] = False one_result["messages"].append( f'The following UI tests failed: {", ".join(failed_test_names)}.' + "See allure report for details." ) lh.section("uninstall") new_inst.un_install_server_package() lh.section("check system") new_inst.check_uninstall_cleanup() lh.section("remove residuals") try: old_inst.cleanup_system() except Exception: print("Ignoring old cleanup error!") try: print("Ignoring new cleanup error!") new_inst.cleanup_system() except Exception: print("Ignoring general cleanup error!") except Exception as ex: print("Caught. " + str(ex)) one_result["success"] = False one_result["messages"].append(str(ex)) one_result["progress"] += "\naborted outside of testcodes" if self.abort_on_error: print("re-throwing.") raise ex traceback.print_exc() kill_all_processes() if runner: try: runner.cleanup() except Exception as exception: print("Ignoring runner cleanup error! Exception:") print(str(exception)) print("".join(traceback.TracebackException.from_exception(exception).format())) try: print("Cleaning up system after error:") old_inst.un_install_debug_package() old_inst.un_install_server_package() old_inst.cleanup_system() except Exception: print("Ignoring old cleanup error!") try: print("Ignoring new cleanup error!") new_inst.un_install_debug_package() new_inst.un_install_server_package() new_inst.cleanup_system() except Exception: print("Ignoring new cleanup error!") results.append(one_result) return results
def teardown_suite(self): """License manager base test suite: teardown""" self.runner.starter_shutdown() kill_all_processes()
def run_upgrade(old_version, new_version, verbose, package_dir, test_data_dir, enterprise, encryption_at_rest, zip_package, interactive, starter_mode, stress_upgrade, publicip, selenium, selenium_driver_args): """ execute upgrade tests """ lh.section("configuration") print("old version: " + str(old_version)) print("version: " + str(new_version)) print("using enterpise: " + str(enterprise)) print("using encryption at rest: " + str(encryption_at_rest)) print("using zip: " + str(zip_package)) print("package directory: " + str(package_dir)) print("starter mode: " + str(starter_mode)) print("public ip: " + str(publicip)) print("interactive: " + str(interactive)) print("verbose: " + str(verbose)) lh.section("startup") if verbose: logging.info("setting debug level to debug (verbose)") logging.getLogger().setLevel(logging.DEBUG) if starter_mode == 'all': starter_mode = [RunnerType.LEADER_FOLLOWER, RunnerType.ACTIVE_FAILOVER, RunnerType.CLUSTER] if enterprise: starter_mode.append(RunnerType.DC2DC) elif starter_mode == 'LF': starter_mode = [RunnerType.LEADER_FOLLOWER] elif starter_mode == 'AFO': starter_mode = [RunnerType.ACTIVE_FAILOVER] elif starter_mode == 'CL': starter_mode = [RunnerType.CLUSTER] elif starter_mode == 'DC': if enterprise: starter_mode = [RunnerType.DC2DC] else: starter_mode = [None] elif starter_mode == 'none': starter_mode = [None] else: raise Exception("invalid starter mode: " + starter_mode) for runner_type in starter_mode: kill_all_processes() install_config_old = InstallerConfig(old_version, verbose, enterprise, encryption_at_rest, zip_package, Path(package_dir), Path(test_data_dir), 'all', publicip, interactive, stress_upgrade) old_inst = make_installer(install_config_old) install_config_new = InstallerConfig(new_version, verbose, enterprise, encryption_at_rest, zip_package, Path(package_dir), Path(test_data_dir), 'all', publicip, interactive, stress_upgrade) new_inst = make_installer(install_config_new) runner = None if runner_type: runner = make_runner(runner_type, selenium, selenium_driver_args, install_config_old, old_inst, install_config_new, new_inst) if runner: runner.run() lh.section("uninstall") new_inst.un_install_package() lh.section("check system") new_inst.check_uninstall_cleanup() lh.section("remove residuals") new_inst.cleanup_system()