def __init__(self, versions: list, base_config: InstallerBaseConfig):
self.new_version = versions[1]
self.old_version = versions[0]
self.zip_package = base_config.zip_package
self.enc_at_rest = None
self.parent_test_suite_name = None
self.auto_generate_parent_test_suite_name = False
self.suite_name = None
self.runner_type = None
self.installer_type = None
self.use_subsuite = False
self.installers = {}
self.installers["community"] = create_config_installer_set(
versions=versions,
base_config=base_config,
deployment_mode="all",
run_properties=RunProperties(enterprise=False,
encryption_at_rest=False,
ssl=False),
)
self.installers["enterprise"] = create_config_installer_set(
versions=versions,
base_config=base_config,
deployment_mode="all",
run_properties=RunProperties(enterprise=True,
encryption_at_rest=False,
ssl=False),
)
self.old_inst_e = self.installers["enterprise"][0][1]
self.new_inst_e = self.installers["enterprise"][1][1]
self.old_inst_c = self.installers["community"][0][1]
self.new_inst_c = self.installers["community"][1][1]
super().__init__()
def main(**kwargs):
""" main """
kwargs['package_dir'] = Path(kwargs['package_dir'])
kwargs['test_data_dir'] = Path(kwargs['test_data_dir'])
kwargs['alluredir'] = Path(kwargs['alluredir'])
kwargs['hb_cli_cfg'] = HotBackupCliCfg.from_dict(**kwargs)
kwargs['base_config'] = InstallerBaseConfig.from_dict(**kwargs)
test_driver = TestDriver(**kwargs)
test_driver.set_r_limits()
results = test_driver.run_upgrade(
[
semver.VersionInfo.parse(kwargs['old_version']),
semver.VersionInfo.parse(kwargs['new_version'])
],
RunProperties(kwargs['enterprise'], kwargs['encryption_at_rest'],
kwargs['ssl']))
print("V" * 80)
status = True
for one_result in results:
print(one_result)
status = status and one_result["success"]
if not status:
print("exiting with failure")
sys.exit(1)
def __init__(
self,
old_version,
new_version,
installer_base_config,
):
package_type = ".tar.gz" if installer_base_config.zip_package else ".deb/.rpm/NSIS"
self.suite_name = f"Licence manager test suite: ArangoDB v. {str(new_version)} ({package_type})"
self.auto_generate_parent_test_suite_name = False
super().__init__(new_version, installer_base_config)
self.old_version = old_version
run_props = RunProperties(
enterprise=True,
encryption_at_rest=False,
ssl=False,
)
versions = [self.old_version, self.new_version]
self.installer_set = create_config_installer_set(
versions=versions,
base_config=self.base_cfg,
deployment_mode="all",
run_properties=run_props)
self.old_installer = self.installer_set[0][1]
self.new_installer = self.installer_set[1][1]
self.installer = self.new_installer
def main(**kwargs):
""" main """
kwargs['stress_upgrade'] = False
kwargs['package_dir'] = Path(kwargs['package_dir'])
kwargs['test_data_dir'] = Path(kwargs['test_data_dir'])
kwargs['alluredir'] = Path(kwargs['alluredir'])
kwargs['hb_cli_cfg'] = HotBackupCliCfg("disabled", "", "", "", "", "", "")
kwargs['base_config'] = InstallerBaseConfig.from_dict(**kwargs)
test_driver = TestDriver(**kwargs)
test_driver.set_r_limits()
result = test_driver.run_perf_test(
kwargs['mode'],
[semver.VersionInfo.parse(kwargs['new_version'])],
# pylint: disable=too-many-function-args
kwargs['frontends'],
kwargs['scenario'],
RunProperties(kwargs['enterprise'], kwargs['encryption_at_rest'],
kwargs['ssl']))
print("V" * 80)
if not result:
print("exiting with failure")
sys.exit(1)
def get_packaging_shorthand(self):
"""get the [DEB|RPM|EXE|DMG|ZIP|targz] from the installer"""
if self.installer_type:
return self.installer_type
installers = create_config_installer_set(["3.3.3"], self.base_config, "all", RunProperties(False, False, False))
self.installer_type = installers[0][1].installer_type.split(" ")[0].replace(".", "")
return self.installer_type
def __init__(self):
self.test_results = []
self.child_classes = self.get_child_test_suite_classes()
self.children = []
self.parent = None
self.enterprise = False
if not hasattr(self, "new_version"):
self.new_version = None
self.enc_at_rest = False
if not hasattr(self, "old_version"):
self.old_version = None
if not hasattr(self, "parent_test_suite_name"):
self.parent_test_suite_name = None
if not hasattr(self, "auto_generate_parent_test_suite_name"):
self.auto_generate_parent_test_suite_name = None
if not hasattr(self, "suite_name"):
self.suite_name = None
self.runner_type = None
self.installer_type = None
self.ssl = False
if not hasattr(self, "use_subsuite"):
self.use_subsuite = True
versions = []
if self.old_version:
versions.append(self.old_version)
if self.new_version:
versions.append(self.new_version)
if hasattr(self, "generate_custom_suite_name"):
# pylint: disable=no-member
self.suite_name = self.generate_custom_suite_name()
if self.use_subsuite:
self.sub_suite_name = self.__doc__ if self.__doc__ else self.__class__.__name__
else:
self.sub_suite_name = None
self.test_suite_context = AllureTestSuiteContext(
properties=RunProperties(self.enterprise, self.enc_at_rest,
self.ssl),
versions=versions,
parent_test_suite_name=None if not self.parent_test_suite_name else
self.parent_test_suite_name,
auto_generate_parent_test_suite_name=True
if not hasattr(self, "auto_generate_parent_test_suite_name") else
self.auto_generate_parent_test_suite_name,
suite_name=None if not self.suite_name else self.suite_name,
sub_suite_name=None
if not self.sub_suite_name else self.sub_suite_name,
runner_type=None if not self.runner_type else self.runner_type,
installer_type=None
if not self.installer_type else self.installer_type,
)
def main(**kwargs):
"""Wrapper..."""
lh.configure_logging(True)
kwargs["verbose"] = False
kwargs["package_dir"] = Path("")
kwargs["test_data_dir"] = Path("")
kwargs["alluredir"] = Path("")
kwargs["clean_alluredir"] = True
kwargs["hb_mode"] = "disabled"
kwargs["hb_provider"] = ""
kwargs["hb_storage_path_prefix"] = ""
kwargs["hb_aws_access_key_id"] = ""
kwargs["hb_aws_secret_access_key"] = ""
kwargs["hb_aws_region"] = ""
kwargs["hb_aws_acl"] = ""
kwargs["hb_gce_service_account_credentials"] = ""
kwargs["hb_gce_service_account_file"] = ""
kwargs["hb_gce_project_number"] = ""
kwargs["hb_azure_key"] = ""
kwargs["hb_azure_account"] = ""
kwargs["interactive"] = False
kwargs["starter_mode"] = "all"
kwargs["stress_upgrade"] = False
kwargs["abort_on_error"] = False
kwargs["publicip"] = "127.0.0.1"
kwargs["selenium"] = "none"
kwargs["selenium_driver_args"] = []
kwargs["use_auto_certs"] = False
# pylint: disable=broad-except
versions = ["3.3.3"]
enterprise = False
try:
saved_cfg = InstallerBase.load_config_from_file()
kwargs["zip_package"] = saved_cfg.zip_package
kwargs["src_testing"] = saved_cfg.src_testing
kwargs["package_dir"] = saved_cfg.package_dir
versions = [saved_cfg.version]
enterprise = saved_cfg.enterprise
except:
print("Failed to load parameters from pre-saved configuration file.")
kwargs["hb_cli_cfg"] = HotBackupCliCfg.from_dict(**kwargs)
kwargs["base_config"] = InstallerBaseConfig.from_dict(**kwargs)
test_driver = TestDriver(**kwargs)
test_driver.set_r_limits()
test_driver.run_cleanup(RunProperties(enterprise, False, False), versions)
def main(
#very_common_options
new_version, verbose, enterprise, package_dir, zip_package, src_testing,
hot_backup, hb_provider, hb_storage_path_prefix,
hb_aws_access_key_id, hb_aws_secret_access_key, hb_aws_region, hb_aws_acl,
# common_options
old_version, test_data_dir, encryption_at_rest, interactive,
starter_mode, stress_upgrade, abort_on_error, publicip,
selenium, selenium_driver_args, ssl):
""" main trampoline """
return run_upgrade(old_version, new_version, verbose,
package_dir, test_data_dir,
zip_package, hot_backup, hb_provider, hb_storage_path_prefix, interactive,
starter_mode, stress_upgrade, abort_on_error,
publicip, selenium, selenium_driver_args,
RunProperties(enterprise,
encryption_at_rest,
ssl))
def start_clusters(self):
"""start DC2DC setup"""
# pylint: disable=attribute-defined-outside-init
self.runner = make_runner(
runner_type=RunnerType.DC2DC,
abort_on_error=False,
installer_set=self.installer_set,
use_auto_certs=False,
selenium_worker="none",
selenium_driver_args=[],
runner_properties=RunProperties(
enterprise=True,
encryption_at_rest=False,
ssl=False,
),
)
self.runner.starter_prepare_env()
self.runner.starter_run()
self.runner.finish_setup()
self.starter = self.runner.cluster1["instance"]
def start_afo(self):
"""start an AFO setup"""
# pylint: disable=attribute-defined-outside-init
self.runner = make_runner(
runner_type=RunnerType.ACTIVE_FAILOVER,
abort_on_error=False,
selenium_worker="none",
selenium_driver_args=[],
installer_set=self.installer_set,
runner_properties=RunProperties(
enterprise=True,
encryption_at_rest=False,
ssl=False,
),
use_auto_certs=False,
)
self.runner.starter_prepare_env()
self.runner.starter_run()
self.runner.finish_setup()
self.starter = self.runner.leader
def start_leader_follower(self):
"""start a leader-follower setup"""
# pylint: disable=attribute-defined-outside-init
self.runner = make_runner(
runner_type=RunnerType.LEADER_FOLLOWER,
abort_on_error=False,
installer_set=self.installer_set,
use_auto_certs=False,
selenium_worker="none",
selenium_driver_args=[],
runner_properties=RunProperties(
enterprise=True,
encryption_at_rest=False,
ssl=False,
),
)
self.runner.starter_prepare_env()
self.runner.starter_run()
self.runner.finish_setup()
self.runner.leader_starter_instance.detect_arangosh_instances()
self.runner.follower_starter_instance.detect_arangosh_instances()
self.starter = self.runner.leader_starter_instance
def __init__(self, new_version, installer_base_config):
self.new_version = new_version
self.base_cfg = installer_base_config
package_type = ".tar.gz" if installer_base_config.zip_package else ".deb/.rpm/NSIS"
self.suite_name = f"Licence manager test suite: ArangoDB v. {str(new_version)} ({package_type})"
self.auto_generate_parent_test_suite_name = False
super().__init__()
self.use_subsuite = True
run_props = RunProperties(
enterprise=True,
encryption_at_rest=False,
ssl=False,
)
self.installer_set = create_config_installer_set(
versions=[new_version],
base_config=self.base_cfg,
deployment_mode="all",
run_properties=run_props)
self.installer = self.installer_set[0][1]
self.starter = None
self.instance = None
self.runner = None
self.passvoid = "license_manager_tests"
self.publicip = installer_base_config.publicip
def upgrade_package_test(
dl_opts: DownloadOptions, new_version, old_version, new_dlstage, old_dlstage, git_version, editions, test_driver
):
"""process fetch & tests"""
test_driver.set_r_limits()
lh.configure_logging(test_driver.base_config.verbose)
list_all_processes()
test_dir = test_driver.base_config.test_data_dir
versions = {}
fresh_versions = {}
version_state_tar = get_tar_file_path(
test_driver.launch_dir, [old_version, new_version], test_driver.get_packaging_shorthand()
)
read_versions_tar(version_state_tar, versions)
print(versions)
results = []
# do the actual work:
for props in EXECUTION_PLAN:
print("Cleaning up" + props.testrun_name)
test_driver.run_cleanup(props)
print("Cleanup done")
for props in EXECUTION_PLAN:
if props.directory_suffix not in editions:
continue
# pylint: disable=unused-variable
dl_old = Download(
dl_opts,
test_driver.base_config.hb_cli_cfg,
old_version,
props.enterprise,
test_driver.base_config.zip_package,
test_driver.base_config.src_testing,
old_dlstage,
versions,
fresh_versions,
git_version,
)
dl_new = Download(
dl_opts,
test_driver.base_config.hb_cli_cfg,
new_version,
props.enterprise,
test_driver.base_config.zip_package,
test_driver.base_config.src_testing,
new_dlstage,
versions,
fresh_versions,
git_version,
)
if not dl_new.is_different() or not dl_old.is_different():
print("we already tested this version. bye.")
return 0
dl_old.get_packages(dl_old.is_different())
dl_new.get_packages(dl_new.is_different())
this_test_dir = test_dir / props.directory_suffix
test_driver.reset_test_data_dir(this_test_dir)
results.append(test_driver.run_upgrade([dl_old.cfg.version, dl_new.cfg.version], props))
for use_enterprise in [True, False]:
results.append(
test_driver.run_conflict_tests(
[dl_old.cfg.version, dl_new.cfg.version],
enterprise=use_enterprise,
)
)
results.append(
test_driver.run_license_manager_tests(
[semver.VersionInfo.parse(dl_old.cfg.version), semver.VersionInfo.parse(dl_new.cfg.version)]
)
)
results.append(
test_driver.run_debugger_tests(
[semver.VersionInfo.parse(dl_old.cfg.version), semver.VersionInfo.parse(dl_new.cfg.version)],
run_props=RunProperties(True, False, False),
)
)
results.append(
test_driver.run_debugger_tests(
[semver.VersionInfo.parse(dl_old.cfg.version), semver.VersionInfo.parse(dl_new.cfg.version)],
run_props=RunProperties(False, False, False),
)
)
print("V" * 80)
status = True
table = BeautifulTable(maxwidth=140)
for one_suite_result in results:
if len(one_suite_result) > 0:
for one_result in one_suite_result:
if one_result["success"]:
table.rows.append(
[
one_result["testrun name"],
one_result["testscenario"],
# one_result['success'],
"\n".join(one_result["messages"]),
]
)
else:
table.rows.append(
[
one_result["testrun name"],
one_result["testscenario"],
# one_result['success'],
"\n".join(one_result["messages"]) + "\n" + "H" * 40 + "\n" + one_result["progress"],
]
)
status = status and one_result["success"]
table.columns.header = [
"Testrun",
"Test Scenario",
# 'success', we also have this in message.
"Message + Progress",
]
table.columns.alignment["Message + Progress"] = ALIGN_LEFT
tablestr = str(table)
Path("testfailures.txt").write_text(tablestr, encoding="utf8")
if not status:
print("exiting with failure")
sys.exit(1)
if dl_opts.force:
touch_all_tars_in_dir(version_state_tar)
else:
write_version_tar(version_state_tar, fresh_versions)
print(tablestr)
return 0
def run_test(self,
deployment_mode,
versions: list,
run_props: RunProperties):
# fmt: on
""" main """
results = []
do_install = deployment_mode in ["all", "install"]
do_uninstall = deployment_mode in ["all", "uninstall"]
installers = create_config_installer_set(
versions,
self.base_config,
deployment_mode,
run_props
)
lh.section("configuration")
print(
"""
mode: {mode}
{cfg_repr}
""".format(
**{"mode": str(deployment_mode), "cfg_repr": repr(installers[0][0])}
)
)
count = 1
for runner_type in STARTER_MODES[self.base_config.starter_mode]:
with AllureTestSuiteContext(properties=run_props,
versions=versions,
parent_test_suite_name=None,
auto_generate_parent_test_suite_name=True,
suite_name=runner_strings[runner_type],
runner_type=None,
installer_type=installers[0][1].installer_type):
with RtaTestcase(runner_strings[runner_type] + " main flow") as testcase:
if not run_props.supports_dc2dc() and runner_type == RunnerType.DC2DC:
testcase.context.status = Status.SKIPPED
testcase.context.statusDetails = StatusDetails(
message="DC2DC is not applicable to Community packages.\nDC2DC is not supported on Windows.")
continue
one_result = {
"testrun name": run_props.testrun_name,
"testscenario": runner_strings[runner_type],
"success": True,
"messages": [],
"progress": "",
}
runner = make_runner(
runner_type,
self.abort_on_error,
self.selenium,
self.selenium_driver_args,
installers,
run_props,
use_auto_certs=self.use_auto_certs,
)
# install on first run:
runner.do_install = (count == 1) and do_install
# only uninstall after the last test:
runner.do_uninstall = (count == len(
STARTER_MODES[deployment_mode])) and do_uninstall
try:
runner.run()
runner.cleanup()
testcase.context.status = Status.PASSED
# pylint: disable=broad-except
except Exception as ex:
one_result["success"] = False
one_result["messages"].append(str(ex))
one_result["progress"] += runner.get_progress()
runner.take_screenshot()
runner.agency_acquire_dump()
runner.search_for_warnings()
runner.quit_selenium()
kill_all_processes()
runner.zip_test_dir()
self.copy_packages_to_result(installers)
testcase.context.status = Status.FAILED
testcase.context.statusDetails = StatusDetails(message=str(ex),
trace="".join(
traceback.TracebackException.from_exception(
ex).format()))
lh.section("uninstall on error")
installers[0][1].un_install_debug_package()
installers[0][1].un_install_server_package()
installers[0][1].cleanup_system()
if self.abort_on_error:
raise ex
traceback.print_exc()
lh.section("uninstall on error")
try:
runner.cleanup()
finally:
pass
results.append(one_result)
continue
if runner.ui_tests_failed:
failed_test_names = [f'"{row["Name"]}"' for row in
runner.ui_test_results_table if
not row["Result"] == "PASSED"]
one_result["success"] = False
# pylint: disable=line-too-long
one_result[
"messages"].append(
f'The following UI tests failed: {", ".join(failed_test_names)}. See allure report for details.')
kill_all_processes()
count += 1
return results
def run_upgrade(self, versions: list, run_props: RunProperties):
"""execute upgrade tests"""
lh.section("startup")
results = []
for runner_type in STARTER_MODES[self.base_config.starter_mode]:
installers = create_config_installer_set(
versions,
self.base_config,
"all",
run_props,
)
old_inst = installers[0][1]
new_inst = installers[1][1]
with AllureTestSuiteContext(
properties=run_props,
versions=versions,
parent_test_suite_name=None,
auto_generate_parent_test_suite_name=True,
suite_name=runner_strings[runner_type],
runner_type=None,
installer_type=new_inst.installer_type,
):
with RtaTestcase(runner_strings[runner_type] + " main flow") as testcase:
if not run_props.supports_dc2dc() and runner_type == RunnerType.DC2DC:
testcase.context.status = Status.SKIPPED
testcase.context.statusDetails = StatusDetails(
message="DC2DC is not applicable to Community packages.\nDC2DC is not supported on Windows."
)
continue
one_result = {
"testrun name": run_props.testrun_name,
"testscenario": runner_strings[runner_type],
"success": True,
"messages": [],
"progress": "",
}
try:
kill_all_processes()
runner = None
lh.section("configuration")
print(
"""
starter mode: {starter_mode}
old version: {old_version}
{cfg_repr}
""".format(
**{
"starter_mode": str(self.base_config.starter_mode),
"old_version": str(versions[0]),
"cfg_repr": repr(installers[1][0]),
}
)
)
if runner_type:
runner = make_runner(
runner_type,
self.abort_on_error,
self.selenium,
self.selenium_driver_args,
installers,
run_props,
use_auto_certs=self.use_auto_certs,
)
if runner:
try:
runner.run()
runner.cleanup()
testcase.context.status = Status.PASSED
except Exception as ex:
one_result["success"] = False
one_result["messages"].append(str(ex))
one_result["progress"] += runner.get_progress()
runner.take_screenshot()
runner.agency_acquire_dump()
runner.search_for_warnings()
runner.quit_selenium()
kill_all_processes()
runner.zip_test_dir()
self.copy_packages_to_result(installers)
testcase.context.status = Status.FAILED
testcase.context.statusDetails = StatusDetails(
message=str(ex),
trace="".join(traceback.TracebackException.from_exception(ex).format()),
)
if self.abort_on_error:
raise ex
one_result["progress"] += str(ex) + "".join(
traceback.TracebackException.from_exception(ex).format()
)
traceback.print_exc()
lh.section("uninstall on error")
old_inst.un_install_debug_package()
old_inst.un_install_server_package()
old_inst.cleanup_system()
try:
runner.cleanup()
finally:
pass
results.append(one_result)
continue
if runner.ui_tests_failed:
failed_test_names = [
f'"{row["Name"]}"'
for row in runner.ui_test_results_table
if not row["Result"] == "PASSED"
]
one_result["success"] = False
one_result["messages"].append(
f'The following UI tests failed: {", ".join(failed_test_names)}.'
+ "See allure report for details."
)
lh.section("uninstall")
new_inst.un_install_server_package()
lh.section("check system")
new_inst.check_uninstall_cleanup()
lh.section("remove residuals")
try:
old_inst.cleanup_system()
except Exception:
print("Ignoring old cleanup error!")
try:
print("Ignoring new cleanup error!")
new_inst.cleanup_system()
except Exception:
print("Ignoring general cleanup error!")
except Exception as ex:
print("Caught. " + str(ex))
one_result["success"] = False
one_result["messages"].append(str(ex))
one_result["progress"] += "\naborted outside of testcodes"
if self.abort_on_error:
print("re-throwing.")
raise ex
traceback.print_exc()
kill_all_processes()
if runner:
try:
runner.cleanup()
except Exception as exception:
print("Ignoring runner cleanup error! Exception:")
print(str(exception))
print("".join(traceback.TracebackException.from_exception(exception).format()))
try:
print("Cleaning up system after error:")
old_inst.un_install_debug_package()
old_inst.un_install_server_package()
old_inst.cleanup_system()
except Exception:
print("Ignoring old cleanup error!")
try:
print("Ignoring new cleanup error!")
new_inst.un_install_debug_package()
new_inst.un_install_server_package()
new_inst.cleanup_system()
except Exception:
print("Ignoring new cleanup error!")
results.append(one_result)
return results