def starter_run_impl(self):
lh.subsection("instance setup")
#if self.remote:
# logging.info("running remote, skipping")
# return
for manager in self.starter_instances:
logging.info("Spawning instance")
manager.run_starter()
logging.info("waiting for the starters to become alive")
not_started = self.starter_instances[:] #This is a explicit copy
while not_started:
logging.debug("waiting for mananger with logfile:" +
str(not_started[-1].log_file))
if not_started[-1].is_instance_up():
not_started.pop()
progress('.')
time.sleep(1)
logging.info("waiting for the cluster instances to become alive")
for node in self.starter_instances:
node.detect_instances()
node.detect_instance_pids()
#self.basecfg.add_frontend('http', self.basecfg.publicip, str(node.get_frontend_port()))
logging.info("instances are ready")
def starter_run_impl(self):
lh.subsection("instance setup")
for manager in self.starter_instances:
logging.info("Spawning instance")
manager.run_starter()
logging.info("waiting for the starters to become alive")
not_started = self.starter_instances[:] # This is a explicit copy
while not_started:
logging.debug("waiting for mananger with logfile:" +
str(not_started[-1].log_file))
if not_started[-1].is_instance_up():
not_started.pop()
progress(".")
time.sleep(1)
logging.info("waiting for the cluster instances to become alive")
for node in self.starter_instances:
node.detect_instances()
node.detect_instance_pids()
# self.basecfg.add_frontend('http', self.basecfg.publicip, str(node.get_frontend_port()))
logging.info("instances are ready - JWT: " +
self.starter_instances[0].get_jwt_header())
count = 0
for node in self.starter_instances:
node.set_passvoid("cluster", count == 0)
count += 1
self.passvoid = "cluster"
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.test_after_install()
# assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.follower_starter_instance.supports_foxx_tests = False
logging.info("Leader follower testing makedata on follower")
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
if self.selenium:
self.selenium.test_setup()
logging.info("Leader follower setup successfully finished!")
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS'], False):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.connect_server_new_tab(
self.follower_starter_instance.get_frontends(), '_system',
self.cfg)
self.selenium.check_old(self.new_cfg if self.new_cfg else self.cfg,
False)
self.selenium.close_tab_again()
#assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
logging.info("Leader follower setup successfully finished!")
def uninstall(self, inst):
""" uninstall the package from the system """
lh.subsection("{0} - uninstall package".format(str(self.name)))
if self.cfg.have_debug_package:
print('uninstalling debug package')
inst.un_install_debug_package()
print('uninstalling server package')
inst.un_install_package()
inst.check_uninstall_cleanup()
inst.cleanup_system()
def test_setup_impl(self):
logging.info("testing the single server setup")
tries = 30
lh.subsection("single server - check test data", "-")
if self.selenium:
self.selenium.test_after_install()
self.make_data()
if self.selenium:
self.selenium.test_setup()
logging.info("Single setup successfully finished!")
def upgrade_arangod_version(self):
""" upgrade this installation """
lh.subsection("{0} - upgrade setup to newer version".format(
str(self.name)))
logging.info("{1} -> {0}".format(self.new_installer.cfg.version,
self.old_installer.cfg.version))
print("deinstall")
print("install")
print("replace starter")
print("upgrade instances")
self.upgrade_arangod_version_impl()
print("check data in instaces")
def progress(self, is_sub, msg, separator="x", supress_allure=False):
"""report user message, record for error handling."""
if self.selenium:
self.state += self.selenium.get_progress()
if is_sub:
if separator == "x":
separator = "="
lh.subsection(msg, separator)
self.state += " - " + msg
else:
if separator == "x":
separator = "#"
lh.section(msg, separator)
self.state += "*** " + msg
if not supress_allure:
with step("Progress: " + msg):
pass
def install(self, inst):
""" install the package to the system """
lh.subsection("{0} - install package".format(str(self.name)))
kill_all_processes(False)
if self.do_install:
lh.subsubsection("installing package")
inst.install_package()
self.cfg.set_directories(inst.cfg)
lh.subsubsection("checking files")
inst.check_installed_files()
lh.subsubsection("saving config")
inst.save_config()
lh.subsubsection("checking if service is up")
if inst.check_service_up():
lh.subsubsection("stopping service")
inst.stop_service()
inst.broadcast_bind()
lh.subsubsection("starting service")
inst.start_service()
inst.check_installed_paths()
inst.check_engine_file()
if not self.new_installer:
# only install debug package for new package.
lh.subsection('installing debug package:')
self.cfg.have_debug_package = inst.install_debug_package()
if self.cfg.have_debug_package:
lh.subsection('testing debug symbols')
inst.gdb_test()
# start / stop
if inst.check_service_up():
inst.stop_service()
inst.start_service()
sys_arangosh = ArangoshExecutor(inst.cfg, inst.instance)
logging.debug("self test after installation")
if inst.cfg.have_system_service:
sys_arangosh.self_test()
if self.do_system_test:
sys_arangosh.js_version_check()
# TODO: here we should invoke Makedata for the system installation.
logging.debug("stop system service "
"to make ports available for starter")
inst.stop_service()
def starter_run(self):
"""
now launch the starter instance s- at this point the basic setup is done
"""
lh.subsection("{0} - run starter instances".format(str(self.name)))
self.starter_run_impl()
def jam_attempt(self):
""" check resilience of setup by obstructing its instances """
lh.subsection("{0}{1} - try to jam setup".format(
self.versionstr, str(self.name)))
self.jam_attempt_impl()
def finish_setup(self):
""" not finish the setup"""
lh.subsection("{0} - finish setup".format(str(self.name)))
self.finish_setup_impl()
def run(self):
"""run the full lifecycle flow of this deployment"""
# pylint: disable=too-many-statements disable=too-many-branches
if self.do_starter_test and not self.remote:
detect_file_ulimit()
self.progress(False, "Runner of type {0}".format(str(self.name)), "<3")
if self.do_install or self.do_system_test:
self.progress(
False,
"INSTALLATION for {0}".format(str(self.name)),
)
self.install(self.old_installer)
else:
self.basecfg.set_directories(self.old_installer.cfg)
if self.do_starter_test:
self.progress(
False,
"PREPARING DEPLOYMENT of {0}".format(str(self.name)),
)
self.starter_prepare_env()
self.starter_run()
self.finish_setup()
self.make_data()
if self.selenium:
self.set_selenium_instances()
self.selenium.test_empty_ui()
ti.prompt_user(
self.basecfg,
"{0}{1} Deployment started. Please test the UI!".format((self.versionstr), str(self.name)),
)
if self.hot_backup:
self.progress(False, "TESTING HOTBACKUP")
self.backup_name = self.create_backup("thy_name_is_" + self.name)
self.validate_local_backup(self.backup_name)
self.tcp_ping_all_nodes()
self.create_non_backup_data()
backups = self.list_backup()
print(backups)
self.upload_backup(backups[0])
self.tcp_ping_all_nodes()
self.delete_backup(backups[0])
self.tcp_ping_all_nodes()
backups = self.list_backup()
if len(backups) != 0:
raise Exception("expected backup to be gone, " "but its still there: " + str(backups))
self.download_backup(self.backup_name)
self.validate_local_backup(self.backup_name)
self.tcp_ping_all_nodes()
backups = self.list_backup()
if backups[0] != self.backup_name:
raise Exception("downloaded backup has different name? " + str(backups))
self.before_backup()
self.restore_backup(backups[0])
self.tcp_ping_all_nodes()
self.after_backup()
self.check_data_impl()
if not self.check_non_backup_data():
raise Exception("data created after backup" " is still there??")
if self.new_installer:
if self.hot_backup:
self.create_non_backup_data()
self.versionstr = "NEW[" + self.new_cfg.version + "] "
self.progress(
False,
"UPGRADE OF DEPLOYMENT {0}".format(str(self.name)),
)
self.new_installer.calculate_package_names()
self.new_installer.upgrade_server_package(self.old_installer)
lh.subsection("outputting version")
self.new_installer.output_arangod_version()
self.new_installer.get_starter_version()
self.new_installer.get_sync_version()
self.new_installer.stop_service()
self.cfg.set_directories(self.new_installer.cfg)
self.new_cfg.set_directories(self.new_installer.cfg)
self.upgrade_arangod_version() # make sure to pass new version
self.old_installer.un_install_server_package_for_upgrade()
if self.is_minor_upgrade() and self.new_installer.supports_backup():
self.new_installer.check_backup_is_created()
if self.hot_backup:
self.check_data_impl()
self.progress(False, "TESTING HOTBACKUP AFTER UPGRADE")
backups = self.list_backup()
print(backups)
self.upload_backup(backups[0])
self.tcp_ping_all_nodes()
self.delete_backup(backups[0])
self.tcp_ping_all_nodes()
backups = self.list_backup()
if len(backups) != 0:
raise Exception("expected backup to be gone, " "but its still there: " + str(backups))
self.download_backup(self.backup_name)
self.validate_local_backup(self.backup_name)
self.tcp_ping_all_nodes()
backups = self.list_backup()
if backups[0] != self.backup_name:
raise Exception("downloaded backup has different name? " + str(backups))
time.sleep(20) # TODO fix
self.before_backup()
self.restore_backup(backups[0])
self.tcp_ping_all_nodes()
self.after_backup()
if not self.check_non_backup_data():
raise Exception("data created after " "backup is still there??")
self.check_data_impl()
else:
logging.info("skipping upgrade step no new version given")
try:
if self.do_starter_test:
self.progress(
False,
"{0} TESTS FOR {1}".format(self.testrun_name, str(self.name)),
)
self.test_setup()
self.jam_attempt()
self.starter_shutdown()
for starter in self.starter_instances:
starter.detect_fatal_errors()
if self.do_uninstall:
self.uninstall(self.old_installer if not self.new_installer else self.new_installer)
finally:
if self.selenium:
ui_test_results_table = BeautifulTable(maxwidth=160)
for result in self.selenium.test_results:
ui_test_results_table.rows.append(
[result.name, "PASSED" if result.success else "FAILED", result.message, result.traceback]
)
if not result.success:
self.ui_tests_failed = True
ui_test_results_table.columns.header = ["Name", "Result", "Message", "Traceback"]
self.progress(False, "UI test results table:", supress_allure=True)
self.progress(False, "\n" + str(ui_test_results_table), supress_allure=True)
self.ui_test_results_table = ui_test_results_table
self.quit_selenium()
self.progress(False, "Runner of type {0} - Finished!".format(str(self.name)))
def run(self):
""" run the full lifecycle flow of this deployment """
# pylint: disable=R0915 disable=R0912
if self.do_starter_test and not self.remote:
self.detect_file_ulimit()
lh.section("Runner of type {0}".format(str(self.name)), "<3")
if self.do_install or self.do_system_test:
lh.section("INSTALLATION for {0}".format(str(self.name)), )
self.install(self.old_installer)
if self.do_starter_test:
lh.section("PREPARING DEPLOYMENT of {0}".format(str(self.name)), )
self.starter_prepare_env()
self.starter_run()
self.finish_setup()
self.make_data()
if self.selenium:
self.selenium.connect_server(self.get_frontend_instances(),
'_system', self.cfg)
self.selenium.check_old(self.old_installer.cfg)
ti.prompt_user(
self.basecfg,
"{0}{1} Deployment started. Please test the UI!".format(
(self.versionstr), str(self.name)))
if self.hot_backup:
lh.section("TESTING HOTBACKUP")
self.before_backup()
# TODO generate name?
self.backup_name = self.create_backup("thy_name_is")
self.tcp_ping_all_nodes()
self.create_non_backup_data()
backups = self.list_backup()
print(backups)
self.upload_backup(backups[0])
self.tcp_ping_all_nodes()
self.delete_backup(backups[0])
self.tcp_ping_all_nodes()
backups = self.list_backup()
if len(backups) != 0:
raise Exception("expected backup to be gone, "
"but its still there: " + str(backups))
self.download_backup(self.backup_name)
self.tcp_ping_all_nodes()
backups = self.list_backup()
if backups[0] != self.backup_name:
raise Exception("downloaded backup has different name? " +
str(backups))
time.sleep(20) # TODO fix
self.restore_backup(backups[0])
self.tcp_ping_all_nodes()
self.after_backup()
self.check_data_impl()
if not self.check_non_backup_data():
raise Exception("data created after backup"
" is still there??")
self.create_non_backup_data()
if self.new_installer:
self.versionstr = "NEW[" + self.new_cfg.version + "] "
lh.section("UPGRADE OF DEPLOYMENT {0}".format(str(self.name)), )
if self.cfg.have_debug_package:
print('removing *old* debug package in advance')
self.old_installer.un_install_debug_package()
self.new_installer.upgrade_package(self.old_installer)
# only install debug package for new package.
lh.subsection('installing debug package:')
self.cfg.have_debug_package = self.new_installer.install_debug_package(
)
if self.cfg.have_debug_package:
self.new_installer.gdb_test()
self.new_installer.stop_service()
self.cfg.set_directories(self.new_installer.cfg)
self.new_cfg.set_directories(self.new_installer.cfg)
self.old_installer.un_install_package_for_upgrade()
self.upgrade_arangod_version() #make sure to pass new version
self.make_data_after_upgrade()
if self.hot_backup:
lh.section("TESTING HOTBACKUP AFTER UPGRADE")
self.before_backup()
backups = self.list_backup()
print(backups)
self.upload_backup(backups[0])
self.tcp_ping_all_nodes()
self.delete_backup(backups[0])
self.tcp_ping_all_nodes()
backups = self.list_backup()
if len(backups) != 0:
raise Exception("expected backup to be gone, "
"but its still there: " + str(backups))
self.download_backup(self.backup_name)
self.tcp_ping_all_nodes()
backups = self.list_backup()
if backups[0] != self.backup_name:
raise Exception("downloaded backup has different name? " +
str(backups))
time.sleep(20) # TODO fix
self.restore_backup(backups[0])
self.tcp_ping_all_nodes()
self.after_backup()
if not self.check_non_backup_data():
raise Exception("data created after "
"backup is still there??")
self.check_data_impl()
else:
logging.info("skipping upgrade step no new version given")
if self.do_starter_test:
lh.section("TESTS FOR {0}".format(str(self.name)), )
self.test_setup()
self.jam_attempt()
self.starter_shutdown()
if self.do_uninstall:
self.uninstall(self.old_installer
if not self.new_installer else self.new_installer)
self.selenium.disconnect()
lh.section("Runner of type {0} - Finished!".format(str(self.name)))
def make_data(self):
""" check if setup is functional """
lh.subsection("{0} - make data".format(str(self.name)))
self.make_data_impl()
def after_backup(self):
""" HotBackup has happened, prepare the SUT to continue testing """
lh.subsection("{0} - preparing SUT for tests after HotBackup".format(
str(self.name)))
self.after_backup_impl()
def before_backup(self):
""" preparing SUT for the execution of the backup steps """
lh.subsection("{0} - preparing SUT for HotBackup".format(str(
self.name)))
self.before_backup_impl()
def starter_prepare_env(self):
""" base setup; declare instance variables etc """
lh.subsection("{0} - prepare starter launch".format(str(self.name)))
self.starter_prepare_env_impl()
def make_data_after_upgrade(self):
""" check if setup is functional """
lh.subsection("{0} - make data after upgrade".format(str(self.name)))
self.make_data_after_upgrade_impl()
def test_setup(self):
""" setup steps after the basic instances were launched """
lh.subsection("{0} - basic test after startup".format(str(self.name)))
self.test_setup_impl()
def detect_instances(self):
""" see which arangods where spawned and inspect their logfiles"""
lh.subsection("Instance Detection for {0.name}".format(self))
self.all_instances = []
logging.debug("waiting for frontend")
logfiles = set() #logfiles that can be used for debugging
# the more instances we expect to spawn the more patient:
tries = 10 * self.expect_instance_count
# Wait for forntend to become alive.
all_instances_up = False
while not all_instances_up and tries:
self.all_instances = []
detected_instances = []
sys.stdout.write(".")
sys.stdout.flush()
for root, dirs, files in os.walk(self.basedir):
for onefile in files:
#logging.debug("f: " + root + os.path.sep + onefile)
if onefile.endswith("log"):
logfiles.add(str(Path(root) / onefile))
for name in dirs:
#logging.debug("d: " + root + os.path.sep + name)
match = None
instance_class = None
if name.startswith('sync'):
match = re.match(r'(syncmaster|syncworker)(\d*)', name)
instance_class = SyncInstance
else:
match = re.match(
r'(agent|coordinator|dbserver|resilientsingle|single)(\d*)',
name)
instance_class = ArangodInstance
# directory = self.basedir / name
if match:
# we may see a `local-slave-*` directory inbetween,
# hence we need to choose the current directory not
# the starter toplevel dir for this:
instance = instance_class(match.group(1),
match.group(2),
self.cfg.localhost,
self.cfg.publicip,
Path(root) / name,
self.passvoid)
instance.wait_for_logfile(tries)
instance.detect_pid(
ppid=self.instance.pid,
full_binary_path=self.cfg.real_sbin_dir,
offset=0)
detected_instances.append(instance.type)
self.all_instances.append(instance)
print(self.expect_instances)
detected_instances.sort()
print(detected_instances)
if ((self.expect_instances != detected_instances)
or (not self.get_frontends())):
tries -= 1
time.sleep(5)
else:
all_instances_up = True
if not self.get_frontends():
print()
logging.error("STARTER FAILED TO SPAWN ARANGOD")
self.show_all_instances()
logging.error("can not continue without frontend instance")
logging.error("please check logs in" + str(self.basedir))
for logf in logfiles:
logging.debug(logf)
logging.error("if that does not help try to delete: " +
str(self.basedir))
sys.exit(1)
self.show_all_instances()
def starter_shutdown(self):
""" stop everything """
lh.subsection("{0}{1} - shutdown".format(self.versionstr,
str(self.name)))
self.shutdown_impl()
