def starter_run_impl(self): lh.subsection("instance setup") #if self.remote: # logging.info("running remote, skipping") # return for manager in self.starter_instances: logging.info("Spawning instance") manager.run_starter() logging.info("waiting for the starters to become alive") not_started = self.starter_instances[:] #This is a explicit copy while not_started: logging.debug("waiting for mananger with logfile:" + str(not_started[-1].log_file)) if not_started[-1].is_instance_up(): not_started.pop() progress('.') time.sleep(1) logging.info("waiting for the cluster instances to become alive") for node in self.starter_instances: node.detect_instances() node.detect_instance_pids() #self.basecfg.add_frontend('http', self.basecfg.publicip, str(node.get_frontend_port())) logging.info("instances are ready")
def starter_run_impl(self): lh.subsection("instance setup") for manager in self.starter_instances: logging.info("Spawning instance") manager.run_starter() logging.info("waiting for the starters to become alive") not_started = self.starter_instances[:] # This is a explicit copy while not_started: logging.debug("waiting for mananger with logfile:" + str(not_started[-1].log_file)) if not_started[-1].is_instance_up(): not_started.pop() progress(".") time.sleep(1) logging.info("waiting for the cluster instances to become alive") for node in self.starter_instances: node.detect_instances() node.detect_instance_pids() # self.basecfg.add_frontend('http', self.basecfg.publicip, str(node.get_frontend_port())) logging.info("instances are ready - JWT: " + self.starter_instances[0].get_jwt_header()) count = 0 for node in self.starter_instances: node.set_passvoid("cluster", count == 0) count += 1 self.passvoid = "cluster"
def test_setup_impl(self): logging.info("testing the leader/follower setup") tries = 30 if not self.follower_starter_instance.execute_frontend( self.checks["checkReplJS"]): while tries: if self.follower_starter_instance.execute_frontend( self.checks["checkReplJS"]): break progress(".") time.sleep(1) tries -= 1 if not tries: if not self.follower_starter_instance.execute_frontend( self.checks["checkReplJS"]): raise Exception("replication didn't make it in 30s!") lh.subsection("leader/follower - check test data", "-") if self.selenium: self.selenium.test_after_install() # assert that data has been replicated self.follower_starter_instance.arangosh.read_only = True self.follower_starter_instance.supports_foxx_tests = False logging.info("Leader follower testing makedata on follower") self.makedata_instances.append(self.follower_starter_instance) self.make_data() if self.selenium: self.selenium.test_setup() logging.info("Leader follower setup successfully finished!")
def test_setup_impl(self): logging.info("testing the leader/follower setup") tries = 30 if not self.follower_starter_instance.execute_frontend( self.checks['checkReplJS']): while tries: if self.follower_starter_instance.execute_frontend( self.checks['checkReplJS'], False): break progress(".") time.sleep(1) tries -= 1 if not tries: if not self.follower_starter_instance.execute_frontend( self.checks['checkReplJS']): raise Exception("replication didn't make it in 30s!") lh.subsection("leader/follower - check test data", "-") if self.selenium: self.selenium.connect_server_new_tab( self.follower_starter_instance.get_frontends(), '_system', self.cfg) self.selenium.check_old(self.new_cfg if self.new_cfg else self.cfg, False) self.selenium.close_tab_again() #assert that data has been replicated self.follower_starter_instance.arangosh.read_only = True self.makedata_instances.append(self.follower_starter_instance) self.make_data() logging.info("Leader follower setup successfully finished!")
def uninstall(self, inst): """ uninstall the package from the system """ lh.subsection("{0} - uninstall package".format(str(self.name))) if self.cfg.have_debug_package: print('uninstalling debug package') inst.un_install_debug_package() print('uninstalling server package') inst.un_install_package() inst.check_uninstall_cleanup() inst.cleanup_system()
def test_setup_impl(self): logging.info("testing the single server setup") tries = 30 lh.subsection("single server - check test data", "-") if self.selenium: self.selenium.test_after_install() self.make_data() if self.selenium: self.selenium.test_setup() logging.info("Single setup successfully finished!")
def upgrade_arangod_version(self): """ upgrade this installation """ lh.subsection("{0} - upgrade setup to newer version".format( str(self.name))) logging.info("{1} -> {0}".format(self.new_installer.cfg.version, self.old_installer.cfg.version)) print("deinstall") print("install") print("replace starter") print("upgrade instances") self.upgrade_arangod_version_impl() print("check data in instaces")
def progress(self, is_sub, msg, separator="x", supress_allure=False): """report user message, record for error handling.""" if self.selenium: self.state += self.selenium.get_progress() if is_sub: if separator == "x": separator = "=" lh.subsection(msg, separator) self.state += " - " + msg else: if separator == "x": separator = "#" lh.section(msg, separator) self.state += "*** " + msg if not supress_allure: with step("Progress: " + msg): pass
def install(self, inst): """ install the package to the system """ lh.subsection("{0} - install package".format(str(self.name))) kill_all_processes(False) if self.do_install: lh.subsubsection("installing package") inst.install_package() self.cfg.set_directories(inst.cfg) lh.subsubsection("checking files") inst.check_installed_files() lh.subsubsection("saving config") inst.save_config() lh.subsubsection("checking if service is up") if inst.check_service_up(): lh.subsubsection("stopping service") inst.stop_service() inst.broadcast_bind() lh.subsubsection("starting service") inst.start_service() inst.check_installed_paths() inst.check_engine_file() if not self.new_installer: # only install debug package for new package. lh.subsection('installing debug package:') self.cfg.have_debug_package = inst.install_debug_package() if self.cfg.have_debug_package: lh.subsection('testing debug symbols') inst.gdb_test() # start / stop if inst.check_service_up(): inst.stop_service() inst.start_service() sys_arangosh = ArangoshExecutor(inst.cfg, inst.instance) logging.debug("self test after installation") if inst.cfg.have_system_service: sys_arangosh.self_test() if self.do_system_test: sys_arangosh.js_version_check() # TODO: here we should invoke Makedata for the system installation. logging.debug("stop system service " "to make ports available for starter") inst.stop_service()
def starter_run(self): """ now launch the starter instance s- at this point the basic setup is done """ lh.subsection("{0} - run starter instances".format(str(self.name))) self.starter_run_impl()
def jam_attempt(self): """ check resilience of setup by obstructing its instances """ lh.subsection("{0}{1} - try to jam setup".format( self.versionstr, str(self.name))) self.jam_attempt_impl()
def finish_setup(self): """ not finish the setup""" lh.subsection("{0} - finish setup".format(str(self.name))) self.finish_setup_impl()
def run(self): """run the full lifecycle flow of this deployment""" # pylint: disable=too-many-statements disable=too-many-branches if self.do_starter_test and not self.remote: detect_file_ulimit() self.progress(False, "Runner of type {0}".format(str(self.name)), "<3") if self.do_install or self.do_system_test: self.progress( False, "INSTALLATION for {0}".format(str(self.name)), ) self.install(self.old_installer) else: self.basecfg.set_directories(self.old_installer.cfg) if self.do_starter_test: self.progress( False, "PREPARING DEPLOYMENT of {0}".format(str(self.name)), ) self.starter_prepare_env() self.starter_run() self.finish_setup() self.make_data() if self.selenium: self.set_selenium_instances() self.selenium.test_empty_ui() ti.prompt_user( self.basecfg, "{0}{1} Deployment started. Please test the UI!".format((self.versionstr), str(self.name)), ) if self.hot_backup: self.progress(False, "TESTING HOTBACKUP") self.backup_name = self.create_backup("thy_name_is_" + self.name) self.validate_local_backup(self.backup_name) self.tcp_ping_all_nodes() self.create_non_backup_data() backups = self.list_backup() print(backups) self.upload_backup(backups[0]) self.tcp_ping_all_nodes() self.delete_backup(backups[0]) self.tcp_ping_all_nodes() backups = self.list_backup() if len(backups) != 0: raise Exception("expected backup to be gone, " "but its still there: " + str(backups)) self.download_backup(self.backup_name) self.validate_local_backup(self.backup_name) self.tcp_ping_all_nodes() backups = self.list_backup() if backups[0] != self.backup_name: raise Exception("downloaded backup has different name? " + str(backups)) self.before_backup() self.restore_backup(backups[0]) self.tcp_ping_all_nodes() self.after_backup() self.check_data_impl() if not self.check_non_backup_data(): raise Exception("data created after backup" " is still there??") if self.new_installer: if self.hot_backup: self.create_non_backup_data() self.versionstr = "NEW[" + self.new_cfg.version + "] " self.progress( False, "UPGRADE OF DEPLOYMENT {0}".format(str(self.name)), ) self.new_installer.calculate_package_names() self.new_installer.upgrade_server_package(self.old_installer) lh.subsection("outputting version") self.new_installer.output_arangod_version() self.new_installer.get_starter_version() self.new_installer.get_sync_version() self.new_installer.stop_service() self.cfg.set_directories(self.new_installer.cfg) self.new_cfg.set_directories(self.new_installer.cfg) self.upgrade_arangod_version() # make sure to pass new version self.old_installer.un_install_server_package_for_upgrade() if self.is_minor_upgrade() and self.new_installer.supports_backup(): self.new_installer.check_backup_is_created() if self.hot_backup: self.check_data_impl() self.progress(False, "TESTING HOTBACKUP AFTER UPGRADE") backups = self.list_backup() print(backups) self.upload_backup(backups[0]) self.tcp_ping_all_nodes() self.delete_backup(backups[0]) self.tcp_ping_all_nodes() backups = self.list_backup() if len(backups) != 0: raise Exception("expected backup to be gone, " "but its still there: " + str(backups)) self.download_backup(self.backup_name) self.validate_local_backup(self.backup_name) self.tcp_ping_all_nodes() backups = self.list_backup() if backups[0] != self.backup_name: raise Exception("downloaded backup has different name? " + str(backups)) time.sleep(20) # TODO fix self.before_backup() self.restore_backup(backups[0]) self.tcp_ping_all_nodes() self.after_backup() if not self.check_non_backup_data(): raise Exception("data created after " "backup is still there??") self.check_data_impl() else: logging.info("skipping upgrade step no new version given") try: if self.do_starter_test: self.progress( False, "{0} TESTS FOR {1}".format(self.testrun_name, str(self.name)), ) self.test_setup() self.jam_attempt() self.starter_shutdown() for starter in self.starter_instances: starter.detect_fatal_errors() if self.do_uninstall: self.uninstall(self.old_installer if not self.new_installer else self.new_installer) finally: if self.selenium: ui_test_results_table = BeautifulTable(maxwidth=160) for result in self.selenium.test_results: ui_test_results_table.rows.append( [result.name, "PASSED" if result.success else "FAILED", result.message, result.traceback] ) if not result.success: self.ui_tests_failed = True ui_test_results_table.columns.header = ["Name", "Result", "Message", "Traceback"] self.progress(False, "UI test results table:", supress_allure=True) self.progress(False, "\n" + str(ui_test_results_table), supress_allure=True) self.ui_test_results_table = ui_test_results_table self.quit_selenium() self.progress(False, "Runner of type {0} - Finished!".format(str(self.name)))
def run(self): """ run the full lifecycle flow of this deployment """ # pylint: disable=R0915 disable=R0912 if self.do_starter_test and not self.remote: self.detect_file_ulimit() lh.section("Runner of type {0}".format(str(self.name)), "<3") if self.do_install or self.do_system_test: lh.section("INSTALLATION for {0}".format(str(self.name)), ) self.install(self.old_installer) if self.do_starter_test: lh.section("PREPARING DEPLOYMENT of {0}".format(str(self.name)), ) self.starter_prepare_env() self.starter_run() self.finish_setup() self.make_data() if self.selenium: self.selenium.connect_server(self.get_frontend_instances(), '_system', self.cfg) self.selenium.check_old(self.old_installer.cfg) ti.prompt_user( self.basecfg, "{0}{1} Deployment started. Please test the UI!".format( (self.versionstr), str(self.name))) if self.hot_backup: lh.section("TESTING HOTBACKUP") self.before_backup() # TODO generate name? self.backup_name = self.create_backup("thy_name_is") self.tcp_ping_all_nodes() self.create_non_backup_data() backups = self.list_backup() print(backups) self.upload_backup(backups[0]) self.tcp_ping_all_nodes() self.delete_backup(backups[0]) self.tcp_ping_all_nodes() backups = self.list_backup() if len(backups) != 0: raise Exception("expected backup to be gone, " "but its still there: " + str(backups)) self.download_backup(self.backup_name) self.tcp_ping_all_nodes() backups = self.list_backup() if backups[0] != self.backup_name: raise Exception("downloaded backup has different name? " + str(backups)) time.sleep(20) # TODO fix self.restore_backup(backups[0]) self.tcp_ping_all_nodes() self.after_backup() self.check_data_impl() if not self.check_non_backup_data(): raise Exception("data created after backup" " is still there??") self.create_non_backup_data() if self.new_installer: self.versionstr = "NEW[" + self.new_cfg.version + "] " lh.section("UPGRADE OF DEPLOYMENT {0}".format(str(self.name)), ) if self.cfg.have_debug_package: print('removing *old* debug package in advance') self.old_installer.un_install_debug_package() self.new_installer.upgrade_package(self.old_installer) # only install debug package for new package. lh.subsection('installing debug package:') self.cfg.have_debug_package = self.new_installer.install_debug_package( ) if self.cfg.have_debug_package: self.new_installer.gdb_test() self.new_installer.stop_service() self.cfg.set_directories(self.new_installer.cfg) self.new_cfg.set_directories(self.new_installer.cfg) self.old_installer.un_install_package_for_upgrade() self.upgrade_arangod_version() #make sure to pass new version self.make_data_after_upgrade() if self.hot_backup: lh.section("TESTING HOTBACKUP AFTER UPGRADE") self.before_backup() backups = self.list_backup() print(backups) self.upload_backup(backups[0]) self.tcp_ping_all_nodes() self.delete_backup(backups[0]) self.tcp_ping_all_nodes() backups = self.list_backup() if len(backups) != 0: raise Exception("expected backup to be gone, " "but its still there: " + str(backups)) self.download_backup(self.backup_name) self.tcp_ping_all_nodes() backups = self.list_backup() if backups[0] != self.backup_name: raise Exception("downloaded backup has different name? " + str(backups)) time.sleep(20) # TODO fix self.restore_backup(backups[0]) self.tcp_ping_all_nodes() self.after_backup() if not self.check_non_backup_data(): raise Exception("data created after " "backup is still there??") self.check_data_impl() else: logging.info("skipping upgrade step no new version given") if self.do_starter_test: lh.section("TESTS FOR {0}".format(str(self.name)), ) self.test_setup() self.jam_attempt() self.starter_shutdown() if self.do_uninstall: self.uninstall(self.old_installer if not self.new_installer else self.new_installer) self.selenium.disconnect() lh.section("Runner of type {0} - Finished!".format(str(self.name)))
def make_data(self): """ check if setup is functional """ lh.subsection("{0} - make data".format(str(self.name))) self.make_data_impl()
def after_backup(self): """ HotBackup has happened, prepare the SUT to continue testing """ lh.subsection("{0} - preparing SUT for tests after HotBackup".format( str(self.name))) self.after_backup_impl()
def before_backup(self): """ preparing SUT for the execution of the backup steps """ lh.subsection("{0} - preparing SUT for HotBackup".format(str( self.name))) self.before_backup_impl()
def starter_prepare_env(self): """ base setup; declare instance variables etc """ lh.subsection("{0} - prepare starter launch".format(str(self.name))) self.starter_prepare_env_impl()
def make_data_after_upgrade(self): """ check if setup is functional """ lh.subsection("{0} - make data after upgrade".format(str(self.name))) self.make_data_after_upgrade_impl()
def test_setup(self): """ setup steps after the basic instances were launched """ lh.subsection("{0} - basic test after startup".format(str(self.name))) self.test_setup_impl()
def detect_instances(self): """ see which arangods where spawned and inspect their logfiles""" lh.subsection("Instance Detection for {0.name}".format(self)) self.all_instances = [] logging.debug("waiting for frontend") logfiles = set() #logfiles that can be used for debugging # the more instances we expect to spawn the more patient: tries = 10 * self.expect_instance_count # Wait for forntend to become alive. all_instances_up = False while not all_instances_up and tries: self.all_instances = [] detected_instances = [] sys.stdout.write(".") sys.stdout.flush() for root, dirs, files in os.walk(self.basedir): for onefile in files: #logging.debug("f: " + root + os.path.sep + onefile) if onefile.endswith("log"): logfiles.add(str(Path(root) / onefile)) for name in dirs: #logging.debug("d: " + root + os.path.sep + name) match = None instance_class = None if name.startswith('sync'): match = re.match(r'(syncmaster|syncworker)(\d*)', name) instance_class = SyncInstance else: match = re.match( r'(agent|coordinator|dbserver|resilientsingle|single)(\d*)', name) instance_class = ArangodInstance # directory = self.basedir / name if match: # we may see a `local-slave-*` directory inbetween, # hence we need to choose the current directory not # the starter toplevel dir for this: instance = instance_class(match.group(1), match.group(2), self.cfg.localhost, self.cfg.publicip, Path(root) / name, self.passvoid) instance.wait_for_logfile(tries) instance.detect_pid( ppid=self.instance.pid, full_binary_path=self.cfg.real_sbin_dir, offset=0) detected_instances.append(instance.type) self.all_instances.append(instance) print(self.expect_instances) detected_instances.sort() print(detected_instances) if ((self.expect_instances != detected_instances) or (not self.get_frontends())): tries -= 1 time.sleep(5) else: all_instances_up = True if not self.get_frontends(): print() logging.error("STARTER FAILED TO SPAWN ARANGOD") self.show_all_instances() logging.error("can not continue without frontend instance") logging.error("please check logs in" + str(self.basedir)) for logf in logfiles: logging.debug(logf) logging.error("if that does not help try to delete: " + str(self.basedir)) sys.exit(1) self.show_all_instances()
def starter_shutdown(self): """ stop everything """ lh.subsection("{0}{1} - shutdown".format(self.versionstr, str(self.name))) self.shutdown_impl()