class LeaderFollower(Runner):
"""this runs a leader / Follower setup with synchronisation"""
# pylint: disable=too-many-arguments disable=too-many-instance-attributes
def __init__(
self,
runner_type,
abort_on_error,
installer_set,
selenium,
selenium_driver_args,
testrun_name: str,
ssl: bool,
use_auto_certs: bool,
):
super().__init__(
runner_type,
abort_on_error,
installer_set,
RunnerProperties("LeaderFollower", 400, 500, False, ssl,
use_auto_certs),
selenium,
selenium_driver_args,
testrun_name,
)
self.leader_starter_instance = None
self.follower_starter_instance = None
self.success = False
self.checks = {
"beforeReplJS": (
"saving document before",
"""
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
""",
),
"afterReplJS": (
"saving document after the replication",
"""
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
""",
),
"checkReplJS": (
"checking documents",
"""
if (!db.testCollectionBefore) {
throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
throw new Error("after collection does not exist - replication failed");
}
if (!(db.testCollectionBefore.toArray()[0]["hello"] === "world")) {
throw new Error("before not yet there?");
}
if (!(db.testCollectionAfter.toArray()[0]["hello"] === "world")) {
throw new Error("after not yet there?");
}
""",
),
}
def starter_prepare_env_impl(self):
leader_opts = []
follower_opts = []
if self.cfg.ssl and not self.cfg.use_auto_certs:
self.create_tls_ca_cert()
leader_tls_keyfile = self.cert_dir / Path("leader") / "tls.keyfile"
follower_tls_keyfile = self.cert_dir / Path(
"follower") / "tls.keyfile"
self.cert_op([
"tls",
"keyfile",
"--cacert=" + str(self.certificate_auth["cert"]),
"--cakey=" + str(self.certificate_auth["key"]),
"--keyfile=" + str(leader_tls_keyfile),
"--host=" + self.cfg.publicip,
"--host=localhost",
])
self.cert_op([
"tls",
"keyfile",
"--cacert=" + str(self.certificate_auth["cert"]),
"--cakey=" + str(self.certificate_auth["key"]),
"--keyfile=" + str(follower_tls_keyfile),
"--host=" + self.cfg.publicip,
"--host=localhost",
])
leader_opts.append(f"--ssl.keyfile={leader_tls_keyfile}")
follower_opts.append(f"--ssl.keyfile={follower_tls_keyfile}")
self.leader_starter_instance = StarterManager(
self.basecfg,
self.basedir,
"leader",
mode="single",
port=1234,
expect_instances=[InstanceType.SINGLE],
jwt_str="leader",
moreopts=leader_opts,
)
self.leader_starter_instance.is_leader = True
self.follower_starter_instance = StarterManager(
self.basecfg,
self.basedir,
"follower",
mode="single",
port=2345,
expect_instances=[InstanceType.SINGLE],
jwt_str="follower",
moreopts=follower_opts,
)
def starter_run_impl(self):
self.leader_starter_instance.run_starter()
self.follower_starter_instance.run_starter()
self.leader_starter_instance.detect_instances()
self.follower_starter_instance.detect_instances()
self.leader_starter_instance.detect_instance_pids()
self.follower_starter_instance.detect_instance_pids()
self.passvoid = "leader"
self.leader_starter_instance.set_passvoid(self.passvoid)
# the replication will overwrite this passvoid anyways:
self.follower_starter_instance.set_passvoid(self.passvoid)
self.starter_instances = [
self.leader_starter_instance,
self.follower_starter_instance,
]
def finish_setup_impl(self):
# finish setup by starting the replications
self.set_frontend_instances()
self.checks["startReplJS"] = (
"launching replication",
"""
print(
require("@arangodb/replication").setupReplicationGlobal({
endpoint: "%s://127.0.0.1:%s",
username: "******",
password: "******",
verbose: false,
includeSystem: true,
incremental: true,
autoResync: true
}));
print("replication started")
process.exit(0);
""" % (
self.get_protocol(),
str(self.leader_starter_instance.get_frontend_port()),
self.leader_starter_instance.get_passvoid(),
),
)
lh.subsubsection("prepare leader follower replication")
arangosh_script = self.checks["beforeReplJS"]
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
lh.subsubsection("start leader follwer replication")
arangosh_script = self.checks["startReplJS"]
retval = self.follower_starter_instance.execute_frontend(
arangosh_script)
if not retval:
raise Exception("Failed to start the replication using: %s %s" %
(retval, str(self.checks["startReplJS"])))
logging.info("Replication started successfully")
logging.info("save document")
arangosh_script = self.checks["afterReplJS"]
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
self.makedata_instances.append(self.leader_starter_instance)
@step
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.test_after_install()
# assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.follower_starter_instance.supports_foxx_tests = False
logging.info("Leader follower testing makedata on follower")
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
if self.selenium:
self.selenium.test_setup()
logging.info("Leader follower setup successfully finished!")
@step
def upgrade_arangod_version_impl(self):
"""rolling upgrade this installation"""
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.replace_binary_for_upgrade(self.new_cfg)
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.command_upgrade()
node.wait_for_upgrade()
node.wait_for_upgrade_done_in_log()
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def upgrade_arangod_version_manual_impl(self):
"""manual upgrade this installation"""
self.progress(True, "step 1 - shut down instances")
instances = [
self.leader_starter_instance, self.follower_starter_instance
]
for node in instances:
node.replace_binary_setup_for_upgrade(self.new_cfg)
node.terminate_instance(True)
self.progress(
True, "step 2 - launch instances with the upgrade options set")
for node in instances:
print("launch")
node.manually_launch_instances(
[InstanceType.SINGLE],
[
"--database.auto-upgrade",
"true",
"--javascript.copy-installation",
"true",
],
)
self.progress(True, "step 3 - launch instances again")
for node in instances:
node.respawn_instance()
self.progress(True, "step 4 - detect system state")
for node in instances:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def jam_attempt_impl(self):
"""run the replication fuzzing test"""
logging.info("running the replication fuzzing test")
# add instace where makedata will be run on
self.tcp_ping_all_nodes()
ret = self.leader_starter_instance.arangosh.run_in_arangosh(
(self.cfg.test_data_dir /
Path("tests/js/server/replication/fuzz/replication-fuzz-global.js"
)),
[],
[
self.follower_starter_instance.get_frontend().get_public_url(
"root:%s@" % self.passvoid)
],
)
if not ret[0]:
if not self.cfg.verbose:
print(ret[1])
raise Exception("replication fuzzing test failed")
prompt_user(self.basecfg, "please test the installation.")
if self.selenium:
self.selenium.test_jam_attempt()
@step
def shutdown_impl(self):
self.leader_starter_instance.terminate_instance()
self.follower_starter_instance.terminate_instance()
pslist = get_all_processes(False)
if len(pslist) > 0:
raise Exception("Not all processes terminated! [%s]" % str(pslist))
logging.info("test ended")
def before_backup_impl(self):
"""nothing to see here"""
def after_backup_impl(self):
"""nothing to see here"""
def set_selenium_instances(self):
"""set instances in selenium runner"""
self.selenium.set_instances(
self.cfg,
self.leader_starter_instance.arango_importer,
self.leader_starter_instance.arango_restore,
self.leader_starter_instance.all_instances[0],
)
class LeaderFollower(Runner):
""" this runs a leader / Follower setup with synchronisation """
# pylint: disable=R0913 disable=R0902
def __init__(self, runner_type, cfg, old_inst, new_cfg, new_inst, selenium,
selenium_driver_args):
super().__init__(runner_type, cfg, old_inst, new_cfg, new_inst, 'lf',
400, 500, selenium, selenium_driver_args)
self.leader_starter_instance = None
self.follower_starter_instance = None
self.success = False
self.checks = {
"beforeReplJS": ("saving document before", """
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
"""),
"afterReplJS": ("saving document after the replication", """
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
"""),
"checkReplJS": ("checking documents", """
if (!db.testCollectionBefore) {
throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
throw new Error("after collection does not exist - replication failed");
}
if (!db.testCollectionBefore.toArray()[0]["hello"] === "world") {
throw new Error("before not yet there?");
}
if (!db.testCollectionAfter.toArray()[0]["hello"] === "world") {
throw new Error("after not yet there?");
}
""")
}
def starter_prepare_env_impl(self):
self.leader_starter_instance = StarterManager(
self.cfg,
self.basedir,
'leader',
mode='single',
port=1234,
expect_instances=[InstanceType.single],
jwtStr="leader",
moreopts=[])
self.leader_starter_instance.is_leader = True
self.follower_starter_instance = StarterManager(
self.cfg,
self.basedir,
'follower',
mode='single',
port=2345,
expect_instances=[InstanceType.single],
jwtStr="follower",
moreopts=[])
def starter_run_impl(self):
self.leader_starter_instance.run_starter()
self.follower_starter_instance.run_starter()
self.leader_starter_instance.detect_instances()
self.follower_starter_instance.detect_instances()
self.leader_starter_instance.detect_instance_pids()
self.follower_starter_instance.detect_instance_pids()
self.passvoid = 'leader'
self.leader_starter_instance.set_passvoid(self.passvoid)
# the replication will overwrite this passvoid anyways:
self.follower_starter_instance.set_passvoid(self.passvoid)
self.starter_instances = [
self.leader_starter_instance, self.follower_starter_instance
]
def finish_setup_impl(self):
# finish setup by starting the replications
self.set_frontend_instances()
self.checks['startReplJS'] = ("launching replication", """
print(
require("@arangodb/replication").setupReplicationGlobal({
endpoint: "tcp://127.0.0.1:%s",
username: "******",
password: "******",
verbose: false,
includeSystem: true,
incremental: true,
autoResync: true
}));
print("replication started")
process.exit(0);
""" % (str(self.leader_starter_instance.get_frontend_port()),
self.leader_starter_instance.get_passvoid()))
lh.subsubsection("prepare leader follower replication")
arangosh_script = self.checks['beforeReplJS']
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
lh.subsubsection("start leader follwer replication")
arangosh_script = self.checks['startReplJS']
retval = self.follower_starter_instance.execute_frontend(
arangosh_script)
if not retval:
raise Exception("Failed to start the replication using: %s %s" %
(retval, str(self.checks['startReplJS'])))
logging.info("Replication started successfully")
logging.info("save document")
arangosh_script = self.checks['afterReplJS']
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
self.makedata_instances.append(self.leader_starter_instance)
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS'], False):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.connect_server_new_tab(
self.follower_starter_instance.get_frontends(), '_system',
self.cfg)
self.selenium.check_old(self.new_cfg if self.new_cfg else self.cfg,
False)
self.selenium.close_tab_again()
#assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
logging.info("Leader follower setup successfully finished!")
def supports_backup_impl(self):
return False
def upgrade_arangod_version_impl(self):
""" upgrade this installation """
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.replace_binary_for_upgrade(self.new_cfg)
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.command_upgrade()
node.wait_for_upgrade()
node.wait_for_upgrade_done_in_log()
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.web.refresh()
self.selenium.check_old(self.new_cfg, True)
self.selenium.connect_server_new_tab(
self.follower_starter_instance.get_frontends(), '_system',
self.cfg)
self.selenium.check_old(self.new_cfg, False)
self.selenium.close_tab_again()
def jam_attempt_impl(self):
""" run the replication fuzzing test """
logging.info("running the replication fuzzing test")
# add instace where makedata will be run on
self.tcp_ping_all_nodes()
ret = self.leader_starter_instance.arangosh.run_in_arangosh((
self.cfg.test_data_dir /
Path('tests/js/server/replication/fuzz/replication-fuzz-global.js')
), [], [
self.follower_starter_instance.get_frontend().get_public_url(
'root:%s@' % self.passvoid)
])
if not ret[0]:
if not self.cfg.verbose:
print(ret[1])
raise Exception("replication fuzzing test failed")
prompt_user(self.basecfg, "please test the installation.")
if self.selenium:
self.selenium.jam_step_1(self.cfg if self.cfg else self.new_cfg)
def shutdown_impl(self):
self.leader_starter_instance.terminate_instance()
self.follower_starter_instance.terminate_instance()
pslist = get_all_processes(False)
if len(pslist) > 0:
raise Exception("Not all processes terminated! [%s]" % str(pslist))
logging.info('test ended')
def before_backup_impl(self):
pass
def after_backup_impl(self):
pass