def create_arangod_dump(installer, starter_dir: str, dump_file_dir: str):
"""create arangod memory dump file"""
starter = StarterManager(
basecfg=installer.cfg,
install_prefix=Path(starter_dir),
instance_prefix="single",
expect_instances=[InstanceType.SINGLE],
mode="single",
jwt_str="single",
)
dump_filename = None
try:
with step("Start a single server deployment"):
starter.run_starter()
starter.detect_instances()
starter.detect_instance_pids()
starter.set_passvoid("")
pid = starter.all_instances[0].pid
with step("Create a dump of arangod process"):
cmd = ["procdump", "-ma", str(pid), dump_file_dir]
lh.log_cmd(cmd)
with psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
(procdump_out, procdump_err) = proc.communicate()
procdump_str = str(procdump_out, "UTF-8")
attach(procdump_str, "procdump sdtout")
attach(str(procdump_err), "procdump stderr")
success_string = "Dump 1 complete"
filename_regex = re.compile(
r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$",
re.MULTILINE)
match = re.search(filename_regex, procdump_str)
if procdump_str.find(success_string) < 0 or not match:
raise Exception(
"procdump wasn't able to create a dump file: " +
procdump_str)
dump_filename = match.group("filename")
finally:
starter.terminate_instance()
kill_all_processes()
return dump_filename
def test_debug_symbols_attach_to_process_windows(self):
"""Debug arangod executable by attaching debugger to a running process (Windows)"""
starter = StarterManager(
basecfg=self.installer.cfg,
install_prefix=Path(DebuggerTestSuite.STARTER_DIR),
instance_prefix="single",
expect_instances=[InstanceType.SINGLE],
mode="single",
jwt_str="single",
)
try:
with step("Start a single server deployment"):
starter.run_starter()
starter.detect_instances()
starter.detect_instance_pids()
starter.set_passvoid("")
pid = starter.all_instances[0].pid
pdb_dir = str(self.installer.cfg.debug_install_prefix)
with step(
"Check that stack trace with function names and line numbers can be acquired from cdb"
):
cmd = " ".join([
"cdb", "-pv", "-p",
str(pid), "-y", pdb_dir, "-lines", "-n"
])
attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
cdb = wexpect.spawn(cmd)
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
cdb.sendline("k")
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
stack = cdb.before
cdb.sendline("q")
attach(stack,
"Stacktrace from cdb output",
attachment_type=AttachmentType.TEXT)
assert "arangod!main" in stack, "Stack must contain real function names."
assert "arangod.cpp" in stack, "Stack must contain real source file names."
finally:
starter.terminate_instance()
kill_all_processes()
class LeaderFollower(Runner):
"""this runs a leader / Follower setup with synchronisation"""
# pylint: disable=too-many-arguments disable=too-many-instance-attributes
def __init__(
self,
runner_type,
abort_on_error,
installer_set,
selenium,
selenium_driver_args,
testrun_name: str,
ssl: bool,
use_auto_certs: bool,
):
super().__init__(
runner_type,
abort_on_error,
installer_set,
RunnerProperties("LeaderFollower", 400, 500, False, ssl,
use_auto_certs),
selenium,
selenium_driver_args,
testrun_name,
)
self.leader_starter_instance = None
self.follower_starter_instance = None
self.success = False
self.checks = {
"beforeReplJS": (
"saving document before",
"""
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
""",
),
"afterReplJS": (
"saving document after the replication",
"""
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
""",
),
"checkReplJS": (
"checking documents",
"""
if (!db.testCollectionBefore) {
throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
throw new Error("after collection does not exist - replication failed");
}
if (!(db.testCollectionBefore.toArray()[0]["hello"] === "world")) {
throw new Error("before not yet there?");
}
if (!(db.testCollectionAfter.toArray()[0]["hello"] === "world")) {
throw new Error("after not yet there?");
}
""",
),
}
def starter_prepare_env_impl(self):
leader_opts = []
follower_opts = []
if self.cfg.ssl and not self.cfg.use_auto_certs:
self.create_tls_ca_cert()
leader_tls_keyfile = self.cert_dir / Path("leader") / "tls.keyfile"
follower_tls_keyfile = self.cert_dir / Path(
"follower") / "tls.keyfile"
self.cert_op([
"tls",
"keyfile",
"--cacert=" + str(self.certificate_auth["cert"]),
"--cakey=" + str(self.certificate_auth["key"]),
"--keyfile=" + str(leader_tls_keyfile),
"--host=" + self.cfg.publicip,
"--host=localhost",
])
self.cert_op([
"tls",
"keyfile",
"--cacert=" + str(self.certificate_auth["cert"]),
"--cakey=" + str(self.certificate_auth["key"]),
"--keyfile=" + str(follower_tls_keyfile),
"--host=" + self.cfg.publicip,
"--host=localhost",
])
leader_opts.append(f"--ssl.keyfile={leader_tls_keyfile}")
follower_opts.append(f"--ssl.keyfile={follower_tls_keyfile}")
self.leader_starter_instance = StarterManager(
self.basecfg,
self.basedir,
"leader",
mode="single",
port=1234,
expect_instances=[InstanceType.SINGLE],
jwt_str="leader",
moreopts=leader_opts,
)
self.leader_starter_instance.is_leader = True
self.follower_starter_instance = StarterManager(
self.basecfg,
self.basedir,
"follower",
mode="single",
port=2345,
expect_instances=[InstanceType.SINGLE],
jwt_str="follower",
moreopts=follower_opts,
)
def starter_run_impl(self):
self.leader_starter_instance.run_starter()
self.follower_starter_instance.run_starter()
self.leader_starter_instance.detect_instances()
self.follower_starter_instance.detect_instances()
self.leader_starter_instance.detect_instance_pids()
self.follower_starter_instance.detect_instance_pids()
self.passvoid = "leader"
self.leader_starter_instance.set_passvoid(self.passvoid)
# the replication will overwrite this passvoid anyways:
self.follower_starter_instance.set_passvoid(self.passvoid)
self.starter_instances = [
self.leader_starter_instance,
self.follower_starter_instance,
]
def finish_setup_impl(self):
# finish setup by starting the replications
self.set_frontend_instances()
self.checks["startReplJS"] = (
"launching replication",
"""
print(
require("@arangodb/replication").setupReplicationGlobal({
endpoint: "%s://127.0.0.1:%s",
username: "******",
password: "******",
verbose: false,
includeSystem: true,
incremental: true,
autoResync: true
}));
print("replication started")
process.exit(0);
""" % (
self.get_protocol(),
str(self.leader_starter_instance.get_frontend_port()),
self.leader_starter_instance.get_passvoid(),
),
)
lh.subsubsection("prepare leader follower replication")
arangosh_script = self.checks["beforeReplJS"]
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
lh.subsubsection("start leader follwer replication")
arangosh_script = self.checks["startReplJS"]
retval = self.follower_starter_instance.execute_frontend(
arangosh_script)
if not retval:
raise Exception("Failed to start the replication using: %s %s" %
(retval, str(self.checks["startReplJS"])))
logging.info("Replication started successfully")
logging.info("save document")
arangosh_script = self.checks["afterReplJS"]
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
self.makedata_instances.append(self.leader_starter_instance)
@step
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks["checkReplJS"]):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.test_after_install()
# assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.follower_starter_instance.supports_foxx_tests = False
logging.info("Leader follower testing makedata on follower")
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
if self.selenium:
self.selenium.test_setup()
logging.info("Leader follower setup successfully finished!")
@step
def upgrade_arangod_version_impl(self):
"""rolling upgrade this installation"""
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.replace_binary_for_upgrade(self.new_cfg)
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.command_upgrade()
node.wait_for_upgrade()
node.wait_for_upgrade_done_in_log()
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def upgrade_arangod_version_manual_impl(self):
"""manual upgrade this installation"""
self.progress(True, "step 1 - shut down instances")
instances = [
self.leader_starter_instance, self.follower_starter_instance
]
for node in instances:
node.replace_binary_setup_for_upgrade(self.new_cfg)
node.terminate_instance(True)
self.progress(
True, "step 2 - launch instances with the upgrade options set")
for node in instances:
print("launch")
node.manually_launch_instances(
[InstanceType.SINGLE],
[
"--database.auto-upgrade",
"true",
"--javascript.copy-installation",
"true",
],
)
self.progress(True, "step 3 - launch instances again")
for node in instances:
node.respawn_instance()
self.progress(True, "step 4 - detect system state")
for node in instances:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def jam_attempt_impl(self):
"""run the replication fuzzing test"""
logging.info("running the replication fuzzing test")
# add instace where makedata will be run on
self.tcp_ping_all_nodes()
ret = self.leader_starter_instance.arangosh.run_in_arangosh(
(self.cfg.test_data_dir /
Path("tests/js/server/replication/fuzz/replication-fuzz-global.js"
)),
[],
[
self.follower_starter_instance.get_frontend().get_public_url(
"root:%s@" % self.passvoid)
],
)
if not ret[0]:
if not self.cfg.verbose:
print(ret[1])
raise Exception("replication fuzzing test failed")
prompt_user(self.basecfg, "please test the installation.")
if self.selenium:
self.selenium.test_jam_attempt()
@step
def shutdown_impl(self):
self.leader_starter_instance.terminate_instance()
self.follower_starter_instance.terminate_instance()
pslist = get_all_processes(False)
if len(pslist) > 0:
raise Exception("Not all processes terminated! [%s]" % str(pslist))
logging.info("test ended")
def before_backup_impl(self):
"""nothing to see here"""
def after_backup_impl(self):
"""nothing to see here"""
def set_selenium_instances(self):
"""set instances in selenium runner"""
self.selenium.set_instances(
self.cfg,
self.leader_starter_instance.arango_importer,
self.leader_starter_instance.arango_restore,
self.leader_starter_instance.all_instances[0],
)
class Single(Runner):
"""this runs a single server setup"""
# pylint: disable=too-many-arguments disable=too-many-instance-attributes
def __init__(
self,
runner_type,
abort_on_error,
installer_set,
selenium,
selenium_driver_args,
testrun_name: str,
ssl: bool,
use_auto_certs: bool,
):
super().__init__(
runner_type,
abort_on_error,
installer_set,
RunnerProperties("Single", 400, 500, True, ssl, use_auto_certs),
selenium,
selenium_driver_args,
testrun_name,
)
self.starter_instance = None
self.backup_instance_count = 1
self.success = False
def starter_prepare_env_impl(self):
opts = []
if self.cfg.ssl and not self.cfg.use_auto_certs:
self.create_tls_ca_cert()
tls_keyfile = self.cert_dir / Path("single") / "tls.keyfile"
self.cert_op([
"tls",
"keyfile",
"--cacert=" + str(self.certificate_auth["cert"]),
"--cakey=" + str(self.certificate_auth["key"]),
"--keyfile=" + str(tls_keyfile),
"--host=" + self.cfg.publicip,
"--host=localhost",
])
opts.append(f"--ssl.keyfile={tls_keyfile}")
self.starter_instance = StarterManager(
self.basecfg,
self.basedir,
"single",
mode="single",
port=1234,
expect_instances=[InstanceType.SINGLE],
jwt_str="single",
moreopts=opts,
)
self.starter_instance.is_leader = True
def starter_run_impl(self):
self.starter_instance.run_starter()
self.starter_instance.detect_instances()
self.starter_instance.detect_instance_pids()
self.passvoid = "single"
self.starter_instance.set_passvoid(self.passvoid)
self.starter_instances = [
self.starter_instance,
]
def finish_setup_impl(self):
# finish setup by starting the replications
self.set_frontend_instances()
self.makedata_instances.append(self.starter_instance)
@step
def test_setup_impl(self):
logging.info("testing the single server setup")
tries = 30
lh.subsection("single server - check test data", "-")
if self.selenium:
self.selenium.test_after_install()
self.make_data()
if self.selenium:
self.selenium.test_setup()
logging.info("Single setup successfully finished!")
@step
def upgrade_arangod_version_impl(self):
"""rolling upgrade this installation"""
self.starter_instance.replace_binary_for_upgrade(self.new_cfg)
self.starter_instance.command_upgrade()
self.starter_instance.wait_for_upgrade()
self.starter_instance.wait_for_upgrade_done_in_log()
self.starter_instance.detect_instances()
self.starter_instance.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def upgrade_arangod_version_manual_impl(self):
"""manual upgrade this installation"""
self.progress(True, "step 1 - shut down instances")
instances = [self.starter_instance]
self.starter_instance.replace_binary_setup_for_upgrade(self.new_cfg)
self.starter_instance.terminate_instance(True)
self.progress(
True, "step 2 - launch instances with the upgrade options set")
print("launch")
self.starter_instance.manually_launch_instances(
[InstanceType.SINGLE],
[
"--database.auto-upgrade",
"true",
"--javascript.copy-installation",
"true",
],
)
self.progress(True, "step 3 - launch instances again")
self.starter_instance.respawn_instance()
self.progress(True, "step 4 - detect system state")
self.starter_instance.detect_instances()
self.starter_instance.wait_for_version_reply()
if self.selenium:
self.selenium.test_after_install()
@step
def jam_attempt_impl(self):
"""run the replication fuzzing test"""
prompt_user(self.basecfg, "please test the installation.")
if self.selenium:
self.selenium.test_jam_attempt()
@step
def shutdown_impl(self):
self.starter_instance.terminate_instance()
pslist = get_all_processes(False)
if len(pslist) > 0:
raise Exception("Not all processes terminated! [%s]" % str(pslist))
logging.info("test ended")
def before_backup_impl(self):
"""nothing to see here"""
def after_backup_impl(self):
"""nothing to see here"""
def set_selenium_instances(self):
"""set instances in selenium runner"""
self.selenium.set_instances(
self.cfg,
sself.starter_instance.arango_importer,
self.starter_instance.arango_restore,
self.starter_instance.all_instances[0],
)
class LeaderFollower(Runner):
""" this runs a leader / Follower setup with synchronisation """
# pylint: disable=R0913 disable=R0902
def __init__(self, runner_type, cfg, old_inst, new_cfg, new_inst, selenium,
selenium_driver_args):
super().__init__(runner_type, cfg, old_inst, new_cfg, new_inst, 'lf',
400, 500, selenium, selenium_driver_args)
self.leader_starter_instance = None
self.follower_starter_instance = None
self.success = False
self.checks = {
"beforeReplJS": ("saving document before", """
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
"""),
"afterReplJS": ("saving document after the replication", """
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
"""),
"checkReplJS": ("checking documents", """
if (!db.testCollectionBefore) {
throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
throw new Error("after collection does not exist - replication failed");
}
if (!db.testCollectionBefore.toArray()[0]["hello"] === "world") {
throw new Error("before not yet there?");
}
if (!db.testCollectionAfter.toArray()[0]["hello"] === "world") {
throw new Error("after not yet there?");
}
""")
}
def starter_prepare_env_impl(self):
self.leader_starter_instance = StarterManager(
self.cfg,
self.basedir,
'leader',
mode='single',
port=1234,
expect_instances=[InstanceType.single],
jwtStr="leader",
moreopts=[])
self.leader_starter_instance.is_leader = True
self.follower_starter_instance = StarterManager(
self.cfg,
self.basedir,
'follower',
mode='single',
port=2345,
expect_instances=[InstanceType.single],
jwtStr="follower",
moreopts=[])
def starter_run_impl(self):
self.leader_starter_instance.run_starter()
self.follower_starter_instance.run_starter()
self.leader_starter_instance.detect_instances()
self.follower_starter_instance.detect_instances()
self.leader_starter_instance.detect_instance_pids()
self.follower_starter_instance.detect_instance_pids()
self.passvoid = 'leader'
self.leader_starter_instance.set_passvoid(self.passvoid)
# the replication will overwrite this passvoid anyways:
self.follower_starter_instance.set_passvoid(self.passvoid)
self.starter_instances = [
self.leader_starter_instance, self.follower_starter_instance
]
def finish_setup_impl(self):
# finish setup by starting the replications
self.set_frontend_instances()
self.checks['startReplJS'] = ("launching replication", """
print(
require("@arangodb/replication").setupReplicationGlobal({
endpoint: "tcp://127.0.0.1:%s",
username: "******",
password: "******",
verbose: false,
includeSystem: true,
incremental: true,
autoResync: true
}));
print("replication started")
process.exit(0);
""" % (str(self.leader_starter_instance.get_frontend_port()),
self.leader_starter_instance.get_passvoid()))
lh.subsubsection("prepare leader follower replication")
arangosh_script = self.checks['beforeReplJS']
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
lh.subsubsection("start leader follwer replication")
arangosh_script = self.checks['startReplJS']
retval = self.follower_starter_instance.execute_frontend(
arangosh_script)
if not retval:
raise Exception("Failed to start the replication using: %s %s" %
(retval, str(self.checks['startReplJS'])))
logging.info("Replication started successfully")
logging.info("save document")
arangosh_script = self.checks['afterReplJS']
logging.info(
str(self.leader_starter_instance.execute_frontend(
arangosh_script)))
self.makedata_instances.append(self.leader_starter_instance)
def test_setup_impl(self):
logging.info("testing the leader/follower setup")
tries = 30
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
while tries:
if self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS'], False):
break
progress(".")
time.sleep(1)
tries -= 1
if not tries:
if not self.follower_starter_instance.execute_frontend(
self.checks['checkReplJS']):
raise Exception("replication didn't make it in 30s!")
lh.subsection("leader/follower - check test data", "-")
if self.selenium:
self.selenium.connect_server_new_tab(
self.follower_starter_instance.get_frontends(), '_system',
self.cfg)
self.selenium.check_old(self.new_cfg if self.new_cfg else self.cfg,
False)
self.selenium.close_tab_again()
#assert that data has been replicated
self.follower_starter_instance.arangosh.read_only = True
self.makedata_instances.append(self.follower_starter_instance)
self.make_data()
logging.info("Leader follower setup successfully finished!")
def supports_backup_impl(self):
return False
def upgrade_arangod_version_impl(self):
""" upgrade this installation """
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.replace_binary_for_upgrade(self.new_cfg)
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.command_upgrade()
node.wait_for_upgrade()
node.wait_for_upgrade_done_in_log()
for node in [
self.leader_starter_instance, self.follower_starter_instance
]:
node.detect_instances()
node.wait_for_version_reply()
if self.selenium:
self.selenium.web.refresh()
self.selenium.check_old(self.new_cfg, True)
self.selenium.connect_server_new_tab(
self.follower_starter_instance.get_frontends(), '_system',
self.cfg)
self.selenium.check_old(self.new_cfg, False)
self.selenium.close_tab_again()
def jam_attempt_impl(self):
""" run the replication fuzzing test """
logging.info("running the replication fuzzing test")
# add instace where makedata will be run on
self.tcp_ping_all_nodes()
ret = self.leader_starter_instance.arangosh.run_in_arangosh((
self.cfg.test_data_dir /
Path('tests/js/server/replication/fuzz/replication-fuzz-global.js')
), [], [
self.follower_starter_instance.get_frontend().get_public_url(
'root:%s@' % self.passvoid)
])
if not ret[0]:
if not self.cfg.verbose:
print(ret[1])
raise Exception("replication fuzzing test failed")
prompt_user(self.basecfg, "please test the installation.")
if self.selenium:
self.selenium.jam_step_1(self.cfg if self.cfg else self.new_cfg)
def shutdown_impl(self):
self.leader_starter_instance.terminate_instance()
self.follower_starter_instance.terminate_instance()
pslist = get_all_processes(False)
if len(pslist) > 0:
raise Exception("Not all processes terminated! [%s]" % str(pslist))
logging.info('test ended')
def before_backup_impl(self):
pass
def after_backup_impl(self):
pass
