def upgrade_when_license_is_expired(self):
"""Check that upgrade can be performed with expired license"""
with step("Create test data"):
self.runner.leader_starter_instance.arangosh.run_command(
("create collection",
'db._create("upgrade_when_license_is_expired");'),
True,
expect_to_fail=False)
self.runner.leader_starter_instance.arangosh.run_command(
(
"create documents",
'for(let i = 0; i < 100; ++i){ db.upgrade_when_license_is_expired.save({"id": i, "a": Math.random(1)})};',
),
True,
expect_to_fail=False,
)
self.expire_license()
self.upgrade_leader_follower()
with step("check that data is present after the upgrade"):
try:
self.runner.leader_starter_instance.arangosh.run_command(
(
"check that data is present",
'console.assert(db._query("for d in upgrade_when_license_is_expired collect with count into l return l==100").data.result[0])',
),
True,
expect_to_fail=False,
)
except CliExecutionException as ex:
raise Exception(
"Can't read data that was created before the upgrade."
) from ex
self.check_readonly()
def create_arangosh_dump(installer, dump_file_dir: str):
"""create arangosh memory dump file"""
dump_filename = None
with step("Start arangosh process"):
exe_file = installer.cfg.bin_dir / "arangosh.exe"
cmd = [str(exe_file)]
arangosh_proc = psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
arangosh_pid = arangosh_proc.pid
with step("Create a dump of arangosh process"):
cmd = ["procdump", "-ma", str(arangosh_pid), dump_file_dir]
lh.log_cmd(cmd)
with psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
(procdump_out, procdump_err) = proc.communicate()
procdump_str = str(procdump_out, "UTF-8")
attach(procdump_str, "procdump sdtout")
attach(str(procdump_err), "procdump stderr")
success_string = "Dump 1 complete"
filename_regex = re.compile(
r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$",
re.MULTILINE)
match = re.search(filename_regex, procdump_str)
if procdump_str.find(success_string) < 0 or not match:
raise Exception(
"procdump wasn't able to create a dump file: " +
procdump_str)
dump_filename = match.group("filename")
with step("Kill arangosh process"):
arangosh_proc.kill()
return dump_filename
def create_dump_for_exe(exe_file: str, dump_file_dir: str):
"""run given executable with \"-?\" command line parameter and create a memory dump when it terminates"""
exe_file = Path(exe_file)
exe_name = exe_file.name
with step(f"Create a memory dump of the program: {exe_name}"):
dump_filename = None
cmd = [
"procdump", "-ma", "-t", "-x", dump_file_dir,
str(exe_file), "-?"
]
lh.log_cmd(cmd)
with psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
(procdump_out, procdump_err) = proc.communicate()
procdump_str = str(procdump_out, "UTF-8")
attach(procdump_str, "procdump sdtout")
attach(str(procdump_err), "procdump stderr")
success_string = "Dump 1 complete"
filename_regex = re.compile(
r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$",
re.MULTILINE)
match = re.search(filename_regex, procdump_str)
if procdump_str.find(success_string) < 0 or not match:
raise Exception(
"procdump wasn't able to create a dump file: " +
procdump_str)
dump_filename = match.group("filename")
return dump_filename
def run_command(self,
cmd,
verbose=True,
timeout=300,
result_line=dummy_line_result,
expect_to_fail=False,
use_default_auth=True):
"""run a command in arangosh"""
title = f"run a command in arangosh: {cmd[0]}"
with step(title):
executable = self.cfg.bin_dir / "arangosh"
arangosh_args = self.cfg.default_arangosh_args + [
"--log.level", "v8=debug",
"--javascript.execute-string"
]
arangosh_args += cmd[1:]
return self.run_arango_tool_monitored(
executeable=executable,
more_args=arangosh_args,
timeout=timeout,
result_line=result_line,
verbose=verbose,
expect_to_fail=expect_to_fail,
use_default_auth=use_default_auth
)
def test_debug_network_symbol_server_windows(self):
"""Check that debug symbols can be found on the ArangoDB symbol server and then used to debug arangod executable"""
# This testcase is needed to check the state of the symbol server and is not meant to run during ArangoDB product testing.
version = semver.VersionInfo.parse(self.installer.cfg.version)
symsrv_dir = "\\\\symbol.arangodb.biz\\symbol\\symsrv_arangodb" + str(
version.major) + str(version.minor)
dump_file = create_arangod_dump(self.installer,
str(DebuggerTestSuite.STARTER_DIR),
str(DebuggerTestSuite.DUMP_FILES_DIR))
with step(
"Check that stack trace with function names and line numbers can be acquired from cdb"
):
cmd = " ".join([
"cdb",
"-z",
dump_file,
"-y",
f"srv*{DebuggerTestSuite.SYMSRV_CACHE_DIR}*{symsrv_dir}",
"-lines",
"-n",
])
attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
cdb = wexpect.spawn(cmd)
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=900)
cdb.sendline("k")
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=900)
stack = cdb.before
cdb.sendline("q")
attach(stack,
"Stacktrace from cdb output",
attachment_type=AttachmentType.TEXT)
assert "arangod!main" in stack, "Stack must contain real function names."
assert "arangod.cpp" in stack, "Stack must contain real source file names."
def test_debug_symbols_windows(self, executable):
"""Check that debug symbols can be used to debug arango executable using a memory dump file (Windows)"""
exe_file = [
str(file.path) for file in self.installer.arango_binaries
if file.path.name == executable + ".exe"
][0]
dump_file = create_dump_for_exe(exe_file,
DebuggerTestSuite.DUMP_FILES_DIR)
pdb_dir = str(self.installer.cfg.debug_install_prefix)
with step(
"Check that stack trace with function names and line numbers can be acquired from cdb"
):
cmd = " ".join(
["cdb", "-z", dump_file, "-y", pdb_dir, "-lines", "-n"])
attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
cdb = wexpect.spawn(cmd)
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=180)
cdb.sendline("k")
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=180)
stack = cdb.before
cdb.sendline("q")
attach(stack,
"Stacktrace from cdb output",
attachment_type=AttachmentType.TEXT)
assert f"{executable}!main" in stack, "Stack must contain real function names."
assert f"{executable}.cpp" in stack, "Stack must contain real source file names."
def test_debug_symbols_symsrv_windows(self):
"""Debug arangod executable using symbol server (Windows)"""
symsrv_dir = str(DebuggerTestSuite.SYMSRV_DIR)
store(str(self.installer.cfg.debug_install_prefix / "arangod.pdb"),
symsrv_dir)
exe_file = [
str(file.path) for file in self.installer.arango_binaries
if file.path.name == "arangod.exe"
][0]
dump_file = create_dump_for_exe(exe_file,
DebuggerTestSuite.DUMP_FILES_DIR)
with step(
"Check that stack trace with function names and line numbers can be acquired from cdb"
):
cmd = " ".join([
"cdb",
"-z",
dump_file,
"-y",
f"srv*{DebuggerTestSuite.SYMSRV_CACHE_DIR}*{DebuggerTestSuite.SYMSRV_DIR}",
"-lines",
"-n",
])
attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
cdb = wexpect.spawn(cmd)
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=900)
cdb.sendline("k")
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=900)
stack = cdb.before
cdb.sendline("q")
attach(stack,
"Stacktrace from cdb output",
attachment_type=AttachmentType.TEXT)
assert "arangod!main" in stack, "Stack must contain real function names."
assert "arangod.cpp" in stack, "Stack must contain real source file names."
def progress(self, msg):
"""add something to the state..."""
logging.info("UI-Test: " + msg)
with step("UI test progress: " + msg):
pass
if len(self.state) > 0:
self.state += "\n"
self.state += "UI: " + msg
def create_arangod_dump(installer, starter_dir: str, dump_file_dir: str):
"""create arangod memory dump file"""
starter = StarterManager(
basecfg=installer.cfg,
install_prefix=Path(starter_dir),
instance_prefix="single",
expect_instances=[InstanceType.SINGLE],
mode="single",
jwt_str="single",
)
dump_filename = None
try:
with step("Start a single server deployment"):
starter.run_starter()
starter.detect_instances()
starter.detect_instance_pids()
starter.set_passvoid("")
pid = starter.all_instances[0].pid
with step("Create a dump of arangod process"):
cmd = ["procdump", "-ma", str(pid), dump_file_dir]
lh.log_cmd(cmd)
with psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
(procdump_out, procdump_err) = proc.communicate()
procdump_str = str(procdump_out, "UTF-8")
attach(procdump_str, "procdump sdtout")
attach(str(procdump_err), "procdump stderr")
success_string = "Dump 1 complete"
filename_regex = re.compile(
r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$",
re.MULTILINE)
match = re.search(filename_regex, procdump_str)
if procdump_str.find(success_string) < 0 or not match:
raise Exception(
"procdump wasn't able to create a dump file: " +
procdump_str)
dump_filename = match.group("filename")
finally:
starter.terminate_instance()
kill_all_processes()
return dump_filename
def store(pdb_filename: str, target_dir: str):
"""store pdb file in symbol server directory"""
with step(f"Store pdb file {pdb_filename} in symbol server directory"):
print(f"Storing file {pdb_filename}")
command = [
"symstore.exe", "add", "/f", pdb_filename, "/s", target_dir, "/t",
"ArangoDB", "/compress"
]
run_cmd_and_log_stdout(command)
print(f"File {pdb_filename} successfully stored to {target_dir}")
def replace_binary_for_upgrade(self, new_install_cfg, relaunch=True):
"""
- replace the parts of the installation with information
after an upgrade
- kill the starter processes of the old version
- revalidate that the old arangods are still running and alive
- replace the starter binary with a new one.
this has not yet spawned any children
"""
# On windows the install prefix may change,
# since we can't overwrite open files:
self.replace_binary_setup_for_upgrade(new_install_cfg)
with step("kill the starter processes of the old version"):
logging.info("StarterManager: Killing my instance [%s]", str(self.instance.pid))
self.kill_instance()
with step("revalidate that the old arangods are still running and alive"):
self.detect_instance_pids_still_alive()
if relaunch:
with step("replace the starter binary with a new one," + " this has not yet spawned any children"):
self.respawn_instance()
logging.info("StarterManager: respawned instance as [%s]", str(self.instance.pid))
def test_debug_symbols_attach_to_process_windows(self):
"""Debug arangod executable by attaching debugger to a running process (Windows)"""
starter = StarterManager(
basecfg=self.installer.cfg,
install_prefix=Path(DebuggerTestSuite.STARTER_DIR),
instance_prefix="single",
expect_instances=[InstanceType.SINGLE],
mode="single",
jwt_str="single",
)
try:
with step("Start a single server deployment"):
starter.run_starter()
starter.detect_instances()
starter.detect_instance_pids()
starter.set_passvoid("")
pid = starter.all_instances[0].pid
pdb_dir = str(self.installer.cfg.debug_install_prefix)
with step(
"Check that stack trace with function names and line numbers can be acquired from cdb"
):
cmd = " ".join([
"cdb", "-pv", "-p",
str(pid), "-y", pdb_dir, "-lines", "-n"
])
attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
cdb = wexpect.spawn(cmd)
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
cdb.sendline("k")
cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
stack = cdb.before
cdb.sendline("q")
attach(stack,
"Stacktrace from cdb output",
attachment_type=AttachmentType.TEXT)
assert "arangod!main" in stack, "Stack must contain real function names."
assert "arangod.cpp" in stack, "Stack must contain real source file names."
finally:
starter.terminate_instance()
kill_all_processes()
def expire_license_on_follower_cluster(self):
"""Check that follower cluster goes to read-only mode when license is expired"""
with step("Expire license on follower"):
# pylint: disable=attribute-defined-outside-init
self.starter = self.runner.cluster1["instance"]
self.expire_license()
self.starter = self.runner.cluster2["instance"]
with step("Create collection on leader cluster"):
self.starter.arangosh.run_command(
("create collection", 'db._create("checkExpireLicenseOnFollower");'), True, expect_to_fail=False
)
with step("Check that collection wasn't replicated to follower cluster"):
try:
self.runner.cluster1["instance"].arangosh.run_command(
("try to read collection", "db._query('FOR doc IN checkExpireLicenseOnFollower RETURN doc');"),
True,
expect_to_fail=True,
)
except CliExecutionException as ex:
raise Exception(
"Collection was replicated to follower cluster after license expiry. Follower must be in read-only mode!"
) from ex
def progress(self, is_sub, msg, separator="x", supress_allure=False):
"""report user message, record for error handling."""
if self.selenium:
self.state += self.selenium.get_progress()
if is_sub:
if separator == "x":
separator = "="
lh.subsection(msg, separator)
self.state += " - " + msg
else:
if separator == "x":
separator = "#"
lh.section(msg, separator)
self.state += "*** " + msg
if not supress_allure:
with step("Progress: " + msg):
pass
def run_before_fixtures(self, funcs):
"""run a set of fixtures before the test suite or test case"""
for func in funcs:
name = func.__doc__ if func.__doc__ else func.__name__
with step(name):
fixture_uuid = str(uuid4())
self.test_suite_context.test_listener.start_before_fixture(
fixture_uuid, name)
exc_type = None
exc_val = None
exc_tb = None
try:
func()
# pylint: disable=bare-except
except:
exc_type, exc_val, exc_tb = sys.exc_info()
self.test_suite_context.test_listener.stop_before_fixture(
fixture_uuid, exc_type, exc_val, exc_tb)
if exc_val:
raise Exception("Fixture failed.") from exc_val