Example #1
0
def create_arangod_dump(installer, starter_dir: str, dump_file_dir: str):
    """create arangod memory dump file"""
    starter = StarterManager(
        basecfg=installer.cfg,
        install_prefix=Path(starter_dir),
        instance_prefix="single",
        expect_instances=[InstanceType.SINGLE],
        mode="single",
        jwt_str="single",
    )
    dump_filename = None
    try:
        with step("Start a single server deployment"):
            starter.run_starter()
            starter.detect_instances()
            starter.detect_instance_pids()
            starter.set_passvoid("")
            pid = starter.all_instances[0].pid
        with step("Create a dump of arangod process"):
            cmd = ["procdump", "-ma", str(pid), dump_file_dir]
            lh.log_cmd(cmd)
            with psutil.Popen(cmd,
                              bufsize=-1,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE) as proc:
                (procdump_out, procdump_err) = proc.communicate()
                procdump_str = str(procdump_out, "UTF-8")
                attach(procdump_str, "procdump sdtout")
                attach(str(procdump_err), "procdump stderr")
                success_string = "Dump 1 complete"
                filename_regex = re.compile(
                    r"^(\[\d{2}:\d{2}:\d{2}\] Dump 1 initiated: )(?P<filename>.*)$",
                    re.MULTILINE)
                match = re.search(filename_regex, procdump_str)
                if procdump_str.find(success_string) < 0 or not match:
                    raise Exception(
                        "procdump wasn't able to create a dump file: " +
                        procdump_str)
                dump_filename = match.group("filename")
    finally:
        starter.terminate_instance()
        kill_all_processes()
    return dump_filename
 def test_debug_symbols_attach_to_process_windows(self):
     """Debug arangod executable by attaching debugger to a running process (Windows)"""
     starter = StarterManager(
         basecfg=self.installer.cfg,
         install_prefix=Path(DebuggerTestSuite.STARTER_DIR),
         instance_prefix="single",
         expect_instances=[InstanceType.SINGLE],
         mode="single",
         jwt_str="single",
     )
     try:
         with step("Start a single server deployment"):
             starter.run_starter()
             starter.detect_instances()
             starter.detect_instance_pids()
             starter.set_passvoid("")
             pid = starter.all_instances[0].pid
         pdb_dir = str(self.installer.cfg.debug_install_prefix)
         with step(
                 "Check that stack trace with function names and line numbers can be acquired from cdb"
         ):
             cmd = " ".join([
                 "cdb", "-pv", "-p",
                 str(pid), "-y", pdb_dir, "-lines", "-n"
             ])
             attach(cmd, "CDB command", attachment_type=AttachmentType.TEXT)
             cdb = wexpect.spawn(cmd)
             cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
             cdb.sendline("k")
             cdb.expect(DebuggerTestSuite.CDB_PROMPT, timeout=300)
             stack = cdb.before
             cdb.sendline("q")
             attach(stack,
                    "Stacktrace from cdb output",
                    attachment_type=AttachmentType.TEXT)
             assert "arangod!main" in stack, "Stack must contain real function names."
             assert "arangod.cpp" in stack, "Stack must contain real source file names."
     finally:
         starter.terminate_instance()
         kill_all_processes()
Example #3
0
class LeaderFollower(Runner):
    """this runs a leader / Follower setup with synchronisation"""

    # pylint: disable=too-many-arguments disable=too-many-instance-attributes
    def __init__(
        self,
        runner_type,
        abort_on_error,
        installer_set,
        selenium,
        selenium_driver_args,
        testrun_name: str,
        ssl: bool,
        use_auto_certs: bool,
    ):
        super().__init__(
            runner_type,
            abort_on_error,
            installer_set,
            RunnerProperties("LeaderFollower", 400, 500, False, ssl,
                             use_auto_certs),
            selenium,
            selenium_driver_args,
            testrun_name,
        )

        self.leader_starter_instance = None
        self.follower_starter_instance = None

        self.success = False
        self.checks = {
            "beforeReplJS": (
                "saving document before",
                """
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
""",
            ),
            "afterReplJS": (
                "saving document after the replication",
                """
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
""",
            ),
            "checkReplJS": (
                "checking documents",
                """
if (!db.testCollectionBefore) {
  throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
  throw new Error("after collection does not exist - replication failed");
}
if (!(db.testCollectionBefore.toArray()[0]["hello"] === "world")) {
  throw new Error("before not yet there?");
}
if (!(db.testCollectionAfter.toArray()[0]["hello"] === "world")) {
  throw new Error("after not yet there?");
}
""",
            ),
        }

    def starter_prepare_env_impl(self):
        leader_opts = []
        follower_opts = []
        if self.cfg.ssl and not self.cfg.use_auto_certs:
            self.create_tls_ca_cert()
            leader_tls_keyfile = self.cert_dir / Path("leader") / "tls.keyfile"
            follower_tls_keyfile = self.cert_dir / Path(
                "follower") / "tls.keyfile"
            self.cert_op([
                "tls",
                "keyfile",
                "--cacert=" + str(self.certificate_auth["cert"]),
                "--cakey=" + str(self.certificate_auth["key"]),
                "--keyfile=" + str(leader_tls_keyfile),
                "--host=" + self.cfg.publicip,
                "--host=localhost",
            ])
            self.cert_op([
                "tls",
                "keyfile",
                "--cacert=" + str(self.certificate_auth["cert"]),
                "--cakey=" + str(self.certificate_auth["key"]),
                "--keyfile=" + str(follower_tls_keyfile),
                "--host=" + self.cfg.publicip,
                "--host=localhost",
            ])
            leader_opts.append(f"--ssl.keyfile={leader_tls_keyfile}")
            follower_opts.append(f"--ssl.keyfile={follower_tls_keyfile}")

        self.leader_starter_instance = StarterManager(
            self.basecfg,
            self.basedir,
            "leader",
            mode="single",
            port=1234,
            expect_instances=[InstanceType.SINGLE],
            jwt_str="leader",
            moreopts=leader_opts,
        )
        self.leader_starter_instance.is_leader = True

        self.follower_starter_instance = StarterManager(
            self.basecfg,
            self.basedir,
            "follower",
            mode="single",
            port=2345,
            expect_instances=[InstanceType.SINGLE],
            jwt_str="follower",
            moreopts=follower_opts,
        )

    def starter_run_impl(self):
        self.leader_starter_instance.run_starter()
        self.follower_starter_instance.run_starter()

        self.leader_starter_instance.detect_instances()
        self.follower_starter_instance.detect_instances()

        self.leader_starter_instance.detect_instance_pids()
        self.follower_starter_instance.detect_instance_pids()

        self.passvoid = "leader"
        self.leader_starter_instance.set_passvoid(self.passvoid)
        # the replication will overwrite this passvoid anyways:
        self.follower_starter_instance.set_passvoid(self.passvoid)

        self.starter_instances = [
            self.leader_starter_instance,
            self.follower_starter_instance,
        ]

    def finish_setup_impl(self):
        # finish setup by starting the replications
        self.set_frontend_instances()

        self.checks["startReplJS"] = (
            "launching replication",
            """
print(
require("@arangodb/replication").setupReplicationGlobal({
    endpoint: "%s://127.0.0.1:%s",
    username: "******",
    password: "******",
    verbose: false,
    includeSystem: true,
    incremental: true,
    autoResync: true
    }));
print("replication started")
process.exit(0);
""" % (
                self.get_protocol(),
                str(self.leader_starter_instance.get_frontend_port()),
                self.leader_starter_instance.get_passvoid(),
            ),
        )
        lh.subsubsection("prepare leader follower replication")
        arangosh_script = self.checks["beforeReplJS"]
        logging.info(
            str(self.leader_starter_instance.execute_frontend(
                arangosh_script)))

        lh.subsubsection("start leader follwer replication")
        arangosh_script = self.checks["startReplJS"]
        retval = self.follower_starter_instance.execute_frontend(
            arangosh_script)
        if not retval:
            raise Exception("Failed to start the replication using: %s %s" %
                            (retval, str(self.checks["startReplJS"])))

        logging.info("Replication started successfully")

        logging.info("save document")
        arangosh_script = self.checks["afterReplJS"]
        logging.info(
            str(self.leader_starter_instance.execute_frontend(
                arangosh_script)))
        self.makedata_instances.append(self.leader_starter_instance)

    @step
    def test_setup_impl(self):
        logging.info("testing the leader/follower setup")
        tries = 30
        if not self.follower_starter_instance.execute_frontend(
                self.checks["checkReplJS"]):
            while tries:
                if self.follower_starter_instance.execute_frontend(
                        self.checks["checkReplJS"]):
                    break
                progress(".")
                time.sleep(1)
                tries -= 1

        if not tries:
            if not self.follower_starter_instance.execute_frontend(
                    self.checks["checkReplJS"]):
                raise Exception("replication didn't make it in 30s!")

        lh.subsection("leader/follower - check test data", "-")

        if self.selenium:
            self.selenium.test_after_install()
        # assert that data has been replicated
        self.follower_starter_instance.arangosh.read_only = True
        self.follower_starter_instance.supports_foxx_tests = False
        logging.info("Leader follower testing makedata on follower")
        self.makedata_instances.append(self.follower_starter_instance)
        self.make_data()
        if self.selenium:
            self.selenium.test_setup()

        logging.info("Leader follower setup successfully finished!")

    @step
    def upgrade_arangod_version_impl(self):
        """rolling upgrade this installation"""
        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.replace_binary_for_upgrade(self.new_cfg)
        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.command_upgrade()
            node.wait_for_upgrade()
            node.wait_for_upgrade_done_in_log()

        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.detect_instances()
            node.wait_for_version_reply()
        if self.selenium:
            self.selenium.test_after_install()

    @step
    def upgrade_arangod_version_manual_impl(self):
        """manual upgrade this installation"""
        self.progress(True, "step 1 - shut down instances")
        instances = [
            self.leader_starter_instance, self.follower_starter_instance
        ]
        for node in instances:
            node.replace_binary_setup_for_upgrade(self.new_cfg)
            node.terminate_instance(True)
        self.progress(
            True, "step 2 - launch instances with the upgrade options set")
        for node in instances:
            print("launch")
            node.manually_launch_instances(
                [InstanceType.SINGLE],
                [
                    "--database.auto-upgrade",
                    "true",
                    "--javascript.copy-installation",
                    "true",
                ],
            )
        self.progress(True, "step 3 - launch instances again")
        for node in instances:
            node.respawn_instance()
        self.progress(True, "step 4 - detect system state")
        for node in instances:
            node.detect_instances()
            node.wait_for_version_reply()
        if self.selenium:
            self.selenium.test_after_install()

    @step
    def jam_attempt_impl(self):
        """run the replication fuzzing test"""
        logging.info("running the replication fuzzing test")
        # add instace where makedata will be run on
        self.tcp_ping_all_nodes()
        ret = self.leader_starter_instance.arangosh.run_in_arangosh(
            (self.cfg.test_data_dir /
             Path("tests/js/server/replication/fuzz/replication-fuzz-global.js"
                  )),
            [],
            [
                self.follower_starter_instance.get_frontend().get_public_url(
                    "root:%s@" % self.passvoid)
            ],
        )
        if not ret[0]:
            if not self.cfg.verbose:
                print(ret[1])
            raise Exception("replication fuzzing test failed")

        prompt_user(self.basecfg, "please test the installation.")
        if self.selenium:
            self.selenium.test_jam_attempt()

    @step
    def shutdown_impl(self):
        self.leader_starter_instance.terminate_instance()
        self.follower_starter_instance.terminate_instance()
        pslist = get_all_processes(False)
        if len(pslist) > 0:
            raise Exception("Not all processes terminated! [%s]" % str(pslist))
        logging.info("test ended")

    def before_backup_impl(self):
        """nothing to see here"""

    def after_backup_impl(self):
        """nothing to see here"""

    def set_selenium_instances(self):
        """set instances in selenium runner"""
        self.selenium.set_instances(
            self.cfg,
            self.leader_starter_instance.arango_importer,
            self.leader_starter_instance.arango_restore,
            self.leader_starter_instance.all_instances[0],
        )
Example #4
0
class Single(Runner):
    """this runs a single server setup"""

    # pylint: disable=too-many-arguments disable=too-many-instance-attributes
    def __init__(
        self,
        runner_type,
        abort_on_error,
        installer_set,
        selenium,
        selenium_driver_args,
        testrun_name: str,
        ssl: bool,
        use_auto_certs: bool,
    ):
        super().__init__(
            runner_type,
            abort_on_error,
            installer_set,
            RunnerProperties("Single", 400, 500, True, ssl, use_auto_certs),
            selenium,
            selenium_driver_args,
            testrun_name,
        )

        self.starter_instance = None
        self.backup_instance_count = 1
        self.success = False

    def starter_prepare_env_impl(self):
        opts = []
        if self.cfg.ssl and not self.cfg.use_auto_certs:
            self.create_tls_ca_cert()
            tls_keyfile = self.cert_dir / Path("single") / "tls.keyfile"
            self.cert_op([
                "tls",
                "keyfile",
                "--cacert=" + str(self.certificate_auth["cert"]),
                "--cakey=" + str(self.certificate_auth["key"]),
                "--keyfile=" + str(tls_keyfile),
                "--host=" + self.cfg.publicip,
                "--host=localhost",
            ])
            opts.append(f"--ssl.keyfile={tls_keyfile}")

        self.starter_instance = StarterManager(
            self.basecfg,
            self.basedir,
            "single",
            mode="single",
            port=1234,
            expect_instances=[InstanceType.SINGLE],
            jwt_str="single",
            moreopts=opts,
        )
        self.starter_instance.is_leader = True

    def starter_run_impl(self):
        self.starter_instance.run_starter()

        self.starter_instance.detect_instances()

        self.starter_instance.detect_instance_pids()

        self.passvoid = "single"
        self.starter_instance.set_passvoid(self.passvoid)

        self.starter_instances = [
            self.starter_instance,
        ]

    def finish_setup_impl(self):
        # finish setup by starting the replications
        self.set_frontend_instances()

        self.makedata_instances.append(self.starter_instance)

    @step
    def test_setup_impl(self):
        logging.info("testing the single server setup")
        tries = 30
        lh.subsection("single server - check test data", "-")

        if self.selenium:
            self.selenium.test_after_install()
        self.make_data()
        if self.selenium:
            self.selenium.test_setup()

        logging.info("Single setup successfully finished!")

    @step
    def upgrade_arangod_version_impl(self):
        """rolling upgrade this installation"""
        self.starter_instance.replace_binary_for_upgrade(self.new_cfg)
        self.starter_instance.command_upgrade()
        self.starter_instance.wait_for_upgrade()
        self.starter_instance.wait_for_upgrade_done_in_log()

        self.starter_instance.detect_instances()
        self.starter_instance.wait_for_version_reply()
        if self.selenium:
            self.selenium.test_after_install()

    @step
    def upgrade_arangod_version_manual_impl(self):
        """manual upgrade this installation"""
        self.progress(True, "step 1 - shut down instances")
        instances = [self.starter_instance]
        self.starter_instance.replace_binary_setup_for_upgrade(self.new_cfg)
        self.starter_instance.terminate_instance(True)
        self.progress(
            True, "step 2 - launch instances with the upgrade options set")
        print("launch")
        self.starter_instance.manually_launch_instances(
            [InstanceType.SINGLE],
            [
                "--database.auto-upgrade",
                "true",
                "--javascript.copy-installation",
                "true",
            ],
        )
        self.progress(True, "step 3 - launch instances again")
        self.starter_instance.respawn_instance()
        self.progress(True, "step 4 - detect system state")
        self.starter_instance.detect_instances()
        self.starter_instance.wait_for_version_reply()
        if self.selenium:
            self.selenium.test_after_install()

    @step
    def jam_attempt_impl(self):
        """run the replication fuzzing test"""
        prompt_user(self.basecfg, "please test the installation.")
        if self.selenium:
            self.selenium.test_jam_attempt()

    @step
    def shutdown_impl(self):
        self.starter_instance.terminate_instance()
        pslist = get_all_processes(False)
        if len(pslist) > 0:
            raise Exception("Not all processes terminated! [%s]" % str(pslist))
        logging.info("test ended")

    def before_backup_impl(self):
        """nothing to see here"""

    def after_backup_impl(self):
        """nothing to see here"""

    def set_selenium_instances(self):
        """set instances in selenium runner"""
        self.selenium.set_instances(
            self.cfg,
            sself.starter_instance.arango_importer,
            self.starter_instance.arango_restore,
            self.starter_instance.all_instances[0],
        )
class LicenseManagerSingleServerTestSuite(LicenseManagerBaseTestSuite):
    """License manager tests: single server"""

    # pylint: disable=dangerous-default-value
    def __init__(self, new_version, installer_base_config):
        super().__init__(
            new_version,
            installer_base_config,
        )
        self.short_name = "SingleServer"

    def get_default_instance_type(self):
        """get the instance type we should communicate with"""
        return InstanceType.SINGLE

    @collect_crash_data
    def save_data_dir(self):
        """save data dir and logs in case a test failed"""
        kill_all_processes()
        if self.starter.basedir.exists():
            archive = shutil.make_archive(
                f"LicenseManagerSingleServerTestSuite(v. {self.base_cfg.version})",
                "bztar", self.starter.basedir)
            attach.file(archive, "test dir archive", "application/x-bzip2",
                        "tar.bz2")
        else:
            print("test basedir doesn't exist, won't create report tar")

    @run_before_suite
    def start(self):
        """clean up the system before running license manager tests on a single server setup"""
        self.cleanup()
        self.start_single_server()

    @run_after_suite
    def teardown_suite(self):
        """Teardown suite environment: single server"""
        self.starter.terminate_instance()
        kill_all_processes()
        self.cleanup()

    def get_server_id(self):
        """read server ID from data directory"""
        datadir = self.starter.all_instances[0].basedir / "data"
        server_file_content = json.load(open(datadir / "SERVER"))
        server_id = server_file_content["serverId"]
        return server_id

    # pylint: disable=redefined-builtin
    def set_license(self, license):
        """set new license"""
        datadir = self.starter.all_instances[0].basedir / "data"
        with open(datadir / ".license", "w") as license_file:
            license_file.truncate()
            license_file.write(license)
        self.starter.terminate_instance()
        self.starter.respawn_instance()

    def cleanup(self):
        """remove all directories created by previous run of this test"""
        testdir = self.base_cfg.test_data_dir / self.short_name
        if testdir.exists():
            shutil.rmtree(testdir)

    @step
    def start_single_server(self):
        """start a single server setup"""
        # pylint: disable=attribute-defined-outside-init
        self.starter = StarterManager(
            basecfg=self.installer.cfg,
            install_prefix=Path(self.short_name),
            instance_prefix="single",
            expect_instances=[InstanceType.SINGLE],
            mode="single",
            jwt_str="single",
        )
        self.starter.run_starter()
        self.starter.detect_instances()
        self.starter.detect_instance_pids()
        self.starter.set_passvoid(self.passvoid)
        self.instance = self.starter.instance

    @testcase
    def clean_install_temp_license(self):
        """Check that server gets a 60-minute license after installation on a clean system"""
        self.check_that_license_is_not_expired(50 * 60)

    @testcase
    def goto_read_only_mode_when_license_expired(self):
        """Check that system goes to read-only mode when license is expired"""
        self.expire_license()
        self.check_readonly()
Example #6
0
class LeaderFollower(Runner):
    """ this runs a leader / Follower setup with synchronisation """

    # pylint: disable=R0913 disable=R0902
    def __init__(self, runner_type, cfg, old_inst, new_cfg, new_inst, selenium,
                 selenium_driver_args):
        super().__init__(runner_type, cfg, old_inst, new_cfg, new_inst, 'lf',
                         400, 500, selenium, selenium_driver_args)

        self.leader_starter_instance = None
        self.follower_starter_instance = None

        self.success = False
        self.checks = {
            "beforeReplJS": ("saving document before", """
db._create("testCollectionBefore");
db.testCollectionBefore.save({"hello": "world"})
"""),
            "afterReplJS": ("saving document after the replication", """
db._create("testCollectionAfter");
db.testCollectionAfter.save({"hello": "world"})
"""),
            "checkReplJS": ("checking documents", """
if (!db.testCollectionBefore) {
  throw new Error("before collection does not exist");
}
if (!db.testCollectionAfter) {
  throw new Error("after collection does not exist - replication failed");
}
if (!db.testCollectionBefore.toArray()[0]["hello"] === "world") {
  throw new Error("before not yet there?");
}
if (!db.testCollectionAfter.toArray()[0]["hello"] === "world") {
  throw new Error("after not yet there?");
}
""")
        }

    def starter_prepare_env_impl(self):
        self.leader_starter_instance = StarterManager(
            self.cfg,
            self.basedir,
            'leader',
            mode='single',
            port=1234,
            expect_instances=[InstanceType.single],
            jwtStr="leader",
            moreopts=[])
        self.leader_starter_instance.is_leader = True

        self.follower_starter_instance = StarterManager(
            self.cfg,
            self.basedir,
            'follower',
            mode='single',
            port=2345,
            expect_instances=[InstanceType.single],
            jwtStr="follower",
            moreopts=[])

    def starter_run_impl(self):
        self.leader_starter_instance.run_starter()
        self.follower_starter_instance.run_starter()

        self.leader_starter_instance.detect_instances()
        self.follower_starter_instance.detect_instances()

        self.leader_starter_instance.detect_instance_pids()
        self.follower_starter_instance.detect_instance_pids()

        self.passvoid = 'leader'
        self.leader_starter_instance.set_passvoid(self.passvoid)
        # the replication will overwrite this passvoid anyways:
        self.follower_starter_instance.set_passvoid(self.passvoid)

        self.starter_instances = [
            self.leader_starter_instance, self.follower_starter_instance
        ]

    def finish_setup_impl(self):
        # finish setup by starting the replications
        self.set_frontend_instances()

        self.checks['startReplJS'] = ("launching replication", """
print(
require("@arangodb/replication").setupReplicationGlobal({
    endpoint: "tcp://127.0.0.1:%s",
    username: "******",
    password: "******",
    verbose: false,
    includeSystem: true,
    incremental: true,
    autoResync: true
    }));
print("replication started")
process.exit(0);
""" % (str(self.leader_starter_instance.get_frontend_port()),
        self.leader_starter_instance.get_passvoid()))
        lh.subsubsection("prepare leader follower replication")
        arangosh_script = self.checks['beforeReplJS']
        logging.info(
            str(self.leader_starter_instance.execute_frontend(
                arangosh_script)))

        lh.subsubsection("start leader follwer replication")
        arangosh_script = self.checks['startReplJS']
        retval = self.follower_starter_instance.execute_frontend(
            arangosh_script)
        if not retval:
            raise Exception("Failed to start the replication using: %s %s" %
                            (retval, str(self.checks['startReplJS'])))

        logging.info("Replication started successfully")

        logging.info("save document")
        arangosh_script = self.checks['afterReplJS']
        logging.info(
            str(self.leader_starter_instance.execute_frontend(
                arangosh_script)))
        self.makedata_instances.append(self.leader_starter_instance)

    def test_setup_impl(self):
        logging.info("testing the leader/follower setup")
        tries = 30
        if not self.follower_starter_instance.execute_frontend(
                self.checks['checkReplJS']):
            while tries:
                if self.follower_starter_instance.execute_frontend(
                        self.checks['checkReplJS'], False):
                    break
                progress(".")
                time.sleep(1)
                tries -= 1

        if not tries:
            if not self.follower_starter_instance.execute_frontend(
                    self.checks['checkReplJS']):
                raise Exception("replication didn't make it in 30s!")

        lh.subsection("leader/follower - check test data", "-")

        if self.selenium:
            self.selenium.connect_server_new_tab(
                self.follower_starter_instance.get_frontends(), '_system',
                self.cfg)
            self.selenium.check_old(self.new_cfg if self.new_cfg else self.cfg,
                                    False)
            self.selenium.close_tab_again()

        #assert that data has been replicated
        self.follower_starter_instance.arangosh.read_only = True
        self.makedata_instances.append(self.follower_starter_instance)
        self.make_data()

        logging.info("Leader follower setup successfully finished!")

    def supports_backup_impl(self):
        return False

    def upgrade_arangod_version_impl(self):
        """ upgrade this installation """
        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.replace_binary_for_upgrade(self.new_cfg)
        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.command_upgrade()
            node.wait_for_upgrade()
            node.wait_for_upgrade_done_in_log()

        for node in [
                self.leader_starter_instance, self.follower_starter_instance
        ]:
            node.detect_instances()
            node.wait_for_version_reply()

        if self.selenium:
            self.selenium.web.refresh()
            self.selenium.check_old(self.new_cfg, True)

            self.selenium.connect_server_new_tab(
                self.follower_starter_instance.get_frontends(), '_system',
                self.cfg)
            self.selenium.check_old(self.new_cfg, False)
            self.selenium.close_tab_again()

    def jam_attempt_impl(self):
        """ run the replication fuzzing test """
        logging.info("running the replication fuzzing test")
        # add instace where makedata will be run on
        self.tcp_ping_all_nodes()
        ret = self.leader_starter_instance.arangosh.run_in_arangosh((
            self.cfg.test_data_dir /
            Path('tests/js/server/replication/fuzz/replication-fuzz-global.js')
        ), [], [
            self.follower_starter_instance.get_frontend().get_public_url(
                'root:%s@' % self.passvoid)
        ])
        if not ret[0]:
            if not self.cfg.verbose:
                print(ret[1])
            raise Exception("replication fuzzing test failed")

        prompt_user(self.basecfg, "please test the installation.")
        if self.selenium:
            self.selenium.jam_step_1(self.cfg if self.cfg else self.new_cfg)

    def shutdown_impl(self):
        self.leader_starter_instance.terminate_instance()
        self.follower_starter_instance.terminate_instance()
        pslist = get_all_processes(False)
        if len(pslist) > 0:
            raise Exception("Not all processes terminated! [%s]" % str(pslist))
        logging.info('test ended')

    def before_backup_impl(self):
        pass

    def after_backup_impl(self):
        pass