示例#1
0
文件: processes.py 项目: sYnfo/samba
    def run(self, sambaopts, versionopts, section_name=None,
            name=None, pid=None):

        lp = sambaopts.get_loadparm()
        logger = self.get_logger("processes")

        msg_ctx = Messaging()

        if name is not None:
            ids = msg_ctx.irpc_servers_byname(name)
            for server_id in ids:
                self.outf.write("%d\n" % server_id.pid)
        elif pid is not None:
            names = msg_ctx.irpc_all_servers()
            for name in names:
                for server_id in name.ids:
                    if server_id.pid == int(pid):
                        self.outf.write("%s\n" % name.name)
        else:
            names = msg_ctx.irpc_all_servers()
            self.outf.write(" Service:                PID \n")
            self.outf.write("-----------------------------\n")
            for name in names:
                for server_id in name.ids:
                    self.outf.write("%-16s      %6d\n" % (name.name, server_id.pid))
示例#2
0
文件: processes.py 项目: oss-qm/samba
    def run(self,
            sambaopts,
            versionopts,
            section_name=None,
            name=None,
            pid=None):

        lp = sambaopts.get_loadparm()
        logger = self.get_logger("processes")

        msg_ctx = Messaging()

        if name is not None:
            try:
                ids = msg_ctx.irpc_servers_byname(name)
            except KeyError:
                ids = []

            for server_id in ids:
                self.outf.write("%d\n" % server_id.pid)
        elif pid is not None:
            names = msg_ctx.irpc_all_servers()
            for name in names:
                for server_id in name.ids:
                    if server_id.pid == int(pid):
                        self.outf.write("%s\n" % name.name)
        else:
            names = msg_ctx.irpc_all_servers()
            self.outf.write(" Service:                PID \n")
            self.outf.write("-----------------------------\n")
            for name in names:
                for server_id in name.ids:
                    self.outf.write("%-16s      %6d\n" %
                                    (name.name, server_id.pid))
示例#3
0
    def run(self, sambaopts, versionopts, section_name=None,
            name=None, pid=None):

        lp = sambaopts.get_loadparm()
        logger = self.get_logger("processes")

        msg_ctx = Messaging()

        if name is not None:
            ids = msg_ctx.irpc_servers_byname(name)
            for server_id in ids:
                print "%d\n" % server_id.pid
        elif pid is not None:
            names = msg_ctx.irpc_all_servers()
            for name in names:
                for server_id in name.ids:
                    if server_id.pid == int(pid):
                        print "%s\n" % name.name
        else:
            names = msg_ctx.irpc_all_servers()
            for name in names:
                print "%s: " % name.name
                for server_id in name.ids:
                    print "%d " % server_id.pid
                print "\n"
示例#4
0
class SmbcontrolBlockboxTests(BlackboxTestCase):

    def setUp(self):
        super(SmbcontrolBlockboxTests, self).setUp()
        lp_ctx = self.get_loadparm()
        self.msg_ctx = Messaging(lp_ctx=lp_ctx)

    def test_expected_processes(self):
        """
        Test that the expected samba processes are running, currently we only
        check that at least one process is running
        """
        processes = self.msg_ctx.irpc_all_servers()
        if not processes:
            self.fail("No samba processes returned")

    def test_ping(self):
        """Test that all the samba processes can be pinged"""

        processes = self.msg_ctx.irpc_all_servers()

        for p in processes:
            for id in p.ids:
                if p.name != "samba":
                    try:
                        self.check_run("%s %d %s" % (COMMAND, id.pid, PING),
                                       msg="trying to ping %s" % p.name)
                    except BlackboxProcessError as e:
                        # This process could not be pinged, which is
                        # expected (occasionally) if the ldap_server
                        # is using the "standard process model" and
                        # forking a short-lived child for each
                        # connection. We don't care about this, so we
                        # list the processes again and assume that
                        # only those that remain are relevant to the
                        # ping test. Additionally we ensure that at
                        # least one process of each name remains -- in
                        # the ldap_server case, we expect at least the
                        # parent to be there.
                        name_exists = False
                        surviving_processes = self.msg_ctx.irpc_all_servers()
                        for q in surviving_processes:
                            if q.name == p.name:
                                name_exists = True
                                if id.pid in [x.pid for x in q.ids]:
                                    # the unpingable server is still
                                    # listed, meaning it is broken
                                    raise

                        if not name_exists:
                            # it looks like the service genuinely died
                            # just at this moment
                            raise

                        print("Ignoring error %s:" % e)
                        print("the process probably died before our ping")
                        continue
示例#5
0
class SmbcontrolBlackboxTests(BlackboxTestCase):
    def setUp(self):
        super(SmbcontrolBlackboxTests, self).setUp()
        lp_ctx = self.get_loadparm()
        self.msg_ctx = Messaging(lp_ctx=lp_ctx)

    def test_expected_processes(self):
        """
        Test that the expected samba processes are running, currently we only
        check that at least one process is running
        """
        processes = self.msg_ctx.irpc_all_servers()
        if not processes:
            self.fail("No samba processes returned")

    def test_ping(self):
        """Test that all the samba processes can be pinged"""

        processes = self.msg_ctx.irpc_all_servers()

        for p in processes:
            for id in p.ids:
                if p.name != "samba":
                    try:
                        self.check_run("%s %d %s" % (COMMAND, id.pid, PING),
                                       msg="trying to ping %s" % p.name)
                    except BlackboxProcessError as e:
                        # This process could not be pinged, which is
                        # expected (occasionally) if the ldap_server
                        # is using the "standard process model" and
                        # forking a short-lived child for each
                        # connection. We don't care about this, so we
                        # list the processes again and assume that
                        # only those that remain are relevant to the
                        # ping test. Additionally we ensure that at
                        # least one process of each name remains -- in
                        # the ldap_server case, we expect at least the
                        # parent to be there.
                        name_exists = False
                        surviving_processes = self.msg_ctx.irpc_all_servers()
                        for q in surviving_processes:
                            if q.name == p.name:
                                name_exists = True
                                if id.pid in [x.pid for x in q.ids]:
                                    # the unpingable server is still
                                    # listed, meaning it is broken
                                    raise

                        if not name_exists:
                            # it looks like the service genuinely died
                            # just at this moment
                            raise

                        print("Ignoring error %s:" % e)
                        print("the process probably died before our ping")
                        continue
示例#6
0
    def run(self,
            sambaopts,
            versionopts,
            section_name=None,
            name=None,
            pid=None):

        lp = sambaopts.get_loadparm()
        logger = self.get_logger("processes")

        msg_ctx = Messaging()

        if name is not None:
            try:
                ids = msg_ctx.irpc_servers_byname(name)
            except KeyError:
                ids = []

            for server_id in ids:
                self.outf.write("%d\n" % server_id.pid)
        elif pid is not None:
            names = msg_ctx.irpc_all_servers()
            for name in names:
                for server_id in name.ids:
                    if server_id.pid == int(pid):
                        self.outf.write("%s\n" % name.name)
        else:
            seen = {}  # Service entries already printed, service names can
            #               be registered multiple times against a process
            #               but we should only display them once.
            prefork = {}  # Services running in the prefork process model
            #               want to ensure that the master process and workers
            #               are grouped to together.
            (services, masters, workers) = self.get_service_data(msg_ctx)
            self.outf.write(" Service:                          PID\n")
            self.outf.write("--------------------------------------\n")

            for service in sorted(services, key=lambda x: x.name):
                if service.name in masters:
                    # If this service is running in a pre-forked process we
                    # want to print the master process followed by all the
                    # worker processes
                    pid = masters[service.name]
                    if pid not in prefork:
                        prefork[pid] = True
                        self.outf.write("%-26s      %6d\n" %
                                        (service.name, pid))
                        if service.name in workers:
                            ws = workers[service.name]
                            for w in ws:
                                (instance, pid) = ws[w]
                                sn = "{0}(worker {1})".format(
                                    service.name, instance)
                                self.outf.write("%-26s      %6d\n" % (sn, pid))
                else:
                    for server_id in service.ids:
                        if (service.name, server_id.pid) not in seen:
                            self.outf.write("%-26s      %6d\n" %
                                            (service.name, server_id.pid))
                            seen[(service.name, server_id.pid)] = True
示例#7
0
class SmbcontrolBlockboxTests(BlackboxTestCase):
    def setUp(self):
        super(SmbcontrolBlockboxTests, self).setUp()
        lp_ctx = self.get_loadparm()
        self.msg_ctx = Messaging(lp_ctx=lp_ctx)

    def test_expected_processes(self):
        """
        Test that the expected samba processes are running, currently we only
        check that at least one process is running
        """
        processes = self.msg_ctx.irpc_all_servers()
        if not processes:
            self.fail("No samba processes returned")

    def test_ping(self):
        """Test that all the samba processes can be pinged"""

        processes = self.msg_ctx.irpc_all_servers()
        for p in processes:
            for id in p.ids:
                if p.name != "samba":
                    self.check_run("%s %d %s" % (COMMAND, id.pid, PING))
示例#8
0
class PreforkProcessRestartTests(TestCase):
    def setUp(self):
        super(PreforkProcessRestartTests, self).setUp()
        lp_ctx = self.get_loadparm()
        self.msg_ctx = Messaging(lp_ctx=lp_ctx)

    def tearDown(self):
        super(PreforkProcessRestartTests, self).tearDown()

    def get_process_data(self):
        services = self.msg_ctx.irpc_all_servers()

        processes = []
        for service in services:
            for id in service.ids:
                processes.append((service.name, id.pid))
        return processes

    def get_process(self, name):
        processes = self.get_process_data()
        for pname, pid in processes:
            if name == pname:
                return pid
        return None

    def get_worker_pids(self, name, workers):
        pids = []
        for x in range(workers):
            process_name = "prefork-worker-{0}-{1}".format(name, x)
            pids.append(self.get_process(process_name))
            self.assertIsNotNone(pids[x])
        return pids

    def wait_for_workers(self, name, workers):
        num_workers = len(workers)
        for x in range(num_workers):
            process_name = "prefork-worker-{0}-{1}".format(name, x)
            self.wait_for_process(process_name, workers[x], 0, 1, 30)

    def wait_for_process(self, name, pid, initial_delay, wait, timeout):
        time.sleep(initial_delay)
        delay = initial_delay
        while delay < timeout:
            p = self.get_process(name)
            if p is not None and p != pid:
                # process has restarted
                return
            time.sleep(wait)
            delay += wait
        self.fail(
            "Times out after {0} seconds waiting for {1} to restart".format(
                delay, name))

    def check_for_duplicate_processes(self):
        processes = self.get_process_data()
        process_map = {}
        for name, p in processes:
            if (name.startswith("prefork-") or name.endswith("_server")
                    or name.endswith("srv")):

                if name in process_map:
                    if p != process_map[name]:
                        self.fail(
                            "Duplicate process for {0}, pids {1} and {2}".
                            format(name, p, process_map[name]))

    def simple_bind(self):
        creds = self.insta_creds(template=self.get_credentials())
        creds.set_bind_dn("%s\\%s" %
                          (creds.get_domain(), creds.get_username()))

        self.samdb = SamDB(url="ldaps://%s" % os.environ["SERVER"],
                           lp=self.get_loadparm(),
                           credentials=creds)

    def rpc_echo(self):
        conn = echo.rpcecho("ncalrpc:", self.get_loadparm())
        self.assertEqual([1, 2, 3], conn.EchoData([1, 2, 3]))

    def netlogon(self):
        server = os.environ["SERVER"]
        host = os.environ["SERVER_IP"]
        lp = self.get_loadparm()

        credentials = self.get_credentials()

        session = system_session()
        ldb = SamDB(url="ldap://%s" % host,
                    session_info=session,
                    credentials=credentials,
                    lp=lp)
        machine_pass = samba.generate_random_password(32, 32)
        machine_name = MACHINE_NAME
        machine_dn = "cn=%s,%s" % (machine_name, ldb.domain_dn())

        delete_force(ldb, machine_dn)

        utf16pw = ('"%s"' % get_string(machine_pass)).encode('utf-16-le')
        ldb.add({
            "dn":
            machine_dn,
            "objectclass":
            "computer",
            "sAMAccountName":
            "%s$" % machine_name,
            "userAccountControl":
            str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PASSWD_NOTREQD),
            "unicodePwd":
            utf16pw
        })

        machine_creds = Credentials()
        machine_creds.guess(lp)
        machine_creds.set_secure_channel_type(SEC_CHAN_WKSTA)
        machine_creds.set_kerberos_state(DONT_USE_KERBEROS)
        machine_creds.set_password(machine_pass)
        machine_creds.set_username(machine_name + "$")
        machine_creds.set_workstation(machine_name)

        netlogon.netlogon("ncacn_ip_tcp:%s[schannel,seal]" % server, lp,
                          machine_creds)

        delete_force(ldb, machine_dn)

    def test_ldap_master_restart(self):
        # check ldap connection, do a simple bind
        self.simple_bind()

        # get ldap master process
        pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("ldap", NUM_WORKERS)

        # kill it
        os.kill(pid, signal.SIGTERM)

        # wait for the process to restart
        self.wait_for_process("prefork-master-ldap", pid, 1, 1, 30)

        # restarting the master restarts the workers as well, so make sure
        # they have finished restarting
        self.wait_for_workers("ldap", workers)

        # get ldap master process
        new_pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(new_pid)

        # check that the pid has changed
        self.assertNotEquals(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
        for x in range(NUM_WORKERS):
            self.assertNotEquals(workers[x], new_workers[x])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check ldap connection, another simple bind
        self.simple_bind()

    def test_ldap_worker_restart(self):
        # check ldap connection, do a simple bind
        self.simple_bind()

        # get ldap master process
        pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("ldap", NUM_WORKERS)

        # kill worker 0
        os.kill(workers[0], signal.SIGTERM)

        # wait for the process to restart
        self.wait_for_process("prefork-worker-ldap-0", pid, 1, 1, 30)

        # get ldap master process
        new_pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(new_pid)

        # check that the pid has not changed
        self.assertEqual(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
        # process 0 should have a new pid the others should be unchanged
        self.assertNotEquals(workers[0], new_workers[0])
        self.assertEqual(workers[1], new_workers[1])
        self.assertEqual(workers[2], new_workers[2])
        self.assertEqual(workers[3], new_workers[3])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check ldap connection, another simple bind
        self.simple_bind()

    #
    # Kill all the ldap worker processes and ensure that they are restarted
    # correctly
    #
    def test_ldap_all_workers_restart(self):
        # check ldap connection, do a simple bind
        self.simple_bind()

        # get ldap master process
        pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("ldap", NUM_WORKERS)

        # kill all the worker processes
        for x in workers:
            os.kill(x, signal.SIGTERM)

        # wait for the worker processes to restart
        self.wait_for_workers("ldap", workers)

        # get ldap master process
        new_pid = self.get_process("prefork-master-ldap")
        self.assertIsNotNone(new_pid)

        # check that the pid has not changed
        self.assertEqual(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
        for x in range(NUM_WORKERS):
            self.assertNotEquals(workers[x], new_workers[x])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check ldap connection, another simple bind
        self.simple_bind()

    def test_rpc_master_restart(self):
        # check rpc connection, make a rpc echo request
        self.rpc_echo()

        # get rpc master process
        pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("rpc", NUM_WORKERS)

        # kill it
        os.kill(pid, signal.SIGTERM)

        # wait for the process to restart
        self.wait_for_process("prefork-master-rpc", pid, 1, 1, 30)

        # wait for workers to restart as well
        self.wait_for_workers("rpc", workers)

        # get ldap master process
        new_pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(new_pid)

        # check that the pid has changed
        self.assertNotEquals(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
        for x in range(NUM_WORKERS):
            self.assertNotEquals(workers[x], new_workers[x])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check rpc connection, another rpc echo request
        self.rpc_echo()

    def test_rpc_worker_zero_restart(self):
        # check rpc connection, make a rpc echo request and a netlogon request
        self.rpc_echo()
        self.netlogon()

        # get rpc master process
        pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("rpc", NUM_WORKERS)

        # kill worker 0
        os.kill(workers[0], signal.SIGTERM)

        # wait for the process to restart
        self.wait_for_process("prefork-worker-rpc-0", workers[0], 1, 1, 30)

        # get rpc master process
        new_pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(new_pid)

        # check that the pid has not changed
        self.assertEqual(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
        # process 0 should have a new pid the others should be unchanged
        self.assertNotEquals(workers[0], new_workers[0])
        self.assertEqual(workers[1], new_workers[1])
        self.assertEqual(workers[2], new_workers[2])
        self.assertEqual(workers[3], new_workers[3])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check rpc connection, another rpc echo request, and netlogon request
        self.rpc_echo()
        self.netlogon()

    def test_rpc_all_workers_restart(self):
        # check rpc connection, make a rpc echo request, and a netlogon request
        self.rpc_echo()
        self.netlogon()

        # get rpc master process
        pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(pid)

        # Get the worker processes
        workers = self.get_worker_pids("rpc", NUM_WORKERS)

        # kill all the worker processes
        for x in workers:
            os.kill(x, signal.SIGTERM)

        # wait for the worker processes to restart
        for x in range(NUM_WORKERS):
            self.wait_for_process("prefork-worker-rpc-{0}".format(x),
                                  workers[x], 0, 1, 30)

        # get rpc master process
        new_pid = self.get_process("prefork-master-rpc")
        self.assertIsNotNone(new_pid)

        # check that the pid has not changed
        self.assertEqual(pid, new_pid)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
        for x in range(NUM_WORKERS):
            self.assertNotEquals(workers[x], new_workers[x])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

        # check rpc connection, another rpc echo request and netlogon
        self.rpc_echo()
        self.netlogon()

    def test_master_restart_backoff(self):

        # get kdc master process
        pid = self.get_process("prefork-master-echo")
        self.assertIsNotNone(pid)

        #
        # Check that the processes get backed off as expected
        #
        # have prefork backoff increment = 5
        #      prefork maximum backoff   = 10
        backoff_increment = 5
        for expected in [0, 5, 10, 10]:
            # Get the worker processes
            workers = self.get_worker_pids("kdc", NUM_WORKERS)

            process = self.get_process("prefork-master-echo")
            os.kill(process, signal.SIGTERM)
            # wait for the process to restart
            start = time.time()
            self.wait_for_process("prefork-master-echo", process, 0, 1, 30)
            # wait for the workers to restart as well
            self.wait_for_workers("echo", workers)
            end = time.time()
            duration = end - start

            # process restart will take some time. Check that the elapsed
            # duration falls somewhere in the expected range, i.e. we haven't
            # taken longer than the backoff increment
            self.assertLess(duration, expected + backoff_increment)
            self.assertGreaterEqual(duration, expected)

        # check that the worker processes have restarted
        new_workers = self.get_worker_pids("echo", NUM_WORKERS)
        for x in range(NUM_WORKERS):
            self.assertNotEquals(workers[x], new_workers[x])

        # check that the previous server entries have been removed.
        self.check_for_duplicate_processes()

    def test_worker_restart_backoff(self):
        #
        # Check that the processes get backed off as expected
        #
        # have prefork backoff increment = 5
        #      prefork maximum backoff   = 10
        backoff_increment = 5
        for expected in [0, 5, 10, 10]:
            process = self.get_process("prefork-worker-echo-2")
            self.assertIsNotNone(process)
            os.kill(process, signal.SIGTERM)
            # wait for the process to restart
            start = time.time()
            self.wait_for_process("prefork-worker-echo-2", process, 0, 1, 30)
            end = time.time()
            duration = end - start

            # process restart will take some time. Check that the elapsed
            # duration falls somewhere in the expected range, i.e. we haven't
            # taken longer than the backoff increment
            self.assertLess(duration, expected + backoff_increment)
            self.assertGreaterEqual(duration, expected)

        self.check_for_duplicate_processes()
示例#9
0
class SmbcontrolProcessBlockboxTests(BlackboxTestCase):
    def setUp(self):
        super(SmbcontrolProcessBlockboxTests, self).setUp()
        lp_ctx = self.get_loadparm()
        self.msg_ctx = Messaging(lp_ctx=lp_ctx)

    def get_process_data(self):
        services = self.msg_ctx.irpc_all_servers()

        processes = []
        for service in services:
            for id in service.ids:
                processes.append((service.name, id.pid))
        return processes

    def get_process(self, name):
        processes = self.get_process_data()
        for pname, pid in processes:
            if name == pname:
                return pid
        return None

    def test_inject_fault(self):
        INJECT = "inject"
        FAULT = "segv"
        #
        # Note that this process name needs to be different to the one used
        # in the sleep test to avoid a race condition
        #
        pid = self.get_process("rpc_server")

        #
        # Ensure we can ping the process before injecting a fault.
        #
        try:
            self.check_run("%s %s %s" % (COMMAND, pid, PING),
                           msg="trying to ping rpc_server")
        except BlackboxProcessError as e:
            self.fail("Unable to ping rpc_server process")

        #
        # Now inject a fault.
        #
        try:
            self.check_run("%s %s %s %s" % (COMMAND, pid, INJECT, FAULT),
                           msg="injecting fault into rpc_server")
        except BlackboxProcessError as e:
            print(e)
            self.fail("Unable to inject a fault into the rpc_server process")

        #
        # The process should have died, so we should not be able to ping it
        #
        try:
            self.check_run("%s %s %s" % (COMMAND, pid, PING),
                           msg="trying to ping rpc_server")
            self.fail("Could ping rpc_server process")
        except BlackboxProcessError as e:
            pass

    def test_sleep(self):
        SLEEP = "sleep"  # smbcontrol sleep command
        DURATION = 5  # duration to sleep server for
        DELTA = 1  # permitted error for the sleep duration

        #
        # Note that this process name needs to be different to the one used
        # in the inject fault test to avoid a race condition
        #
        pid = self.get_process("ldap_server")
        #
        # Ensure we can ping the process before getting it to sleep
        #
        try:
            self.check_run("%s %s %s" % (COMMAND, pid, PING),
                           msg="trying to ping rpc_server")
        except BlackboxProcessError as e:
            self.fail("Unable to ping rpc_server process")

        #
        # Now ask it to sleep
        #
        start = time.time()
        try:
            self.check_run("%s %s %s %s" % (COMMAND, pid, SLEEP, DURATION),
                           msg="putting rpc_server to sleep for %d" % DURATION)
        except BlackboxProcessError as e:
            print(e)
            self.fail("Failed to get rpc_server to sleep for %d" % DURATION)

        #
        # The process should be sleeping and not respond until it wakes
        #
        try:
            self.check_run("%s %s %s" % (COMMAND, pid, PING),
                           msg="trying to ping rpc_server")
            end = time.time()
            duration = end - start
            self.assertGreater(duration + DELTA, DURATION)
        except BlackboxProcessError as e:
            self.fail("Unable to ping rpc_server process")