Exemplo n.º 1
0
    def try_mcrouter(self, config):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.binPath('mcrouter'),
            '-L', '/tmp/test.log',
            '--config', 'file:' + config,
            '--listen-sock-fd', str(listen_sock.fileno()),
            '--stats-logging-interval', '0',
            '--config-dump-root', '',
            '--validate-config',
        ] + self.extra_args)
        proc = Popen(cmd)
        for _i in range(50):
            if proc.poll() is not None:
                break
            time.sleep(0.1)

        ret = proc.returncode

        if proc is not None and proc.poll() is None:
            proc.terminate()
        listen_sock.close()

        return ret
    def try_mcrouter(self, config):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.binPath('mcrouter'),
            '-L',
            '/tmp/test.log',
            '--config',
            'file:' + config,
            '--listen-sock-fd',
            str(listen_sock.fileno()),
            '--stats-logging-interval',
            '0',
            '--config-dump-root',
            '',
            '--validate-config',
        ] + self.extra_args)
        proc = Popen(cmd)
        for _i in range(50):
            if proc.poll() is not None:
                break
            time.sleep(0.1)

        ret = proc.returncode

        if proc is not None and proc.poll() is None:
            proc.terminate()
        listen_sock.close()

        return ret
Exemplo n.º 3
0
    def __init__(self, port=None, ssl_port=None, extra_args=None):
        args = [McrouterGlobals.binPath('prodmc')]
        listen_sock = None

        # if mockmc is used here, we initialize the same way as MockMemcached
        if McrouterGlobals.binPath('mockmc') == args[0]:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['-t', str(listen_sock.fileno())])
            else:
                args.extend(['-P', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()
        else:
            args.extend([
                '-A',
                '-g',
                '-t', '1',
                '--enable_hash_aliases',
                '--enable_unchecked_l1_sentinel_reads',
                '--reaper_throttle=100',
                '--ini_hashpower=16',
                '--num_listening_sockets=1',
            ])
            if (extra_args):
                args.extend(extra_args)
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['--listen_sock_fd', str(listen_sock.fileno())])
            else:
                args.extend(['-p', str(port)])

            if ssl_port:
                self.ssl_port = ssl_port
                args.extend(['--ssl_port', str(self.ssl_port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()

            # delay here until the server goes up
            self.ensure_connected()
            tries = 10
            s = self.stats()
            while ((not s or len(s) == 0) and tries > 0):
                # Note, we need to reconnect, because it's possible the
                # server is still going to process previous requests.
                self.ensure_connected()
                s = self.stats()
                time.sleep(0.5)
                tries -= 1
            self.disconnect()
Exemplo n.º 4
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,  # noqa: C901
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None, flavor=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')

        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            (_, config) = tempfile.mkstemp(dir=base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            (_, config) = tempfile.mkstemp(dir=base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        if (flavor):
            args = [McrouterGlobals.binPath('mcrouter'), 'file:' + flavor,
                    '--config', 'file:' + config]
        else:
            args = [McrouterGlobals.binPath('mcrouter'),
                    '--config', 'file:' + config]

        if default_route:
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        if '-b' in args:
            def get_pid():
                stats = self.stats()
                if stats:
                    return int(stats['pid'])
                return None

            def terminate():
                pid = get_pid()
                if pid:
                    os.kill(pid, signal.SIGTERM)

            def is_alive():
                pid = get_pid()
                if pid:
                    return os.path.exists("/proc/{}".format(pid))
                return False

            self.terminate = terminate
            self.is_alive = is_alive

        McrouterBase.__init__(self, args, port, base_dir)
Exemplo n.º 5
0
    def test_linenumbers(self):
        args = McrouterGlobals.preprocessArgs([
            McrouterGlobals.binPath('mcrouter'), '--config',
            'file:' + self.config, '-p',
            str(5721)
        ])

        self.spawn(args)
        self.assertTrue(self.check_for_error_line_number())
Exemplo n.º 6
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('prodmc')]
        listen_sock = None

        # if mockmc is used here, we initialize the same way as MockMemcached
        if McrouterGlobals.binPath('mockmc') == args[0]:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['-t', str(listen_sock.fileno())])
            else:
                args.extend(['-P', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()
        else:
            args.extend([
                '-A',
                '-g',
                '-t',
                '1',
                '--enable-hash-alias',
                '--enable-unchecked-l1-sentinel-reads',
                '--use-asmcs',
                '--reaper_throttle=100',
                '--ini-hashpower=16',
            ])
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
            else:
                args.extend(['-p', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()

            # delay here until the server goes up
            self.ensure_connected()
            l = 10
            s = self.stats()
            while ((not s or len(s) == 0) and l > 0):
                # Note, we need to reconnect, because it's possible the
                # server is still going to process previous requests.
                self.ensure_connected()
                s = self.stats()
                time.sleep(0.5)
                l = l - 1
            self.disconnect()
Exemplo n.º 7
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('prodmc')]
        listen_sock = None

        # if mockmc is used here, we initialize the same way as MockMemcached
        if McrouterGlobals.binPath('mockmc') == args[0]:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['-t', str(listen_sock.fileno())])
            else:
                args.extend(['-P', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()
        else:
            args.extend([
                '-A',
                '-g',
                '-t', '1',
                '--enable_hash_aliases',
                '--enable_unchecked_l1_sentinel_reads',
                '--reaper_throttle=100',
                '--ini_hashpower=16',
            ])
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['--listen_sock_fd', str(listen_sock.fileno())])
            else:
                args.extend(['-p', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()

            # delay here until the server goes up
            self.ensure_connected()
            tries = 10
            s = self.stats()
            while ((not s or len(s) == 0) and tries > 0):
                # Note, we need to reconnect, because it's possible the
                # server is still going to process previous requests.
                self.ensure_connected()
                s = self.stats()
                time.sleep(0.5)
                tries -= 1
            self.disconnect()
Exemplo n.º 8
0
    def test_bad_config(self):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        args = McrouterGlobals.preprocessArgs([
            McrouterGlobals.binPath('mcrouter'), '-f',
            "/dev/null/doesnotexist", '--listen-sock-fd',
            str(listen_sock.fileno())
        ])

        self.spawn(args)
        self.check_for_message(good='Can not read config',
                               bad='reconfigured with',
                               timeout=10)

        listen_sock.close()
Exemplo n.º 9
0
    def __init__(self, args, port=None, base_dir=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')

        self.log = os.path.join(base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(base_dir.path, 'spool.mcrouter')
        os.mkdir(self.async_spool)
        self.stats_dir = os.path.join(base_dir.path, 'stats')
        os.mkdir(self.stats_dir)
        self.fifos_dir = os.path.join(base_dir.path, 'fifos')
        os.mkdir(self.fifos_dir)

        args.extend(['-L', self.log,
                     '-a', self.async_spool,
                     '--stats-root', self.stats_dir,
                     '--debug-fifo-root', self.fifos_dir])

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 10
0
    def __init__(self, thriftPort=None, asyncPort=None):
        args = [McrouterGlobals.binPath('mockmcdual')]
        listen_sock = None
        pass_fds = []
        if thriftPort is None:
            self.listenSocketThrift = create_listen_socket()
            thriftPort = self.listenSocketThrift.getsockname()[1]
            sock_fd = self.listenSocketThrift.fileno()
            args.extend(['-t', str(sock_fd)])
            pass_fds.append(sock_fd)
        else:
            args.extend(['-p', str(thriftPort)])

        if asyncPort is None:
            self.listenSocketAsyncMc = create_listen_socket()
            asyncPort = self.listenSocketAsyncMc.getsockname()[1]
            sock_fd = self.listenSocketAsyncMc.fileno()
            args.extend(['-T', str(sock_fd)])
            pass_fds.append(sock_fd)
        else:
            args.extend(['-P', str(asyncPort)])

        MCProcess.__init__(self, args, asyncPort, pass_fds=pass_fds)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 11
0
    def test_multiple_ports(self):
        first_port = 11106
        num_ports = 3
        num_passes = 100
        ports = range(first_port, first_port + num_ports)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
            '-L', '/tmp/test.log',
            '-f', 'mcrouter/test/test_ascii.json',
            '-p', ','.join(map(str, ports)),
            '--proxy-threads', '2'
        ])
        proc = Popen(cmd)
        time.sleep(1)  # magic
        for i in range(num_passes):
            for port in ports:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                address = ('localhost', port)
                sock.connect(address)
                fd = sock.makefile()
                sock.send("get multiple-ports-junk-key\r\n")
                self.assertTrue(fd.readline().strip() == "END")
                sock.close()

        proc.terminate()
Exemplo n.º 12
0
    def test_multiple_ports(self):
        first_port = 11108
        num_ports = 3
        num_passes = 100
        ports = range(first_port, first_port + num_ports)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
            '-L', '/tmp/test.log',
            '-f', 'mcrouter/test/test_ascii.json',
            '-p', ','.join(map(str, ports)),
            '--proxy-threads', '2'
        ])
        proc = Popen(cmd)
        time.sleep(1)  # magic
        for i in range(num_passes):
            for port in ports:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                address = ('localhost', port)
                sock.connect(address)
                fd = sock.makefile()
                sock.send("get multiple-ports-junk-key\r\n")
                self.assertTrue(fd.readline().strip() == "END")
                sock.close()

        proc.terminate()
Exemplo n.º 13
0
    def __init__(self, thriftPort=None, asyncPort=None, extra_args=None, mcrouterUseThrift=True):
        args = [McrouterGlobals.binPath('mockmcdual')]
        pass_fds = []
        if thriftPort is None:
            listenSocketThrift = create_listen_socket()
            thriftPort = listenSocketThrift.getsockname()[1]
            sock_fd = listenSocketThrift.fileno()
            args.extend(['-t', str(sock_fd)])
            pass_fds.append(sock_fd)
        else:
            args.extend(['-p', str(thriftPort)])
        self.thriftPort = thriftPort
        self.mcrouterUseThrift = mcrouterUseThrift

        if asyncPort is None:
            listenSocketAsyncMc = create_listen_socket()
            asyncPort = listenSocketAsyncMc.getsockname()[1]
            sock_fd = listenSocketAsyncMc.fileno()
            args.extend(['-T', str(sock_fd)])
            pass_fds.append(sock_fd)
        else:
            args.extend(['-P', str(asyncPort)])
        self.asyncPort = asyncPort

        if extra_args:
            args.extend(extra_args)

        MCProcess.__init__(self, args, asyncPort, pass_fds=pass_fds)

        if listenSocketThrift is not None:
            listenSocketThrift.close()
        if listenSocketAsyncMc is not None:
            listenSocketAsyncMc.close()
Exemplo n.º 14
0
    def test_bad_config(self):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        args = McrouterGlobals.preprocessArgs([
            McrouterGlobals.binPath('mcrouter'),
            '-f', "/dev/null/doesnotexist",
            '--listen-sock-fd', str(listen_sock.fileno())
        ])

        self.spawn(args)
        self.check_for_message(
                good='Can not read config',
                bad='reconfigured with',
                timeout=10)

        listen_sock.close()
Exemplo n.º 15
0
    def __init__(self, args, port=None, base_dir=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')

        self.log = os.path.join(base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(base_dir.path, 'spool.mcrouter')
        os.mkdir(self.async_spool)
        self.stats_dir = os.path.join(base_dir.path, 'stats')
        os.mkdir(self.stats_dir)
        self.debug_fifo_root = os.path.join(base_dir.path, 'fifos')
        os.mkdir(self.debug_fifo_root)

        args.extend(['-L', self.log,
                     '-a', self.async_spool,
                     '--stats-root', self.stats_dir,
                     '--debug-fifo-root', self.debug_fifo_root])

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 16
0
    def __init__(self,
                 config,
                 port=None,
                 default_route=None,
                 extra_args=None,
                 base_dir=None,
                 substitute_config_ports=None,
                 substitute_port_map=None,
                 replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')
        self.base_dir = base_dir

        self.log = os.path.join(self.base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(self.base_dir.path, 'spool')
        os.mkdir(self.async_spool)
        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            config = tempfile.mktemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            config = tempfile.mktemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d', '-f',
            config, '-L', self.log, '-a', self.async_spool
        ]

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        if (default_route):
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 17
0
    def __init__(self, fifos_dir, extra_args=None):
        base_dir = BaseDirectory('mcpiper')
        args = [McrouterGlobals.binPath('mcpiper'), '--fifo-root', fifos_dir]

        if extra_args:
            args.extend(extra_args)

        super().__init__(args, base_dir)
Exemplo n.º 18
0
    def __init__(self, fifos_dir, extra_args=None):
        base_dir = BaseDirectory('mcpiper')
        args = [McrouterGlobals.binPath('mcpiper'), '--fifo-root', fifos_dir]

        if extra_args:
            args.extend(extra_args)

        ProcessBase.__init__(self, args, base_dir)
Exemplo n.º 19
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('prodmc')]
        listen_sock = None

        # if mockmc is used here, we initialize the same way as MockMemcached
        if McrouterGlobals.binPath('mockmc') == args[0]:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['-t', str(listen_sock.fileno())])
            else:
                args.extend(['-P', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()
        else:
            args.extend(['--ht_lease_token_power', '0'])
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['--listen_sock', str(listen_sock.fileno())])
            else:
                args.extend(['--port', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()

            # delay here until the server goes up
            self.ensure_connected()
            l = 10
            s = self.stats()
            while ((not s or len(s) == 0) and l > 0):
                # Note, we need to reconnect, because it's possible the
                # server is still going to process previous requests.
                self.ensure_connected()
                s = self.stats()
                time.sleep(0.5)
                l = l - 1
            self.disconnect()
Exemplo n.º 20
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')

        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            (_, config) = tempfile.mkstemp(dir=base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            (_, config) = tempfile.mkstemp(dir=base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [McrouterGlobals.binPath('mcrouter'),
                '--config', 'file:' + config]

        if default_route:
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        if '-b' in args:
            def get_pid():
                stats = self.stats()
                if stats:
                    return int(stats['pid'])
                return None

            def terminate():
                pid = get_pid()
                if pid:
                    os.kill(pid, signal.SIGTERM)

            def is_alive():
                pid = get_pid()
                if pid:
                    return os.path.exists("/proc/{}".format(pid))
                return False

            self.terminate = terminate
            self.is_alive = is_alive

        McrouterBase.__init__(self, args, port, base_dir)
Exemplo n.º 21
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')
        self.base_dir = base_dir

        self.log = os.path.join(self.base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(self.base_dir.path, 'spool')
        os.mkdir(self.async_spool)
        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            config = tempfile.mktemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            config = tempfile.mktemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
                '-f', config,
                '-L', self.log,
                '-a', self.async_spool]

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        if (default_route):
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 22
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('prodmc')]
        listen_sock = None

        # if mockmc is used here, we initialize the same way as MockMemcached
        if McrouterGlobals.binPath('mockmc') == args[0]:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['-t', str(listen_sock.fileno())])
            else:
                args.extend(['-P', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()
        else:
            if port is None:
                listen_sock = create_listen_socket()
                port = listen_sock.getsockname()[1]
                args.extend(['--listen_sock', str(listen_sock.fileno())])
            else:
                args.extend(['--port', str(port)])

            MCProcess.__init__(self, args, port)

            if listen_sock is not None:
                listen_sock.close()

            # delay here until the server goes up
            self.ensure_connected()
            l = 20
            s = self.stats()
            while (len(s) == 0 and l > 0):
                s = self.stats()
                time.sleep(0.5)
                l = l - 1
            self.disconnect()
Exemplo n.º 23
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('mockmc')]
        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['-t', str(listen_sock.fileno())])
        else:
            args.extend(['-P', str(port)])

        MCProcess.__init__(self, args, port)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 24
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('mockmc')]
        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['-t', str(listen_sock.fileno())])
        else:
            args.extend(['-P', str(port)])

        MCProcess.__init__(self, args, port)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 25
0
    def __init__(self, port=None):
        args = [McrouterGlobals.binPath('mockmcthrift')]
        listen_sock = None
        pass_fds = []
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            listen_sock_fd = listen_sock.fileno()
            args.extend(['-t', str(listen_sock.fileno())])
            pass_fds.append(listen_sock_fd)
        else:
            args.extend(['-P', str(port)])

        MCProcess.__init__(self, args, port, pass_fds=pass_fds)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 26
0
    def test_bad_tko_param(self):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        args = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d', '-f',
            'mcrouter/test/test_ascii.json', '--listen-sock-fd',
            str(listen_sock.fileno()), '--fibers-max-pool-size', 'uu'
        ])

        self.spawn(args)
        self.check_for_message(
            good="Option parse error: fibers_max_pool_size=uu,"
            " Couldn't convert value to integer",
            bad='mcrouter .* startup',
            timeout=10)

        listen_sock.close()
Exemplo n.º 27
0
    def test_bad_tko_param(self):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        args = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
            '-f', 'mcrouter/test/test_ascii.json',
            '--listen-sock-fd', str(listen_sock.fileno()),
            '--fibers-max-pool-size', 'uu'
        ])

        self.spawn(args)
        self.check_for_message(
                good="Option parse error: fibers_max_pool_size=uu,"
                " Couldn't convert value to integer",
                bad='mcrouter .* startup',
                timeout=10)

        listen_sock.close()
Exemplo n.º 28
0
    def try_mcrouter(self, config):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-L',
            '/tmp/test.log', '-f', config, '--listen-sock-fd',
            str(listen_sock.fileno()), '--validate-config'
        ] + self.extra_args)
        proc = Popen(cmd)
        for i in range(50):
            if proc.poll() is not None:
                break
            time.sleep(0.1)

        ret = proc.returncode

        if proc is not None and proc.poll() is None:
            proc.terminate()
        listen_sock.close()

        return ret
Exemplo n.º 29
0
    def try_mcrouter(self, config):
        listen_sock = socket.socket()
        listen_sock.listen(100)
        cmd = McrouterGlobals.preprocessArgs([
            McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
            '-L', '/tmp/test.log',
            '-f', config,
            '--listen-sock-fd', str(listen_sock.fileno()),
            '--validate-config'
        ] + self.extra_args)
        proc = Popen(cmd)
        for i in range(50):
            if proc.poll() is not None:
                break
            time.sleep(0.1)

        ret = proc.returncode

        if proc is not None and proc.poll() is None:
            proc.terminate()
        listen_sock.close()

        return ret
Exemplo n.º 30
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')
        self.base_dir = base_dir

        self.log = os.path.join(self.base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(self.base_dir.path, 'spool.mcrouter')
        os.mkdir(self.async_spool)
        self.stats_dir = os.path.join(self.base_dir.path, 'stats')
        os.mkdir(self.stats_dir)
        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
                '-f', config,
                '-L', self.log,
                '-a', self.async_spool,
                '--stats-root', self.stats_dir]

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        if default_route:
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        if '-b' in args:
            def get_pid():
                stats = self.stats()
                if stats:
                    return int(stats['child_pid'])
                return None

            def terminate():
                pid = get_pid()
                if pid:
                    os.kill(pid, signal.SIGTERM)

            def is_alive():
                pid = get_pid()
                if pid:
                    return os.path.exists("/proc/{}".format(pid))
                return False

            self.terminate = terminate
            self.is_alive = is_alive

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 31
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')
        self.base_dir = base_dir

        self.log = os.path.join(self.base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(self.base_dir.path, 'spool')
        os.mkdir(self.async_spool)
        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
                '-f', config,
                '-L', self.log,
                '-a', self.async_spool]

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        if default_route:
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        if '-b' in args:
            pid_file = os.path.join(self.base_dir.path, 'mcrouter.pid')
            args.extend(['-P', pid_file])

            def get_pid():
                with open(pid_file, 'r') as pid_f:
                    return int(pid_f.read().strip())

            self.terminate = lambda: os.kill(get_pid(), signal.SIGTERM)
            self.is_alive = lambda: os.path.exists("/proc/%d" % (get_pid()))

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()
Exemplo n.º 32
0
    def __init__(self, config, port=None, default_route=None, extra_args=None,
                 base_dir=None, substitute_config_ports=None,
                 substitute_port_map=None, replace_map=None):
        if base_dir is None:
            base_dir = BaseDirectory('mcrouter')
        self.base_dir = base_dir

        self.log = os.path.join(self.base_dir.path, 'mcrouter.log')

        self.async_spool = os.path.join(self.base_dir.path, 'spool.mcrouter')
        os.mkdir(self.async_spool)
        self.stats_dir = os.path.join(self.base_dir.path, 'stats')
        os.mkdir(self.stats_dir)
        if replace_map:
            with open(config, 'r') as config_file:
                replaced_config = replace_strings(config_file.read(),
                                                  replace_map)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        if substitute_config_ports:
            with open(config, 'r') as config_file:
                replaced_config = replace_ports(config_file.read(),
                                                substitute_config_ports)
            (_, config) = tempfile.mkstemp(dir=self.base_dir.path)
            with open(config, 'w') as config_file:
                config_file.write(replaced_config)

        self.config = config
        args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
                '-f', config,
                '-L', self.log,
                '-a', self.async_spool,
                '--stats-root', self.stats_dir]

        listen_sock = None
        if port is None:
            listen_sock = create_listen_socket()
            port = listen_sock.getsockname()[1]
            args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
        else:
            args.extend(['-p', str(port)])

        if default_route:
            args.extend(['-R', default_route])

        if extra_args:
            args.extend(extra_args)

        if '-b' in args:
            def get_pid():
                stats = self.stats()
                if stats:
                    return int(stats['pid'])
                return None

            def terminate():
                pid = get_pid()
                if pid:
                    os.kill(pid, signal.SIGTERM)

            def is_alive():
                pid = get_pid()
                if pid:
                    return os.path.exists("/proc/{}".format(pid))
                return False

            self.terminate = terminate
            self.is_alive = is_alive

        args = McrouterGlobals.preprocessArgs(args)

        MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)

        if listen_sock is not None:
            listen_sock.close()