def test_multiple_ports(self):
first_port = 11106
num_ports = 3
num_passes = 100
ports = range(first_port, first_port + num_ports)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
'-L', '/tmp/test.log',
'-f', 'mcrouter/test/test_ascii.json',
'-p', ','.join(map(str, ports)),
'--proxy-threads', '2'
])
proc = Popen(cmd)
time.sleep(1) # magic
for i in range(num_passes):
for port in ports:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
address = ('localhost', port)
sock.connect(address)
fd = sock.makefile()
sock.send("get multiple-ports-junk-key\r\n")
self.assertTrue(fd.readline().strip() == "END")
sock.close()
proc.terminate()
def __init__(self, args, port=None, base_dir=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.log = os.path.join(base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(base_dir.path, 'spool.mcrouter')
os.mkdir(self.async_spool)
self.stats_dir = os.path.join(base_dir.path, 'stats')
os.mkdir(self.stats_dir)
self.debug_fifo_root = os.path.join(base_dir.path, 'fifos')
os.mkdir(self.debug_fifo_root)
args.extend(['-L', self.log,
'-a', self.async_spool,
'--stats-root', self.stats_dir,
'--debug-fifo-root', self.debug_fifo_root])
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def __init__(self, args, port=None, base_dir=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.log = os.path.join(base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(base_dir.path, 'spool.mcrouter')
os.mkdir(self.async_spool)
self.stats_dir = os.path.join(base_dir.path, 'stats')
os.mkdir(self.stats_dir)
self.fifos_dir = os.path.join(base_dir.path, 'fifos')
os.mkdir(self.fifos_dir)
args.extend(['-L', self.log,
'-a', self.async_spool,
'--stats-root', self.stats_dir,
'--debug-fifo-root', self.fifos_dir])
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def try_mcrouter(self, config):
listen_sock = socket.socket()
listen_sock.listen(100)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.binPath('mcrouter'),
'-L', '/tmp/test.log',
'--config', 'file:' + config,
'--listen-sock-fd', str(listen_sock.fileno()),
'--stats-logging-interval', '0',
'--config-dump-root', '',
'--validate-config',
] + self.extra_args)
proc = Popen(cmd)
for _i in range(50):
if proc.poll() is not None:
break
time.sleep(0.1)
ret = proc.returncode
if proc is not None and proc.poll() is None:
proc.terminate()
listen_sock.close()
return ret
def try_mcrouter(self, config):
listen_sock = socket.socket()
listen_sock.listen(100)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.binPath('mcrouter'),
'-L',
'/tmp/test.log',
'--config',
'file:' + config,
'--listen-sock-fd',
str(listen_sock.fileno()),
'--stats-logging-interval',
'0',
'--config-dump-root',
'',
'--validate-config',
] + self.extra_args)
proc = Popen(cmd)
for _i in range(50):
if proc.poll() is not None:
break
time.sleep(0.1)
ret = proc.returncode
if proc is not None and proc.poll() is None:
proc.terminate()
listen_sock.close()
return ret
def test_multiple_ports(self):
first_port = 11108
num_ports = 3
num_passes = 100
ports = range(first_port, first_port + num_ports)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
'-L', '/tmp/test.log',
'-f', 'mcrouter/test/test_ascii.json',
'-p', ','.join(map(str, ports)),
'--proxy-threads', '2'
])
proc = Popen(cmd)
time.sleep(1) # magic
for i in range(num_passes):
for port in ports:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
address = ('localhost', port)
sock.connect(address)
fd = sock.makefile()
sock.send("get multiple-ports-junk-key\r\n")
self.assertTrue(fd.readline().strip() == "END")
sock.close()
proc.terminate()
def __init__(self,
config,
port=None,
default_route=None,
extra_args=None,
base_dir=None,
substitute_config_ports=None,
substitute_port_map=None,
replace_map=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.base_dir = base_dir
self.log = os.path.join(self.base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(self.base_dir.path, 'spool')
os.mkdir(self.async_spool)
if replace_map:
with open(config, 'r') as config_file:
replaced_config = replace_strings(config_file.read(),
replace_map)
config = tempfile.mktemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
if substitute_config_ports:
with open(config, 'r') as config_file:
replaced_config = replace_ports(config_file.read(),
substitute_config_ports)
config = tempfile.mktemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
self.config = config
args = [
McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d', '-f',
config, '-L', self.log, '-a', self.async_spool
]
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
if (default_route):
args.extend(['-R', default_route])
if extra_args:
args.extend(extra_args)
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def test_linenumbers(self):
args = McrouterGlobals.preprocessArgs([
McrouterGlobals.binPath('mcrouter'), '--config',
'file:' + self.config, '-p',
str(5721)
])
self.spawn(args)
self.assertTrue(self.check_for_error_line_number())
def __init__(self, config, port=None, default_route=None, extra_args=None,
base_dir=None, substitute_config_ports=None,
substitute_port_map=None, replace_map=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.base_dir = base_dir
self.log = os.path.join(self.base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(self.base_dir.path, 'spool')
os.mkdir(self.async_spool)
if replace_map:
with open(config, 'r') as config_file:
replaced_config = replace_strings(config_file.read(),
replace_map)
config = tempfile.mktemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
if substitute_config_ports:
with open(config, 'r') as config_file:
replaced_config = replace_ports(config_file.read(),
substitute_config_ports)
config = tempfile.mktemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
self.config = config
args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
'-f', config,
'-L', self.log,
'-a', self.async_spool]
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
if (default_route):
args.extend(['-R', default_route])
if extra_args:
args.extend(extra_args)
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def test_bad_config(self):
listen_sock = socket.socket()
listen_sock.listen(100)
args = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d', '-f',
"/dev/null/doesnotexist", '--listen-sock-fd',
str(listen_sock.fileno())
])
self.spawn(args)
self.check_for_message(good='Can not read config file',
bad='reconfigured with',
timeout=10)
listen_sock.close()
def test_bad_config(self):
listen_sock = socket.socket()
listen_sock.listen(100)
args = McrouterGlobals.preprocessArgs([
McrouterGlobals.binPath('mcrouter'),
'-f', "/dev/null/doesnotexist",
'--listen-sock-fd', str(listen_sock.fileno())
])
self.spawn(args)
self.check_for_message(
good='Can not read config',
bad='reconfigured with',
timeout=10)
listen_sock.close()
def test_bad_tko_param(self):
listen_sock = socket.socket()
listen_sock.listen(100)
args = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d', '-f',
'mcrouter/test/test_ascii.json', '--listen-sock-fd',
str(listen_sock.fileno()), '--fibers-max-pool-size', 'uu'
])
self.spawn(args)
self.check_for_message(
good="Option parse error: fibers_max_pool_size=uu,"
" Couldn't convert value to integer",
bad='mcrouter .* startup',
timeout=10)
listen_sock.close()
def test_bad_tko_param(self):
listen_sock = socket.socket()
listen_sock.listen(100)
args = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
'-f', 'mcrouter/test/test_ascii.json',
'--listen-sock-fd', str(listen_sock.fileno()),
'--fibers-max-pool-size', 'uu'
])
self.spawn(args)
self.check_for_message(
good="Option parse error: fibers_max_pool_size=uu,"
" Couldn't convert value to integer",
bad='mcrouter .* startup',
timeout=10)
listen_sock.close()
def try_mcrouter(self, config):
listen_sock = socket.socket()
listen_sock.listen(100)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-L',
'/tmp/test.log', '-f', config, '--listen-sock-fd',
str(listen_sock.fileno()), '--validate-config'
] + self.extra_args)
proc = Popen(cmd)
for i in range(50):
if proc.poll() is not None:
break
time.sleep(0.1)
ret = proc.returncode
if proc is not None and proc.poll() is None:
proc.terminate()
listen_sock.close()
return ret
def try_mcrouter(self, config):
listen_sock = socket.socket()
listen_sock.listen(100)
cmd = McrouterGlobals.preprocessArgs([
McrouterGlobals.InstallDir + '/mcrouter/mcrouter',
'-L', '/tmp/test.log',
'-f', config,
'--listen-sock-fd', str(listen_sock.fileno()),
'--validate-config'
] + self.extra_args)
proc = Popen(cmd)
for i in range(50):
if proc.poll() is not None:
break
time.sleep(0.1)
ret = proc.returncode
if proc is not None and proc.poll() is None:
proc.terminate()
listen_sock.close()
return ret
def __init__(self, config, port=None, default_route=None, extra_args=None,
base_dir=None, substitute_config_ports=None,
substitute_port_map=None, replace_map=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.base_dir = base_dir
self.log = os.path.join(self.base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(self.base_dir.path, 'spool.mcrouter')
os.mkdir(self.async_spool)
self.stats_dir = os.path.join(self.base_dir.path, 'stats')
os.mkdir(self.stats_dir)
if replace_map:
with open(config, 'r') as config_file:
replaced_config = replace_strings(config_file.read(),
replace_map)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
if substitute_config_ports:
with open(config, 'r') as config_file:
replaced_config = replace_ports(config_file.read(),
substitute_config_ports)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
self.config = config
args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
'-f', config,
'-L', self.log,
'-a', self.async_spool,
'--stats-root', self.stats_dir]
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
if default_route:
args.extend(['-R', default_route])
if extra_args:
args.extend(extra_args)
if '-b' in args:
def get_pid():
stats = self.stats()
if stats:
return int(stats['child_pid'])
return None
def terminate():
pid = get_pid()
if pid:
os.kill(pid, signal.SIGTERM)
def is_alive():
pid = get_pid()
if pid:
return os.path.exists("/proc/{}".format(pid))
return False
self.terminate = terminate
self.is_alive = is_alive
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def __init__(self, config, port=None, default_route=None, extra_args=None,
base_dir=None, substitute_config_ports=None,
substitute_port_map=None, replace_map=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.base_dir = base_dir
self.log = os.path.join(self.base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(self.base_dir.path, 'spool')
os.mkdir(self.async_spool)
if replace_map:
with open(config, 'r') as config_file:
replaced_config = replace_strings(config_file.read(),
replace_map)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
if substitute_config_ports:
with open(config, 'r') as config_file:
replaced_config = replace_ports(config_file.read(),
substitute_config_ports)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
self.config = config
args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
'-f', config,
'-L', self.log,
'-a', self.async_spool]
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
if default_route:
args.extend(['-R', default_route])
if extra_args:
args.extend(extra_args)
if '-b' in args:
pid_file = os.path.join(self.base_dir.path, 'mcrouter.pid')
args.extend(['-P', pid_file])
def get_pid():
with open(pid_file, 'r') as pid_f:
return int(pid_f.read().strip())
self.terminate = lambda: os.kill(get_pid(), signal.SIGTERM)
self.is_alive = lambda: os.path.exists("/proc/%d" % (get_pid()))
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()
def __init__(self, config, port=None, default_route=None, extra_args=None,
base_dir=None, substitute_config_ports=None,
substitute_port_map=None, replace_map=None):
if base_dir is None:
base_dir = BaseDirectory('mcrouter')
self.base_dir = base_dir
self.log = os.path.join(self.base_dir.path, 'mcrouter.log')
self.async_spool = os.path.join(self.base_dir.path, 'spool.mcrouter')
os.mkdir(self.async_spool)
self.stats_dir = os.path.join(self.base_dir.path, 'stats')
os.mkdir(self.stats_dir)
if replace_map:
with open(config, 'r') as config_file:
replaced_config = replace_strings(config_file.read(),
replace_map)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
if substitute_config_ports:
with open(config, 'r') as config_file:
replaced_config = replace_ports(config_file.read(),
substitute_config_ports)
(_, config) = tempfile.mkstemp(dir=self.base_dir.path)
with open(config, 'w') as config_file:
config_file.write(replaced_config)
self.config = config
args = [McrouterGlobals.InstallDir + '/mcrouter/mcrouter', '-d',
'-f', config,
'-L', self.log,
'-a', self.async_spool,
'--stats-root', self.stats_dir]
listen_sock = None
if port is None:
listen_sock = create_listen_socket()
port = listen_sock.getsockname()[1]
args.extend(['--listen-sock-fd', str(listen_sock.fileno())])
else:
args.extend(['-p', str(port)])
if default_route:
args.extend(['-R', default_route])
if extra_args:
args.extend(extra_args)
if '-b' in args:
def get_pid():
stats = self.stats()
if stats:
return int(stats['pid'])
return None
def terminate():
pid = get_pid()
if pid:
os.kill(pid, signal.SIGTERM)
def is_alive():
pid = get_pid()
if pid:
return os.path.exists("/proc/{}".format(pid))
return False
self.terminate = terminate
self.is_alive = is_alive
args = McrouterGlobals.preprocessArgs(args)
MCProcess.__init__(self, args, port, self.base_dir, junk_fill=True)
if listen_sock is not None:
listen_sock.close()