def tearDown(self):
tf_cfg.dbg(3, "\tTeardown")
for cid in self.__clients:
client = self.__clients[cid]
client.stop()
self.__tempesta.stop()
for sid in self.__servers:
server = self.__servers[sid]
server.stop()
self.deproxy_manager.stop()
try:
deproxy_manager.finish_all_deproxy()
except:
print('Unknown exception in stopping deproxy')
tf_cfg.dbg(3, "Removing interfaces")
interface = tf_cfg.cfg.get('Server', 'aliases_interface')
remove_routes(interface, self.__ips)
remove_interfaces(interface, self.__ips)
self.__ips = []
self.oops.update()
for err in ["Oops", "WARNING", "ERROR"]:
if err in self.oops_ignore:
continue
if self.oops._warn_count(err) > 0:
self.oops_ignore = []
raise Exception("%s happened during test on Tempesta" % err)
# Drop the list of ignored errors to allow set different errors masks
# for different tests.
self.oops_ignore = []
def test_cookie(self):
self.start_all()
client = self.get_client('client-with-cookies')
req = ("GET / HTTP/1.1\r\n" "Host: localhost\r\n" "\r\n")
response = self.client_send_req(client, req)
self.assertEqual(response.status, '302',
("Unexpected redirect status code: %s, expected 302" %
response.status))
cookie = self.extract_cookie(response)
self.assertIsNotNone(cookie, "Can't find cookie in response")
req = ("GET / HTTP/1.1\r\n"
"Host: localhost\r\n"
"Cookie: %s=%s\r\n"
"\r\n" % (cookie[0], cookie[1]))
response = self.client_send_req(client, req)
self.assertEqual(response.status, '200',
("Unexpected redirect status code: %s, expected 200" %
response.status))
# Cookies are enforced, only the first response (redirect) has
# Set-Cookie header, following responses has no such header.
self.assertIsNone(
response.headers.get('Set-Cookie', None),
"Set-Cookie header is mistakenly set in the response")
tf_cfg.dbg(3, "Sleep until session get expired...")
time.sleep(5)
req = ("GET / HTTP/1.1\r\n"
"Host: localhost\r\n"
"Cookie: %s=%s\r\n"
"\r\n" % (cookie[0], cookie[1]))
response = self.client_send_req(client, req)
self.assertEqual(response.status, '302',
"Unexpected redirect status code")
def test(self):
""" Clients sends the validating request after reload just in time and
passes the challenge.
"""
self.start_all()
# Reloading Tempesta config with JS challenge enabled
config = tempesta.Config()
config.set_defconfig("""
server %s:8000;
sticky {
cookie enforce name=cname;
js_challenge resp_code=503 delay_min=1000 delay_range=1500
delay_limit=3000 %s/js1.html;
}
""" % (tf_cfg.cfg.get('Server',
'ip'), tf_cfg.cfg.get('Tempesta', 'workdir')))
self.get_tempesta().config = config
self.get_tempesta().reload()
tf_cfg.dbg(3, "Send request to vhost 1 with timeout 2s...")
client = self.get_client('client-1')
self.process_js_challenge(client,
'vh1.com',
delay_min=1000,
delay_range=1500,
status_code=503,
expect_pass=True,
req_delay=2.5)
def test_pipelined(self):
# Check that responses goes in the same order as requests
request = "GET /0 HTTP/1.1\r\n" \
"Host: localhost\r\n" \
"\r\n" \
"GET /1 HTTP/1.1\r\n" \
"Host: localhost\r\n" \
"\r\n" \
"GET /2 HTTP/1.1\r\n" \
"Host: localhost\r\n" \
"\r\n" \
"GET /3 HTTP/1.1\r\n" \
"Host: localhost\r\n" \
"\r\n"
deproxy_srv = self.get_server('deproxy')
deproxy_srv.start()
self.assertEqual(0, len(deproxy_srv.requests))
self.start_tempesta()
deproxy_cl = self.get_client('deproxy')
deproxy_cl.start()
self.deproxy_manager.start()
self.assertTrue(deproxy_srv.wait_for_connections(timeout=1))
deproxy_cl.make_requests(request)
resp = deproxy_cl.wait_for_response(timeout=5)
self.assertTrue(resp, "Response not received")
self.assertEqual(4, len(deproxy_cl.responses))
self.assertEqual(4, len(deproxy_srv.requests))
for i in range(len(deproxy_cl.responses)):
tf_cfg.dbg(3, "Resp %i: %s" % (i, deproxy_cl.responses[i].msg))
for i in range(len(deproxy_cl.responses)):
self.assertEqual(deproxy_cl.responses[i].body, "/" + str(i))
def is_busy(self):
busy = not self.exit_event.is_set()
if busy:
tf_cfg.dbg(4, "\tClient is running")
else:
tf_cfg.dbg(4, "\tClient is not running")
return busy
def start_all(self):
tf_cfg.dbg(3, 'Start servers and TempestaFW')
ka_timeout = 'keepalive_timeout %s;\n' % self.timeout
self.tempesta.config.set_defconfig(ka_timeout + self.config)
self.configure_tempesta()
control.servers_start(self.servers)
self.tempesta.start()
def setUp(self):
tf_cfg.dbg(2, "Creating interfaces")
self.interface = tf_cfg.cfg.get('Server', 'aliases_interface')
self.base_ip = tf_cfg.cfg.get('Server', 'aliases_base_ip')
self.ips = sysnet.create_interfaces(self.interface, self.base_ip,
self.num_extra_interfaces + 1)
stress.StressTest.setUp(self)
def test_load_distribution(self):
""" All servers must receive almost the same number of requests.
"""
wrk = self.get_client('wrk')
self.start_all_servers()
self.start_tempesta()
self.start_all_clients()
self.wait_while_busy(wrk)
tempesta = self.get_tempesta()
servers = self.get_servers()
tempesta.get_stats()
servers_get_stats(servers)
cl_reqs = tempesta.stats.cl_msg_forwarded
s_reqs_expected = cl_reqs / len(servers)
# On short running tests too small delta leads to false negatives.
delta = max(self.precision * s_reqs_expected, self.min_delta)
for srv in servers:
tf_cfg.dbg(
3, "Server %s received %d requests, [%d, %d] was expected" %
(srv.get_name(), srv.requests, s_reqs_expected - delta,
s_reqs_expected + delta))
self.assertAlmostEqual(
srv.requests,
s_reqs_expected,
delta=delta,
msg=("Server %s received %d requests, but [%d, %d] "
"was expected" %
(srv.get_name(), srv.requests, s_reqs_expected - delta,
s_reqs_expected + delta)))
def parse_out(self, stdout, stderr):
m = re.search(r'(\d+) requests in ', stdout.decode())
if m:
self.requests = int(m.group(1))
m = re.search(r'Non-2xx or 3xx responses: (\d+)', stdout.decode())
if m:
self.errors = int(m.group(1))
m = re.search(r'Requests\/sec:\s+(\d+)', stdout.decode())
if m:
self.rate = int(m.group(1))
matches = re.findall(r'Status (\d{3}) : (\d+) times', stdout.decode())
for match in matches:
status = match[0]
status = int(status)
amount = match[1]
amount = int(amount)
self.statuses[status] = amount
sock_err_msg = "Socket errors on wrk. Too many concurrent connections?"
m = re.search(r'(Socket errors:.+)', stdout.decode())
if self.FAIL_ON_SOCK_ERR:
assert not m, sock_err_msg
if m:
tf_cfg.dbg(1, "WARNING! %s" % sock_err_msg)
err_m = re.search(r'\w+ (\d+), \w+ (\d+), \w+ (\d+), \w+ (\d+)',
m.group(1))
self.errors += (int(err_m.group(1)) + int(err_m.group(2)) +
int(err_m.group(3)) + int(err_m.group(4)))
# this is wrk-dependent results
self.statuses["connect_error"] = int(err_m.group(1))
self.statuses["read_error"] = int(err_m.group(2))
self.statuses["write_error"] = int(err_m.group(3))
self.statuses["timeout_error"] = int(err_m.group(4))
return True
def create_content(self, length):
""" Create content file """
content = body_generator.generate_body(length)
location = tf_cfg.cfg.get('Server', 'resources')
self.fullname = os.path.join(location, self.filename)
tf_cfg.dbg(3, "Copy %s to %s" % (self.filename, self.fullname))
remote.server.copy_file(self.fullname, content)
def generic_test_routine(self, tempesta_defconfig, message_chains):
""" Make necessary updates to configs of servers, create tempesta config
and run the routine in you `test_*()` function.
"""
if message_chains and message_chains != []:
self.tester.message_chains = message_chains
# Set defconfig for Tempesta.
self.tempesta.config.set_defconfig(tempesta_defconfig)
self.configure_tempesta()
tf_cfg.dbg(3, "Starting %i servers" % len(self.servers))
for server in self.servers:
server.start()
self.tempesta.start()
self.client.start()
self.tester.start()
tf_cfg.dbg(3, "\tStarting completed")
try:
self.tester.run()
except ParseError as err:
self.assertTrue(False, msg=err)
self.tempesta.get_stats()
self.assert_tempesta()
def test(self):
self.start_all()
client = self.get_client('deproxy')
uri = '/'
uri, _ = self.client_send_first_req(client, uri)
uri, _ = self.client_send_first_req(client, uri)
uri, _ = self.client_send_first_req(client, uri)
uri, _ = self.client_send_first_req(client, uri)
uri, _ = self.client_send_first_req(client, uri)
tf_cfg.dbg(3, "Sleep until cookie timeout get expired...")
time.sleep(3)
uri, cookie = self.client_send_first_req(client, uri)
uri, _ = self.client_send_custom_req(client, uri, cookie)
hostname = tf_cfg.cfg.get('Tempesta', 'hostname')
self.assertEqual(uri, 'http://%s/' % hostname)
req = ("GET %s HTTP/1.1\r\n"
"Host: localhost\r\n"
"Cookie: %s=%s\r\n"
"\r\n" % (uri, cookie[0], cookie[1]))
response = self.client_send_req(client, req)
self.assertEqual(response.status, '200',
"unexpected response status code")
def test_pass_challenge(self):
""" Clients send the validating request just in time and pass the
challenge.
"""
self.start_all()
tf_cfg.dbg(3, "Send request to vhost 1 with timeout 2s...")
client = self.get_client('client-1')
self.process_js_challenge(client,
'vh1.com',
delay_min=1000,
delay_range=1500,
status_code=503,
expect_pass=True,
req_delay=2.5)
tf_cfg.dbg(3, "Send request to vhost 2 with timeout 4s...")
client = self.get_client('client-2')
self.process_js_challenge(client,
'vh2.com',
delay_min=2000,
delay_range=1200,
status_code=302,
expect_pass=True,
req_delay=3.5)
def check_dynamic_lb(self, servers, group_name):
""" Group of servers use ratio dynamic load balancer. For details
see test_ratio_dynamic.py .
"""
tot_weight = len(servers) * 50 # for weight normalisation.
tot_reqs = 0
for srv in servers:
tot_reqs += srv.requests
weights = [(srv.get_name(), 1.0 * srv.requests / tot_reqs * tot_weight)
for srv in servers]
weights.sort()
prev_name, prev_weight = weights[0]
for name, weight in weights:
self.assertLessEqual(
weight,
prev_weight,
msg=("Faster server %s got less weight than slower %s" %
(prev_name, name)))
if weight <= self.min_server_weight:
break
prev_weight = weight
prev_name = name
tf_cfg.dbg(
3, "Server group %s uses 'ratio dynamic' scheduler" % (group_name))
def remove_route(interface_name, ip):
""" Remove route """
template = "LANG=C ip route del %s dev %s"
try:
tf_cfg.dbg(3, "Removing route for %s" % ip)
remote.tempesta.run_cmd(template % (ip, interface_name))
except:
tf_cfg.dbg(3, "Route already removed")
def run_start(self):
tf_cfg.dbg(3, "Running deproxy")
self.exit_event.clear()
self.proc = threading.Thread(target=run_deproxy_server,
args=(self, self.exit_event,
self.polling_lock,
self.thread_expts))
self.proc.start()
def handle_close(self):
tf_cfg.dbg(6, '\tDeproxy: SrvConnection: Close connection.')
self.close()
if self.server:
try:
self.server.connections.remove(self)
except ValueError:
pass
def remove_interface(interface_name, iface_ip):
""" Remove interface """
template = "LANG=C ip address del %s/24 dev %s"
try:
tf_cfg.dbg(3, "Removing ip %s" % iface_ip)
remote.server.run_cmd(template % (iface_ip, interface_name))
except:
tf_cfg.dbg(3, "Interface alias already removed")
def __init__(self, server, sock=None, keep_alive=None):
asyncore.dispatcher_with_send.__init__(self, sock)
self.server = server
self.keep_alive = keep_alive
self.last_segment_time = 0
self.responses_done = 0
self.request_buffer = ''
tf_cfg.dbg(6, '\tDeproxy: SrvConnection: New server connection.')
def setUp(self):
self.client = None
self.tempesta = None
self.servers = []
self.tester = None
tf_cfg.dbg(3) # Step to the next line after name of test case.
tf_cfg.dbg(3, '\tInit test case...')
self.create_tempesta()
def setUp(self):
# Init members used in tearDown function.
self.tempesta = None
self.servers = []
tf_cfg.dbg(3) # Step to the next line after name of test case.
tf_cfg.dbg(3, '\tInit test case...')
self.create_clients()
self.create_servers()
self.create_tempesta()
def run_start(self):
""" Run client """
tf_cfg.dbg(3, "Running client")
self.exit_event.clear()
self.prepare()
self.proc = multiprocessing.Process(target=_run_client,
args=(self, self.exit_event,
self.resq))
self.proc.start()
def reboot(self):
sleep(self.warm_timeout)
for i in range(self.restart_cycles):
sleep(self.restart_timeout)
tf_cfg.dbg(3, '\tReboot %d of %d' % (i + 1, self.restart_cycles))
self.tempesta.stop()
# Run random command on remote node to see if it is still alive.
remote.tempesta.run_cmd('uname')
self.tempesta.start()
def read_kmemleaks():
""" Get amount of kmemleak records """
kmemleakfile = "/sys/kernel/debug/kmemleak"
if not has_kmemleak():
tf_cfg.dbg(1, "kmemleak file does not exists")
return -1
cmd = "cat %s | grep \"unreferenced object\" | wc -l" % kmemleakfile
[stdout, stderr] = remote.tempesta.run_cmd(cmd)
return int(stdout)
def setUp(self):
tf_cfg.dbg(3, '\tInit test case...')
if not remote.wait_available():
raise Exception("Tempesta node is unavaliable")
self.oops = dmesg.DmesgFinder()
self.oops_ignore = []
self.__create_servers()
self.__create_tempesta()
self.__create_clients()
def test_deproxy_srvclient_direct(self):
""" Simple test with deproxy server """
dsrv = self.get_server('deproxy')
dsrv.start()
cl = self.get_client('deproxy_direct')
cl.start()
self.deproxy_manager.start()
cl.make_request('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
cl.wait_for_response(timeout=5)
tf_cfg.dbg(3, "deproxy response:\n%s" % str(cl.last_response))
def test_deproxy_client_direct(self):
""" Simple test with deproxy client """
nginx = self.get_server('nginx')
nginx.start()
deproxy = self.get_client('deproxy_direct')
deproxy.start()
self.deproxy_manager.start()
deproxy.make_request('GET / HTTP/1.1\r\nHost: localhost\r\n\r\n')
deproxy.wait_for_response(timeout=5)
tf_cfg.dbg(3, "nginx response:\n%s" % str(deproxy.last_response))
def stop_nginx(self):
tf_cfg.dbg(3, '\tStopping Nginx on %s' % self.get_name())
pid_file = os.path.join(self.workdir, self.config.pidfile_name)
cmd = ' && '.join([
'[ -e \'%s\' ]' % pid_file,
'pid=$(cat %s)' % pid_file, 'kill -s TERM $pid',
'while [ -e \'/proc/$pid\' ]; do sleep 1; done'
])
self.node.run_cmd(cmd,
ignore_stderr=True,
err_msg=(self.err_msg % ('stop', self.get_name())))
def create_route(base_iface_name, ip, gateway_ip):
""" Create route """
command = "LANG=C ip route add %s via %s dev %s" % \
(ip, gateway_ip, base_iface_name)
try:
tf_cfg.dbg(3, "Adding route for %s" % ip)
remote.tempesta.run_cmd(command)
except:
tf_cfg.dbg(3, "Route already added")
return
def test_cookie(self):
self.start_all()
vhost = 'localhost'
tf_cfg.dbg(3, "Send request from client without cookie support...")
self.assertFalse(self.client_get('client-no-cookies', vhost),
"Client accessed resource without cookie challenge")
tf_cfg.dbg(3, "Send request from client with cookie support...")
self.assertTrue(self.client_get('client-with-cookies', vhost),
"Client couldn't access resource")
