def curl_available():
popen_args = ['curl', '--version']
try:
process = Process(popen_args,
name='curl_check',
stdout=PIPE,
expect=None,
universal_newlines=True)
out = process.communicate()[0]
return True
except:
return False
def quart_available():
"""
Checks if quart version is greater than 0.13
"""
popen_args = ['quart', '--version']
try:
process = Process(popen_args,
name='quart_check',
stdout=PIPE,
expect=None,
universal_newlines=True)
out = process.communicate()[0]
parts = out.split(".")
major_version = parts[0]
if int(major_version[-1]) > 0 or int(parts[1]) >= 13:
return True
return False
except Exception as e:
print(e)
print("quart_not_available")
return False
def setUpClass(cls):
"""Start a router"""
super(TcpAdaptor, cls).setUpClass()
if DISABLE_SELECTOR_TESTS:
return
def router(name, mode, connection, extra=None):
"""
Launch a router through the system_test framework.
For each router:
* normal listener first
#* http listener for console connections
* tcp listener for 'nodest', which will never exist
* tcp connector to echo server whose address is the same as this router's name
* six tcp listeners, one for each server on each router on the network
:param name: router name
:param mode: router mode: interior or edge
:param connection: list of router-level connection/listener tuples
:param extra: yet more configuation tuples. unused for now
:return:
"""
config = [
('router', {
'mode': mode,
'id': name
}),
('listener', {
'port': cls.amqp_listener_ports[name]
}),
# ('listener', {'port': cls.http_listener_ports[name], 'http': 'yes'}),
('tcpListener', {
'host': "0.0.0.0",
'port': cls.nodest_listener_ports[name],
'address': 'nodest',
'siteId': cls.site
}),
('tcpConnector', {
'host': "127.0.0.1",
'port': cls.tcp_server_listener_ports[name],
'address': 'ES_' + name,
'siteId': cls.site
})
]
if connection:
config.extend(connection)
listeners = []
for rtr in cls.router_order:
listener = {
'host': "0.0.0.0",
'port': cls.tcp_client_listener_ports[name][rtr],
'address': 'ES_' + rtr,
'siteId': cls.site
}
tup = [(('tcpListener', listener))]
listeners.extend(tup)
config.extend(listeners)
if extra:
config.extend(extra)
config = Qdrouterd.Config(config)
cls.routers.append(cls.tester.qdrouterd(name, config, wait=True))
cls.routers = []
# define logging levels
cls.print_logs_server = False
cls.print_logs_client = True
cls.logger = Logger(title="TcpAdaptor-testClass",
print_to_console=True,
save_for_dump=False,
ofilename='../setUpClass/TcpAdaptor.log')
# Write a dummy log line for scraper.
cls.logger.log("SERVER (info) Container Name: TCP_TEST")
# Allocate echo server ports first
for rtr in cls.router_order:
cls.tcp_server_listener_ports[rtr] = cls.tester.get_port()
# start echo servers immediately after the echo server
# ports are assigned.
for rtr in cls.router_order:
test_name = "TcpAdaptor"
server_prefix = "ECHO_SERVER %s ES_%s" % (test_name, rtr)
server_logger = Logger(
title=test_name,
print_to_console=cls.print_logs_server,
save_for_dump=False,
ofilename="../setUpClass/TcpAdaptor_echo_server_%s.log" % rtr)
cls.logger.log("TCP_TEST Launching echo server '%s'" %
server_prefix)
server = TcpEchoServer(prefix=server_prefix,
port=cls.tcp_server_listener_ports[rtr],
logger=server_logger)
assert server.is_running
cls.echo_servers[rtr] = server
cls.EC2_conn_stall_connector_port = cls.tester.get_port()
# start special naughty servers that misbehave on purpose
server_prefix = "ECHO_SERVER TcpAdaptor NS_EC2_CONN_STALL"
server_logger = Logger(
title="TcpAdaptor",
print_to_console=cls.print_logs_server,
save_for_dump=False,
ofilename="../setUpClass/TcpAdaptor_echo_server_NS_CONN_STALL.log")
cls.logger.log("TCP_TEST Launching echo server '%s'" % server_prefix)
server = TcpEchoServer(prefix=server_prefix,
port=cls.EC2_conn_stall_connector_port,
logger=server_logger,
conn_stall=Q2_DELAY_SECONDS)
assert server.is_running
cls.echo_server_NS_CONN_STALL = server
# Allocate a sea of router ports
for rtr in cls.router_order:
cls.amqp_listener_ports[rtr] = cls.tester.get_port()
tl_ports = {}
for tcp_listener in cls.router_order:
tl_ports[tcp_listener] = cls.tester.get_port()
cls.tcp_client_listener_ports[rtr] = tl_ports
cls.nodest_listener_ports[rtr] = cls.tester.get_port()
inter_router_port_AB = cls.tester.get_port()
cls.INTA_edge_port = cls.tester.get_port()
cls.INTA_conn_stall_listener_port = cls.tester.get_port()
# Launch the routers using the sea of router ports
router('INTA', 'interior',
[('listener', {
'role': 'inter-router',
'port': inter_router_port_AB
}),
('listener', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTA_edge_port
}),
('tcpListener', {
'host': "0.0.0.0",
'port': cls.INTA_conn_stall_listener_port,
'address': 'NS_EC2_CONN_STALL',
'siteId': cls.site
})])
inter_router_port_BC = cls.tester.get_port()
cls.INTB_edge_port = cls.tester.get_port()
router('INTB', 'interior', [('connector', {
'role': 'inter-router',
'port': inter_router_port_AB
}), ('listener', {
'role': 'inter-router',
'port': inter_router_port_BC
}),
('listener', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTB_edge_port
})])
cls.INTC_edge_port = cls.tester.get_port()
router('INTC', 'interior', [('connector', {
'role': 'inter-router',
'port': inter_router_port_BC
}),
('listener', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTC_edge_port
})])
router('EA1', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTA_edge_port
})])
router('EA2', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTA_edge_port
})])
router('EB1', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTB_edge_port
})])
router('EB2', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTB_edge_port
})])
router('EC1', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTC_edge_port
})])
cls.EC2_conn_stall_listener_port = cls.tester.get_port()
router('EC2', 'edge', [('connector', {
'name': 'uplink',
'role': 'edge',
'port': cls.INTC_edge_port
}),
('tcpConnector', {
'host': "127.0.0.1",
'port': cls.EC2_conn_stall_connector_port,
'address': 'NS_EC2_CONN_STALL',
'siteId': cls.site
}),
('tcpListener', {
'host': "0.0.0.0",
'port': cls.EC2_conn_stall_listener_port,
'address': 'NS_EC2_CONN_STALL',
'siteId': cls.site
})])
cls.INTA = cls.routers[0]
cls.INTB = cls.routers[1]
cls.INTC = cls.routers[2]
cls.EA1 = cls.routers[3]
cls.EA2 = cls.routers[4]
cls.EB1 = cls.routers[5]
cls.EB2 = cls.routers[6]
cls.EC1 = cls.routers[7]
cls.EC2 = cls.routers[8]
cls.router_dict = {}
cls.router_dict['INTA'] = cls.INTA
cls.router_dict['INTB'] = cls.INTB
cls.router_dict['INTC'] = cls.INTC
cls.router_dict['EA1'] = cls.EA1
cls.router_dict['EA2'] = cls.EA2
cls.router_dict['EB1'] = cls.EB1
cls.router_dict['EB2'] = cls.EB2
cls.router_dict['EC1'] = cls.EC1
cls.router_dict['EC2'] = cls.EC2
cls.logger.log("TCP_TEST INTA waiting for connection to INTB")
cls.INTA.wait_router_connected('INTB')
cls.logger.log("TCP_TEST INTB waiting for connection to INTA")
cls.INTB.wait_router_connected('INTA')
cls.logger.log("TCP_TEST INTB waiting for connection to INTC")
cls.INTB.wait_router_connected('INTC')
cls.logger.log("TCP_TEST INTC waiting for connection to INTB")
cls.INTC.wait_router_connected('INTB')
# Create a scoreboard for the ports
p_out = []
for rtr in cls.router_order:
p_out.append("%s_amqp=%d" % (rtr, cls.amqp_listener_ports[rtr]))
p_out.append("%s_echo_server=%d" %
(rtr, cls.tcp_server_listener_ports[rtr]))
for tcp_listener in cls.router_order:
p_out.append(
"%s_echo_listener_for_%s=%d" %
(rtr, tcp_listener,
cls.tcp_client_listener_ports[rtr][tcp_listener]))
p_out.append("%s_nodest_listener=%d" %
(rtr, cls.nodest_listener_ports[rtr]))
# p_out.append("%s_http_listener=%d" %
# (rtr, cls.http_listener_ports[rtr]))
p_out.append("inter_router_port_AB=%d" % inter_router_port_AB)
p_out.append("inter_router_port_BC=%d" % inter_router_port_BC)
p_out.append("INTA_edge_port=%d" % cls.INTA_edge_port)
p_out.append("INTB_edge_port=%d" % cls.INTB_edge_port)
p_out.append("INTC_edge_port=%d" % cls.INTC_edge_port)
p_out.append("EC2_conn_stall_connector_port%d" %
cls.EC2_conn_stall_connector_port)
p_out.append("INTA_conn_stall_listener_port%d" %
cls.INTA_conn_stall_listener_port)
p_out.append("EC2_conn_stall_listener_port%d" %
cls.EC2_conn_stall_listener_port)
# write to log
for line in p_out:
cls.logger.log("TCP_TEST %s" % line)
# write to shell script
with open("../setUpClass/TcpAdaptor-ports.sh", 'w') as o_file:
for line in p_out:
o_file.write("set %s\n" % line)
# Write a script to run scraper on this test's log files
scraper_abspath = os.path.join(os.environ.get('BUILD_DIR'), 'tests',
'scraper', 'scraper.py')
logs_dir = os.path.abspath("../setUpClass")
main_log = "TcpAdaptor.log"
echo_logs = "TcpAdaptor_echo*"
big_test_log = "TcpAdaptor_all.log"
int_logs = "I*.log"
edge_logs = "E*.log"
log_modules_spec = "--log-modules TCP_ADAPTOR,TCP_TEST,ECHO_SERVER,ECHO_CLIENT"
html_output = "TcpAdaptor.html"
with open("../setUpClass/TcpAdaptor-run-scraper.sh", 'w') as o_file:
o_file.write("#!/bin/bash\n\n")
o_file.write(
"# Script to run scraper on test class TcpAdaptor test result\n"
)
o_file.write("# cd into logs directory\n")
o_file.write("cd %s\n\n" % logs_dir)
o_file.write("# Concatenate test class logs into single file\n")
o_file.write("cat %s %s > %s\n\n" %
(main_log, echo_logs, big_test_log))
o_file.write("# run scraper\n")
o_file.write("python %s %s -f %s %s %s > %s\n\n" %
(scraper_abspath, log_modules_spec, int_logs,
edge_logs, big_test_log, html_output))
o_file.write("echo View the results by opening the html file\n")
o_file.write("echo firefox %s" %
(os.path.join(logs_dir, html_output)))
# wait for server addresses (mobile ES_<rtr>) to propagate to all interior routers
interior_rtrs = [
rtr for rtr in cls.router_order if rtr.startswith('I')
]
found_all = False
while not found_all:
found_all = True
cls.logger.log(
"TCP_TEST Poll wait for echo server addresses to propagate")
for rtr in interior_rtrs:
# query each interior for addresses
p = Process([
'qdstat', '-b',
str(cls.router_dict[rtr].addresses[0]), '-a'
],
name='qdstat-snap1',
stdout=PIPE,
expect=None,
universal_newlines=True)
out = p.communicate()[0]
# examine what this router can see; signal poll loop to continue or not
lines = out.split("\n")
server_lines = [
line for line in lines
if "mobile" in line and "ES_" in line
]
if not len(server_lines) == len(cls.router_order):
found_all = False
seen = []
for line in server_lines:
flds = line.split()
seen.extend(
[fld for fld in flds if fld.startswith("ES_")])
unseen = [
srv for srv in cls.router_order
if "ES_" + srv not in seen
]
cls.logger.log(
"TCP_TEST Router %s sees only %d of %d addresses. Waiting for %s"
% (rtr, len(server_lines), len(
cls.router_order), unseen))
cls.logger.log("TCP_TEST Done poll wait")