def curl_available(): popen_args = ['curl', '--version'] try: process = Process(popen_args, name='curl_check', stdout=PIPE, expect=None, universal_newlines=True) out = process.communicate()[0] return True except: return False
def quart_available(): """ Checks if quart version is greater than 0.13 """ popen_args = ['quart', '--version'] try: process = Process(popen_args, name='quart_check', stdout=PIPE, expect=None, universal_newlines=True) out = process.communicate()[0] parts = out.split(".") major_version = parts[0] if int(major_version[-1]) > 0 or int(parts[1]) >= 13: return True return False except Exception as e: print(e) print("quart_not_available") return False
def setUpClass(cls): """Start a router""" super(TcpAdaptor, cls).setUpClass() if DISABLE_SELECTOR_TESTS: return def router(name, mode, connection, extra=None): """ Launch a router through the system_test framework. For each router: * normal listener first #* http listener for console connections * tcp listener for 'nodest', which will never exist * tcp connector to echo server whose address is the same as this router's name * six tcp listeners, one for each server on each router on the network :param name: router name :param mode: router mode: interior or edge :param connection: list of router-level connection/listener tuples :param extra: yet more configuation tuples. unused for now :return: """ config = [ ('router', { 'mode': mode, 'id': name }), ('listener', { 'port': cls.amqp_listener_ports[name] }), # ('listener', {'port': cls.http_listener_ports[name], 'http': 'yes'}), ('tcpListener', { 'host': "0.0.0.0", 'port': cls.nodest_listener_ports[name], 'address': 'nodest', 'siteId': cls.site }), ('tcpConnector', { 'host': "127.0.0.1", 'port': cls.tcp_server_listener_ports[name], 'address': 'ES_' + name, 'siteId': cls.site }) ] if connection: config.extend(connection) listeners = [] for rtr in cls.router_order: listener = { 'host': "0.0.0.0", 'port': cls.tcp_client_listener_ports[name][rtr], 'address': 'ES_' + rtr, 'siteId': cls.site } tup = [(('tcpListener', listener))] listeners.extend(tup) config.extend(listeners) if extra: config.extend(extra) config = Qdrouterd.Config(config) cls.routers.append(cls.tester.qdrouterd(name, config, wait=True)) cls.routers = [] # define logging levels cls.print_logs_server = False cls.print_logs_client = True cls.logger = Logger(title="TcpAdaptor-testClass", print_to_console=True, save_for_dump=False, ofilename='../setUpClass/TcpAdaptor.log') # Write a dummy log line for scraper. cls.logger.log("SERVER (info) Container Name: TCP_TEST") # Allocate echo server ports first for rtr in cls.router_order: cls.tcp_server_listener_ports[rtr] = cls.tester.get_port() # start echo servers immediately after the echo server # ports are assigned. for rtr in cls.router_order: test_name = "TcpAdaptor" server_prefix = "ECHO_SERVER %s ES_%s" % (test_name, rtr) server_logger = Logger( title=test_name, print_to_console=cls.print_logs_server, save_for_dump=False, ofilename="../setUpClass/TcpAdaptor_echo_server_%s.log" % rtr) cls.logger.log("TCP_TEST Launching echo server '%s'" % server_prefix) server = TcpEchoServer(prefix=server_prefix, port=cls.tcp_server_listener_ports[rtr], logger=server_logger) assert server.is_running cls.echo_servers[rtr] = server cls.EC2_conn_stall_connector_port = cls.tester.get_port() # start special naughty servers that misbehave on purpose server_prefix = "ECHO_SERVER TcpAdaptor NS_EC2_CONN_STALL" server_logger = Logger( title="TcpAdaptor", print_to_console=cls.print_logs_server, save_for_dump=False, ofilename="../setUpClass/TcpAdaptor_echo_server_NS_CONN_STALL.log") cls.logger.log("TCP_TEST Launching echo server '%s'" % server_prefix) server = TcpEchoServer(prefix=server_prefix, port=cls.EC2_conn_stall_connector_port, logger=server_logger, conn_stall=Q2_DELAY_SECONDS) assert server.is_running cls.echo_server_NS_CONN_STALL = server # Allocate a sea of router ports for rtr in cls.router_order: cls.amqp_listener_ports[rtr] = cls.tester.get_port() tl_ports = {} for tcp_listener in cls.router_order: tl_ports[tcp_listener] = cls.tester.get_port() cls.tcp_client_listener_ports[rtr] = tl_ports cls.nodest_listener_ports[rtr] = cls.tester.get_port() inter_router_port_AB = cls.tester.get_port() cls.INTA_edge_port = cls.tester.get_port() cls.INTA_conn_stall_listener_port = cls.tester.get_port() # Launch the routers using the sea of router ports router('INTA', 'interior', [('listener', { 'role': 'inter-router', 'port': inter_router_port_AB }), ('listener', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTA_edge_port }), ('tcpListener', { 'host': "0.0.0.0", 'port': cls.INTA_conn_stall_listener_port, 'address': 'NS_EC2_CONN_STALL', 'siteId': cls.site })]) inter_router_port_BC = cls.tester.get_port() cls.INTB_edge_port = cls.tester.get_port() router('INTB', 'interior', [('connector', { 'role': 'inter-router', 'port': inter_router_port_AB }), ('listener', { 'role': 'inter-router', 'port': inter_router_port_BC }), ('listener', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTB_edge_port })]) cls.INTC_edge_port = cls.tester.get_port() router('INTC', 'interior', [('connector', { 'role': 'inter-router', 'port': inter_router_port_BC }), ('listener', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTC_edge_port })]) router('EA1', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTA_edge_port })]) router('EA2', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTA_edge_port })]) router('EB1', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTB_edge_port })]) router('EB2', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTB_edge_port })]) router('EC1', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTC_edge_port })]) cls.EC2_conn_stall_listener_port = cls.tester.get_port() router('EC2', 'edge', [('connector', { 'name': 'uplink', 'role': 'edge', 'port': cls.INTC_edge_port }), ('tcpConnector', { 'host': "127.0.0.1", 'port': cls.EC2_conn_stall_connector_port, 'address': 'NS_EC2_CONN_STALL', 'siteId': cls.site }), ('tcpListener', { 'host': "0.0.0.0", 'port': cls.EC2_conn_stall_listener_port, 'address': 'NS_EC2_CONN_STALL', 'siteId': cls.site })]) cls.INTA = cls.routers[0] cls.INTB = cls.routers[1] cls.INTC = cls.routers[2] cls.EA1 = cls.routers[3] cls.EA2 = cls.routers[4] cls.EB1 = cls.routers[5] cls.EB2 = cls.routers[6] cls.EC1 = cls.routers[7] cls.EC2 = cls.routers[8] cls.router_dict = {} cls.router_dict['INTA'] = cls.INTA cls.router_dict['INTB'] = cls.INTB cls.router_dict['INTC'] = cls.INTC cls.router_dict['EA1'] = cls.EA1 cls.router_dict['EA2'] = cls.EA2 cls.router_dict['EB1'] = cls.EB1 cls.router_dict['EB2'] = cls.EB2 cls.router_dict['EC1'] = cls.EC1 cls.router_dict['EC2'] = cls.EC2 cls.logger.log("TCP_TEST INTA waiting for connection to INTB") cls.INTA.wait_router_connected('INTB') cls.logger.log("TCP_TEST INTB waiting for connection to INTA") cls.INTB.wait_router_connected('INTA') cls.logger.log("TCP_TEST INTB waiting for connection to INTC") cls.INTB.wait_router_connected('INTC') cls.logger.log("TCP_TEST INTC waiting for connection to INTB") cls.INTC.wait_router_connected('INTB') # Create a scoreboard for the ports p_out = [] for rtr in cls.router_order: p_out.append("%s_amqp=%d" % (rtr, cls.amqp_listener_ports[rtr])) p_out.append("%s_echo_server=%d" % (rtr, cls.tcp_server_listener_ports[rtr])) for tcp_listener in cls.router_order: p_out.append( "%s_echo_listener_for_%s=%d" % (rtr, tcp_listener, cls.tcp_client_listener_ports[rtr][tcp_listener])) p_out.append("%s_nodest_listener=%d" % (rtr, cls.nodest_listener_ports[rtr])) # p_out.append("%s_http_listener=%d" % # (rtr, cls.http_listener_ports[rtr])) p_out.append("inter_router_port_AB=%d" % inter_router_port_AB) p_out.append("inter_router_port_BC=%d" % inter_router_port_BC) p_out.append("INTA_edge_port=%d" % cls.INTA_edge_port) p_out.append("INTB_edge_port=%d" % cls.INTB_edge_port) p_out.append("INTC_edge_port=%d" % cls.INTC_edge_port) p_out.append("EC2_conn_stall_connector_port%d" % cls.EC2_conn_stall_connector_port) p_out.append("INTA_conn_stall_listener_port%d" % cls.INTA_conn_stall_listener_port) p_out.append("EC2_conn_stall_listener_port%d" % cls.EC2_conn_stall_listener_port) # write to log for line in p_out: cls.logger.log("TCP_TEST %s" % line) # write to shell script with open("../setUpClass/TcpAdaptor-ports.sh", 'w') as o_file: for line in p_out: o_file.write("set %s\n" % line) # Write a script to run scraper on this test's log files scraper_abspath = os.path.join(os.environ.get('BUILD_DIR'), 'tests', 'scraper', 'scraper.py') logs_dir = os.path.abspath("../setUpClass") main_log = "TcpAdaptor.log" echo_logs = "TcpAdaptor_echo*" big_test_log = "TcpAdaptor_all.log" int_logs = "I*.log" edge_logs = "E*.log" log_modules_spec = "--log-modules TCP_ADAPTOR,TCP_TEST,ECHO_SERVER,ECHO_CLIENT" html_output = "TcpAdaptor.html" with open("../setUpClass/TcpAdaptor-run-scraper.sh", 'w') as o_file: o_file.write("#!/bin/bash\n\n") o_file.write( "# Script to run scraper on test class TcpAdaptor test result\n" ) o_file.write("# cd into logs directory\n") o_file.write("cd %s\n\n" % logs_dir) o_file.write("# Concatenate test class logs into single file\n") o_file.write("cat %s %s > %s\n\n" % (main_log, echo_logs, big_test_log)) o_file.write("# run scraper\n") o_file.write("python %s %s -f %s %s %s > %s\n\n" % (scraper_abspath, log_modules_spec, int_logs, edge_logs, big_test_log, html_output)) o_file.write("echo View the results by opening the html file\n") o_file.write("echo firefox %s" % (os.path.join(logs_dir, html_output))) # wait for server addresses (mobile ES_<rtr>) to propagate to all interior routers interior_rtrs = [ rtr for rtr in cls.router_order if rtr.startswith('I') ] found_all = False while not found_all: found_all = True cls.logger.log( "TCP_TEST Poll wait for echo server addresses to propagate") for rtr in interior_rtrs: # query each interior for addresses p = Process([ 'qdstat', '-b', str(cls.router_dict[rtr].addresses[0]), '-a' ], name='qdstat-snap1', stdout=PIPE, expect=None, universal_newlines=True) out = p.communicate()[0] # examine what this router can see; signal poll loop to continue or not lines = out.split("\n") server_lines = [ line for line in lines if "mobile" in line and "ES_" in line ] if not len(server_lines) == len(cls.router_order): found_all = False seen = [] for line in server_lines: flds = line.split() seen.extend( [fld for fld in flds if fld.startswith("ES_")]) unseen = [ srv for srv in cls.router_order if "ES_" + srv not in seen ] cls.logger.log( "TCP_TEST Router %s sees only %d of %d addresses. Waiting for %s" % (rtr, len(server_lines), len( cls.router_order), unseen)) cls.logger.log("TCP_TEST Done poll wait")