def LaunchWorkers(path, nProcess, proxy, replay_type, nThread): ms1 = time.time() s = sv.SessionValidator(path) sessions = s.getSessionList() sessions.sort(key=lambda session: session._timestamp) Processes = [] Qsize = 25000 # int (1.5 * len(sessions)/(nProcess)) QList = [Queue(Qsize) for i in range(nProcess)] print("Dropped {0} sessions for being malformed. Number of correct sessions {1}".format( len(s.getBadSessionList()), len(sessions))) print(range(nProcess)) OutputQ = Queue() #======================================== Pre-load queues for session in sessions: # if nProcess == 1: # QList[0].put(session) # else: QList[random.randint(0, nProcess - 1)].put(session) # if QList[0].qsize() > 10 : # break #=============================================== Launch Processes print("size", QList[0].qsize()) for i in range(nProcess): QList[i].put('STOP') for i in range(nProcess): p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, replay_type, nThread]) p.daemon = False Processes.append(p) p.start() for p in Processes: p.join() ms2 = time.time() print("OK enough, it is time to exit, running time in seconds", (ms2 - ms1))
def main(): global test_mode_enabled parser = argparse.ArgumentParser() parser.add_argument("--data-dir","-d", type=lambda x: _path(True,x), required=True, help="Directory with data file" ) parser.add_argument("--public","-P", type=_bool, default=False, help="Bind server to public IP 0.0.0.0 vs private IP of 127.0.0.1" ) parser.add_argument("--port","-p", type=int, default=SERVER_PORT, help="Port to use") parser.add_argument("--timeout","-t", type=float, default=None, help="socket time out in seconds") parser.add_argument('-V','--version', action='version', version='%(prog)s {0}'.format(__version__)) parser.add_argument("--mode","-m", type=str, default="test", help="Mode of operation") args=parser.parse_args() # set up global dictionary of {uuid (string): response (Response object)} s = sv.SessionValidator(args.data_dir) populate_global_replay_dictionary(s.getSessionIter()) print("Dropped {0} sessions for being malformed".format(len(s.getBadSessionList()))) # start server try: server_port = args.port socket_timeout = args.timeout test_mode_enabled = args.mode=="test" MyHandler.protocol_version = HTTP_VERSION server = ThreadingServer(('', server_port), MyHandler) server.timeout = socket_timeout or 5 print("=== started httpserver ===") server_thread = threading.Thread(target=server.serve_forever()) server_thread.daemon=True server_thread.start() #server.serve_forever() except KeyboardInterrupt: print("\n=== ^C received, shutting down httpserver ===") server.socket.close() sys.exit(0)
def LaunchWorkers(path, nProcess, proxy, replay_type, nThread): ms1 = time.time() s = sv.SessionValidator(path, allow_custom=True) sessions = s.getSessionList() sessions.sort(key=lambda session: session._timestamp) Processes = [] Qsize = 25000 # int (1.5 * len(sessions)/(nProcess)) QList = [Queue(Qsize) for i in range(nProcess)] print( "Dropped {0} sessions for being malformed. Number of correct sessions {1}" .format(len(s.getBadSessionList()), len(sessions))) print(range(nProcess)) OutputQ = Queue() #======================================== Pre-load queues for session in sessions: if replay_type == 'mixed': if nProcess < 2: raise ValueError( "For mixed replay type, there should be at least 2 processes." ) # odd Qs for SSL sessions, even Qs for nonSSL sessions num = random.randint(0, nProcess - 1) # get the first transaction in each session, which is indictive if session is over SSL or not if "https" in session.returnFirstTransaction().getRequest( ).getHeaders(): # spin until we get an odd number while num & 1 == 0: num = random.randint(0, nProcess - 1) else: # nonSSL sessions get put here into even Qs while num & 1 == 1: num = random.randint(0, nProcess - 1) QList[num].put(session) else: # if nProcess == 1: # QList[0].put(session) # else: QList[random.randint(0, nProcess - 1)].put(session) # if QList[0].qsize() > 10 : # break #=============================================== Launch Processes # for i in range(nProcess): # QList[i].put('STOP') for i in range(nProcess): QList[i].put('STOP') if replay_type == 'mixed': if i & 1: # odd/SSL p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'ssl', nThread]) else: # even/nonSSL p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, 'nossl', nThread]) else: p = Process(target=WorkerTask.worker, args=[QList[i], OutputQ, proxy, replay_type, nThread]) p.daemon = False Processes.append(p) p.start() for p in Processes: p.join() ms2 = time.time() print("OK enough, it is time to exit, running time in seconds", (ms2 - ms1))
def main(): global test_mode_enabled parser = argparse.ArgumentParser() parser.add_argument("--data-dir", "-d", type=lambda x: _path(True, x), required=True, help="Directory with data file") parser.add_argument("--ip_address", "-ip", type=str, default='INADDR_LOOPBACK', help="IP address of the interface to serve on") parser.add_argument("--port", "-p", type=int, default=SERVER_PORT, help="Port to use") parser.add_argument("--delay", "-dy", type=float, default=SERVER_DELAY, help="Response delay") parser.add_argument("--timeout", "-t", type=float, default=None, help="socket time out in seconds") parser.add_argument('-V', '--version', action='version', version='%(prog)s {0}'.format(__version__)) parser.add_argument("--mode", "-m", type=str, default="test", help="Mode of operation") parser.add_argument("--ssl", "-ssl", type=str, default="False", help="SSL port") parser.add_argument("--key", "-k", type=str, default="ssl/server.pem", help="key for ssl connnection") parser.add_argument("--cert", "-cert", type=str, default="ssl/server.crt", help="certificate") parser.add_argument("--clientCA", type=str, default="", help="CA for client certificates") parser.add_argument("--clientverify", "-cverify", type=_argparse_bool, default=False, help="verify client cert") parser.add_argument("--load", dest='load', type=str, default='', help="A file which will install observers on hooks") parser.add_argument( "--lookupkey", type=str, default="{PATH}", help="format string used as a key for response lookup: \ example: \"{%%Host}{%%Server}{PATH}\", \"{HOST}{PATH}\", \"{PATH}\"\ All the args preceded by %% are header fields in the request\ The only two acceptable arguments which are not header fields are : fqdn (represented by HOST) and the url path (represented by PATH) in a request line.\ Example: given a client request as << GET /some/resource/location HTTP/1.1\nHost: hahaha.com\n\n >>, if the user wishes the host field and the path to be used for the response lookup\ then the required format will be {%%Host}{PATH}") args = parser.parse_args() global time_delay time_delay = args.delay # set up global dictionary of {uuid (string): response (Response object)} s = sv.SessionValidator(args.data_dir) populate_global_replay_dictionary(s.getSessionIter()) print("Dropped {0} sessions for being malformed".format( len(s.getBadSessionList()))) # start server try: socket_timeout = args.timeout test_mode_enabled = args.mode == "test" global lookup_key_ lookup_key_ = args.lookupkey MyHandler.protocol_version = HTTP_VERSION if IPConstants.isIPv6(args.ip_address): print("Server running on IPv6") HTTPServer.address_family = socket.AF_INET6 if args.ssl == "True" or args.ssl == "true": server = SSLServer((IPConstants.getIP(args.ip_address), args.port), MyHandler, args) else: server = ThreadingServer( (IPConstants.getIP(args.ip_address), args.port), MyHandler, args) server.timeout = 5 print("Started server on port {0}".format(args.port)) server_thread = threading.Thread(target=server.serve_forever) server_thread.daemon = True server_thread.start() try: while 1: time.sleep(1) sys.stderr.flush() sys.stdout.flush() except KeyboardInterrupt: print("\n=== ^C received, shutting down microservers ===") server.shutdown() server_thread.join() except KeyboardInterrupt: print("\n=== ^C received, shutting down httpserver ===") server.socket.close() # s_server.socket.close() sys.exit(0)
def main(): global test_mode_enabled parser = argparse.ArgumentParser() parser.add_argument("--data-dir", "-d", type=lambda x: _path(True, x), required=True, help="Directory with data file") parser.add_argument( "--public", "-P", type=_bool, default=False, help="Bind server to public IP 0.0.0.0 vs private IP of 127.0.0.1") parser.add_argument("--ip_address", "-ip", type=str, default='', help="IP address of the interface to serve on") parser.add_argument("--port", "-p", type=int, default=SERVER_PORT, help="Port to use") parser.add_argument("--timeout", "-t", type=float, default=None, help="socket time out in seconds") parser.add_argument('-V', '--version', action='version', version='%(prog)s {0}'.format(__version__)) parser.add_argument("--mode", "-m", type=str, default="test", help="Mode of operation") parser.add_argument("--ssl", "-ssl", type=str, default="False", help="SSL port") parser.add_argument("--key", "-k", type=str, default="ssl/server.pem", help="key for ssl connnection") parser.add_argument("--cert", "-cert", type=str, default="ssl/server.crt", help="certificate") parser.add_argument("--clientverify", "-cverify", type=bool, default=False, help="verify client cert") parser.add_argument("--load", dest='load', type=str, default='', help="A file which will install observers on hooks") args = parser.parse_args() options = args # set up global dictionary of {uuid (string): response (Response object)} s = sv.SessionValidator(args.data_dir) populate_global_replay_dictionary(s.getSessionIter()) print("Dropped {0} sessions for being malformed".format( len(s.getBadSessionList()))) # start server try: socket_timeout = args.timeout test_mode_enabled = args.mode == "test" MyHandler.protocol_version = HTTP_VERSION if options.ssl == "True" or options.ssl == "true": server = SSLServer((options.ip_address, options.port), MyHandler, options) else: server = ThreadingServer((options.ip_address, options.port), MyHandler, options) server.timeout = 5 print("started server") server_thread = threading.Thread(target=server.serve_forever()) server_thread.daemon = True server_thread.start() except KeyboardInterrupt: print("\n=== ^C received, shutting down httpserver ===") server.socket.close() # s_server.socket.close() sys.exit(0)