def txn_replay(session_filename, txn, proxy, result_queue, request_session): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() responseDict = {} # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] #print("Replaying session") try: # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None body = None content = None if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] body = gen() if 'Content-Length' in txn_req_headers_dict: nBytes = int(txn_req_headers_dict['Content-Length']) body = createDummyBodywithLength(nBytes) #print("request session is",id(request_session)) if method == 'GET': request_session.request('GET', 'https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseHeaders = r1.getheaders() responseContent = r1.read() elif method == 'POST': request_session.request('POST', 'https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseHeaders = r1.getheaders() responseContent = r1.read() elif method == 'HEAD': request_session.request('HEAD', 'https://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseHeaders = r1.getheaders() responseContent = r1.read() for key, value in responseHeaders: responseDict[key.lower()] = value expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) # print(responseDict) if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) r = result.Result(session_filename, expected_output[0], r1.status) print(r.getResultString(responseDict, expected, colorize=True)) r.Compare(responseDict, expected) # result_queue.put(r) except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except requests.exceptions.ContentDecodingError as e: print("ContentDecodingError", e) except: e = sys.exc_info() print("ERROR in requests: ", e, response, session_filename)
def txn_replay(session_filename, txn, proxy, result_queue, request_session): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] #print("Replaying session") try: #response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None body=None content=None if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] body = gen() if 'Content-Length' in txn_req_headers_dict: nBytes=int(txn_req_headers_dict['Content-Length']) body = createDummyBodywithLength(nBytes) #print("request session is",id(request_session)) if method == 'GET': response = request_session.get('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, stream=False, allow_redirects=False,data=body) if 'Content-Length' in response.headers: content = response.raw #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) elif method == 'POST': response = request_session.post('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, stream=False, data=body, allow_redirects=False) if 'Content-Length' in response.headers: content = response.raw #print("reading==========>>>>>>>>>>>>>.") #print(content.data) #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) elif method == 'HEAD': response = request_session.head('http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, stream=False) #gzip_file = gzip.GzipFile(fileobj=content) #shutil.copyfileobj(gzip_file, f) expected=extractHeader.responseHeader_to_dict(resp.getHeaders()) #print(expected) if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[ 0].split(' ', 2) expected_output = (int(expected_output_split[1]), str( expected_output_split[2])) r = result.Result(session_filename, expected_output[0], response.status_code) print(r.getResultString(response.headers,expected,colorize=True)) r.Compare(response.headers,expected) #result_queue.put(r) except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except requests.exceptions.ContentDecodingError as e: print("ContentDecodingError",e) except: e=sys.exc_info() print("ERROR in requests: ",e,response, session_filename)
def txn_replay(session_filename, txn, proxy, result_queue, h2conn, request_IDs): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict[ 'Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] responseID = -1 #print("Replaying session") try: # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None mbody = None #txn_req_headers_dict['Host'] = "localhost" if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? #del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] mbody = gen() if 'Content-Length' in txn_req_headers_dict: nBytes = int(txn_req_headers_dict['Content-Length']) mbody = createDummyBodywithLength(nBytes) if 'Connection' in txn_req_headers_dict: del txn_req_headers_dict['Connection'] #str2 = extractHeader.extract_host(txn_req_headers)+ extractHeader.extract_GET_path(txn_req_headers) # print(str2) if method == 'GET': responseID = h2conn.request( 'GET', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=mbody) #print("get response", responseID) return responseID # request_IDs.append(responseID) #response = h2conn.get_response(id) # print(response.headers) # if 'Content-Length' in response.headers: # content = response.read() #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) elif method == 'POST': responseID = h2conn.request( 'POST', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=mbody) print("get response", responseID) return responseID elif method == 'HEAD': responseID = h2conn.request( 'HEAD', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict) print("get response", responseID) return responseID # print(response.headers) #print("logged respose") expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) # print(expected) if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[0].split( ' ', 2) expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) r = result.Result(session_filename, expected_output[0], response.status_code) print(r.getResultString(response.headers, expected, colorize=True)) # return responseID except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except: e = sys.exc_info() print("ERROR in requests: ", e, response, session_filename)
def txn_replay(session_filename, txn, proxy, result_queue, request_session): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] #print("Replaying session") try: # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None body = None content = None if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] body = gen() if 'Content-Length' in txn_req_headers_dict: nBytes = int(txn_req_headers_dict['Content-Length']) body = createDummyBodywithLength(nBytes) #print("request session is",id(request_session)) if method == 'GET': r1 = request_session.request('GET', extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, data=body) responseHeaders = r1.headers responseContent = r1.content # byte array #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) elif method == 'POST': r1 = request_session.request('POST', extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, data=body) responseHeaders = r1.headers responseContent = r1.content #print("len: {0} received {1}".format(responseHeaders['Content-Length'], responseContent)) elif method == 'HEAD': r1 = request_session.request('HEAD', extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, data=body) responseHeaders = r1.headers responseContent = r1.content else: # EXPERIMENTAL r1 = request_session.request(method, extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, data=body) responseHeaders = r1.headers responseContent = r1.content #gzip_file = gzip.GzipFile(fileobj=responseContent) #shutil.copyfileobj(gzip_file, f) expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) # print("------------EXPECTED-----------") # print(expected) # print("------------RESP--------------") # print(responseHeaders) # print() if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) r = result.Result(session_filename, expected_output[0], r1.status_code, responseContent) b_res, res = r.getResult(responseHeaders, expected, colorize=True) print(res) if not b_res: print("Received response") print(responseHeaders) print("Expected response") print(expected) # result_queue.put(r) except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except requests.exceptions.ContentDecodingError as e: print("ContentDecodingError", e) except: e = sys.exc_info() print("ERROR in NonSSLReplay: ", e, response, session_filename)
def txn_replay(session_filename, txn, proxy, result_queue, h2conn, request_IDs): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict['Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] responseID = -1 #print("Replaying session") try: # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None mbody = None #txn_req_headers_dict['Host'] = "localhost" if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? #del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] mbody = gen() if 'Content-Length' in txn_req_headers_dict: nBytes = int(txn_req_headers_dict['Content-Length']) mbody = createDummyBodywithLength(nBytes) if 'Connection' in txn_req_headers_dict: del txn_req_headers_dict['Connection'] #str2 = extractHeader.extract_host(txn_req_headers)+ extractHeader.extract_GET_path(txn_req_headers) # print(str2) if method == 'GET': responseID = h2conn.request('GET', url=extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, body=mbody) #print("get response", responseID) return responseID # request_IDs.append(responseID) #response = h2conn.get_response(id) # print(response.headers) # if 'Content-Length' in response.headers: # content = response.read() #print("len: {0} received {1}".format(response.headers['Content-Length'],content)) elif method == 'POST': responseID = h2conn.request('POST', url=extractHeader.extract_GET_path( txn_req_headers), headers=txn_req_headers_dict, body=mbody) print("get response", responseID) return responseID elif method == 'HEAD': responseID = h2conn.request('HEAD', url=extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict) print("get response", responseID) return responseID # print(response.headers) #print("logged respose") expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) # print(expected) if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[0].split(' ', 2) expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) r = result.Result(session_filename, expected_output[0], response.status_code) print(r.getResultString(response.headers, expected, colorize=True)) # return responseID except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except: e = sys.exc_info() print("ERROR in requests: ", e, response, session_filename)
def txn_replay(session_filename, txn, proxy, result_queue, request_session): """ Replays a single transaction :param request_session: has to be a valid requests session""" req = txn.getRequest() resp = txn.getResponse() responseDict = {} # Construct HTTP request & fire it off txn_req_headers = req.getHeaders() txn_req_headers_dict = extractHeader.header_to_dict(txn_req_headers) txn_req_headers_dict[ 'Content-MD5'] = txn._uuid # used as unique identifier if 'body' in txn_req_headers_dict: del txn_req_headers_dict['body'] #print("Replaying session") try: # response = request_session.request(extractHeader.extract_txn_req_method(txn_req_headers), # 'http://' + extractHeader.extract_host(txn_req_headers) + extractHeader.extract_GET_path(txn_req_headers), # headers=txn_req_headers_dict,stream=False) # making stream=False raises contentdecoding exception? kill me method = extractHeader.extract_txn_req_method(txn_req_headers) response = None body = None content = None if 'Transfer-Encoding' in txn_req_headers_dict: # deleting the host key, since the STUPID post/get functions are going to add host field anyway, so there will be multiple host fields in the header # This confuses the ATS and it returns 400 "Invalid HTTP request". I don't believe this # BUT, this is not a problem if the data is not chunked encoded.. Strange, huh? del txn_req_headers_dict['Host'] if 'Content-Length' in txn_req_headers_dict: #print("ewww !") del txn_req_headers_dict['Content-Length'] body = gen() if 'Content-Length' in txn_req_headers_dict: nBytes = int(txn_req_headers_dict['Content-Length']) body = createDummyBodywithLength(nBytes) #print("request session is",id(request_session)) # NOTE: request_session here is actually python's HTTPSConnection, which is different from that in NonSSL, which uses the requests library -_- if method == 'GET': request_session.request( 'GET', extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseContent = r1.read() # byte array elif method == 'POST': request_session.request( 'POST', extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseContent = r1.read() elif method == 'HEAD': request_session.request( 'HEAD', extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseContent = r1.read() else: # EXPERIMENTAL request_session.request( method, extractHeader.extract_GET_path(txn_req_headers), headers=txn_req_headers_dict, body=body) r1 = request_session.getresponse() responseContent = r1.read() responseHeaders = extractHeader.responseHeaderTuple_to_dict( r1.getheaders()) expected = extractHeader.responseHeader_to_dict(resp.getHeaders()) # print("------------EXPECTED-----------") # print(expected) # print("------------RESP--------------") # print(responseHeaders) # print() if mainProcess.verbose: expected_output_split = resp.getHeaders().split('\r\n')[0].split( ' ', 2) expected_output = (int(expected_output_split[1]), str(expected_output_split[2])) r = result.Result(session_filename, expected_output[0], r1.status, responseContent) b_res, res = r.getResult(responseHeaders, expected, colorize=True) print(res) if not res: print("Received response") print(responseHeaders) print("Expected response") print(expected) # result_queue.put(r) except UnicodeEncodeError as e: # these unicode errors are due to the interaction between Requests and our wiretrace data. # TODO fix print("UnicodeEncodeError exception") except requests.exceptions.ContentDecodingError as e: print("ContentDecodingError", e) except: e = sys.exc_info() print("ERROR in SSLReplay: ", e, response, session_filename)