def get_response(request_uri=None):
"""
Description: Gets the next response data for the server
input_param: request_uri - Request Uri present in client request
input_type: string
out_param: response_data - Response data as a string
out_type: String
sample output: "HTTP/1.1 200 OK\r\nConnection:Close\r\n\r\n"
"""
if not request_uri:
return {}
test_id, req_id = [ int(parts) for parts in request_uri.strip().split("/")[:3] if len(parts) ]
running_test_row = session.query(HttpTestResults)\
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.request_id==req_id)\
.filter(HttpTestResults.is_running==True).first()
if running_test_row and running_test_row.sub_response_id:
sub_response = session.query(HttpSubResponse).get(running_test_row.sub_response_id)
if not sub_response:
failure_data = "Proxy sent one extra request. The Request should have been served from cache"
server_failure_reason = HttpServerTestFailureReason(reason=failure_data)
session.add(server_failure_reason)
session.flush()
running_test_row.request_result=False
running_test_row.server_failure_id = server_failure_reason.id
session.commit()
return "HTTP/1.1 404 Not Found\r\nConnection:Close\r\nContent-Length:0\r\n\r\n"
else:
failure_data = "Proxy sent one extra request. The Request should have been served from cache"
server_failure_reason = HttpServerTestFailureReason(reason=failure_data)
session.add(server_failure_reason)
session.flush()
running_test_row.request_result=False
running_test_row.server_failure_id = server_failure_reason.id
session.commit()
return "HTTP/1.1 404 Not Found\r\nConnection:Close\r\nContent-Length:0\r\n\r\n"
response_data = "HTTP/" + sub_response.version + " "
response_data = response_data + sub_response.response_code.code_name + "\r\n"
for response_header in sub_response.response_hdrs:
response_data = response_data + response_header.header_name + ":"
value_list = eval(response_header.server_value)
if response_header.single_value_hdr:
response_data = response_data + value_list[0]+ "\r\n"
else:
response_data = response_data + + ";".join(value_list) + "\r\n"
if sub_response.data_id:
response_data = response_data + "Content-Length:" + str( len(sub_response.data.data) )
response_data = response_data + "\r\n\r\n"
response_data = response_data + sub_response.data.data
else:
response_data = response_data + "Content-Length:0\r\n\r\n"
logger.debug("Response From Server : " + response_data)
return str(response_data)
def set_test_completion(test_id, req_id):
"""
Description: Before closing the socket, verify the closing socket requested
completed successfuly or not
input_param: test_id - Test Id of the given test
input_type: int
out_param: req_id - Request Id of the closing socket
out_type: int
sample output : None
"""
running_test_row = session.query(HttpTestResults)\
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.request_id==req_id)\
.filter(HttpTestResults.is_running==True)\
.filter(HttpTestResults.is_completed==False).first()
if running_test_row:
reason = "Unexpected Event Happened in while receiving client response"
client_failure_reason = HttpClientTestFailureReason(reason=reason)
session.add(client_failure_reason)
session.flush()
running_test_row.response_result=False
running_test_row.client_failure_id = client_failure_reason.id
running_test_row.is_completed=True
running_test_row.is_running=False
session.commit()
def update_static_types(types):
"""
Description : View function for specific static data Updation
"""
form_data = request.json if request.json else request.form
edited_items = form_data.get('edited')
added_items = form_data.get('added')
type_detail = update_details.get(types)
if type_detail:
update_change_item(edited_items,
type_detail.get('table_name'),
type_detail.get('attr_map')
)
session.commit()
response_data = {
'post_response': {
'response_text': 'Data have beed saved successfully'
}
}
resp = make_response(jsonify(response_data), 200)
else:
response_data = {'error': "unknown Static Data update"}
resp = make_response(jsonify(response_data), 400)
return resp
def update_request(request_id):
"""
Description : View function for Request Updation
"""
form_data = request.json if request.json else request.form
if form_data.get('request') and form_data.get('response'):
# clean_previous_requests, to keep the logic simple
# we delete all the details of this request group
# and re-generate them using the details present in post request
session.query(HttpSubResponse).filter(HttpSubResponse.request_id==request_id).delete()
session.query(HttpSubRequest).filter(HttpSubRequest.request_id==request_id).delete()
session.commit()
request_info_dict = format_request_details(form_data)
new_request_insert([request_info_dict], True)
session.commit()
response_data = {
'post_response': {
'response_text': 'Data have beed saved successfully'
}
}
resp = make_response(jsonify(response_data), 200)
else:
response_data = {'error': "unknown request"}
resp = make_response(jsonify(response_data), 400)
return resp
def new_request():
"""
Description : View function for Request Configuration
"""
if request.method == 'GET':
response_data = {
'form' : render_template('configure_request.html'),
'response_data' : get_static_datas()
}
resp = make_response(jsonify(response_data), 200)
return resp
else:
form_data = request.json if request.json else request.form
if form_data.get('request') and form_data.get('response'):
request_info_dict = format_request_details(form_data)
request_id = new_request_insert([request_info_dict])
session.commit()
response_text = """Your new request have beed saved with
<b>Id : {0}""".format(request_id)
response_data = {
'post_response': {
'response_text': response_text
}
}
resp = make_response(jsonify(response_data), 200)
else:
response_data = {'error': "Unknown Data"}
resp = make_response(jsonify(response_data), 400)
return resp
def verify_server_response(req_info, resp_info):
"""
Description: Verifies the Server response for
1. Expected status line
2. Response Header
3. Data checksum if present
and inserts the result in respective tables
input_param: req_info - Request information Dictionary
input_type: dict
out_param: resp_info - Response information Dictionary
out_type: dict
sample output : None
"""
result = True
result_reason = {'error':""}
if req_info.get('uri'):
test_id, req_id = [ int(parts) \
for parts in req_info.get('uri').strip().split("/")[:3] \
if len(parts) ]
running_test_row = session.query(HttpTestResults)\
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.request_id==req_id)\
.filter(HttpTestResults.is_running==True)\
.filter(HttpTestResults.is_completed==False).first()
if running_test_row:
verification_details = session.query(HttpResponseVerification)\
.filter(HttpResponseVerification.request_id==running_test_row.request_id)\
.filter(HttpResponseVerification.sub_request_id==running_test_row.sub_request_id).first()
if verification_details:
status_line = "HTTP/" + verification_details.version + " " + verification_details.http_response_codes.code_name
header_verification_result = verify_headers(verification_details.response_hdrs, resp_info, result_reason)
if resp_info.get('status_line') == status_line and header_verification_result:
if verification_details.data_id:
if verification_details.http_data.cksum != find_checksum(resp_info['data']):
result=False
result_reason['error'] = "CheckSum differes between expected and calculated from Response"
else:
if not result_reason['error']:
result_reason['error'] = "Expected Status Line:" + status_line
result_reason['error'] = result_reason['error'] + " Does not Match with response: " + resp_info.get('status_line')
result=False
running_test_row.is_completed=True
running_test_row.is_running=False
if not result:
client_failure_reason = HttpClientTestFailureReason(reason=result_reason['error'])
session.add(client_failure_reason)
session.flush()
running_test_row.response_result=False
running_test_row.client_failure_id = client_failure_reason.id
else:
running_test_row.response_result=True
session.commit()
def load_tests(name,
description,
scheduled_by,
category_id):
"""
Description: Inserts the new record in the global Test Table
and copies the new tests requires into HttpTestResults
Table
input_param: category_id - Test Type Id ( minor, major, critical)
input_type: int
out_param: test_id - Newly inserted test primary Id
out_type: int
sample output : 1
"""
test_id = 0
new_test = HttpTest(name=name, description=description, category_id=category_id, scheduled_by=scheduled_by)
session.add(new_test)
session.flush()
session.commit()
test_id = new_test.id
requests = session.query(HttpRequest).filter(HttpRequest.category_id==category_id).all()
for request in requests:
sub_requests = session.query(HttpSubRequest).filter(HttpSubRequest.request_id==request.id).all()
for sub_request in sub_requests:
http_sub_response = session.query(HttpSubResponse).filter(HttpSubResponse.request_id==request.id).filter(HttpSubResponse.sub_request_id==sub_request.id).first()
http_test = HttpTestResults(test_id=test_id,
request_id=request.id,
sub_request_id=sub_request.id,
response_id= http_sub_response.response_id if http_sub_response else None,
sub_response_id= http_sub_response.id if http_sub_response else None,
request_result=None,
response_result=None,
is_running=False,
is_completed=False,
server_failure_id=None,
client_failure_id=None,
no_of_times=1
)
session.add(http_test)
session.commit()
return test_id
def load_test_details(test_id=0):
"""
Description : View function for loading the specific test details
"""
test = session.query(HttpTest).get(test_id)
if request.method == 'GET':
response_data = { 'form' : render_template('test_details.html'),
'response_data' : {'test_details': {}}
}
if test:
response_data['response_data']['test_details'] = format_test_data(test)
resp = make_response(jsonify(response_data), 200)
return resp
else:
form_data = request.json if request.json else request.form
test.paused = form_data.get('pause')
test.running = not form_data.get('pause')
session.commit()
response_data = { 'post_response': { 'test_details' : format_test_data(test) }}
resp = make_response(jsonify(response_data), 200)
return resp
{
"id": 1,
"header_name": "Connection",
"server_value": "['Keep-Alive']",
"proxy_value": "['Keep-Alive', 'Close']",
"single_value_hdr": True,
"is_active": True,
}
]
for response_header in response_headers:
result = session.query(HttpResponseHeaders).filter(HttpResponseHeaders.id == response_header["id"]).first()
if not result:
new_item = HttpResponseHeaders(**response_header)
session.add(new_item)
session.commit()
http_details = [
{
"request": {
"is_active": True,
"category_id": 1,
"pipe_line": False,
"total_requests": 2,
"description": "Sample Test to verify Server Connection",
},
"request_details": [
{
"method_id": 3,
"request_hdr_ids": "1",
"version": 1.1,
def new_request_insert(http_details, update=None):
"""
Description: Inserts the new record in the Request/Response
RequestVerification/ResponseVerification tables
input_param: http_details - list of dictionary contains information
for the new request details
input_type: list
"""
new_request = None
for details in http_details:
request = details.get('request')
request_details = details.get('request_details')
response_verifications = details.get('response_verification')
response = details.get('response')
response_details = details.get('response_details')
request_verifications = details.get('request_verification')
if request:
if not update:
new_request = HttpRequest(**request)
session.add(new_request)
session.flush()
new_response = HttpResponse(request_id=new_request.id, **response)
session.add(new_response)
session.flush()
else:
new_request = session.query(HttpRequest).get(request['id'])
new_request.is_active = True
new_request.pipe_line = False
new_request.category_id = request['category_id']
new_request.description = request['description']
new_request.total_request = request['total_requests']
new_response = session.query(HttpResponse).get(response['id'])
new_response.pipe_line = False
new_response.description = response['description']
new_response.total_response = response['total_response']
new_response.is_active = True
session.commit()
if request_details:
all_requests = [ HttpSubRequest(request_id=new_request.id,
**request_detail )
for request_detail in request_details
]
session.add_all(all_requests)
session.flush()
response_index = 0
all_responses = [
HttpSubResponse(request_id=new_request.id,
response_id=new_response.id,
sub_request_id=request.id,
**response_details[response_index])
for response_index, request in enumerate(all_requests)
if request.reach_server and response_details[response_index]
]
session.add_all(all_responses)
session.flush()
if response_verifications:
all_response_verification = [ HttpResponseVerification(
request_id=new_request.id,
sub_request_id=all_requests[index].id,
**response_verification)
for index, response_verification in enumerate(response_verifications)
if response_verification
]
session.add_all(all_response_verification)
session.flush()
if request_verifications:
all_request_verification = []
index = 0
for request_verification in request_verifications:
if request_verification and all_requests[index].reach_server:
all_request_verification.append(HttpRequestVerification(
request_id=new_request.id,
sub_request_id=all_responses[index].sub_request_id,
sub_response_id=all_responses[index].id,
**request_verification)
)
index = index + 1
session.add_all(all_request_verification)
session.flush()
return new_request.id if new_request else ''
def get_next_request(test_id=0, request_id=None):
"""
Description: Gets the next request data for the client
input_param: test_id - Test Id of the client running test
input_type: int
out_param: request_data - Request data as a string
out_type: String
sample output: "GET /test HTTP/1.1\r\nConnection:Close\r\n\r\n"
"""
# Find the Request Ids that is already running
running_request_ids = [ test_result.request_id for test_result in session.query(HttpTestResults.request_id) \
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.is_running==True)\
.filter(HttpTestResults.is_completed==False)\
.order_by(HttpTestResults.request_id).all() ]
next_test_result_row_query = session.query(HttpTestResults) \
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.is_running==False)\
.filter(HttpTestResults.is_completed==False)
if running_request_ids:
next_test_result_row_query = next_test_result_row_query.filter(not_(HttpTestResults.request_id.in_(running_request_ids)))
# If Request Id is given, we need to return the sub request
# for the given request Id
if request_id:
next_test_result_row_query = session.query(HttpTestResults)\
.filter(HttpTestResults.test_id==test_id)\
.filter(HttpTestResults.request_id==request_id)\
.filter(HttpTestResults.is_running==False)\
.filter(HttpTestResults.is_completed==False)
next_test_result_row = next_test_result_row_query.order_by(HttpTestResults.request_id).first()
if not next_test_result_row:
logger.debug("All the requests are completed\n")
return {}
next_sub_request = session.query(HttpSubRequest)\
.filter(\
and_(HttpSubRequest.request_id==next_test_result_row.request_id, \
HttpSubRequest.id==next_test_result_row.sub_request_id
)
).first()
request_data = next_sub_request.method.method_name + " "
request_data = request_data + "/" + str(test_id) + "/" + str(next_sub_request.request_id)
request_data = request_data + " HTTP/"
request_data = request_data + next_sub_request.version + "\r\n"
for request_header in next_sub_request.request_hdrs:
request_data = request_data + request_header.header_name + ":"
value_list = eval(request_header.client_value)
if request_header.single_value_hdr:
request_data = request_data + value_list[0]+ "\r\n"
else:
request_data = request_data + ";".join(value_list) + "\r\n"
if next_sub_request.data_id:
request_data = request_data + "Content-Length:" + len(next_sub_request.data.data)
request_data = request_data + "\r\n"
request_data = request_data + next_sub_request.data.data
else:
request_data = request_data + "\r\n"
next_test_result_row.is_running=True
session.commit()
logger.debug("Next Request : %s\r\n", request_data)
return {'data': request_data,
'id': next_test_result_row.request_id
}
def handle_client_socket_events(test_id, epoll, client_connections_info, client_requests, client_responses):
"""
Description: Handles all the events happens in client side sockets
input_param: epoll - Epoll listener bucket for the client socket
input_type: epoll
out_param:
out_type:
sample output: None
"""
try:
while True:
parallel_clients = len(client_connections_info.keys())
if parallel_clients == 0:
logger.debug("Test Completed Id : " + str(test_id))
test = session.query(HttpTest).get(test_id)
test.completed=True
test.running=False
test.completed_time=datetime.datetime.now()
session.commit()
break
events = epoll.poll(parallel_clients)
for fileno, event in events:
conn_info = client_connections_info[fileno]
resp_info = client_responses[fileno]
req_info = client_requests[fileno]
conn_info['last_accessed_time'] = datetime.datetime.now()
if event & select.EPOLLERR:
logger.error("EPOLL ERROR in client response handling")
if event & select.EPOLLIN:
new_data = None
read_err = None
# Receive the response from server
if resp_info['resp_start'] or resp_info['rem_bytes_to_read']:
new_data = read_data(client_connections_info[fileno]['socket'])
# If any read failure happens
if not new_data:
logger.error("Read failed so closing the socket\n")
read_err = True
resp_info['rem_bytes_to_read'] = 1
modify_socket_epoll_event(epoll, fileno, 0)
shut_down_socket(conn_info['socket'])
# Before receiving the all response headers
if new_data and not resp_info.get('full_header_received'):
resp_info['resp_start'] = False
resp_info['header_data'] = resp_info['header_data'] + new_data
if resp_info['header_data'].find(EOL1) != -1 or resp_info['header_data'].find(EOL2) != -1:
logger.debug("Full Header Received\n")
client_responses[fileno] = handle_server_response_data(resp_info)
resp_info = client_responses[fileno]
resp_info['full_header_received'] = True
# After receiving the all the response headers
# And when the response data is not chunked
elif new_data and resp_info.get('full_header_received') and not resp_info['is_chunked']:
resp_info['received_bytes'] = resp_info['received_bytes'] + len(new_data)
resp_info['rem_bytes_to_read'] = resp_info['rem_bytes_to_read'] - len(new_data)
resp_info['data'] = resp_info['data'] + new_data
# Once all response is received, verify the data
if not resp_info['rem_bytes_to_read'] and not read_err:
logger.info('-'*40 + '\n' + resp_info['header_data'])
logger.info(resp_info['data'])
verify_server_response(req_info, resp_info)
# For a non-persistent connection
if resp_info['not_persistent']:
logger.debug("Not Persistent Connection, So closing it\n")
modify_socket_epoll_event(epoll, fileno, 0)
shut_down_socket(conn_info['socket'])
else:
# For persistent connection, if there is any
# sub-requests pending. Re-set the previous request, response dats
if conn_info['remaining_requests']:
logger.debug("Starting new request in persistent connection\n")
modify_socket_epoll_event(epoll, fileno, select.EPOLLOUT)
next_request_info = get_next_request(test_id, conn_info['request_id'])
conn_info['remaining_requests'] = conn_info['remaining_requests'] - 1
client_requests[fileno] = intialize_client_request_info(next_request_info['data'])
client_responses[fileno] = intialize_client_response_info()
else:
# For new request set, start the request sending in
# already opened persistent connection
logger.debug("All Sub Request have completed for the request " + (conn_info['request_id']))
next_request_info = get_next_request(test_id)
# If any request set is pending for the test, start it
if next_request_info.get('id'):
logger.debug("Opening New Request in persistent Connection \n")
request_info = session.query(HttpRequest).filter(HttpRequest.id==int(next_request_info['id'])).first()
conn_info['tot_requests_per_connection'] = request_info.total_requests
conn_info['remaining_requests'] = request_info.total_requests
conn_info['last_accessed_time'] = datetime.datetime.now(),
conn_info['request_id'] = next_request_info['id']
client_requests[fileno] = intialize_client_request_info(next_request_info['data']) # change the 1 to category id
client_responses[fileno] = intialize_client_response_info()
client_connections_info[fileno]['remaining_requests'] = client_connections_info[new_client_no]['remaining_requests'] - 1
modify_socket_epoll_event(epoll, fileno, select.EPOLLOUT)
else:
# If there is not any request for test, close the connection
modify_socket_epoll_event(epoll, fileno, 0)
shut_down_socket(conn_info['socket'])
elif event & select.EPOLLOUT:
# Send the Request to server
if req_info.get('rem_bytes_to_send'):
tot_sent = req_info['sent_bytes']
tot_sent = tot_sent + send_data(conn_info['socket'], req_info['request_data'][tot_sent:])
req_info['sent_bytes'] = tot_sent
req_info['rem_bytes_to_send'] = req_info['tot_bytes_to_send'] - req_info['sent_bytes']
else:
modify_socket_epoll_event(epoll, fileno, select.EPOLLIN)
elif event & select.EPOLLHUP:
logger.warning("EPOLLHUP from client\n")
set_test_completion(test_id, conn_info['request_id'])
# If there is any remaining sub-request on closing socket
# open the new socket and assign closing socket's
# request set properties to it.
if conn_info['remaining_requests']:
prepare_client_connection(epoll, client_connections_info, client_requests, client_responses, test_id, conn_info['request_id'])
else:
prepare_client_connection(epoll, client_connections_info, client_requests, client_responses, test_id)
close_socket(epoll, fileno, client_connections_info)
del client_requests[fileno]
del client_responses[fileno]
except:
epoll.close()
raise
