def save_request(self, **args):
# the the ip from database
try:
flow = Flow.objects.get(hash_value=args['hash_value'])
flow_details = flow.details
for detail in flow_details:
# create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_orig.dat
source_str = ":".join([detail.src_ip, str(detail.sport)])
destination_str = ":".join([detail.dst_ip, str(detail.dport)])
flow_str = "-".join([source_str, destination_str])
orig_file = "_".join(["contents", flow_str,"orig.dat"])
file_path = "/".join([args['path'], orig_file])
file_path = str(file_path)
strings = ["GET", "PUT", "POST"]
file_handler = FileHandler()
requests = []
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
requests.append(item[0])
# i am making a hacky thing here, finding empty lines, each request is separeted with an empty line
empty_lines = []
strings = ["\r\n\r\n"]
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
empty_lines.append(item[0])
for x in range(len(requests)):
# here i have the request header
data = file_handler.data
request = data[requests[x]:empty_lines[x]]
request_li = request.split("\n")
for entry in request_li:
# the first line is method and uri with version information
info = entry.split(":")
if len(info) == 1:
info = info[0].split()
method = info[0]
uri = info[1]
version = info[2].split("/")[1]
try:
http_details = HTTPDetails.objects.get(http_type="request", method=method, uri=uri, headers=request_li, version=version, flow_deatils=detail)
except:
http_details = HTTPDetails(http_type="request", method=method, uri=uri, headers=request_li, version=version, flow_details=detail)
http_details.save()
return True
except Exception, ex:
return False
def save_response_files(self, path, hash_value):
try:
flow = Flow.objects.get(hash_value=hash_value)
flow_details = flow.details
for detail in flow_details:
# create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_resp.dat
source_str = ":".join([detail.src_ip, str(detail.sport)])
destination_str = ":".join([detail.dst_ip, str(detail.dport)])
flow_str = "-".join([source_str, destination_str])
resp_file = "_".join(["contents", flow_str,"resp.dat"])
file_path = "/".join([path, resp_file])
# path is created as unicode, convert it a regular string for hachoir operation
file_path = str(file_path)
strings = ["Content-Type: text/html", "Content-Type: application/x-javascript", "Content-Type: text/css"]
file_handler = FileHandler()
responses = []
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
responses.append(item[0])
empty_lines = []
strings = ["\r\n\r\n"]
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
empty_lines.append(item[0])
http_lines = []
strings = ["HTTP/1.1"]
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
http_lines.append(item[0])
try:
stream = FileInputStream(unicodeFilename(file_path), real_filename=file_path)
except NullStreamError:
continue
subfile = SearchSubfile(stream, 0, None)
subfile.loadParsers()
root = "/".join([path, "html-files"])
if not os.path.exists(root):
os.makedirs(root)
output = "/".join([root, flow_str])
output = str(output)
subfile.setOutput(output)
for x in range(len(responses)):
# here i have the request header
data = file_handler.data
#f = data[empty_lines[x]:http_lines[x+1]]
file_ext = ".txt"
#if ("html" in f or "body" in f):
# file_ext = ".html"
#elif ("script" in f):
# file_ext = ".js"
#else:
# select the closest empty line
empty_lines.append(responses[x])
empty_lines.sort()
index = empty_lines.index(responses[x])
offset = empty_lines[index+1]
size = None
try:
size = http_lines[x+1]-2
except IndexError:
size = stream.size
f = data[offset+4:size]
filename = subfile.output.createFilename(file_ext)
w = open("/".join([output, filename]), "w")
w.write(f)
w.close()
# saving the hachoir saved binaries to the db with the created txt files
if detail.protocol == "http":
http_files = os.listdir(output)
#http_files = filter(lambda x: x.split(".")[-1] != 'txt', http_files) # no need to take the txt files
if len(http_files) > 0:
http_li = filter(lambda x: x.flow_details.id == detail.id, HTTPDetails.objects.all())
for http in http_li:
http.files = http_files
http.save()
return True
except Exception, ex:
print ex
return False
def save_response_headers(self, path, hash_value):
try:
flow = Flow.objects.get(hash_value=hash_value)
flow_details = flow.details
for detail in flow_details:
# create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_resp.dat
source_str = ":".join([detail.src_ip, str(detail.sport)])
destination_str = ":".join([detail.dst_ip, str(detail.dport)])
flow_str = "-".join([source_str, destination_str])
resp_file = "_".join(["contents", flow_str,"resp.dat"])
file_path = "/".join([path, resp_file])
# path is created as unicode, convert it a regular string for hachoir operation
file_path = str(file_path)
strings = ["HTTP/1.1"]
file_handler = FileHandler()
responses = []
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
responses.append(item[0])
empty_lines = []
strings = ["\r\n\r\n"]
search_li = file_handler.search(file_path, strings)
if not search_li: continue
for item in search_li:
empty_lines.append(item[0])
for x in range(len(responses)):
# here i have the request header
data = file_handler.data
response = data[responses[x]:empty_lines[x]]
response_li = response.split("\n")
header = info = version = status = content_type = content_encoding = None
for entry in response_li:
# the first line is method and uri with version information
info = entry.split(":")
if len(info) == 1:
info = info[0].split()
version = info[0].split("/")[1]
status = info[1]
header = response_li
else:
if "Content-Type" in info:
content_type = info[1]
if filter(lambda x: "gzip" in x, info):
content_encoding = "gzip"
http_li = filter(lambda x: x.flow_details.id == detail.id, HTTPDetails.objects.filter(http_type="response",
version=version, headers=header, status=status,
content_type=content_type, content_encoding=content_encoding))
#http_details = HTTPDetails.objects.get(http_type="response", version=version, headers=header, status=status, content_type=content_type, content_encoding=content_encoding, flow_details=detail)
if len(http_li) == 1:
http_details = http_li[0]
file_path = os.path.join(path, "html-files")
http_details.file_path = file_path
http_details.save()
else: # encoding error is fixed
file_path = os.path.join(path, "html-files")
http_details = HTTPDetails(http_type="response", version=version, headers=header,
status=status, content_type=content_type, content_encoding=content_encoding,
file_path=file_path, flow_details=detail)
http_details.save()
return True
except:
return False
