def save_request(self, **args): # the the ip from database try: flow = Flow.objects.get(hash_value=args['hash_value']) flow_details = flow.details for detail in flow_details: # create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_orig.dat source_str = ":".join([detail.src_ip, str(detail.sport)]) destination_str = ":".join([detail.dst_ip, str(detail.dport)]) flow_str = "-".join([source_str, destination_str]) orig_file = "_".join(["contents", flow_str,"orig.dat"]) file_path = "/".join([args['path'], orig_file]) file_path = str(file_path) strings = ["GET", "PUT", "POST"] file_handler = FileHandler() requests = [] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: requests.append(item[0]) # i am making a hacky thing here, finding empty lines, each request is separeted with an empty line empty_lines = [] strings = ["\r\n\r\n"] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: empty_lines.append(item[0]) for x in range(len(requests)): # here i have the request header data = file_handler.data request = data[requests[x]:empty_lines[x]] request_li = request.split("\n") for entry in request_li: # the first line is method and uri with version information info = entry.split(":") if len(info) == 1: info = info[0].split() method = info[0] uri = info[1] version = info[2].split("/")[1] try: http_details = HTTPDetails.objects.get(http_type="request", method=method, uri=uri, headers=request_li, version=version, flow_deatils=detail) except: http_details = HTTPDetails(http_type="request", method=method, uri=uri, headers=request_li, version=version, flow_details=detail) http_details.save() return True except Exception, ex: return False
def upload(request): log = Logger("Upload form", "DEBUG") context = { 'page_title': 'Upload your pcap file here', 'upload_status': False, 'message': request.session.get('message', False) } if request.method == "POST": form = UploadPcapForm(request.POST, request.FILES) if form.is_valid(): user_id = request.user.id context['form'] = form file_handler = FileHandler() file_handler.create_dir() mem_file = request.FILES['pcap_file'] log.message("file: %s" % mem_file.name) file_handler.save_file(mem_file) context['upload_status'] = True #save the file name to the db pcap_name = mem_file.name upload_path = file_handler.upload_dir # evey pcap file is saved as a flow container, there may or may not be flows, the pcaps colon will give the flow pcaps hash_handler = HashHandler() hash_value = hash_handler.get_hash(os.path.join(upload_path, pcap_name)) request.session['uploaded_hash'] = hash_value request.session['uploaded_file_name'] = pcap_name # send the file to the defined protocol handler so that it can detect protocol_handler = settings.PROTOCOL_HANDLER package = "ovizart.modules.traffic.detector" module_name = ".".join([package, protocol_handler]) # from ovizart.modules.traffic.detector.x import handler as imported_module traffic_detector_module = getattr(__import__(module_name, fromlist=["handler"]), "handler") traffic_detector_handler = traffic_detector_module.Handler() traffic_detector_handler.create_reassemble_information(file_handler.file_path, upload_path) output = traffic_detector_handler.detect_proto(file_handler.file_path, upload_path) if output == False: request.session['message'] = "Error occured. Please try again." return redirect('/pcap/upload') file_type = get_file_type(file_handler.file_path) file_size = get_file_size(file_handler.file_path) flow_file, created = Flow.objects.get_or_create(user_id=user_id, hash_value=hash_value,file_name=pcap_name, path=upload_path, upload_time=datetime.datetime.now(), file_type=file_type, file_size=file_size) if "tcp" in output: log.message("protocol detected: %s" % "TCP") # run tcp flow extractor p_read_handler = PcapHandler() p_read_handler.open_file(file_handler.file_path) p_read_handler.open_pcap() f_handler = FlowHandler(p_read_handler) flow, direction = f_handler.get_tcp_flows() p_write_handler = PcapHandler() files = f_handler.save_flow(flow, p_write_handler, save_path=upload_path) # save the flow pcap names to the mongo db pcap_list = map(lambda x: Pcap.objects.create(hash_value=hash_handler.get_hash(os.path.join(upload_path, x)), file_name=x, path=upload_path), files.values()[0]) if flow_file.pcaps: pre_li = flow_file.pcaps pre_li.extend(pcap_list) flow_file.pcaps = pre_li else: flow_file.pcaps = pcap_list flow_file.save() p_read_handler.close_file() p_write_handler.close_file() # now i should hook a protocol detector # before that i should detect the application level protocol for f in files.values()[0]: packets = [] # better to save tcp level information to db here full_path = os.path.join(upload_path, f) p_read_handler.open_file(full_path) p_read_handler.open_pcap() pcap = p_read_handler.get_pcap() # the list that will keep the tcp part of the packet tcp_list = [] for ts, buf in pcap: tcp_handler = TcpHandler() tcp = tcp_handler.read_tcp(ts, buf) # this list will be used at the layers above tcp if tcp: tcp_list.append((tcp, tcp_handler.ident)) else: continue tcp_data = u"." if tcp_handler.data: tcp_data = tcp_handler.data # some requests include hexadecimal info, most probably some binary info that can not be # converted to the utf8, for now i better remove them, #TODO should handle them, though # try with 4, tcp.data has binary request # def get_tcp(n): # count = 1 # f = file("milliyet.pcap", "rb") # reader = dpkt.pcap.Reader(f) # for ts, buf in reader: # if count == n: # f.close() # return buf # count += 1 data_li = tcp_data.split("\r\n") tmp = [] for data in data_li: try: data.encode("utf-8") tmp.append(data) except: tmp.append("data that can not be encoded to utf-8") tcp_data = " \n".join(tmp) packet = PacketDetails.objects.create(ident=tcp_handler.ident, flow_hash=hash_value, timestamp=tcp_handler.timestamp, length=tcp_handler.length, protocol=tcp_handler.proto, src_ip=tcp_handler.src_ip, dst_ip=tcp_handler.dst_ip, sport=tcp_handler.sport, dport=tcp_handler.dport, data=str(tcp_data)) packets.append(packet) # get the pcap object p = Pcap.objects.get(hash_value=hash_handler.get_hash(os.path.join(upload_path, f))) log.message("pcap for packet update detected: %s" % p) # update its packets p.packets = list(packets) # converting a queryset to list p.save() p_read_handler.close_file() if "udp" in output: log.message("protocol detected: %s" % "UDP") p_read_handler = PcapHandler() file_path = os.path.join(upload_path, pcap_name) p_read_handler.open_file(file_path) p_read_handler.open_pcap() udp_handler = UDPHandler() pcap = Pcap.objects.create(hash_value=hash_handler.get_hash(os.path.join(upload_path, pcap_name)), file_name=pcap_name, path=upload_path) pcap_list = list([pcap]) if flow_file.pcaps: pre_li = flow_file.pcaps pre_li.extend(pcap_list) flow_file.pcaps = pre_li else: flow_file.pcaps = pcap_list flow_file.save() packets = [] for ts, buf in p_read_handler.get_reader(): udp = udp_handler.read_udp(ts, buf) if udp: udp_data = u"." if udp_handler.data: udp_data = udp_handler.data try: udp_data = udp_data.encode("utf-8") except: udp_data = "data that can not be encoded to utf-8" packet = PacketDetails.objects.create(ident=udp_handler.ident, flow_hash=hash_value, timestamp=udp_handler.timestamp, length = udp_handler.length, protocol=udp_handler.proto, src_ip=udp_handler.src_ip, dst_ip=udp_handler.dst_ip, sport=udp_handler.sport, dport=udp_handler.dport, data=str(udp_data)) packets.append(packet) # get the pcap object p = Pcap.objects.get(hash_value=hash_handler.get_hash(os.path.join(upload_path, pcap_name))) # update its packets p.packets = list(packets) # converting a queryset to list p.save() p_read_handler.close_file() # starting the bro related issues for the reassembled data output = traffic_detector_handler.detect_appproto(file_handler.file_path, upload_path) log.message("protocol detected: %s" % output) if output and "http" in output: log.message("protocol detected: %s" % "HTTP") # save the reassembled http session IPs to FlowDetails # this part is checking the http handler module name and importing the handler http_protocol_handler = settings.HTTP_HANDLER package = "ovizart.modules.traffic.parser.tcp" module_name = ".".join([package, http_protocol_handler]) # from ovizart.modules.traffic.parser.tcp.x import handler as imported_module http_handler_module = getattr(__import__(module_name, fromlist=["handler"]), "handler") http_handler = http_handler_module.Handler() # define a get_flow_ips function for the custom handler if required # TODO: save the timestamps of the flows flow_ips = http_handler.get_flow_ips(path=upload_path) flow_detail_li = [] for detail in flow_ips: flow_detail, create = FlowDetails.objects.get_or_create(parent_hash_value=request.session['uploaded_hash'], user_id=user_id, src_ip=detail[0], sport=int(detail[1]), dst_ip=detail[2], dport=int(detail[3]), protocol="http", timestamp = detail[4]) flow_detail_li.append(flow_detail) if flow_file.details: pre_li = flow_file.details pre_li.extend(flow_detail_li) flow_file.details = pre_li else: flow_file.details = flow_detail_li flow_file.save() # then call functions that will save request and responses that will parse dat files, save the headers and files #http_handler.save_request(path=upload_path, hash_value=request.session['uploaded_hash']) #http_handler.save_response(path=upload_path, hash_value=request.session['uploaded_hash']) http_handler.save_request_response(path=upload_path, hash_value=request.session['uploaded_hash']) # dns realted issues starts here if output and "dns" in output: log.message("protocol detected: %s" % "DNS") dns_protocol_handler = settings.DNS_HANDLER package = "ovizart.modules.traffic.parser.udp" module_name = ".".join([package, dns_protocol_handler]) # from ovizart.modules.traffic.parser.udp.x import handler as imported_module dns_handler_module = getattr(__import__(module_name, fromlist=["handler"]), "handler") dns_handler = dns_handler_module.Handler() # define a get_flow_ips function for the custom handler if required flow_ips = dns_handler.get_flow_ips(path=upload_path, file_name=request.session['uploaded_file_name']) flow_detail_li = [] for detail in flow_ips: flow_detail, create = FlowDetails.objects.get_or_create(parent_hash_value=request.session['uploaded_hash'], user_id=user_id, src_ip=detail[0], sport=int(detail[1]), dst_ip=detail[2], dport=int(detail[3]), protocol="dns", timestamp = detail[4]) flow_detail_li.append(flow_detail) if flow_file.details: pre_li = flow_file.details pre_li.extend(flow_detail_li) flow_file.details = pre_li else: flow_file.details = flow_detail_li flow_file.save() dns_handler.save_request_response() if output and "smtp" in output: log.message("protocol detected: %s" % "SMTP") smtp_protocol_handler = settings.SMTP_HANDLER package = "ovizart.modules.traffic.parser.tcp" module_name = ".".join([package, smtp_protocol_handler]) # from ovizart.modules.traffic.parser.tcp.x import handler as imported_module smtp_handler_module = getattr(__import__(module_name, fromlist=["handler"]), "handler") smtp_handler = smtp_handler_module.Handler() # define a get_flow_ips function for the custom handler if required smtp_handler.set_flow(flow_file) # i need this, to get the timestamp from a packet belongs to the flow flow_ips = smtp_handler.get_flow_ips(path=upload_path, file_name=request.session['uploaded_file_name']) flow_detail_li = [] for detail in flow_ips: flow_detail, create = FlowDetails.objects.get_or_create(parent_hash_value=request.session['uploaded_hash'], user_id=user_id, src_ip=detail[0], sport=int(detail[1]), dst_ip=detail[2], dport=int(detail[3]), protocol="smtp", timestamp = detail[4]) flow_detail_li.append(flow_detail) if flow_file.details: pre_li = flow_file.details pre_li.extend(flow_detail_li) flow_file.details = pre_li else: flow_file.details = flow_detail_li flow_file.save() smtp_handler.save_request_response(upload_path=upload_path) else: log.message("protocol detected: %s" % "Unknown") unknown_protocol_handler = settings.UNKNOWN_HANDLER package = "ovizart.modules.traffic.parser" module_name = ".".join([package, unknown_protocol_handler]) unknown_handler_module = getattr(__import__(module_name, fromlist=["handler"]), "handler") unknown_handler = unknown_handler_module.Handler() flow_ips = unknown_handler.get_flow_ips(path=upload_path, file_name=request.session['uploaded_file_name'], parent_hash_value=request.session['uploaded_hash'], user_id=user_id) flow_detail_li = [] for detail in flow_ips: flow_detail, create = FlowDetails.objects.get_or_create(parent_hash_value=request.session['uploaded_hash'], user_id=user_id, src_ip=detail[0], sport=int(detail[1]), dst_ip=detail[2], dport=int(detail[3]), protocol="unknown", timestamp = detail[4]) if created: flow_detail_li.append(flow_detail) if flow_file.details: pre_li = flow_file.details pre_li.extend(flow_detail_li) flow_file.details = pre_li else: flow_file.details = flow_detail_li flow_file.save() else: form = UploadPcapForm() context['form'] = form request.session['message'] = False return render_to_response("pcap/upload.html", context_instance=RequestContext(request, context))
def save_response_files(self, path, hash_value): try: flow = Flow.objects.get(hash_value=hash_value) flow_details = flow.details for detail in flow_details: # create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_resp.dat source_str = ":".join([detail.src_ip, str(detail.sport)]) destination_str = ":".join([detail.dst_ip, str(detail.dport)]) flow_str = "-".join([source_str, destination_str]) resp_file = "_".join(["contents", flow_str,"resp.dat"]) file_path = "/".join([path, resp_file]) # path is created as unicode, convert it a regular string for hachoir operation file_path = str(file_path) strings = ["Content-Type: text/html", "Content-Type: application/x-javascript", "Content-Type: text/css"] file_handler = FileHandler() responses = [] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: responses.append(item[0]) empty_lines = [] strings = ["\r\n\r\n"] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: empty_lines.append(item[0]) http_lines = [] strings = ["HTTP/1.1"] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: http_lines.append(item[0]) try: stream = FileInputStream(unicodeFilename(file_path), real_filename=file_path) except NullStreamError: continue subfile = SearchSubfile(stream, 0, None) subfile.loadParsers() root = "/".join([path, "html-files"]) if not os.path.exists(root): os.makedirs(root) output = "/".join([root, flow_str]) output = str(output) subfile.setOutput(output) for x in range(len(responses)): # here i have the request header data = file_handler.data #f = data[empty_lines[x]:http_lines[x+1]] file_ext = ".txt" #if ("html" in f or "body" in f): # file_ext = ".html" #elif ("script" in f): # file_ext = ".js" #else: # select the closest empty line empty_lines.append(responses[x]) empty_lines.sort() index = empty_lines.index(responses[x]) offset = empty_lines[index+1] size = None try: size = http_lines[x+1]-2 except IndexError: size = stream.size f = data[offset+4:size] filename = subfile.output.createFilename(file_ext) w = open("/".join([output, filename]), "w") w.write(f) w.close() # saving the hachoir saved binaries to the db with the created txt files if detail.protocol == "http": http_files = os.listdir(output) #http_files = filter(lambda x: x.split(".")[-1] != 'txt', http_files) # no need to take the txt files if len(http_files) > 0: http_li = filter(lambda x: x.flow_details.id == detail.id, HTTPDetails.objects.all()) for http in http_li: http.files = http_files http.save() return True except Exception, ex: print ex return False
def save_response_headers(self, path, hash_value): try: flow = Flow.objects.get(hash_value=hash_value) flow_details = flow.details for detail in flow_details: # create the orig file ex: contents_192.168.1.5:42825-62.212.84.227:80_resp.dat source_str = ":".join([detail.src_ip, str(detail.sport)]) destination_str = ":".join([detail.dst_ip, str(detail.dport)]) flow_str = "-".join([source_str, destination_str]) resp_file = "_".join(["contents", flow_str,"resp.dat"]) file_path = "/".join([path, resp_file]) # path is created as unicode, convert it a regular string for hachoir operation file_path = str(file_path) strings = ["HTTP/1.1"] file_handler = FileHandler() responses = [] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: responses.append(item[0]) empty_lines = [] strings = ["\r\n\r\n"] search_li = file_handler.search(file_path, strings) if not search_li: continue for item in search_li: empty_lines.append(item[0]) for x in range(len(responses)): # here i have the request header data = file_handler.data response = data[responses[x]:empty_lines[x]] response_li = response.split("\n") header = info = version = status = content_type = content_encoding = None for entry in response_li: # the first line is method and uri with version information info = entry.split(":") if len(info) == 1: info = info[0].split() version = info[0].split("/")[1] status = info[1] header = response_li else: if "Content-Type" in info: content_type = info[1] if filter(lambda x: "gzip" in x, info): content_encoding = "gzip" http_li = filter(lambda x: x.flow_details.id == detail.id, HTTPDetails.objects.filter(http_type="response", version=version, headers=header, status=status, content_type=content_type, content_encoding=content_encoding)) #http_details = HTTPDetails.objects.get(http_type="response", version=version, headers=header, status=status, content_type=content_type, content_encoding=content_encoding, flow_details=detail) if len(http_li) == 1: http_details = http_li[0] file_path = os.path.join(path, "html-files") http_details.file_path = file_path http_details.save() else: # encoding error is fixed file_path = os.path.join(path, "html-files") http_details = HTTPDetails(http_type="response", version=version, headers=header, status=status, content_type=content_type, content_encoding=content_encoding, file_path=file_path, flow_details=detail) http_details.save() return True except: return False