def __init__(self, packetdispatcher):
'''
parses http.flows from packetdispatcher, and parses those for HAR info
'''
# parse http flows
self.flows= []
for flow in packetdispatcher.tcp.flowdict.itervalues():
try:
self.flows.append(http.Flow(flow))
except (http.Error,):
error = sys.exc_info()[1]
log.warning(error)
except (dpkt.dpkt.Error,):
error = sys.exc_info()[1]
log.warning(error)
# combine the messages into a list
pairs = reduce(lambda p, f: p+f.pairs, self.flows, [])
# set-up
self.user_agents = UserAgentTracker()
if settings.process_pages:
self.page_tracker = PageTracker()
else:
self.page_tracker = None
self.entries = []
# sort pairs on request.ts_connect
pairs.sort(
key=lambda pair: pair.request.ts_connect
)
# iter through messages and do important stuff
for msg in pairs:
entry = Entry(msg.request, msg.response)
# if msg.request has a user-agent, add it to our list
if 'user-agent' in msg.request.msg.headers:
self.user_agents.add(msg.request.msg.headers['user-agent'])
# if msg.request has a referer, keep track of that, too
if self.page_tracker:
entry.pageref = self.page_tracker.getref(entry)
# add it to the list
self.entries.append(entry)
self.user_agent = self.user_agents.dominant_user_agent()
# handle DNS AFTER sorting
# this algo depends on first appearance of a name
# being the actual first mention
names_mentioned = set()
dns = packetdispatcher.udp.dns
for entry in self.entries:
name = entry.request.host
# if this is the first time seeing the name
if name not in names_mentioned:
if name in dns.by_hostname:
# TODO: handle multiple DNS queries for now just use last one
entry.add_dns(dns.by_hostname[name][-1])
names_mentioned.add(name)
def convert(pcap_in, har_out, options):
flows = pcap.TCPFlowsFromString(pcap_in, options)
# generate HTTP Flows
httpflows = []
flow_count = 0
for flow in sorted(flows.flowdict.itervalues(),
cmp=lambda x, y: cmp(x.start(), y.start())):
try:
httpflows.append(http.Flow(flow))
flow_count += 1
except http.Error, error:
logging.warning(error)
except Exception, error:
logging.warning(error)