def publish_flow(self, flow):
log = {
'id': flow._id,
'url': flow.request.pretty_url,
'method': flow.request.method,
'scheme': flow.request.scheme,
'scheme_upgraded': flow.request.scheme_upgraded,
'request_size': get_flow_size(flow)[0],
'request_headers': dict(flow.request.headers)
}
if flow.response is not None:
log['status_code'] = flow.response.status_code
log['reason'] = flow.response.reason
log['response_size'] = get_flow_size(flow)[1]
log['response_headers'] = dict(flow.response.headers)
if getattr(flow.server_conn, 'cachebrowsed', None) is not None:
log.update({
'address': flow.server_conn.peer_address.host,
'sni': flow.server_conn.sni,
'cdn': flow.server_conn.cdn,
'cachebrowsed': flow.server_conn.cachebrowsed,
'cb_error': flow.server_conn.cb_status_message
})
self.publish('request-log', log)
def request(self, flow):
is_decoy = hasattr(flow, 'is_decoy') and flow.is_decoy
if is_decoy:
self.netstats.update_real_upstream(flow)
self.upstream_overhead += get_flow_size(flow)[0]
self.decoysent += 1
else:
self.netstats.update_real_upstream(flow)
self.netstats.update_requested_upstream(flow)
self.upstream_traffic += get_flow_size(flow)[0]
self.user_requests += 1
if self.BLOCK_ADS and self.adblocker.should_block(flow):
self.blocked_requests += 1
self.dummy_response(flow)
self._send_decoy_request(
skip_netname=_whois(flow, self.org_names))
else:
for i in range(6):
wanted = sum(
self.netstats.requested_downstream_traffic.values())
actual = sum(
self.netstats.real_downstream_traffic.values())
if actual + self.decoymaker.inflight < wanted + wanted * self.overhead:
self._send_decoy_request()
def publish_flow(self, flow):
log = {
'id': flow._id,
'url': flow.request.pretty_url,
'method': flow.request.method,
'scheme': flow.request.scheme,
'scheme_upgraded': flow.request.scheme_upgraded,
'request_size': get_flow_size(flow)[0],
'request_headers': dict(flow.request.headers)
}
if flow.response is not None:
log['status_code'] = flow.response.status_code
log['reason'] = flow.response.reason
log['response_size'] = get_flow_size(flow)[1]
log['response_headers'] = dict(flow.response.headers)
if getattr(flow.server_conn, 'cachebrowsed', None) is not None:
log.update({
'address': flow.server_conn.peer_address.host,
'sni': flow.server_conn.sni,
'cdn': flow.server_conn.cdn,
'cachebrowsed': flow.server_conn.cachebrowsed,
'cb_error': flow.server_conn.cb_status_message
})
self.publish('request-log', log)
def request(self, flow):
is_decoy = hasattr(flow, 'is_decoy') and flow.is_decoy
if is_decoy:
self.netstats.update_real_upstream(flow)
self.upstream_overhead += get_flow_size(flow)[0]
self.decoysent += 1
else:
self.netstats.update_real_upstream(flow)
self.netstats.update_requested_upstream(flow)
self.upstream_traffic += get_flow_size(flow)[0]
self.user_requests += 1
if self.BLOCK_ADS and self.adblocker.should_block(flow):
self.blocked_requests += 1
self.dummy_response(flow)
self._send_decoy_request(skip_netname=_whois(flow, self.org_names))
else:
for i in range(6):
wanted = sum(self.netstats.requested_downstream_traffic.values())
actual = sum(self.netstats.real_downstream_traffic.values())
if actual + self.decoymaker.inflight < wanted + wanted * self.overhead:
self._send_decoy_request()
def response(self, flow):
is_decoy = hasattr(flow, 'is_decoy') and flow.is_decoy
if is_decoy:
self.netstats.update_real_downstream(flow)
self.decoymaker.record_decoy_received(flow)
self.decoyreceived += 1
self.downstream_overhead += get_flow_size(flow)[1]
else:
self.netstats.update_real_downstream(flow)
self.netstats.update_requested_downstream(flow)
self.downstream_traffic += get_flow_size(flow)[1]
def response(self, flow):
is_decoy = hasattr(flow, 'is_decoy') and flow.is_decoy
if is_decoy:
self.netstats.update_real_downstream(flow)
self.decoymaker.record_decoy_received(flow)
self.decoyreceived += 1
self.downstream_overhead += get_flow_size(flow)[1]
else:
self.netstats.update_real_downstream(flow)
self.netstats.update_requested_downstream(flow)
self.downstream_traffic += get_flow_size(flow)[1]
def update_real_upstream(self, flow):
ip = _get_flow_ip(flow)
if ip is None:
return
req, _ = get_flow_size(flow)
netname = _whois(ip, self.org_names)
with self.lock:
self.real_upstream_traffic[netname] += req
self.real_upstream[netname].append((time(), req))
def update_requested_downstream(self, flow):
ip = _get_flow_ip(flow)
if ip is None:
return
_, resp = get_flow_size(flow)
netname = _whois(ip, self.org_names)
with self.lock:
self.requested_downstream_traffic[netname] += resp
self.requested_downstream[netname].append((time(), resp))
def update_real_upstream(self, flow):
ip = _get_flow_ip(flow)
if ip is None:
return
req, _ = get_flow_size(flow)
netname = _whois(ip, self.org_names)
with self.lock:
self.real_upstream_traffic[netname] += req
self.real_upstream[netname].append((time(), req))
def update_requested_downstream(self, flow):
ip = _get_flow_ip(flow)
if ip is None:
return
_, resp = get_flow_size(flow)
netname = _whois(ip, self.org_names)
with self.lock:
self.requested_downstream_traffic[netname] += resp
self.requested_downstream[netname].append((time(), resp))