def cache_messages(streams):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
redis_pipe = REDIS_CONN.pipeline()
count = 0
serializer = Serializer()
stream = Stream()
for stream_id, stream.segments in streams.iteritems():
data = stream.data()
_data = data.next()
while True:
try:
(msg, _data) = serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data = data.next()
except StopIteration:
break
else:
node = (stream_id[0], stream_id[1])
count += cache_message(redis_pipe, node, stream.timestamp, msg)
redis_pipe.execute()
return count
def cache_messages(streams):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
redis_pipe = REDIS_CONN.pipeline()
count = 0
serializer = Serializer()
stream = Stream()
for stream_id, stream.segments in streams.iteritems():
data = stream.data()
_data = data.next()
while True:
try:
(msg, _data) = serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data = data.next()
except StopIteration:
break
else:
node = (stream_id[0], stream_id[1])
count += cache_message(redis_pipe, node, stream.timestamp, msg)
redis_pipe.execute()
return count
def get_invs(filepath):
"""
Extracts inv messages from the specified pcap file.
"""
serializer = Serializer()
pcap_file = open(filepath)
pcap_reader = dpkt.pcap.Reader(pcap_file)
for timestamp, buf in pcap_reader:
frame = dpkt.ethernet.Ethernet(buf)
ip_packet = frame.data
if isinstance(ip_packet.data, dpkt.tcp.TCP):
tcp_packet = ip_packet.data
payload = tcp_packet.data
if len(payload) > 0:
try:
(msg, _) = serializer.deserialize_msg(payload)
except ProtocolError as err:
pass
else:
if msg['command'] == "inv":
if ip_packet.v == 6:
address = socket.inet_ntop(socket.AF_INET6,
ip_packet.src)
else:
address = socket.inet_ntop(socket.AF_INET,
ip_packet.src)
node = (address, tcp_packet.sport)
save_invs(timestamp, node, msg['inventory'])
pcap_file.close()
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer()
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.keys = set() # ping:ADDRESS-PORT:NONCE
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer(magic_number=CONF['magic_number'])
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.ping_keys = set() # ping:ADDRESS-PORT:NONCE
self.invs = defaultdict(list)
def __init__(self, pipe, token='', treefile='', serializer=None, debug=0):
super(ClientWorker, self).__init__('Client')
self.secure = pipe
self.s = serializer or Serializer()
self.token = token
self.treefile = treefile
self._debug = debug
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer()
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.keys = set() # ping:ADDRESS-PORT:NONCE
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer(magic_number=CONF['magic_number'])
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.ping_keys = set() # ping:ADDRESS-PORT:NONCE
self.invs = defaultdict(list)
def __init__(self, controller, filemanager, serializer=None):
super(ServerLogics, self).__init__('Backend')
self.s = serializer or Serializer()
self.filelogics = filemanager
self.controller = controller
self.resources = {
'users': self.users,
'files': self.files,
}
def __init__(self, filepath):
self.filepath = filepath
self.serializer = Serializer(magic_number=unhexlify('daa5bef9'))
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
class Cache(object):
"""
Implements caching mechanic to cache messages from pcap file in Redis.
"""
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer()
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.keys = set() # ping:ADDRESS-PORT:NONCE
def cache_messages(self):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
self.extract_streams()
for stream_id, self.stream.segments in self.streams.iteritems():
data = self.stream.data()
_data = data.next()
while True:
try:
(msg, _data) = self.serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("{}: {}".format(stream_id, err))
try:
_data = data.next()
except StopIteration:
break
else:
node = (stream_id[0], stream_id[1])
self.cache_message(node, self.stream.timestamp, msg)
self.redis_pipe.execute()
self.cache_rtt()
def extract_streams(self):
"""
Extracts TCP streams with data from the pcap file. TCP segments in
each stream are queued according to their sequence number.
"""
with open(self.filepath) as pcap_file:
pcap_reader = dpkt.pcap.Reader(pcap_file)
for timestamp, buf in pcap_reader:
frame = dpkt.ethernet.Ethernet(buf)
ip_pkt = frame.data
if isinstance(ip_pkt.data, dpkt.tcp.TCP):
ip_ver = socket.AF_INET
if ip_pkt.v == 6:
ip_ver = socket.AF_INET6
tcp_pkt = ip_pkt.data
stream_id = (socket.inet_ntop(ip_ver,
ip_pkt.src), tcp_pkt.sport,
socket.inet_ntop(ip_ver,
ip_pkt.dst), tcp_pkt.dport)
if len(tcp_pkt.data) > 0:
timestamp = int(timestamp * 1000) # in ms
self.streams[stream_id].put(
(tcp_pkt.seq, (timestamp, tcp_pkt)))
logging.info("Streams: {}".format(len(self.streams)))
def cache_message(self, node, timestamp, msg):
"""
Caches inv/pong message from the specified node.
"""
if msg['command'] == "inv":
for inv in msg['inventory']:
key = "inv:{}:{}".format(inv['type'], inv['hash'])
if inv['type'] == 2:
# Redis key for reference (first seen) block inv
rkey = "r{}".format(key)
rkey_ms = REDIS_CONN.get(rkey)
if rkey_ms is None:
REDIS_CONN.set(rkey, timestamp)
self.redis_pipe.set("lastblockhash", inv['hash'])
elif (timestamp - int(rkey_ms)) / 1000 > SETTINGS['ttl']:
# Ignore block inv first seen more than 3 hours ago
logging.debug("Skip: {}".format(key))
continue
self.redis_pipe.zadd(key, timestamp, node)
self.redis_pipe.expire(key, SETTINGS['ttl'])
self.count += msg['count']
elif msg['command'] == "pong":
key = "ping:{}-{}:{}".format(node[0], node[1], msg['nonce'])
self.redis_pipe.rpushx(key, timestamp)
self.keys.add(key)
self.count += 1
def cache_rtt(self):
"""
Calculates round-trip time (RTT) values and caches them in Redis.
"""
for key in self.keys:
timestamps = REDIS_CONN.lrange(key, 0, 1)
if len(timestamps) > 1:
rtt_key = "rtt:{}".format(':'.join(key.split(":")[1:-1]))
rtt = int(timestamps[1]) - int(timestamps[0]) # pong - ping
logging.debug("{}: {}".format(rtt_key, rtt))
self.redis_pipe.lpush(rtt_key, rtt)
self.redis_pipe.ltrim(rtt_key, 0, SETTINGS['rtt_count'] - 1)
self.redis_pipe.expire(rtt_key, SETTINGS['ttl'])
self.redis_pipe.execute()
class Cache(object):
"""
Implements caching mechanic to cache messages from pcap file in Redis.
"""
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer(magic_number=CONF['magic_number'])
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.ping_keys = set() # ping:ADDRESS-PORT:NONCE
self.invs = defaultdict(list)
def cache_messages(self):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
try:
self.extract_streams()
except dpkt.dpkt.NeedData:
logging.warning("Need data: %s", self.filepath)
for stream_id, self.stream.segments in self.streams.iteritems():
data = self.stream.data()
_data = data.next()
while True:
try:
(msg, _data) = self.serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("%s: %s", stream_id, err)
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("%s: %s", stream_id, err)
try:
_data = data.next()
except StopIteration:
break
else:
src = (stream_id[0], stream_id[1])
dst = (stream_id[2], stream_id[3])
node = src
is_tor = False
if src in CONF['tor_proxies']:
# dst port will be used to restore .onion node.
node = dst
is_tor = True
self.cache_message(node,
self.stream.timestamp,
msg,
is_tor=is_tor)
self.redis_pipe.execute()
self.cache_rtt()
def extract_streams(self):
"""
Extracts TCP streams with data from the pcap file. TCP segments in
each stream are queued according to their sequence number.
"""
with open(self.filepath) as pcap_file:
pcap_reader = dpkt.pcap.Reader(pcap_file)
for timestamp, buf in pcap_reader:
try:
frame = dpkt.ethernet.Ethernet(buf)
except dpkt.dpkt.UnpackError:
continue
ip_pkt = frame.data
if (not isinstance(ip_pkt, dpkt.ip.IP)
and not isinstance(ip_pkt, dpkt.ip6.IP6)):
continue
if not isinstance(ip_pkt.data, dpkt.tcp.TCP):
continue
ip_ver = socket.AF_INET
if ip_pkt.v == 6:
ip_ver = socket.AF_INET6
tcp_pkt = ip_pkt.data
stream_id = (socket.inet_ntop(ip_ver,
ip_pkt.src), tcp_pkt.sport,
socket.inet_ntop(ip_ver,
ip_pkt.dst), tcp_pkt.dport)
if len(tcp_pkt.data) > 0:
timestamp = int(timestamp * 1000) # in ms
self.streams[stream_id].put(
(tcp_pkt.seq, (timestamp, tcp_pkt)))
logging.debug("Streams: %d", len(self.streams))
def cache_message(self, node, timestamp, msg, is_tor=False):
"""
Caches inv/pong message from the specified node.
"""
if msg['command'] not in ["inv", "pong"]:
return
# Restore .onion node using port info from node
if is_tor:
onion_node = REDIS_CONN.get("onion:{}".format(node[1]))
if onion_node:
node = eval(onion_node)
if msg['command'] == "inv":
invs = 0
for inv in msg['inventory']:
key = "inv:{}:{}".format(inv['type'], inv['hash'])
if (len(self.invs[key]) >= CONF['inv_count']
and timestamp > self.invs[key][0]):
logging.debug("Skip: %s (%d)", key, timestamp)
continue
bisect.insort(self.invs[key], timestamp)
if inv['type'] == 2:
# Redis key for reference (first seen) block inv
rkey = "r{}".format(key)
rkey_ms = REDIS_CONN.get(rkey)
if rkey_ms is None:
REDIS_CONN.set(rkey, timestamp)
self.redis_pipe.set("lastblockhash", inv['hash'])
elif (timestamp - int(rkey_ms)) / 1000 > CONF['ttl']:
# Ignore block inv first seen more than 3 hours ago
logging.debug("Skip: %s (%d)", key, timestamp)
continue
invs += 1
self.redis_pipe.zadd(key, timestamp, self.node_hash(node))
self.redis_pipe.expire(key, CONF['ttl'])
self.count += invs
elif msg['command'] == "pong":
key = "ping:{}-{}:{}".format(node[0], node[1], msg['nonce'])
self.redis_pipe.rpushx(key, timestamp)
self.ping_keys.add(key)
self.count += 1
def node_hash(self, node):
"""
Encodes a tuple of address and port in shorten hash for storage in
Redis.
"""
return hashlib.sha256('%s-%d' % node).hexdigest()[:8]
def cache_rtt(self):
"""
Calculates round-trip time (RTT) values and caches them in Redis.
"""
for key in self.ping_keys:
timestamps = REDIS_CONN.lrange(key, 0, 1)
if len(timestamps) > 1:
rtt_key = "rtt:{}".format(':'.join(key.split(":")[1:-1]))
rtt = int(timestamps[1]) - int(timestamps[0]) # pong - ping
logging.debug("%s: %d", rtt_key, rtt)
self.redis_pipe.lpush(rtt_key, rtt)
self.redis_pipe.ltrim(rtt_key, 0, CONF['rtt_count'] - 1)
self.redis_pipe.expire(rtt_key, CONF['ttl'])
self.redis_pipe.execute()
class Cache(object):
"""
Implements caching mechanic to cache messages from pcap file in Redis.
"""
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer()
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.keys = set() # ping:ADDRESS-PORT:NONCE
def cache_messages(self):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
self.extract_streams()
for stream_id, self.stream.segments in self.streams.iteritems():
data = self.stream.data()
_data = data.next()
while True:
try:
(msg, _data) = self.serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("%s: %s", stream_id, err)
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("%s: %s", stream_id, err)
try:
_data = data.next()
except StopIteration:
break
else:
node = (stream_id[0], stream_id[1])
self.cache_message(node, self.stream.timestamp, msg)
self.redis_pipe.execute()
self.cache_rtt()
def extract_streams(self):
"""
Extracts TCP streams with data from the pcap file. TCP segments in
each stream are queued according to their sequence number.
"""
with open(self.filepath) as pcap_file:
pcap_reader = dpkt.pcap.Reader(pcap_file)
for timestamp, buf in pcap_reader:
frame = dpkt.ethernet.Ethernet(buf)
ip_pkt = frame.data
if isinstance(ip_pkt.data, dpkt.tcp.TCP):
ip_ver = socket.AF_INET
if ip_pkt.v == 6:
ip_ver = socket.AF_INET6
tcp_pkt = ip_pkt.data
stream_id = (
socket.inet_ntop(ip_ver, ip_pkt.src),
tcp_pkt.sport,
socket.inet_ntop(ip_ver, ip_pkt.dst),
tcp_pkt.dport
)
if len(tcp_pkt.data) > 0:
timestamp = int(timestamp * 1000) # in ms
self.streams[stream_id].put(
(tcp_pkt.seq, (timestamp, tcp_pkt)))
logging.info("Streams: %d", len(self.streams))
def cache_message(self, node, timestamp, msg):
"""
Caches inv/pong message from the specified node.
"""
if msg['command'] == "inv":
for inv in msg['inventory']:
key = "inv:{}:{}".format(inv['type'], inv['hash'])
if inv['type'] == 2:
# Redis key for reference (first seen) block inv
rkey = "r{}".format(key)
rkey_ms = REDIS_CONN.get(rkey)
if rkey_ms is None:
REDIS_CONN.set(rkey, timestamp)
self.redis_pipe.set("lastblockhash", inv['hash'])
elif (timestamp - int(rkey_ms)) / 1000 > SETTINGS['ttl']:
# Ignore block inv first seen more than 3 hours ago
logging.debug("Skip: %s", key)
continue
self.redis_pipe.zadd(key, timestamp, node)
self.redis_pipe.expire(key, SETTINGS['ttl'])
self.count += msg['count']
elif msg['command'] == "pong":
key = "ping:{}-{}:{}".format(node[0], node[1], msg['nonce'])
self.redis_pipe.rpushx(key, timestamp)
self.keys.add(key)
self.count += 1
def cache_rtt(self):
"""
Calculates round-trip time (RTT) values and caches them in Redis.
"""
for key in self.keys:
timestamps = REDIS_CONN.lrange(key, 0, 1)
if len(timestamps) > 1:
rtt_key = "rtt:{}".format(':'.join(key.split(":")[1:-1]))
rtt = int(timestamps[1]) - int(timestamps[0]) # pong - ping
logging.debug("%s: %d", rtt_key, rtt)
self.redis_pipe.lpush(rtt_key, rtt)
self.redis_pipe.ltrim(rtt_key, 0, SETTINGS['rtt_count'] - 1)
self.redis_pipe.expire(rtt_key, SETTINGS['ttl'])
self.redis_pipe.execute()
class Cache(object):
"""
Implements caching mechanic to cache messages from pcap file in Redis.
"""
def __init__(self, filepath):
self.filepath = filepath
self.redis_pipe = REDIS_CONN.pipeline()
self.serializer = Serializer(magic_number=CONF['magic_number'])
self.streams = defaultdict(PriorityQueue)
self.stream = Stream()
self.count = 0
self.ping_keys = set() # ping:ADDRESS-PORT:NONCE
self.invs = defaultdict(list)
def cache_messages(self):
"""
Reconstructs messages from TCP streams and caches them in Redis.
"""
try:
self.extract_streams()
except dpkt.dpkt.NeedData:
logging.warning("Need data: %s", self.filepath)
for stream_id, self.stream.segments in self.streams.iteritems():
data = self.stream.data()
_data = data.next()
while True:
try:
(msg, _data) = self.serializer.deserialize_msg(_data)
except (HeaderTooShortError, PayloadTooShortError) as err:
logging.debug("%s: %s", stream_id, err)
try:
_data += data.next()
except StopIteration:
break
except ProtocolError as err:
logging.debug("%s: %s", stream_id, err)
try:
_data = data.next()
except StopIteration:
break
else:
src = (stream_id[0], stream_id[1])
dst = (stream_id[2], stream_id[3])
node = src
if src == CONF['tor_proxy']:
node = dst
self.cache_message(node, self.stream.timestamp, msg)
self.redis_pipe.execute()
self.cache_rtt()
def extract_streams(self):
"""
Extracts TCP streams with data from the pcap file. TCP segments in
each stream are queued according to their sequence number.
"""
with open(self.filepath) as pcap_file:
pcap_reader = dpkt.pcap.Reader(pcap_file)
for timestamp, buf in pcap_reader:
try:
frame = dpkt.ethernet.Ethernet(buf)
except dpkt.dpkt.UnpackError:
continue
ip_pkt = frame.data
if (not isinstance(ip_pkt, dpkt.ip.IP) and
not isinstance(ip_pkt, dpkt.ip6.IP6)):
continue
if not isinstance(ip_pkt.data, dpkt.tcp.TCP):
continue
ip_ver = socket.AF_INET
if ip_pkt.v == 6:
ip_ver = socket.AF_INET6
tcp_pkt = ip_pkt.data
stream_id = (
socket.inet_ntop(ip_ver, ip_pkt.src),
tcp_pkt.sport,
socket.inet_ntop(ip_ver, ip_pkt.dst),
tcp_pkt.dport
)
if len(tcp_pkt.data) > 0:
timestamp = int(timestamp * 1000) # in ms
self.streams[stream_id].put(
(tcp_pkt.seq, (timestamp, tcp_pkt)))
logging.info("Streams: %d", len(self.streams))
def cache_message(self, node, timestamp, msg):
"""
Caches inv/pong message from the specified node.
"""
if msg['command'] not in ["inv", "pong"]:
return
if node[0] == "127.0.0.1":
# Restore .onion node
onion_node = REDIS_CONN.get("onion:{}".format(node[1]))
if onion_node:
node = eval(onion_node)
if msg['command'] == "inv":
invs = 0
for inv in msg['inventory']:
key = "inv:{}:{}".format(inv['type'], inv['hash'])
if (len(self.invs[key]) >= CONF['inv_count'] and
timestamp > self.invs[key][0]):
logging.debug("Skip: %s (%d)", key, timestamp)
continue
bisect.insort(self.invs[key], timestamp)
if inv['type'] == 2:
# Redis key for reference (first seen) block inv
rkey = "r{}".format(key)
rkey_ms = REDIS_CONN.get(rkey)
if rkey_ms is None:
REDIS_CONN.set(rkey, timestamp)
self.redis_pipe.set("lastblockhash", inv['hash'])
elif (timestamp - int(rkey_ms)) / 1000 > CONF['ttl']:
# Ignore block inv first seen more than 3 hours ago
logging.debug("Skip: %s (%d)", key, timestamp)
continue
invs += 1
self.redis_pipe.zadd(key, timestamp, self.node_hash(node))
self.redis_pipe.expire(key, CONF['ttl'])
self.count += invs
elif msg['command'] == "pong":
key = "ping:{}-{}:{}".format(node[0], node[1], msg['nonce'])
self.redis_pipe.rpushx(key, timestamp)
self.ping_keys.add(key)
self.count += 1
def node_hash(self, node):
"""
Encodes a tuple of address and port in shorten hash for storage in
Redis.
"""
return hashlib.sha256('%s-%d' % node).hexdigest()[:8]
def cache_rtt(self):
"""
Calculates round-trip time (RTT) values and caches them in Redis.
"""
for key in self.ping_keys:
timestamps = REDIS_CONN.lrange(key, 0, 1)
if len(timestamps) > 1:
rtt_key = "rtt:{}".format(':'.join(key.split(":")[1:-1]))
rtt = int(timestamps[1]) - int(timestamps[0]) # pong - ping
logging.debug("%s: %d", rtt_key, rtt)
self.redis_pipe.lpush(rtt_key, rtt)
self.redis_pipe.ltrim(rtt_key, 0, CONF['rtt_count'] - 1)
self.redis_pipe.expire(rtt_key, CONF['ttl'])
self.redis_pipe.execute()