def analytics(self):
debug_output("(url analytics for %s)" % self["value"])
new = []
# link with hostname
# host = toolbox.url_get_host(self['value'])
# if host == None:
# self['hostname'] = "No hostname"
# else:
# self['hostname'] = host
# find path
path, scheme, hostname = toolbox.split_url(self["value"])
self["path"] = path
self["scheme"] = scheme
self["hostname"] = hostname
if toolbox.is_ip(self["hostname"]):
new.append(("host", Ip(toolbox.is_ip(self["hostname"]))))
elif toolbox.is_hostname(self["hostname"]):
new.append(("host", Hostname(toolbox.is_hostname(self["hostname"]))))
else:
debug_output("No hostname found for %s" % self["value"], type="error")
return
self["last_analysis"] = datetime.datetime.utcnow()
return new
def get_pcap(self):
debug_output("Generating PCAP (length: %s)" % len(self.pkts))
if len(self.pkts) == 0:
return ""
wrpcap("/tmp/temp.cap", self.pkts)
pcap = open('/tmp/temp.cap').read()
return pcap
def analytics(self):
debug_output("(url analytics for %s)" % self['value'])
new = []
#link with hostname
# host = toolbox.url_get_host(self['value'])
# if host == None:
# self['hostname'] = "No hostname"
# else:
# self['hostname'] = host
# find path
path, scheme, hostname = toolbox.split_url(self['value'])
self['path'] = path
self['scheme'] = scheme
self['hostname'] = hostname
if toolbox.is_ip(self['hostname']):
new.append(('host', Ip(self['hostname'])))
elif toolbox.is_hostname(self['hostname']):
new.append(('host', Hostname(self['hostname'])))
else:
debug_output("No hostname found for %s" % self['value'],
type='error')
return
self['last_analysis'] = datetime.datetime.utcnow()
return new
def analytics(self):
debug_output("(url analytics for %s)" % self['value'])
new = []
#link with hostname
# host = toolbox.url_get_host(self['value'])
# if host == None:
# self['hostname'] = "No hostname"
# else:
# self['hostname'] = host
# find path
path, scheme, hostname = toolbox.split_url(self['value'])
self['path'] = path
self['scheme'] = scheme
self['hostname'] = hostname
if toolbox.is_ip(self['hostname']):
new.append(('host', Ip(self['hostname'])))
elif toolbox.is_hostname(self['hostname']):
new.append(('host', Hostname(self['hostname'])))
self['last_analysis'] = datetime.datetime.utcnow()
return new
def get_pcap(self):
debug_output("Generating PCAP (length: %s)" % len(self.pkts))
if len(self.pkts) == 0:
return ""
wrpcap("/tmp/temp.cap", self.pkts)
pcap = open('/tmp/temp.cap').read()
return pcap
def send_nodes(self, elts=[], edges=[]):
data = { 'querya': {}, 'nodes':elts, 'edges': edges, 'type': 'nodeupdate'}
try:
if (len(elts) > 0 or len(edges) > 0) and self.ws:
self.ws.send(dumps(data))
except Exception, e:
debug_output("Could not send nodes: %s" % e)
def run_scheduled_feeds(self):
debug_output("Running scheduled feeds")
for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
for t in self.threads:
self.threads[t].join()
def run_scheduled_feeds(self):
for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t].join()
def run_all_feeds(self):
debug_output("Running all feeds")
for feed_name in [f for f in self.feeds if self.feeds[f].enabled]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
for t in self.threads:
self.threads[t].join()
def send_flow_statistics(self, flow):
data = {}
data['flow'] = flow.get_statistics()
data['type'] = 'flow_statistics_update'
if self.ws:
try:
self.ws.send(dumps(data))
except Exception, e:
debug_output("Could not send flow statistics: %s" % e)
def send_flow_statistics(self, flow):
data = {}
data['flow'] = flow.get_statistics()
data['type'] = 'flow_statistics_update'
if self.ws:
try:
self.ws.send(dumps(data))
except Exception, e:
debug_output("Could not send flow statistics: %s" % e)
def analytics(self):
debug_output( "(ip analytics for %s)" % self['value'])
# get geolocation info
try:
gi = pygeoip.GeoIP('geoIP/GeoLiteCity.dat')
geoinfo = gi.record_by_addr(self.value)
for key in geoinfo:
self[key] = geoinfo[key]
except Exception, e:
debug_output( "Could not get IP info for %s: %s" %(self.value, e), 'error')
def send_nodes(self, elts=[], edges=[]):
data = {
'querya': {},
'nodes': elts,
'edges': edges,
'type': 'nodeupdate'
}
try:
if (len(elts) > 0 or len(edges) > 0) and self.ws:
self.ws.send(dumps(data))
except Exception, e:
debug_output("Could not send nodes: %s" % e)
def analytics(self):
debug_output("(ip analytics for %s)" % self['value'])
# get geolocation info
try:
gi = pygeoip.GeoIP('geoIP/GeoLiteCity.dat')
geoinfo = gi.record_by_addr(self.value)
for key in geoinfo:
self[key] = geoinfo[key]
except Exception, e:
debug_output("Could not get IP info for %s: %s" % (self.value, e),
'error')
def analytics(self):
debug_output("(ip analytics for %s)" % self["value"])
# get geolocation info
try:
file = os.path.abspath(__file__)
datatypes_directory = os.path.dirname(file)
gi = pygeoip.GeoIP(datatypes_directory + "/../geoIP/GeoLiteCity.dat")
geoinfo = gi.record_by_addr(self.value)
for key in geoinfo:
self[key] = geoinfo[key]
except Exception, e:
debug_output("Could not get IP info for %s: %s" % (self.value, e), "error")
def load_pcap(self, pcap):
debug_output("Loading PCAP from file...")
timestamp = str(time.mktime(time.gmtime())).split('.')[0]
filename = '/tmp/load-%s.cap' % timestamp
#try:
f = open(filename, 'wb')
f.write(pcap)
f.close()
self.pkts += self.sniff(stopper=self.stop_sniffing, filter=self.filter, prn=self.handlePacket, stopperTimeout=1, offline=filename)
debug_output("Loaded %s packets from file." % len(self.pkts))
#except Exception, e:
# return e
return True
def run_all_feeds(self):
debug_output("Running all feeds")
for feed_name in [f for f in self.feeds if self.feeds[f].enabled]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t].join()
self.a.notify_progress("Working")
self.a.process()
self.a.data.rebuild_indexes()
def run_scheduled_feeds(self):
self.a.notify_progress("Feeding")
for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]:
debug_output('Starting thread for feed %s...' % feed_name)
self.run_feed(feed_name)
for t in self.threads:
if self.threads[t].is_alive():
self.threads[t].join()
self.a.notify_progress("Working")
self.a.process()
self.a.data.rebuild_indexes()
def load_feeds(self):
globals_, locals_ = globals(), locals()
file = os.path.abspath(__file__)
malcom_directory = os.path.dirname(file)
package_name = 'feeds'
feeds_dir = malcom_directory + '/' + package_name
feeds_dir = malcom_directory
debug_output("Loading feeds in %s" % feeds_dir)
for filename in os.listdir(feeds_dir):
export_names = []
export_classes = []
modulename, ext = os.path.splitext(filename)
if modulename[0] != "_" and ext in ['.py']:
subpackage = '%s.%s' % (package_name, modulename)
module = __import__(subpackage, globals_, locals_,
[modulename])
modict = module.__dict__
names = [name for name in modict if name[0] != '_']
for n in names:
if n == 'Feed':
continue
class_n = modict.get(n)
try:
if issubclass(class_n,
Feed) and class_n not in globals_:
sys.stderr.write(" + Loading %s..." % n)
new_feed = class_n(n) # create new feed object
self.feeds[n] = new_feed
# this may be for show for now
export_names.append(n)
export_classes.append(class_n)
except Exception, e:
pass
def load_pcap(self, pcap):
debug_output("Loading PCAP from file...")
timestamp = str(time.mktime(time.gmtime())).split('.')[0]
filename = '/tmp/load-%s.cap' % timestamp
#try:
f = open(filename, 'wb')
f.write(pcap)
f.close()
self.pkts += self.sniff(stopper=self.stop_sniffing,
filter=self.filter,
prn=self.handlePacket,
stopperTimeout=1,
offline=filename)
debug_output("Loaded %s packets from file." % len(self.pkts))
#except Exception, e:
# return e
return True
def load_feeds(self):
globals_, locals_ = globals(), locals()
file = os.path.abspath(__file__)
malcom_directory = os.path.dirname(file)
package_name = 'feeds'
feeds_dir = malcom_directory + '/' + package_name
feeds_dir = malcom_directory
debug_output("Loading feeds in %s" % feeds_dir)
for filename in os.listdir(feeds_dir):
export_names = []
export_classes = []
modulename, ext = os.path.splitext(filename)
if modulename[0] != "_" and ext in ['.py']:
subpackage = '%s.%s' % (package_name, modulename)
module = __import__(subpackage, globals_, locals_, [modulename])
modict = module.__dict__
names = [name for name in modict if name[0] != '_']
for n in names:
if n == 'Feed':
continue
class_n = modict.get(n)
try:
if issubclass(class_n, Feed) and class_n not in globals_:
new_feed = class_n(n) # create new feed object
new_feed.analytics = self.a # attach analytics instance to feed
self.feeds[n] = new_feed
# this may be for show for now
export_names.append(n)
export_classes.append(class_n)
sys.stderr.write(" + Loaded %s...\n" % n)
except Exception, e:
pass
def analytics(self):
debug_output( "(host analytics for %s)" % self.value)
# this should get us a couple of IP addresses, or other hostnames
self['dns_info'] = toolbox.dns_dig_records(self.value)
new = []
#get Whois
self['whois'] = toolbox.whois(self['value'])
# get DNS info
for record in self.dns_info:
if record in ['MX', 'A', 'NS', 'CNAME']:
for entry in self['dns_info'][record]:
art = toolbox.find_artifacts(entry) #do this
for t in art:
for findings in art[t]:
if t == 'hostnames':
new.append((record, Hostname(findings)))
if t == 'urls':
new.append((record, Url(findings)))
if t == 'ips':
new.append((record, Ip(findings)))
# is _hostname a subdomain ?
if len(self.value.split(".")) > 2:
domain = toolbox.is_subdomain(self.value)
if domain:
new.append(('domain', Hostname(domain)))
self['last_analysis'] = datetime.datetime.utcnow()
return new
def analytics(self):
debug_output("(host analytics for %s)" % self.value)
# this should get us a couple of IP addresses, or other hostnames
self['dns_info'] = toolbox.dns_dig_records(self.value)
new = []
#get Whois
self['whois'] = toolbox.whois(self['value'])
# get DNS info
for record in self.dns_info:
if record in ['MX', 'A', 'NS', 'CNAME']:
for entry in self['dns_info'][record]:
art = toolbox.find_artifacts(entry) #do this
for t in art:
for findings in art[t]:
if t == 'hostnames':
new.append((record, Hostname(findings)))
if t == 'urls':
new.append((record, Url(findings)))
if t == 'ips':
new.append((record, Ip(findings)))
# is _hostname a subdomain ?
if len(self.value.split(".")) > 2:
domain = toolbox.is_subdomain(self.value)
if domain:
new.append(('domain', Hostname(domain)))
self['last_analysis'] = datetime.datetime.utcnow()
return new
def analytics(self):
debug_output("(host analytics for %s)" % self.value)
# this should get us a couple of IP addresses, or other hostnames
self["dns_info"] = toolbox.dns_dig_records(self.value)
new = []
# get Whois
self["whois"] = toolbox.whois(self["value"])
# get DNS info
for record in self.dns_info:
if record in ["MX", "A", "NS", "CNAME"]:
for entry in self["dns_info"][record]:
art = toolbox.find_artifacts(entry) # do this
for t in art:
for findings in art[t]:
if t == "hostnames":
new.append((record, Hostname(findings)))
if t == "urls":
new.append((record, Url(findings)))
if t == "ips":
new.append((record, Ip(findings)))
# is _hostname a subdomain ?
if len(self.value.split(".")) > 2:
domain = toolbox.is_subdomain(self.value)
if domain:
new.append(("domain", Hostname(domain)))
self["last_analysis"] = datetime.datetime.utcnow()
return new
def run(self):
debug_output("[+] Sniffing session %s started" % self.name)
debug_output("[+] Filter: %s" % self.filter)
self.stopSniffing = False
self.pkts += self.sniff(stopper=self.stop_sniffing, filter=self.filter, prn=self.handlePacket, stopperTimeout=1)
#except Exception, e:
# print e
debug_output("[+] Sniffing session %s stopped" % self.name)
return
def run(self):
debug_output("[+] Sniffing session %s started" % self.name)
debug_output("[+] Filter: %s" % self.filter)
self.stopSniffing = False
self.pkts += self.sniff(stopper=self.stop_sniffing,
filter=self.filter,
prn=self.handlePacket,
stopperTimeout=1)
#except Exception, e:
# print e
debug_output("[+] Sniffing session %s stopped" % self.name)
return
def checkDNS(self, pkt):
new_elts = []
new_edges = []
# intercept DNS responses (these contain names and IPs)
if DNS in pkt and pkt[IP].sport == 53:
debug_output("[+] DNS reply caught (%s answers)" %
pkt[DNS].ancount)
for i in xrange(
pkt[DNS].ancount
): # cycle through responses and add records to graph
if pkt[DNS].an[i].type != 1:
debug_output('No A records in reply')
continue
hname = pkt[DNS].an[i].rrname
ipaddr = pkt[DNS].an[i].rdata
# check if hname ends with '.'
if hname[-1:] == ".":
hname = hname[:-1]
# check if we haven't seen these already
if hname not in self.nodes_values:
_hname = self.analytics.add_text(
[hname],
['sniffer', self.name]) # log every discovery to db
self.nodes_ids.append(_hname['_id'])
self.nodes_values.append(_hname['value'])
self.nodes.append(_hname)
new_elts.append(_hname)
else:
_hname = [e for e in self.nodes if e['value'] == hname][0]
if ipaddr not in self.nodes_values:
_ipaddr = self.analytics.add_text([ipaddr],
['sniffer', self.name])
self.nodes_ids.append(_ipaddr['_id'])
self.nodes_values.append(_ipaddr['value'])
self.nodes.append(_ipaddr)
new_elts.append(_ipaddr)
else:
_ipaddr = [e for e in self.nodes
if e['value'] == ipaddr][0]
debug_output("Caught DNS response %s: %s -> %s" %
(i, _hname['value'], _ipaddr['value']))
debug_output("Added %s, %s" % (hname, ipaddr))
conn = {
'attribs': 'A',
'src': _hname['_id'],
'dst': _ipaddr['_id'],
'_id': {
'$oid': str(_hname['_id']) + str(_ipaddr['_id'])
}
}
if conn not in self.edges:
self.edges.append(conn)
new_edges.append(conn)
#deal with the original DNS request
question = pkt[DNS].qd.qname
if question not in self.nodes_values:
_question = self.analytics.add_text(
[question], ['sniffer', self.name
]) # log it to db (for further reference)
if _question:
self.nodes_ids.append(_question['_id'])
self.nodes_values.append(_question['value'])
self.nodes.append(_question)
new_edges.append(_question)
else:
_question = [e for e in self.nodes
if e['value'] == question][0]
#conn = self.analytics.data.connect(_question, elt, "resolve", True)
# conn = {'attribs': 'query', 'src': _question['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id']) } }
# if conn not in self.edges:
# self.edges.append(conn)
# new_edges.append(conn)
return new_elts, new_edges
def run(self):
self.run_periodically = True
while self.run_periodically:
time.sleep(self.period) # run a new thread every period seconds
debug_output("Checking feeds...")
self.run_scheduled_feeds()
from analytics import Analytics
from toolbox import debug_output
a = Analytics()
# Check if there are elements which have no created date
nocreate = a.data.find({"date_created": None}).count()
type = "error" if nocreate > 0 else "debug"
debug_output("Elements without a date_created: {}".format(nocreate), type)
# Check if there are elements which have no updated date
noupdate = a.data.find({"date_updated": None}).count()
type = "error" if noupdate > 0 else "debug"
debug_output("Elements without a date_updated: {}".format(noupdate), type)
# Check if there are urls that don't have hostnames
nohostname = a.data.find({'type': 'url', 'hostname': None}).count()
type = "error" if nohostname > 0 else "debug"
debug_output("URLs without a hostname: {}".format(nohostname), type)
emptyhostname = a.data.find({'type': 'url', 'hostname': ""}).count()
type = "error" if emptyhostname > 0 else "debug"
debug_output("URLs with empty hostname: {}".format(emptyhostname), type)
def checkDNS(self, pkt):
new_elts = []
new_edges = []
# intercept DNS responses (these contain names and IPs)
if DNS in pkt and pkt[IP].sport == 53:
debug_output("[+] DNS reply caught (%s answers)" % pkt[DNS].ancount)
for i in xrange(pkt[DNS].ancount): # cycle through responses and add records to graph
if pkt[DNS].an[i].type != 1:
debug_output('No A records in reply')
continue
hname = pkt[DNS].an[i].rrname
ipaddr = pkt[DNS].an[i].rdata
# check if hname ends with '.'
if hname[-1:] == ".":
hname = hname[:-1]
# check if we haven't seen these already
if hname not in self.nodes_values:
_hname = self.analytics.add_text([hname], ['sniffer', self.name]) # log every discovery to db
self.nodes_ids.append(_hname['_id'])
self.nodes_values.append(_hname['value'])
self.nodes.append(_hname)
new_elts.append(_hname)
else:
_hname = [e for e in self.nodes if e['value'] == hname][0]
if ipaddr not in self.nodes_values:
_ipaddr = self.analytics.add_text([ipaddr], ['sniffer', self.name]) # log every discovery to db
self.nodes_ids.append(_ipaddr['_id'])
self.nodes_values.append(_ipaddr['value'])
self.nodes.append(_ipaddr)
new_elts.append(_ipaddr)
else:
_ipaddr = [e for e in self.nodes if e['value'] == ipaddr][0]
debug_output("Caught DNS response %s: %s -> %s" % (i, _hname['value'], _ipaddr['value']))
debug_output("Added %s, %s" %(hname, ipaddr))
conn = {'attribs': 'A', 'src': _hname['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id'])}}
if conn not in self.edges:
self.edges.append(conn)
new_edges.append(conn)
#deal with the original DNS request
question = pkt[DNS].qd.qname
if question not in self.nodes_values:
_question = self.analytics.add_text([question], ['sniffer', self.name]) # log it to db (for further reference)
if _question:
self.nodes_ids.append(_question['_id'])
self.nodes_values.append(_question['value'])
self.nodes.append(_question)
new_elts.append(_question)
else:
_question = [e for e in self.nodes if e['value'] == question][0]
# conn = self.analytics.data.connect(_question, elt, "resolve", True)
# conn = {'attribs': 'query', 'src': _question['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id']) } }
# if conn not in self.edges:
# self.edges.append(conn)
# new_edges.append(conn)
return new_elts, new_edges
