def analytics(self): debug_output("(url analytics for %s)" % self["value"]) new = [] # link with hostname # host = toolbox.url_get_host(self['value']) # if host == None: # self['hostname'] = "No hostname" # else: # self['hostname'] = host # find path path, scheme, hostname = toolbox.split_url(self["value"]) self["path"] = path self["scheme"] = scheme self["hostname"] = hostname if toolbox.is_ip(self["hostname"]): new.append(("host", Ip(toolbox.is_ip(self["hostname"])))) elif toolbox.is_hostname(self["hostname"]): new.append(("host", Hostname(toolbox.is_hostname(self["hostname"])))) else: debug_output("No hostname found for %s" % self["value"], type="error") return self["last_analysis"] = datetime.datetime.utcnow() return new
def get_pcap(self): debug_output("Generating PCAP (length: %s)" % len(self.pkts)) if len(self.pkts) == 0: return "" wrpcap("/tmp/temp.cap", self.pkts) pcap = open('/tmp/temp.cap').read() return pcap
def analytics(self): debug_output("(url analytics for %s)" % self['value']) new = [] #link with hostname # host = toolbox.url_get_host(self['value']) # if host == None: # self['hostname'] = "No hostname" # else: # self['hostname'] = host # find path path, scheme, hostname = toolbox.split_url(self['value']) self['path'] = path self['scheme'] = scheme self['hostname'] = hostname if toolbox.is_ip(self['hostname']): new.append(('host', Ip(self['hostname']))) elif toolbox.is_hostname(self['hostname']): new.append(('host', Hostname(self['hostname']))) else: debug_output("No hostname found for %s" % self['value'], type='error') return self['last_analysis'] = datetime.datetime.utcnow() return new
def analytics(self): debug_output("(url analytics for %s)" % self['value']) new = [] #link with hostname # host = toolbox.url_get_host(self['value']) # if host == None: # self['hostname'] = "No hostname" # else: # self['hostname'] = host # find path path, scheme, hostname = toolbox.split_url(self['value']) self['path'] = path self['scheme'] = scheme self['hostname'] = hostname if toolbox.is_ip(self['hostname']): new.append(('host', Ip(self['hostname']))) elif toolbox.is_hostname(self['hostname']): new.append(('host', Hostname(self['hostname']))) self['last_analysis'] = datetime.datetime.utcnow() return new
def send_nodes(self, elts=[], edges=[]): data = { 'querya': {}, 'nodes':elts, 'edges': edges, 'type': 'nodeupdate'} try: if (len(elts) > 0 or len(edges) > 0) and self.ws: self.ws.send(dumps(data)) except Exception, e: debug_output("Could not send nodes: %s" % e)
def run_scheduled_feeds(self): debug_output("Running scheduled feeds") for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) for t in self.threads: self.threads[t].join()
def run_scheduled_feeds(self): for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) for t in self.threads: if self.threads[t].is_alive(): self.threads[t].join()
def run_all_feeds(self): debug_output("Running all feeds") for feed_name in [f for f in self.feeds if self.feeds[f].enabled]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) for t in self.threads: self.threads[t].join()
def send_flow_statistics(self, flow): data = {} data['flow'] = flow.get_statistics() data['type'] = 'flow_statistics_update' if self.ws: try: self.ws.send(dumps(data)) except Exception, e: debug_output("Could not send flow statistics: %s" % e)
def analytics(self): debug_output( "(ip analytics for %s)" % self['value']) # get geolocation info try: gi = pygeoip.GeoIP('geoIP/GeoLiteCity.dat') geoinfo = gi.record_by_addr(self.value) for key in geoinfo: self[key] = geoinfo[key] except Exception, e: debug_output( "Could not get IP info for %s: %s" %(self.value, e), 'error')
def send_nodes(self, elts=[], edges=[]): data = { 'querya': {}, 'nodes': elts, 'edges': edges, 'type': 'nodeupdate' } try: if (len(elts) > 0 or len(edges) > 0) and self.ws: self.ws.send(dumps(data)) except Exception, e: debug_output("Could not send nodes: %s" % e)
def analytics(self): debug_output("(ip analytics for %s)" % self['value']) # get geolocation info try: gi = pygeoip.GeoIP('geoIP/GeoLiteCity.dat') geoinfo = gi.record_by_addr(self.value) for key in geoinfo: self[key] = geoinfo[key] except Exception, e: debug_output("Could not get IP info for %s: %s" % (self.value, e), 'error')
def analytics(self): debug_output("(ip analytics for %s)" % self["value"]) # get geolocation info try: file = os.path.abspath(__file__) datatypes_directory = os.path.dirname(file) gi = pygeoip.GeoIP(datatypes_directory + "/../geoIP/GeoLiteCity.dat") geoinfo = gi.record_by_addr(self.value) for key in geoinfo: self[key] = geoinfo[key] except Exception, e: debug_output("Could not get IP info for %s: %s" % (self.value, e), "error")
def load_pcap(self, pcap): debug_output("Loading PCAP from file...") timestamp = str(time.mktime(time.gmtime())).split('.')[0] filename = '/tmp/load-%s.cap' % timestamp #try: f = open(filename, 'wb') f.write(pcap) f.close() self.pkts += self.sniff(stopper=self.stop_sniffing, filter=self.filter, prn=self.handlePacket, stopperTimeout=1, offline=filename) debug_output("Loaded %s packets from file." % len(self.pkts)) #except Exception, e: # return e return True
def run_all_feeds(self): debug_output("Running all feeds") for feed_name in [f for f in self.feeds if self.feeds[f].enabled]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) for t in self.threads: if self.threads[t].is_alive(): self.threads[t].join() self.a.notify_progress("Working") self.a.process() self.a.data.rebuild_indexes()
def run_scheduled_feeds(self): self.a.notify_progress("Feeding") for feed_name in [f for f in self.feeds if (self.feeds[f].next_run < datetime.utcnow() and self.feeds[f].enabled)]: debug_output('Starting thread for feed %s...' % feed_name) self.run_feed(feed_name) for t in self.threads: if self.threads[t].is_alive(): self.threads[t].join() self.a.notify_progress("Working") self.a.process() self.a.data.rebuild_indexes()
def load_feeds(self): globals_, locals_ = globals(), locals() file = os.path.abspath(__file__) malcom_directory = os.path.dirname(file) package_name = 'feeds' feeds_dir = malcom_directory + '/' + package_name feeds_dir = malcom_directory debug_output("Loading feeds in %s" % feeds_dir) for filename in os.listdir(feeds_dir): export_names = [] export_classes = [] modulename, ext = os.path.splitext(filename) if modulename[0] != "_" and ext in ['.py']: subpackage = '%s.%s' % (package_name, modulename) module = __import__(subpackage, globals_, locals_, [modulename]) modict = module.__dict__ names = [name for name in modict if name[0] != '_'] for n in names: if n == 'Feed': continue class_n = modict.get(n) try: if issubclass(class_n, Feed) and class_n not in globals_: sys.stderr.write(" + Loading %s..." % n) new_feed = class_n(n) # create new feed object self.feeds[n] = new_feed # this may be for show for now export_names.append(n) export_classes.append(class_n) except Exception, e: pass
def load_feeds(self): globals_, locals_ = globals(), locals() file = os.path.abspath(__file__) malcom_directory = os.path.dirname(file) package_name = 'feeds' feeds_dir = malcom_directory + '/' + package_name feeds_dir = malcom_directory debug_output("Loading feeds in %s" % feeds_dir) for filename in os.listdir(feeds_dir): export_names = [] export_classes = [] modulename, ext = os.path.splitext(filename) if modulename[0] != "_" and ext in ['.py']: subpackage = '%s.%s' % (package_name, modulename) module = __import__(subpackage, globals_, locals_, [modulename]) modict = module.__dict__ names = [name for name in modict if name[0] != '_'] for n in names: if n == 'Feed': continue class_n = modict.get(n) try: if issubclass(class_n, Feed) and class_n not in globals_: new_feed = class_n(n) # create new feed object new_feed.analytics = self.a # attach analytics instance to feed self.feeds[n] = new_feed # this may be for show for now export_names.append(n) export_classes.append(class_n) sys.stderr.write(" + Loaded %s...\n" % n) except Exception, e: pass
def analytics(self): debug_output( "(host analytics for %s)" % self.value) # this should get us a couple of IP addresses, or other hostnames self['dns_info'] = toolbox.dns_dig_records(self.value) new = [] #get Whois self['whois'] = toolbox.whois(self['value']) # get DNS info for record in self.dns_info: if record in ['MX', 'A', 'NS', 'CNAME']: for entry in self['dns_info'][record]: art = toolbox.find_artifacts(entry) #do this for t in art: for findings in art[t]: if t == 'hostnames': new.append((record, Hostname(findings))) if t == 'urls': new.append((record, Url(findings))) if t == 'ips': new.append((record, Ip(findings))) # is _hostname a subdomain ? if len(self.value.split(".")) > 2: domain = toolbox.is_subdomain(self.value) if domain: new.append(('domain', Hostname(domain))) self['last_analysis'] = datetime.datetime.utcnow() return new
def analytics(self): debug_output("(host analytics for %s)" % self.value) # this should get us a couple of IP addresses, or other hostnames self['dns_info'] = toolbox.dns_dig_records(self.value) new = [] #get Whois self['whois'] = toolbox.whois(self['value']) # get DNS info for record in self.dns_info: if record in ['MX', 'A', 'NS', 'CNAME']: for entry in self['dns_info'][record]: art = toolbox.find_artifacts(entry) #do this for t in art: for findings in art[t]: if t == 'hostnames': new.append((record, Hostname(findings))) if t == 'urls': new.append((record, Url(findings))) if t == 'ips': new.append((record, Ip(findings))) # is _hostname a subdomain ? if len(self.value.split(".")) > 2: domain = toolbox.is_subdomain(self.value) if domain: new.append(('domain', Hostname(domain))) self['last_analysis'] = datetime.datetime.utcnow() return new
def analytics(self): debug_output("(host analytics for %s)" % self.value) # this should get us a couple of IP addresses, or other hostnames self["dns_info"] = toolbox.dns_dig_records(self.value) new = [] # get Whois self["whois"] = toolbox.whois(self["value"]) # get DNS info for record in self.dns_info: if record in ["MX", "A", "NS", "CNAME"]: for entry in self["dns_info"][record]: art = toolbox.find_artifacts(entry) # do this for t in art: for findings in art[t]: if t == "hostnames": new.append((record, Hostname(findings))) if t == "urls": new.append((record, Url(findings))) if t == "ips": new.append((record, Ip(findings))) # is _hostname a subdomain ? if len(self.value.split(".")) > 2: domain = toolbox.is_subdomain(self.value) if domain: new.append(("domain", Hostname(domain))) self["last_analysis"] = datetime.datetime.utcnow() return new
def run(self): debug_output("[+] Sniffing session %s started" % self.name) debug_output("[+] Filter: %s" % self.filter) self.stopSniffing = False self.pkts += self.sniff(stopper=self.stop_sniffing, filter=self.filter, prn=self.handlePacket, stopperTimeout=1) #except Exception, e: # print e debug_output("[+] Sniffing session %s stopped" % self.name) return
def checkDNS(self, pkt): new_elts = [] new_edges = [] # intercept DNS responses (these contain names and IPs) if DNS in pkt and pkt[IP].sport == 53: debug_output("[+] DNS reply caught (%s answers)" % pkt[DNS].ancount) for i in xrange( pkt[DNS].ancount ): # cycle through responses and add records to graph if pkt[DNS].an[i].type != 1: debug_output('No A records in reply') continue hname = pkt[DNS].an[i].rrname ipaddr = pkt[DNS].an[i].rdata # check if hname ends with '.' if hname[-1:] == ".": hname = hname[:-1] # check if we haven't seen these already if hname not in self.nodes_values: _hname = self.analytics.add_text( [hname], ['sniffer', self.name]) # log every discovery to db self.nodes_ids.append(_hname['_id']) self.nodes_values.append(_hname['value']) self.nodes.append(_hname) new_elts.append(_hname) else: _hname = [e for e in self.nodes if e['value'] == hname][0] if ipaddr not in self.nodes_values: _ipaddr = self.analytics.add_text([ipaddr], ['sniffer', self.name]) self.nodes_ids.append(_ipaddr['_id']) self.nodes_values.append(_ipaddr['value']) self.nodes.append(_ipaddr) new_elts.append(_ipaddr) else: _ipaddr = [e for e in self.nodes if e['value'] == ipaddr][0] debug_output("Caught DNS response %s: %s -> %s" % (i, _hname['value'], _ipaddr['value'])) debug_output("Added %s, %s" % (hname, ipaddr)) conn = { 'attribs': 'A', 'src': _hname['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id']) + str(_ipaddr['_id']) } } if conn not in self.edges: self.edges.append(conn) new_edges.append(conn) #deal with the original DNS request question = pkt[DNS].qd.qname if question not in self.nodes_values: _question = self.analytics.add_text( [question], ['sniffer', self.name ]) # log it to db (for further reference) if _question: self.nodes_ids.append(_question['_id']) self.nodes_values.append(_question['value']) self.nodes.append(_question) new_edges.append(_question) else: _question = [e for e in self.nodes if e['value'] == question][0] #conn = self.analytics.data.connect(_question, elt, "resolve", True) # conn = {'attribs': 'query', 'src': _question['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id']) } } # if conn not in self.edges: # self.edges.append(conn) # new_edges.append(conn) return new_elts, new_edges
def run(self): self.run_periodically = True while self.run_periodically: time.sleep(self.period) # run a new thread every period seconds debug_output("Checking feeds...") self.run_scheduled_feeds()
from analytics import Analytics from toolbox import debug_output a = Analytics() # Check if there are elements which have no created date nocreate = a.data.find({"date_created": None}).count() type = "error" if nocreate > 0 else "debug" debug_output("Elements without a date_created: {}".format(nocreate), type) # Check if there are elements which have no updated date noupdate = a.data.find({"date_updated": None}).count() type = "error" if noupdate > 0 else "debug" debug_output("Elements without a date_updated: {}".format(noupdate), type) # Check if there are urls that don't have hostnames nohostname = a.data.find({'type': 'url', 'hostname': None}).count() type = "error" if nohostname > 0 else "debug" debug_output("URLs without a hostname: {}".format(nohostname), type) emptyhostname = a.data.find({'type': 'url', 'hostname': ""}).count() type = "error" if emptyhostname > 0 else "debug" debug_output("URLs with empty hostname: {}".format(emptyhostname), type)
def checkDNS(self, pkt): new_elts = [] new_edges = [] # intercept DNS responses (these contain names and IPs) if DNS in pkt and pkt[IP].sport == 53: debug_output("[+] DNS reply caught (%s answers)" % pkt[DNS].ancount) for i in xrange(pkt[DNS].ancount): # cycle through responses and add records to graph if pkt[DNS].an[i].type != 1: debug_output('No A records in reply') continue hname = pkt[DNS].an[i].rrname ipaddr = pkt[DNS].an[i].rdata # check if hname ends with '.' if hname[-1:] == ".": hname = hname[:-1] # check if we haven't seen these already if hname not in self.nodes_values: _hname = self.analytics.add_text([hname], ['sniffer', self.name]) # log every discovery to db self.nodes_ids.append(_hname['_id']) self.nodes_values.append(_hname['value']) self.nodes.append(_hname) new_elts.append(_hname) else: _hname = [e for e in self.nodes if e['value'] == hname][0] if ipaddr not in self.nodes_values: _ipaddr = self.analytics.add_text([ipaddr], ['sniffer', self.name]) # log every discovery to db self.nodes_ids.append(_ipaddr['_id']) self.nodes_values.append(_ipaddr['value']) self.nodes.append(_ipaddr) new_elts.append(_ipaddr) else: _ipaddr = [e for e in self.nodes if e['value'] == ipaddr][0] debug_output("Caught DNS response %s: %s -> %s" % (i, _hname['value'], _ipaddr['value'])) debug_output("Added %s, %s" %(hname, ipaddr)) conn = {'attribs': 'A', 'src': _hname['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id'])}} if conn not in self.edges: self.edges.append(conn) new_edges.append(conn) #deal with the original DNS request question = pkt[DNS].qd.qname if question not in self.nodes_values: _question = self.analytics.add_text([question], ['sniffer', self.name]) # log it to db (for further reference) if _question: self.nodes_ids.append(_question['_id']) self.nodes_values.append(_question['value']) self.nodes.append(_question) new_elts.append(_question) else: _question = [e for e in self.nodes if e['value'] == question][0] # conn = self.analytics.data.connect(_question, elt, "resolve", True) # conn = {'attribs': 'query', 'src': _question['_id'], 'dst': _ipaddr['_id'], '_id': { '$oid': str(_hname['_id'])+str(_ipaddr['_id']) } } # if conn not in self.edges: # self.edges.append(conn) # new_edges.append(conn) return new_elts, new_edges