def analyze(self,meta,data): # Analyze the DNS packet, handle distribution to agents. # Meta = [metadata list format] # Data is a dpkt.dns object try: if data.opcode == dns.DNS_QUERY: # operation is Query or Query Response if data.qr == dns.DNS_Q: # DNS Query Analysis doms = {} for i in xrange(len(data.qd)): # for domain in query list try: name = domaintools.tld_detect(data.qd[i].name)) if settings['known_domains'] == 'On': res = self.blip(name[2]) else: res = {'white':False,'hard':False} if name: if not (res['white'] and res['hard']): if settings['name_structure'] == 'On': # send to name structure analysis [meta,name,'Q'] ('Q' = query) self.inq.put([meta,name,'Q']) if settings['time_analysis'] == 'On': # put meta and domain into time feature analysis queue self.icq.put([meta,name]) if settings['known_domains'] == 'On' and name[2] not in doms: report = { 'type' :'event','agent' :'known_domains', 'event' :{'agent' :'known_domains','meta':meta, 'res':res, 'host':name[2]}} if settings['fusion']: self.out_queue.put(report) doms[name[2]] = '' except: pass
def start(self): print "[%s] [Hooves] Running" % (time.ctime()) # get info from queue, and analyze the domain if not self.settings["testing"]: # if testing is turned off, get info from live data while 1: try: meta, dom, type = self.in_queue.get_nowait() # Get dns information from queue self.check(meta, dom, type) # analyze it except: # Nothing in the queue time.sleep(0.5) # sleep .5 sec # print 'Error: Queue Size = %s' %(k) # check if the time has been > 15 minutes, if yes, update global_analysis if self.settings["rainbow"]: if (time.time() - self.curtime) >= self.settings["fusion_interval"]: self.curtime += self.settings["fusion_interval"] if self.settings["fusion"]: print "[%s] [Hooves] Updating Rainbow Love!" % (time.ctime()) report = { "type": "agent", "agent": "Hooves", "event": {"hist": self.hist, "new": self.events}, } self.out_queue.put(report) # send update to Rainbow Love self.events = {} if self.settings["collection"]: if (time.time() - self.t_time) > self.settings["collect_int"]: print "[%s][Hooves] Printing History to Collection File: %s" % ( time.ctime(), self.settings["collect_output_file"], ) outfile = open(self.settings["collect_output_file"], "w+") for i in self.hist: outfile.write(i + "\r\n") if not self.settings["collect_rep"]: self.settings["collection"] = 0 else: self.t_time = time.time() else: # Read in the test file print "Reading test file: " + self.settings["test_input_file"] try: infile = open(self.settings["test_input_file"]) except: print "Unable to open test_input_file, check configuration file: " + self.settings["test_input_file"] sys.exit() raw = infile.readlines() infile.close() # Analyze the domains in the test file print "Analyzing & writing individual feature files" for line in raw: self.analyze(domaintools.tld_detect(line.rstrip())) # Write results to file print "Writing Results Files to: " + self.settings["test_output_dir"] try: outfile = open(self.settings["test_output_dir"] + "combined.csv", "w+") except: print "Unable to open test_output_file, check configuration file: " + self.settings["test_output_file"] sys.exit() for i in self.hist: outfile.write(",".join(map(str, [i, self.hist[i]["score"]])) + "\r\n") outfile.flush() self.vow_out.flush() self.num_out.flush() self.tld_out.flush() self.freq_out.flush() outfile.close() self.vow_out.close() self.num_out.close() self.tld_out.close() self.freq_out.close() popen("chmod 777 -R " + self.settings["test_output_dir"]) print "Finished Testing, view " + self.settings["test_output_dir"] + " to see results"
def start(self): print '[%s] [Hooves] Running' % (time.ctime()) # get info from queue, and analyze the domain if not self.settings['testing']: # if testing is turned off, get info from live data while 1: try: meta, dom, type = self.in_queue.get_nowait( ) # Get dns information from queue self.check(meta, dom, type) # analyze it except: # Nothing in the queue time.sleep(.5) # sleep .5 sec #print 'Error: Queue Size = %s' %(k) # check if the time has been > 15 minutes, if yes, update global_analysis if self.settings['rainbow']: if (time.time() - self.curtime) >= self.settings['fusion_interval']: self.curtime += self.settings['fusion_interval'] if self.settings['fusion']: print '[%s] [Hooves] Updating Rainbow Love!' % ( time.ctime()) report = { 'type': 'agent', 'agent': 'Hooves', 'event': { 'hist': self.hist, 'new': self.events } } self.out_queue.put( report) # send update to Rainbow Love self.events = {} if self.settings['collection']: if (time.time() - self.t_time) > self.settings['collect_int']: print '[%s][Hooves] Printing History to Collection File: %s' % ( time.ctime(), self.settings['collect_output_file']) outfile = open(self.settings['collect_output_file'], 'w+') for i in self.hist: outfile.write(i + '\r\n') if not self.settings['collect_rep']: self.settings['collection'] = 0 else: self.t_time = time.time() else: # Read in the test file print 'Reading test file: ' + self.settings['test_input_file'] try: infile = open(self.settings['test_input_file']) except: print 'Unable to open test_input_file, check configuration file: ' + self.settings[ 'test_input_file'] sys.exit() raw = infile.readlines() infile.close() # Analyze the domains in the test file print 'Analyzing & writing individual feature files' for line in raw: self.analyze(domaintools.tld_detect(line.rstrip())) # Write results to file print 'Writing Results Files to: ' + self.settings[ 'test_output_dir'] try: outfile = open( self.settings['test_output_dir'] + 'combined.csv', 'w+') except: print 'Unable to open test_output_file, check configuration file: ' + self.settings[ 'test_output_file'] sys.exit() for i in self.hist: outfile.write(','.join(map(str, [i, self.hist[i]['score']])) + '\r\n') outfile.flush() self.vow_out.flush() self.num_out.flush() self.tld_out.flush() self.freq_out.flush() outfile.close() self.vow_out.close() self.num_out.close() self.tld_out.close() self.freq_out.close() popen('chmod 777 -R ' + self.settings['test_output_dir']) print 'Finished Testing, view ' + self.settings[ 'test_output_dir'] + ' to see results'
'event' :{'agent' :'known_domains','meta':meta, 'res':res, 'host':name[2]}} if settings['fusion']: self.out_queue.put(report) doms[name[2]] = '' except: pass elif data.qr == dns.DNS_R: # DNS Response analysis ips,doms = {},{} for i in xrange(len(data.an)): # cycle through answers if data.an[i].cls == dns.DNS_IN: if data.an[i].type == dns.DNS_A: try: # if answers, answer(ip) and ttl analysis name = domaintools.tld_detect(data.an[i].name) if settings['known_domains'] == 'On': res = self.blip(name[2]) else: res = {'white':False,'hard':False} if name: ip, ttl = data.an[i].ip, data.an[i].ttl if settings['dns_answer'] == 'On': self.iaq.put([meta,data.an[i].name,ip,'R',ttl]) if not (res['white'] and res['hard']): # known_domains ip support goes here if settings['known_domains'] == 'On':
def analyze(self,meta,data): # Analyze the DNS packet, handle distribution to agents. # Meta = [metadata list format] # Data is a dpkt.dns object try: if data.opcode == dns.DNS_QUERY: # operation is Query or Query Response # DNS Query Analysis if data.qr == dns.DNS_Q: doms = {} for i in xrange(len(data.qd)): # for domain in query list try: name = domaintools.tld_detect(data.qd[i].name)) if settings['blips'] == 'On': res = self.blip(name[2]) else: res = {'white':False,'hard':False} if name: if not (res['white'] and res['hard']): # put [meta,name,'Q'] into derpy.hooves queue ('Q' = query) if settings['hooves'] == 'On': self.inq.put([meta,name,'Q']) # put meta and domain into derpy.dash queue if settings['dash'] == 'On': self.icq.put([meta,name]) # Blips support goes here if settings['blips'] == 'On' and name[2] not in doms: report = { 'type' :'event','agent' :'blips', 'event' :{'agent' :'blips','meta':meta, 'res':res, 'host':name[2]}} if settings['rainbow']: self.out_queue.put(report) doms[name[2]] = '' except: pass # DNS Response analysis elif data.qr == dns.DNS_R: ips,doms = {},{} for i in xrange(len(data.an)): # cycle through answers if data.an[i].cls == dns.DNS_IN: if data.an[i].type == dns.DNS_A: # if answers, answer(ip) and ttl analysis try: name = domaintools.tld_detect(data.an[i].name) if settings['blips'] == 'On': res = self.blip(name[2]) else: res = {'white':False,'hard':False} if name: ip, ttl = data.an[i].ip, data.an[i].ttl if settings['jack'] == 'On': self.iaq.put([meta,data.an[i].name,ip,'R',ttl]) if not (res['white'] and res['hard']): # blips ip support goes here if settings['blips'] == 'On': ires = self.blipip(ip) if not ires['white'] and ires['hard'] and ip not in ips: ires['msg'] = 'Domain Resolution Contained Blacklisted IP Address' report = { 'type' :'event','agent' :'blips', 'event' :{'agent' :'blips','meta':meta, 'res':ires, 'host':dec2ip(unpack('>I',ip)[0])}} if settings['rainbow']: self.out_queue.put(report) ips[ip] = '' if settings['blips'] == 'On' and name[2] not in doms: report = { 'type' :'event','agent' :'blips', 'event' :{'agent' :'blips','meta':meta, 'res':res, 'host':name[2]}} if settings['rainbow']: self.out_queue.put(report) doms[name[2]] = '' except: pass elif data.an[i].type == dns.DNS_CNAME: # if this is in alias, answer(alias) analysis try: name = domaintools.tld_detect(data.an[i].name) cname = domaintools.tld_detect(data.an[i].cname) res = self.blip(cname[2]) if (name and cname) and (data.an[i].name != data.an[i].cname): if settings['jack'] == 'On': self.iaq.put([meta,data.an[i].name,data.an[i].cname,'A']) if res['white'] and res1['hard']: pass else: # [meta,cname,'A'] name analysis ('A'=alias) if settings['hooves'] == 'On': self.inq.put([meta,cname,'A']) except: pass