def analyze(self,meta,data): # Analyze the DNS packet, handle distribution to agents.
# Meta = [metadata list format]
# Data is a dpkt.dns object
try:
if data.opcode == dns.DNS_QUERY: # operation is Query or Query Response
if data.qr == dns.DNS_Q: # DNS Query Analysis
doms = {}
for i in xrange(len(data.qd)): # for domain in query list
try:
name = domaintools.tld_detect(data.qd[i].name))
if settings['known_domains'] == 'On':
res = self.blip(name[2])
else:
res = {'white':False,'hard':False}
if name:
if not (res['white'] and res['hard']):
if settings['name_structure'] == 'On': # send to name structure analysis [meta,name,'Q'] ('Q' = query)
self.inq.put([meta,name,'Q'])
if settings['time_analysis'] == 'On': # put meta and domain into time feature analysis queue
self.icq.put([meta,name])
if settings['known_domains'] == 'On' and name[2] not in doms:
report = { 'type' :'event','agent' :'known_domains',
'event' :{'agent' :'known_domains','meta':meta, 'res':res, 'host':name[2]}}
if settings['fusion']:
self.out_queue.put(report)
doms[name[2]] = ''
except:
pass
def start(self):
print "[%s] [Hooves] Running" % (time.ctime())
# get info from queue, and analyze the domain
if not self.settings["testing"]:
# if testing is turned off, get info from live data
while 1:
try:
meta, dom, type = self.in_queue.get_nowait() # Get dns information from queue
self.check(meta, dom, type) # analyze it
except: # Nothing in the queue
time.sleep(0.5) # sleep .5 sec
# print 'Error: Queue Size = %s' %(k)
# check if the time has been > 15 minutes, if yes, update global_analysis
if self.settings["rainbow"]:
if (time.time() - self.curtime) >= self.settings["fusion_interval"]:
self.curtime += self.settings["fusion_interval"]
if self.settings["fusion"]:
print "[%s] [Hooves] Updating Rainbow Love!" % (time.ctime())
report = {
"type": "agent",
"agent": "Hooves",
"event": {"hist": self.hist, "new": self.events},
}
self.out_queue.put(report) # send update to Rainbow Love
self.events = {}
if self.settings["collection"]:
if (time.time() - self.t_time) > self.settings["collect_int"]:
print "[%s][Hooves] Printing History to Collection File: %s" % (
time.ctime(),
self.settings["collect_output_file"],
)
outfile = open(self.settings["collect_output_file"], "w+")
for i in self.hist:
outfile.write(i + "\r\n")
if not self.settings["collect_rep"]:
self.settings["collection"] = 0
else:
self.t_time = time.time()
else:
# Read in the test file
print "Reading test file: " + self.settings["test_input_file"]
try:
infile = open(self.settings["test_input_file"])
except:
print "Unable to open test_input_file, check configuration file: " + self.settings["test_input_file"]
sys.exit()
raw = infile.readlines()
infile.close()
# Analyze the domains in the test file
print "Analyzing & writing individual feature files"
for line in raw:
self.analyze(domaintools.tld_detect(line.rstrip()))
# Write results to file
print "Writing Results Files to: " + self.settings["test_output_dir"]
try:
outfile = open(self.settings["test_output_dir"] + "combined.csv", "w+")
except:
print "Unable to open test_output_file, check configuration file: " + self.settings["test_output_file"]
sys.exit()
for i in self.hist:
outfile.write(",".join(map(str, [i, self.hist[i]["score"]])) + "\r\n")
outfile.flush()
self.vow_out.flush()
self.num_out.flush()
self.tld_out.flush()
self.freq_out.flush()
outfile.close()
self.vow_out.close()
self.num_out.close()
self.tld_out.close()
self.freq_out.close()
popen("chmod 777 -R " + self.settings["test_output_dir"])
print "Finished Testing, view " + self.settings["test_output_dir"] + " to see results"
def start(self):
print '[%s] [Hooves] Running' % (time.ctime())
# get info from queue, and analyze the domain
if not self.settings['testing']:
# if testing is turned off, get info from live data
while 1:
try:
meta, dom, type = self.in_queue.get_nowait(
) # Get dns information from queue
self.check(meta, dom, type) # analyze it
except: # Nothing in the queue
time.sleep(.5) # sleep .5 sec
#print 'Error: Queue Size = %s' %(k)
# check if the time has been > 15 minutes, if yes, update global_analysis
if self.settings['rainbow']:
if (time.time() -
self.curtime) >= self.settings['fusion_interval']:
self.curtime += self.settings['fusion_interval']
if self.settings['fusion']:
print '[%s] [Hooves] Updating Rainbow Love!' % (
time.ctime())
report = {
'type': 'agent',
'agent': 'Hooves',
'event': {
'hist': self.hist,
'new': self.events
}
}
self.out_queue.put(
report) # send update to Rainbow Love
self.events = {}
if self.settings['collection']:
if (time.time() - self.t_time) > self.settings['collect_int']:
print '[%s][Hooves] Printing History to Collection File: %s' % (
time.ctime(), self.settings['collect_output_file'])
outfile = open(self.settings['collect_output_file'], 'w+')
for i in self.hist:
outfile.write(i + '\r\n')
if not self.settings['collect_rep']:
self.settings['collection'] = 0
else:
self.t_time = time.time()
else:
# Read in the test file
print 'Reading test file: ' + self.settings['test_input_file']
try:
infile = open(self.settings['test_input_file'])
except:
print 'Unable to open test_input_file, check configuration file: ' + self.settings[
'test_input_file']
sys.exit()
raw = infile.readlines()
infile.close()
# Analyze the domains in the test file
print 'Analyzing & writing individual feature files'
for line in raw:
self.analyze(domaintools.tld_detect(line.rstrip()))
# Write results to file
print 'Writing Results Files to: ' + self.settings[
'test_output_dir']
try:
outfile = open(
self.settings['test_output_dir'] + 'combined.csv', 'w+')
except:
print 'Unable to open test_output_file, check configuration file: ' + self.settings[
'test_output_file']
sys.exit()
for i in self.hist:
outfile.write(','.join(map(str, [i, self.hist[i]['score']])) +
'\r\n')
outfile.flush()
self.vow_out.flush()
self.num_out.flush()
self.tld_out.flush()
self.freq_out.flush()
outfile.close()
self.vow_out.close()
self.num_out.close()
self.tld_out.close()
self.freq_out.close()
popen('chmod 777 -R ' + self.settings['test_output_dir'])
print 'Finished Testing, view ' + self.settings[
'test_output_dir'] + ' to see results'
'event' :{'agent' :'known_domains','meta':meta, 'res':res, 'host':name[2]}}
if settings['fusion']:
self.out_queue.put(report)
doms[name[2]] = ''
except:
pass
elif data.qr == dns.DNS_R: # DNS Response analysis
ips,doms = {},{}
for i in xrange(len(data.an)): # cycle through answers
if data.an[i].cls == dns.DNS_IN:
if data.an[i].type == dns.DNS_A:
try: # if answers, answer(ip) and ttl analysis
name = domaintools.tld_detect(data.an[i].name)
if settings['known_domains'] == 'On':
res = self.blip(name[2])
else:
res = {'white':False,'hard':False}
if name:
ip, ttl = data.an[i].ip, data.an[i].ttl
if settings['dns_answer'] == 'On':
self.iaq.put([meta,data.an[i].name,ip,'R',ttl])
if not (res['white'] and res['hard']):
# known_domains ip support goes here
if settings['known_domains'] == 'On':
def analyze(self,meta,data): # Analyze the DNS packet, handle distribution to agents.
# Meta = [metadata list format]
# Data is a dpkt.dns object
try:
if data.opcode == dns.DNS_QUERY:
# operation is Query or Query Response
# DNS Query Analysis
if data.qr == dns.DNS_Q:
doms = {}
for i in xrange(len(data.qd)): # for domain in query list
try:
name = domaintools.tld_detect(data.qd[i].name))
if settings['blips'] == 'On':
res = self.blip(name[2])
else:
res = {'white':False,'hard':False}
if name:
if not (res['white'] and res['hard']):
# put [meta,name,'Q'] into derpy.hooves queue ('Q' = query)
if settings['hooves'] == 'On':
self.inq.put([meta,name,'Q'])
# put meta and domain into derpy.dash queue
if settings['dash'] == 'On':
self.icq.put([meta,name])
# Blips support goes here
if settings['blips'] == 'On' and name[2] not in doms:
report = { 'type' :'event','agent' :'blips',
'event' :{'agent' :'blips','meta':meta, 'res':res, 'host':name[2]}}
if settings['rainbow']: self.out_queue.put(report)
doms[name[2]] = ''
except: pass
# DNS Response analysis
elif data.qr == dns.DNS_R:
ips,doms = {},{}
for i in xrange(len(data.an)): # cycle through answers
if data.an[i].cls == dns.DNS_IN:
if data.an[i].type == dns.DNS_A:
# if answers, answer(ip) and ttl analysis
try:
name = domaintools.tld_detect(data.an[i].name)
if settings['blips'] == 'On':
res = self.blip(name[2])
else:
res = {'white':False,'hard':False}
if name:
ip, ttl = data.an[i].ip, data.an[i].ttl
if settings['jack'] == 'On':
self.iaq.put([meta,data.an[i].name,ip,'R',ttl])
if not (res['white'] and res['hard']):
# blips ip support goes here
if settings['blips'] == 'On':
ires = self.blipip(ip)
if not ires['white'] and ires['hard'] and ip not in ips:
ires['msg'] = 'Domain Resolution Contained Blacklisted IP Address'
report = { 'type' :'event','agent' :'blips',
'event' :{'agent' :'blips','meta':meta, 'res':ires, 'host':dec2ip(unpack('>I',ip)[0])}}
if settings['rainbow']: self.out_queue.put(report)
ips[ip] = ''
if settings['blips'] == 'On' and name[2] not in doms:
report = { 'type' :'event','agent' :'blips',
'event' :{'agent' :'blips','meta':meta, 'res':res, 'host':name[2]}}
if settings['rainbow']: self.out_queue.put(report)
doms[name[2]] = ''
except: pass
elif data.an[i].type == dns.DNS_CNAME:
# if this is in alias, answer(alias) analysis
try:
name = domaintools.tld_detect(data.an[i].name)
cname = domaintools.tld_detect(data.an[i].cname)
res = self.blip(cname[2])
if (name and cname) and (data.an[i].name != data.an[i].cname):
if settings['jack'] == 'On':
self.iaq.put([meta,data.an[i].name,data.an[i].cname,'A'])
if res['white'] and res1['hard']:
pass
else:
# [meta,cname,'A'] name analysis ('A'=alias)
if settings['hooves'] == 'On':
self.inq.put([meta,cname,'A'])
except: pass
