def FSHandler(srcip):
"""
This function creates an object from the farsight script with the values
that are found in the config file. After the object is created, the object
does a request with the param to the farsight database. Next, it checks how
many company's are attached to 1 IP-address, if that is more than 5, it will
send the value "FastFlux" to the Redis database, else it will send the value
"Clean" to the database. Lastly it will get the longitude and latitude from
the scrip package and store these in Redis, which eventually, will be in MongoDB.
:param srcip: IP-address from where the dns package came from
"""
global ipTeller
request = fsReq.DnsdbClient(server=getSetting("dnsdb", "endpoint"),
apikey=getSetting("dnsdb", "key"))
geo = pygeoip.GeoIP("GeoIPASNum.dat")
ipTeller += 1
try:
for rrset in request.query_rdata_ip(srcip):
fsinfo = rrset.get("rdata")
answer = {"_id": ipTeller, "IP": fsinfo}
if r_serv.hget("ipID" + str(ipTeller), "IP") == fsinfo:
pass
else:
r_serv.hmset("ipID" + str(ipTeller), answer)
asnInfo = geo.org_by_addr(
r_serv.hget("ipID" + str(ipTeller), "IP"))
r_serv.hset("asnID" + str(ipTeller), "ASN", asnInfo)
if len(r_serv.hgetall("asnID" + str(ipTeller))) >= 5:
r_serv.hset("_id" + str(teller), "fs", "FastFlux")
else:
r_serv.hset("_id" + str(teller), "fs", "Clean")
except Exception:
pass
try:
global locID
gi = pygeoip.GeoIP('GeoLiteCity.dat')
locatie = gi.record_by_addr(srcip)
lon = str(locatie['longitude'])
lat = str(locatie['latitude'])
latlon = lat + ", " + lon
locID += 1
r_serv.hset("locID" + str(locID), "locID" + str(locID), latlon)
r_serv.hset("_id" + str(teller), "location",
r_serv.hget("locID" + str(locID), "locID" + str(locID)))
except:
pass
def winnow(in_file, out_file, enr_file):
config = ConfigParser.SafeConfigParser(allow_no_value=True)
base_path = os.path.dirname(__file__)
full_path = base_path + '/combine.cfg'
cfg_success = config.read(full_path)
if not cfg_success:
logger.error('Winnower: Could not read combine.cfg.')
logger.error('HINT: edit combine-example.cfg and save as combine.cfg.')
return
server = config.get('Winnower', 'dnsdb_server')
api = config.get('Winnower', 'dnsdb_api')
enrich_ip = config.get('Winnower', 'enrich_ip')
if enrich_ip == '1' or enrich_ip == 'True':
enrich_ip = True
logger.info('Enriching IPv4 indicators: TRUE')
else:
enrich_ip = False
logger.info('Enriching IPv4 indicators: FALSE')
enrich_dns = config.get('Winnower', 'enrich_dns')
if enrich_dns == '1' or enrich_dns == 'True':
enrich_dns = True
logger.info('Enriching DNS indicators: TRUE')
else:
enrich_dns = False
logger.info('Enriching DNS indicators: FALSE')
logger.info('Setting up DNSDB client')
# handle the case where we aren't using DNSDB
dnsdb = dnsdb_query.DnsdbClient(server, api)
if api == 'YOUR_API_KEY_HERE' or len(
dnsdb.query_rdata_name('google.com')) == 0:
dnsdb = None
logger.info('Invalid DNSDB configuration found')
with open(in_file, 'rb') as f:
crop = json.load(f)
# TODO: make these locations configurable?
logger.info('Loading GeoIP data')
gi_org = load_gi_org('data/GeoIPASNum2.csv')
wheat = []
enriched = []
logger.info('Beginning winnowing process')
for each in crop:
(addr, addr_type, direction, source, note, date) = each
# this should be refactored into appropriate functions
if addr_type == 'IPv4' and is_ipv4(addr):
#logger.info('Enriching %s' % addr)
ipaddr = IPAddress(addr)
if not reserved(ipaddr):
wheat.append(each)
if enrich_ip:
e_data = (addr, addr_type, direction, source, note,
date) + enrich_IPv4(ipaddr, dnsdb)
enriched.append(e_data)
else:
e_data = (addr, addr_type, direction, source, note,
date) + enrich_IPv4(ipaddr)
enriched.append(e_data)
else:
logger.error('Found invalid address: %s from: %s' %
(addr, source))
elif addr_type == 'FQDN' and is_fqdn(addr):
#logger.info('Enriching %s' % addr)
wheat.append(each)
if enrich_dns and dnsdb:
# print "Enriching %s" % addr
e_data = enrich_FQDN(addr, date, dnsdb)
if e_data:
for each in e_data:
datum = (each[0], "IPv4", direction, source, note,
date) + each[1:]
enriched.append(datum)
else:
logger.error(
'Could not determine address type for %s listed as %s' %
(addr, addr_type))
logger.info('Dumping results')
with open(out_file, 'wb') as f:
w_data = json.dumps(wheat, indent=2, ensure_ascii=False).encode('utf8')
f.write(w_data)
with open(enr_file, 'wb') as f:
e_data = json.dumps(enriched, indent=2,
ensure_ascii=False).encode('utf8')
f.write(e_data)
def winnow(in_file, out_file, enr_file):
config = ConfigParser.SafeConfigParser(allow_no_value=True)
cfg_success = config.read('combine.cfg')
if not cfg_success:
sys.stderr.write('Winnower: Could not read combine.cfg.\n')
sys.stderr.write(
'HINT: edit combine-example.cfg and save as combine.cfg.\n')
return
server = config.get('Winnower', 'dnsdb_server')
api = config.get('Winnower', 'dnsdb_api')
enrich_ip = config.get('Winnower', 'enrich_ip')
if enrich_ip == '1':
enrich_ip = True
sys.stderr.write('Enriching IPv4 indicators: TRUE\n')
else:
enrich_ip = False
sys.stderr.write('Enriching IPv4 indicators: FALSE\n')
enrich_dns = config.get('Winnower', 'enrich_dns')
if enrich_dns == '1':
enrich_dns = True
sys.stderr.write('Enriching DNS indicators: TRUE\n')
else:
enrich_dns = False
sys.stderr.write('Enriching DNS indicators: FALSE\n')
sys.stderr.write('Setting up DNSDB client\n')
dnsdb = dnsdb_query.DnsdbClient(server, api)
with open(in_file, 'rb') as f:
crop = json.load(f)
# TODO: make these locations configurable?
sys.stderr.write('Loading GeoIP data\n')
org_data = load_gi_org('data/GeoIPASNum2.csv')
geo_data = pygeoip.GeoIP('data/GeoIP.dat')
wheat = []
enriched = []
sys.stderr.write('Beginning winnowing process\n')
for each in crop:
(addr, addr_type, direction, source, note, date) = each
# TODO: enrich DNS indicators as well
if addr_type == 'IPv4':
sys.stderr.write('Enriching %s\n' % addr)
ipaddr = IPAddress(addr)
if not reserved(ipaddr):
wheat.append(each)
if enrich_ip:
e_data = (addr, addr_type, direction, source,
note, date) + enrich_IPv4(
ipaddr, org_data, geo_data, dnsdb)
enriched.append(e_data)
else:
e_data = (addr, addr_type, direction, source, note,
date) + enrich_IPv4(ipaddr, org_data, geo_data)
enriched.append(e_data)
else:
sys.stderr.write('Found invalid address: %s from: %s\n' %
(addr, source))
elif addr_type == 'FQDN':
# TODO: validate these (cf. https://github.com/mlsecproject/combine/issues/15 )
sys.stderr.write('Enriching %s\n' % addr)
wheat.append(each)
if enrich_dns:
e_data = (addr, addr_type, direction, source, note, date,
enrich_FQDN(addr, date, dnsdb))
enriched.append(e_data)
sys.stderr.write('Dumping results\n')
with open(out_file, 'wb') as f:
json.dump(wheat, f, indent=2)
with open(enr_file, 'wb') as f:
json.dump(enriched, f, indent=2)
def setUp(self):
super(DnsdbClientTestCase, self).setUp()
self.client = dnsdb_query.DnsdbClient(
DNSDB_SERVER,
API_KEY,
)