def store_validation(remote_ip, hours):
valdata = load_authorized_ips()
utc = dateutil.tz.tzutc()
now_time = datetime.datetime.now(utc).replace(microsecond=0)
expires = now_time + datetime.timedelta(hours=hours)
logger.info('Adding IP address %s until %s' % (remote_ip, expires.strftime('%c %Z')))
valdata[remote_ip] = {
'added': now_time.isoformat(sep=' '),
'expires': expires.isoformat(sep=' '),
}
# Try to lookup whois info if cymruwhois is available
try:
import cymruwhois
cym = cymruwhois.Client()
res = cym.lookup(remote_ip)
if res.owner and res.cc:
whois = "%s/%s\n" % (res.owner, res.cc)
valdata[remote_ip]['whois'] = whois
logger.info('Whois information for %s: %s' % (remote_ip, whois))
except:
pass
try:
geoip = get_geoip_crc(remote_ip)
if geoip is not None:
valdata[remote_ip]['geoip'] = geoip
logger.info('GeoIP information for %s: %s' % (remote_ip, geoip))
except:
pass
store_authorized_ips(valdata)
def asn_lookup(self, t_ip):
try:
asnClient = cymruwhois.Client()
r = asnClient.lookup(t_ip.value)
return {"number": r.asn, "name": r.owner}
except Exception, e:
print "[%s] ASN Lookup exception: %s" % (self.metadata.module_name,
e)
def test_common():
l = cymruwhois.Client()
places = [
['www.google.com', 'google'],
['www.yahoo.com', 'yahoo'],
['www.albany.edu', 'albany'],
]
for hostname, owner in places:
yield common_case, l, hostname, owner
def main(input):
ips = sorted(
set(ip_re.match(line).group(0) for line in input if ip_re.match(line)))
ips = filter(lambda ip: ip not in private_nets, ips)
public_nets = IpRangeList()
whois = cymruwhois.Client()
nets = (iprangelistappend(whois.lookup(ip), public_nets) for ip in ips
if ip not in public_nets)
for prefix, owner in nets:
print prefix, owner
def test_common():
l = cymruwhois.Client()
places = [
['www.google.com', 'google'],
['www.microsoft.com', 'microsoft'],
#['www.apple.com', 'apple'],
['www.albany.edu', 'albany'],
]
for hostname, owner in places:
yield common_case, l, hostname, owner
def test_normal():
l = cymruwhois.Client(memcache_host=None)
l.socket = FakeSocket()
l.file = FakeFile([
'22990 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany'
])
l._connected = True
rec = l.lookup("169.226.11.11")
assert rec.asn == '22990'
assert rec.cc == 'US'
assert rec.owner == 'ALBANYEDU - The University at Albany'
def store_validation(validated_ip, hours, session):
valdata = load_authorized_ips()
# Get rid of any previously validated IPs matching this one (or this range)
mynetwork = netaddr.IPNetwork(validated_ip)
for authorized_ip in valdata.keys():
if netaddr.IPNetwork(authorized_ip) in mynetwork:
del valdata[authorized_ip]
utc = dateutil.tz.tzutc()
now_time = datetime.datetime.now(utc).replace(microsecond=0)
expires = now_time + datetime.timedelta(hours=hours)
valdata[validated_ip] = {
'added': now_time.isoformat(sep=' '),
'expires': expires.isoformat(sep=' '),
}
# if val-session was used, we store the current XDG_SESSION_ID, if found
# otherwise it's effectively equivalent to 'val'
if session and 'XDG_SESSION_ID' in os.environ:
xdg_session_id = os.environ['XDG_SESSION_ID']
valdata[validated_ip]['sessionid'] = xdg_session_id
logger.info('Adding IP address %s with sessionid %s until %s' %
(validated_ip, xdg_session_id, expires.strftime('%c %Z')))
else:
logger.info('Adding IP address %s until %s' %
(validated_ip, expires.strftime('%c %Z')))
if mynetwork.size == 1:
# Try to lookup whois info if cymruwhois is available
try:
import cymruwhois
cym = cymruwhois.Client()
res = cym.lookup(validated_ip)
if res.owner and res.cc:
whois = "%s/%s\n" % (res.owner, res.cc)
valdata[validated_ip]['whois'] = whois
logger.info('Whois information for %s: %s' %
(validated_ip, whois))
except:
pass
try:
geoip = get_geoip_crc(validated_ip)
if geoip is not None:
valdata[validated_ip]['geoip'] = geoip
logger.info('GeoIP information for %s: %s' %
(validated_ip, geoip))
except:
pass
store_authorized_ips(valdata)
def get_destination_ip_details(self):
ulist = self.quick_unique(self._destination_ips)
c = cymruwhois.Client()
for item in c.lookupmany(ulist):
try:
if item.prefix == None:
tmp = { "ip_address": item.ip, "block": "", "asn": "", "owner": "" }
else:
tmp = { "ip_address": item.ip, "block": item.prefix, "asn": item.asn, "owner": item.owner }
self._destination_ip_details.append(tmp)
except Exception, e:
continue
def test_multiple_returned_for_a_single_ip():
l = cymruwhois.Client(memcache_host=None)
l.socket = FakeSocket()
l.file = FakeFile([
'22990 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany',
'22991 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany',
'15169 | 66.102.1.104 | 66.102.0.0/23 | US | GOOGLE - Google Inc.',
])
l._connected = True
rec = l.lookup("169.226.11.11")
assert rec.asn == '22990'
rec = l.lookup("66.102.1.104")
assert rec.asn == '15169'
def whoisrecord(ip):
try:
currenttime = time.time()
ts = currenttime
if ip in whois:
ASN, ts = whois[ip]
else:
ts = 0
if ((currenttime - ts) > 36000):
c = cymruwhois.Client()
ASN = c.lookup(ip)
whois[ip] = (ASN, currenttime)
return ASN
except Exception as e:
return e
def whois_lookup(ip):
try:
global whois_cache
currenttime = time.time()
ts = currenttime
if ip in whois_cache:
asn, ts = whois_cache[ip]
else:
ts = 0
if (currenttime - ts) > 36000:
c = cymruwhois.Client()
asn = c.lookup(ip)
whois_cache[ip] = (asn, currenttime)
return asn
except Exception as e:
return e
def store_validation(validated_ip, hours):
valdata = load_authorized_ips()
# Get rid of any previously validated IPs matching this one (or this range)
mynetwork = netaddr.IPNetwork(validated_ip)
for authorized_ip in valdata.keys():
if netaddr.IPNetwork(authorized_ip) in mynetwork:
del valdata[authorized_ip]
utc = dateutil.tz.tzutc()
now_time = datetime.datetime.now(utc).replace(microsecond=0)
expires = now_time + datetime.timedelta(hours=hours)
logger.info('Adding IP address %s until %s' %
(validated_ip, expires.strftime('%c %Z')))
valdata[validated_ip] = {
'added': now_time.isoformat(sep=' '),
'expires': expires.isoformat(sep=' '),
}
if mynetwork.size == 1:
# Try to lookup whois info if cymruwhois is available
try:
import cymruwhois
cym = cymruwhois.Client()
res = cym.lookup(validated_ip)
if res.owner and res.cc:
whois = "%s/%s\n" % (res.owner, res.cc)
valdata[validated_ip]['whois'] = whois
logger.info('Whois information for %s: %s' %
(validated_ip, whois))
except:
pass
try:
geoip = get_geoip_crc(validated_ip)
if geoip is not None:
valdata[validated_ip]['geoip'] = geoip
logger.info('GeoIP information for %s: %s' %
(validated_ip, geoip))
except:
pass
store_authorized_ips(valdata)
def asn_lookup(ip, whois_cache) -> (str, dict):
"""
Look up an ASN given teh IP address from cache. If not in cache, lookup from a whois server and update the cache
:param ip: IP Address (str)
:param whois_cache: whois data cache (dict)
:return: AS Number (str), Updated whois cache (dict)
"""
asn = None
try:
currenttime = time.time()
if ip in whois_cache:
asn, ts = whois_cache[ip]
else:
ts = 0
if (currenttime - ts) > 36000:
c = cymruwhois.Client()
asn = c.lookup(ip)
whois_cache[ip] = (asn, currenttime)
except Exception:
pass
return asn, whois_cache
def country_code(ip):
import cymruwhois
c = cymruwhois.Client()
code = c.lookup(str(ip)).cc
c.disconnect()
return code
def check(config, userid, ipaddr, hostname=None, daemon=None, sendmail=True):
if 'mailto' not in config.keys() or not len(config['mailto']):
sendmail = False
else:
from_addr = config['mailfrom']
to_addr = config['mailto']
# check if it's a user we should ignore
logger.info('Checking: userid=%s, ipaddr=%s' % (userid, ipaddr))
if 'ignoreusers' in config.keys():
ignoreusers = config['ignoreusers'].split(',')
for ignoreuser in ignoreusers:
if ignoreuser.strip() == userid:
logger.info('Quick out: %s in ignore list' % userid)
return None
if 'ignoreranges' in config.keys() and len(config['ignoreranges']):
import netaddr
na_ipaddr = netaddr.IPAddress(ipaddr)
for iprange in config['ignoreranges'].split('\n'):
# Most formats should be grokkable by netaddr
if na_ipaddr in netaddr.IPNetwork(iprange):
logger.info('Quick out: %s in ignored ranges (%s)' %
(ipaddr, iprange))
return None
# Check if the IP has changed since last login.
# the last_seen database is anydbm, because it's fast
last_seen = connect_last_seen(config['dbdir'])
prev_ipaddr = None
if userid in last_seen.keys():
prev_ipaddr = last_seen[userid]
if prev_ipaddr == ipaddr:
logger.info('Quick out: %s last seen from %s' % (userid, ipaddr))
return None
gi = connect_geoip(config['geoipcitydb'])
# Record the last_seen ip
last_seen[userid] = ipaddr
last_seen.close()
if 'mindistance' in config.keys() and prev_ipaddr is not None:
# calculate distance between previous and new ips
dist = get_distance_between_ips(gi, prev_ipaddr, ipaddr)
if dist is not None and dist < int(config['mindistance']):
logger.info('Distance between IPs less than %s km, ignoring' %
config['mindistance'])
return None
crc = get_geoip_crc(gi, ipaddr)
if crc is None:
logger.info('GeoIP City database did not return anything for %s' %
ipaddr)
return None
logger.info('Location: %s' % crc)
if 'ignorelocations' in config.keys() and len(config['ignorelocations']):
for entry in config['ignorelocations'].split('\n'):
if crc == entry.strip():
logger.info('Quick out: %s in ignored locations' % crc)
return None
# is it different from the last-seen location?
prev_crc = None
if prev_ipaddr is not None:
prev_crc = get_geoip_crc(gi, prev_ipaddr)
logger.info('Previous location: %s' % prev_crc)
sconn = connect_locations(config['dbdir'])
scursor = sconn.cursor()
if ('hop_detect' in config.keys() and config['hop_detect'] == 'yes'
and (prev_crc is None or prev_crc != crc)):
# Make sure hop_hours and hop_times is sane
if 'hop_hours' not in config.keys():
logger.warning('Please set hop_hours in howler.ini')
hop_hours = 12
else:
hop_hours = int(config['hop_hours'])
logger.debug('hop_hours = %s' % hop_hours)
if 'hop_times' not in config.keys():
logger.warning('Please set hop_times in howler.ini')
hop_times = 4
else:
hop_times = int(config['hop_times'])
logger.debug('hop_times = %s' % hop_times)
tsnow = int(datetime.datetime.utcnow().strftime('%s'))
logger.debug('Creating new entry in hopping')
query = """INSERT INTO hopping (userid, location, seen_ts)
VALUES (?, ?, ?)"""
scursor.execute(query, (userid, crc, tsnow))
logger.debug('Deleting obsolete entries in hopping')
tsold = tsnow - hop_hours * 3600
query = 'DELETE FROM hopping WHERE seen_ts < ?'
scursor.execute(query, (tsold, ))
# Find all entries for this user
query = """SELECT location, seen_ts
FROM hopping
WHERE reported = 0
AND userid = ?
ORDER BY seen_ts DESC"""
rows = scursor.execute(query, (userid, )).fetchall()
if len(rows) >= hop_times:
body = (u"Locations seen for %s in the past %s hours (UTC):\n\n" %
(userid, hop_hours))
query = """UPDATE hopping
SET reported = 1
WHERE userid = ?
AND location = ?
AND seen_ts = ?"""
for (seen_crc, seen_ts) in rows:
scursor.execute(query, (userid, seen_crc, seen_ts))
when = datetime.datetime.fromtimestamp(seen_ts).strftime('%T')
body += u"\t%s: %s\n" % (when, seen_crc)
logger.info('Alert message:\n%s' % body)
if sendmail:
subject = u'Hopping detected for user %s' % userid
send_email_alert(body, subject, from_addr, to_addr)
sconn.commit()
query = """SELECT location, last_seen
FROM locations
WHERE userid = ?
ORDER BY last_seen DESC"""
locations = []
for row in scursor.execute(query, (userid, )):
if row[0] == crc:
logger.info('This location already seen on %s' % row[1])
# Update last_seen
query = """UPDATE locations
SET last_seen = CURRENT_DATE
WHERE userid = ?
AND location = ?"""
scursor.execute(query, (userid, crc))
sconn.commit()
return None
locations.append(row)
# If we got here, this location either has not been seen before,
# or this is a new user that we haven't seen before. Either way, start
# by recording the new data.
query = """INSERT INTO locations (userid, location)
VALUES (?, ?)"""
scursor.execute(query, (userid, crc))
sconn.commit()
sconn.close()
if len(locations) == 0:
# New user. Finish up if we don't notify about new users
if config['alertnew'] != 'True':
logger.info('Quietly added new user %s' % userid)
return None
logger.info('Added new user %s' % userid)
else:
logger.info('Added new location for user %s' % userid)
body = (u"This user logged in from a new location:\n\n" +
u"\tUser : %s\n" % userid + u"\tIP Addr : %s\n" % ipaddr +
u"\tLocation: %s\n" % crc)
# Try to lookup whois info if cymruwhois is available
try:
import cymruwhois
cym = cymruwhois.Client()
res = cym.lookup(ipaddr)
if res.owner and res.cc:
body += u"\tIP Owner: %s/%s\n" % (res.owner, res.cc)
except:
pass
if hostname is not None:
body += u"\tHostname: %s\n" % hostname
if daemon is not None:
body += u"\tDaemon : %s\n" % daemon
if len(locations):
body += u"\nPreviously seen locations for this user:\n"
for row in locations:
body += u"\t%s: %s\n" % (row[1], row[0])
logger.info('Alert message:\n%s' % body)
if sendmail:
subject = u'New login by %s from %s' % (userid, crc)
send_email_alert(body, subject, from_addr, to_addr)
retval = {
'location': crc,
'previous': locations,
}
return retval
from settings import sql
import cymruwhois, time
cur = sql.cursor()
# get all ips where there is no asn
cur.execute("select ip from client where asn is Null")
ips = [x[0].split("/")[0] for x in cur.fetchall()]
cw = cymruwhois.Client(memcache_host=None)
success = False
while not success:
try:
lu = cw.lookupmany(ips)
success = True
except:
time.spleep(10)
for l in lu:
if l.asn:
cur.execute("select asn from asn where asn=%s" % l.asn)
r = cur.fetchone()
if not r:
cur.execute("insert into asn (asn,owner) values(%s,'%s')" %
(l.asn, l.owner))
cur.execute("update client set asn=%s where ip='%s' and asn is Null" %
(l.asn, l.ip))
sql.commit()
def lookup_countries(ips):
import cymruwhois
c = cymruwhois.Client()
ret = c.lookupmany_dict(ips)
c.disconnect()
return ret
continue
exit_fpr = dns_labels[0].lower()
exit_queries[exit_fpr] = (query, packet[scapy.UDP].sport, src_addr)
if len(packets) >= 2:
first, last = packets[0].time, packets[-1].time
log.info("Trace duration: %s" %
str(datetime.timedelta(seconds=last - first)))
return exit_queries
if __name__ == "__main__":
if len(sys.argv) != 2:
log.critical("Usage: %s PCAP_FILE" % sys.argv[0])
sys.exit(1)
pcap_file = sys.argv[1]
before = time.time()
exit_queries = parse_file(pcap_file)
log.info("Parsed file in %ss." % str(time.time() - before))
if len(exit_queries) == 0:
log.critical("Could not extract any queries from pcap.")
sys.exit(2)
analyse_queries(exit_queries, cymruwhois.Client())
sys.exit(0)
def test_asn():
l = cymruwhois.Client()
record = l.lookup("AS15169")
assert 'google' in record.owner.lower()
