def store_validation(remote_ip, hours): valdata = load_authorized_ips() utc = dateutil.tz.tzutc() now_time = datetime.datetime.now(utc).replace(microsecond=0) expires = now_time + datetime.timedelta(hours=hours) logger.info('Adding IP address %s until %s' % (remote_ip, expires.strftime('%c %Z'))) valdata[remote_ip] = { 'added': now_time.isoformat(sep=' '), 'expires': expires.isoformat(sep=' '), } # Try to lookup whois info if cymruwhois is available try: import cymruwhois cym = cymruwhois.Client() res = cym.lookup(remote_ip) if res.owner and res.cc: whois = "%s/%s\n" % (res.owner, res.cc) valdata[remote_ip]['whois'] = whois logger.info('Whois information for %s: %s' % (remote_ip, whois)) except: pass try: geoip = get_geoip_crc(remote_ip) if geoip is not None: valdata[remote_ip]['geoip'] = geoip logger.info('GeoIP information for %s: %s' % (remote_ip, geoip)) except: pass store_authorized_ips(valdata)
def asn_lookup(self, t_ip): try: asnClient = cymruwhois.Client() r = asnClient.lookup(t_ip.value) return {"number": r.asn, "name": r.owner} except Exception, e: print "[%s] ASN Lookup exception: %s" % (self.metadata.module_name, e)
def test_common(): l = cymruwhois.Client() places = [ ['www.google.com', 'google'], ['www.yahoo.com', 'yahoo'], ['www.albany.edu', 'albany'], ] for hostname, owner in places: yield common_case, l, hostname, owner
def main(input): ips = sorted( set(ip_re.match(line).group(0) for line in input if ip_re.match(line))) ips = filter(lambda ip: ip not in private_nets, ips) public_nets = IpRangeList() whois = cymruwhois.Client() nets = (iprangelistappend(whois.lookup(ip), public_nets) for ip in ips if ip not in public_nets) for prefix, owner in nets: print prefix, owner
def test_common(): l = cymruwhois.Client() places = [ ['www.google.com', 'google'], ['www.microsoft.com', 'microsoft'], #['www.apple.com', 'apple'], ['www.albany.edu', 'albany'], ] for hostname, owner in places: yield common_case, l, hostname, owner
def test_normal(): l = cymruwhois.Client(memcache_host=None) l.socket = FakeSocket() l.file = FakeFile([ '22990 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany' ]) l._connected = True rec = l.lookup("169.226.11.11") assert rec.asn == '22990' assert rec.cc == 'US' assert rec.owner == 'ALBANYEDU - The University at Albany'
def store_validation(validated_ip, hours, session): valdata = load_authorized_ips() # Get rid of any previously validated IPs matching this one (or this range) mynetwork = netaddr.IPNetwork(validated_ip) for authorized_ip in valdata.keys(): if netaddr.IPNetwork(authorized_ip) in mynetwork: del valdata[authorized_ip] utc = dateutil.tz.tzutc() now_time = datetime.datetime.now(utc).replace(microsecond=0) expires = now_time + datetime.timedelta(hours=hours) valdata[validated_ip] = { 'added': now_time.isoformat(sep=' '), 'expires': expires.isoformat(sep=' '), } # if val-session was used, we store the current XDG_SESSION_ID, if found # otherwise it's effectively equivalent to 'val' if session and 'XDG_SESSION_ID' in os.environ: xdg_session_id = os.environ['XDG_SESSION_ID'] valdata[validated_ip]['sessionid'] = xdg_session_id logger.info('Adding IP address %s with sessionid %s until %s' % (validated_ip, xdg_session_id, expires.strftime('%c %Z'))) else: logger.info('Adding IP address %s until %s' % (validated_ip, expires.strftime('%c %Z'))) if mynetwork.size == 1: # Try to lookup whois info if cymruwhois is available try: import cymruwhois cym = cymruwhois.Client() res = cym.lookup(validated_ip) if res.owner and res.cc: whois = "%s/%s\n" % (res.owner, res.cc) valdata[validated_ip]['whois'] = whois logger.info('Whois information for %s: %s' % (validated_ip, whois)) except: pass try: geoip = get_geoip_crc(validated_ip) if geoip is not None: valdata[validated_ip]['geoip'] = geoip logger.info('GeoIP information for %s: %s' % (validated_ip, geoip)) except: pass store_authorized_ips(valdata)
def get_destination_ip_details(self): ulist = self.quick_unique(self._destination_ips) c = cymruwhois.Client() for item in c.lookupmany(ulist): try: if item.prefix == None: tmp = { "ip_address": item.ip, "block": "", "asn": "", "owner": "" } else: tmp = { "ip_address": item.ip, "block": item.prefix, "asn": item.asn, "owner": item.owner } self._destination_ip_details.append(tmp) except Exception, e: continue
def test_multiple_returned_for_a_single_ip(): l = cymruwhois.Client(memcache_host=None) l.socket = FakeSocket() l.file = FakeFile([ '22990 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany', '22991 | 169.226.11.11 | 169.226.0.0/16 | US | ALBANYEDU - The University at Albany', '15169 | 66.102.1.104 | 66.102.0.0/23 | US | GOOGLE - Google Inc.', ]) l._connected = True rec = l.lookup("169.226.11.11") assert rec.asn == '22990' rec = l.lookup("66.102.1.104") assert rec.asn == '15169'
def whoisrecord(ip): try: currenttime = time.time() ts = currenttime if ip in whois: ASN, ts = whois[ip] else: ts = 0 if ((currenttime - ts) > 36000): c = cymruwhois.Client() ASN = c.lookup(ip) whois[ip] = (ASN, currenttime) return ASN except Exception as e: return e
def whois_lookup(ip): try: global whois_cache currenttime = time.time() ts = currenttime if ip in whois_cache: asn, ts = whois_cache[ip] else: ts = 0 if (currenttime - ts) > 36000: c = cymruwhois.Client() asn = c.lookup(ip) whois_cache[ip] = (asn, currenttime) return asn except Exception as e: return e
def store_validation(validated_ip, hours): valdata = load_authorized_ips() # Get rid of any previously validated IPs matching this one (or this range) mynetwork = netaddr.IPNetwork(validated_ip) for authorized_ip in valdata.keys(): if netaddr.IPNetwork(authorized_ip) in mynetwork: del valdata[authorized_ip] utc = dateutil.tz.tzutc() now_time = datetime.datetime.now(utc).replace(microsecond=0) expires = now_time + datetime.timedelta(hours=hours) logger.info('Adding IP address %s until %s' % (validated_ip, expires.strftime('%c %Z'))) valdata[validated_ip] = { 'added': now_time.isoformat(sep=' '), 'expires': expires.isoformat(sep=' '), } if mynetwork.size == 1: # Try to lookup whois info if cymruwhois is available try: import cymruwhois cym = cymruwhois.Client() res = cym.lookup(validated_ip) if res.owner and res.cc: whois = "%s/%s\n" % (res.owner, res.cc) valdata[validated_ip]['whois'] = whois logger.info('Whois information for %s: %s' % (validated_ip, whois)) except: pass try: geoip = get_geoip_crc(validated_ip) if geoip is not None: valdata[validated_ip]['geoip'] = geoip logger.info('GeoIP information for %s: %s' % (validated_ip, geoip)) except: pass store_authorized_ips(valdata)
def asn_lookup(ip, whois_cache) -> (str, dict): """ Look up an ASN given teh IP address from cache. If not in cache, lookup from a whois server and update the cache :param ip: IP Address (str) :param whois_cache: whois data cache (dict) :return: AS Number (str), Updated whois cache (dict) """ asn = None try: currenttime = time.time() if ip in whois_cache: asn, ts = whois_cache[ip] else: ts = 0 if (currenttime - ts) > 36000: c = cymruwhois.Client() asn = c.lookup(ip) whois_cache[ip] = (asn, currenttime) except Exception: pass return asn, whois_cache
def country_code(ip): import cymruwhois c = cymruwhois.Client() code = c.lookup(str(ip)).cc c.disconnect() return code
def check(config, userid, ipaddr, hostname=None, daemon=None, sendmail=True): if 'mailto' not in config.keys() or not len(config['mailto']): sendmail = False else: from_addr = config['mailfrom'] to_addr = config['mailto'] # check if it's a user we should ignore logger.info('Checking: userid=%s, ipaddr=%s' % (userid, ipaddr)) if 'ignoreusers' in config.keys(): ignoreusers = config['ignoreusers'].split(',') for ignoreuser in ignoreusers: if ignoreuser.strip() == userid: logger.info('Quick out: %s in ignore list' % userid) return None if 'ignoreranges' in config.keys() and len(config['ignoreranges']): import netaddr na_ipaddr = netaddr.IPAddress(ipaddr) for iprange in config['ignoreranges'].split('\n'): # Most formats should be grokkable by netaddr if na_ipaddr in netaddr.IPNetwork(iprange): logger.info('Quick out: %s in ignored ranges (%s)' % (ipaddr, iprange)) return None # Check if the IP has changed since last login. # the last_seen database is anydbm, because it's fast last_seen = connect_last_seen(config['dbdir']) prev_ipaddr = None if userid in last_seen.keys(): prev_ipaddr = last_seen[userid] if prev_ipaddr == ipaddr: logger.info('Quick out: %s last seen from %s' % (userid, ipaddr)) return None gi = connect_geoip(config['geoipcitydb']) # Record the last_seen ip last_seen[userid] = ipaddr last_seen.close() if 'mindistance' in config.keys() and prev_ipaddr is not None: # calculate distance between previous and new ips dist = get_distance_between_ips(gi, prev_ipaddr, ipaddr) if dist is not None and dist < int(config['mindistance']): logger.info('Distance between IPs less than %s km, ignoring' % config['mindistance']) return None crc = get_geoip_crc(gi, ipaddr) if crc is None: logger.info('GeoIP City database did not return anything for %s' % ipaddr) return None logger.info('Location: %s' % crc) if 'ignorelocations' in config.keys() and len(config['ignorelocations']): for entry in config['ignorelocations'].split('\n'): if crc == entry.strip(): logger.info('Quick out: %s in ignored locations' % crc) return None # is it different from the last-seen location? prev_crc = None if prev_ipaddr is not None: prev_crc = get_geoip_crc(gi, prev_ipaddr) logger.info('Previous location: %s' % prev_crc) sconn = connect_locations(config['dbdir']) scursor = sconn.cursor() if ('hop_detect' in config.keys() and config['hop_detect'] == 'yes' and (prev_crc is None or prev_crc != crc)): # Make sure hop_hours and hop_times is sane if 'hop_hours' not in config.keys(): logger.warning('Please set hop_hours in howler.ini') hop_hours = 12 else: hop_hours = int(config['hop_hours']) logger.debug('hop_hours = %s' % hop_hours) if 'hop_times' not in config.keys(): logger.warning('Please set hop_times in howler.ini') hop_times = 4 else: hop_times = int(config['hop_times']) logger.debug('hop_times = %s' % hop_times) tsnow = int(datetime.datetime.utcnow().strftime('%s')) logger.debug('Creating new entry in hopping') query = """INSERT INTO hopping (userid, location, seen_ts) VALUES (?, ?, ?)""" scursor.execute(query, (userid, crc, tsnow)) logger.debug('Deleting obsolete entries in hopping') tsold = tsnow - hop_hours * 3600 query = 'DELETE FROM hopping WHERE seen_ts < ?' scursor.execute(query, (tsold, )) # Find all entries for this user query = """SELECT location, seen_ts FROM hopping WHERE reported = 0 AND userid = ? ORDER BY seen_ts DESC""" rows = scursor.execute(query, (userid, )).fetchall() if len(rows) >= hop_times: body = (u"Locations seen for %s in the past %s hours (UTC):\n\n" % (userid, hop_hours)) query = """UPDATE hopping SET reported = 1 WHERE userid = ? AND location = ? AND seen_ts = ?""" for (seen_crc, seen_ts) in rows: scursor.execute(query, (userid, seen_crc, seen_ts)) when = datetime.datetime.fromtimestamp(seen_ts).strftime('%T') body += u"\t%s: %s\n" % (when, seen_crc) logger.info('Alert message:\n%s' % body) if sendmail: subject = u'Hopping detected for user %s' % userid send_email_alert(body, subject, from_addr, to_addr) sconn.commit() query = """SELECT location, last_seen FROM locations WHERE userid = ? ORDER BY last_seen DESC""" locations = [] for row in scursor.execute(query, (userid, )): if row[0] == crc: logger.info('This location already seen on %s' % row[1]) # Update last_seen query = """UPDATE locations SET last_seen = CURRENT_DATE WHERE userid = ? AND location = ?""" scursor.execute(query, (userid, crc)) sconn.commit() return None locations.append(row) # If we got here, this location either has not been seen before, # or this is a new user that we haven't seen before. Either way, start # by recording the new data. query = """INSERT INTO locations (userid, location) VALUES (?, ?)""" scursor.execute(query, (userid, crc)) sconn.commit() sconn.close() if len(locations) == 0: # New user. Finish up if we don't notify about new users if config['alertnew'] != 'True': logger.info('Quietly added new user %s' % userid) return None logger.info('Added new user %s' % userid) else: logger.info('Added new location for user %s' % userid) body = (u"This user logged in from a new location:\n\n" + u"\tUser : %s\n" % userid + u"\tIP Addr : %s\n" % ipaddr + u"\tLocation: %s\n" % crc) # Try to lookup whois info if cymruwhois is available try: import cymruwhois cym = cymruwhois.Client() res = cym.lookup(ipaddr) if res.owner and res.cc: body += u"\tIP Owner: %s/%s\n" % (res.owner, res.cc) except: pass if hostname is not None: body += u"\tHostname: %s\n" % hostname if daemon is not None: body += u"\tDaemon : %s\n" % daemon if len(locations): body += u"\nPreviously seen locations for this user:\n" for row in locations: body += u"\t%s: %s\n" % (row[1], row[0]) logger.info('Alert message:\n%s' % body) if sendmail: subject = u'New login by %s from %s' % (userid, crc) send_email_alert(body, subject, from_addr, to_addr) retval = { 'location': crc, 'previous': locations, } return retval
from settings import sql import cymruwhois, time cur = sql.cursor() # get all ips where there is no asn cur.execute("select ip from client where asn is Null") ips = [x[0].split("/")[0] for x in cur.fetchall()] cw = cymruwhois.Client(memcache_host=None) success = False while not success: try: lu = cw.lookupmany(ips) success = True except: time.spleep(10) for l in lu: if l.asn: cur.execute("select asn from asn where asn=%s" % l.asn) r = cur.fetchone() if not r: cur.execute("insert into asn (asn,owner) values(%s,'%s')" % (l.asn, l.owner)) cur.execute("update client set asn=%s where ip='%s' and asn is Null" % (l.asn, l.ip)) sql.commit()
def lookup_countries(ips): import cymruwhois c = cymruwhois.Client() ret = c.lookupmany_dict(ips) c.disconnect() return ret
continue exit_fpr = dns_labels[0].lower() exit_queries[exit_fpr] = (query, packet[scapy.UDP].sport, src_addr) if len(packets) >= 2: first, last = packets[0].time, packets[-1].time log.info("Trace duration: %s" % str(datetime.timedelta(seconds=last - first))) return exit_queries if __name__ == "__main__": if len(sys.argv) != 2: log.critical("Usage: %s PCAP_FILE" % sys.argv[0]) sys.exit(1) pcap_file = sys.argv[1] before = time.time() exit_queries = parse_file(pcap_file) log.info("Parsed file in %ss." % str(time.time() - before)) if len(exit_queries) == 0: log.critical("Could not extract any queries from pcap.") sys.exit(2) analyse_queries(exit_queries, cymruwhois.Client()) sys.exit(0)
def test_asn(): l = cymruwhois.Client() record = l.lookup("AS15169") assert 'google' in record.owner.lower()