def get_geoip_crc(ipaddr):
import GeoIP
if os.path.exists(GEOIP_CITY_DB):
logger.debug("Opening geoip db in %s" % GEOIP_CITY_DB)
gi = GeoIP.open(GEOIP_CITY_DB, GeoIP.GEOIP_STANDARD)
else:
logger.debug("%s does not exist, using basic geoip db" % GEOIP_CITY_DB)
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
ginfo = gi.record_by_addr(ipaddr)
if ginfo is not None:
city = region_name = country_code = "Unknown"
if ginfo["city"] is not None:
city = unicode(ginfo["city"], "iso-8859-1")
if ginfo["region_name"] is not None:
region_name = unicode(ginfo["region_name"], "iso-8859-1")
if ginfo["country_code"] is not None:
country_code = unicode(ginfo["country_code"], "iso-8859-1")
crc = u"%s, %s, %s" % (city, region_name, country_code)
else:
# try just the country code, then
crc = gi.country_code_by_addr(ipaddr)
if not crc:
return None
crc = unicode(crc, "iso-8859-1")
return crc
def do_ips(self):
a = self.conn.cursor()
a.execute('select count(id) from registry_ip')
logger.debug("total hits %i" % a.fetchone()[0])
logger.debug("'SELECT ip FROM registry_ip group by ip' ...")
self.ips = self.conn.cursor()
self.ips.execute('SELECT ip FROM registry_ip group by ip')
result = self.ips.fetchall()
logger.debug("results = %i" % len(result))
for row in result:
ip = row[0]
if self.ip_map.has_key(ip):
logger.debug("that ip already exist in array '%s'" % ip)
continue
try:
GeoIP.gi = GeoIP.open(os.path.join(os.path.join(os.path.dirname(__file__), '../'), 'GeoLiteCity.dat').replace('\\','/'),GeoIP.GEOIP_STANDARD)
gir = GeoIP.gi.record_by_addr(ip)
GeoIP.gi = GeoIP.open(os.path.join(os.path.join(os.path.dirname(__file__), '../'), 'GeoIP.dat').replace('\\','/'),GeoIP.GEOIP_MEMORY_CACHE)
country = GeoIP.gi.country_name_by_addr(ip)
lat = gir["latitude"]
lon = gir["longitude"]
except:
lat = "null"
lon = "null"
country = None
self.IpPk += 1
self.ip_map[ip] = self.IpPk
self.hit_map[ip] = dict()
print """INSERT INTO registry_ip (ip, latitude, longitude, country) VALUES('%s',%s,%s,%s);""" % (ip, lat,lon, (country and "'%s'" % country or 'null'))
logger.debug("total ips %i" % self.IpPk)
def get_geoip_crc(config, ipaddr):
import GeoIP
# Open the GeoIP database and perform the lookup
if os.path.exists(config['geoipcitydb']):
gi = GeoIP.open(config['geoipcitydb'], GeoIP.GEOIP_STANDARD)
ginfo = gi.record_by_addr(ipaddr)
if not ginfo:
return None
city = region_name = country_code = 'Unknown'
if ginfo['city'] is not None:
city = unicode(ginfo['city'], 'iso-8859-1')
if ginfo['region_name'] is not None:
region_name = unicode(ginfo['region_name'], 'iso-8859-1')
if ginfo['country_code'] is not None:
country_code = unicode(ginfo['country_code'], 'iso-8859-1')
crc = u'%s, %s, %s' % (city, region_name, country_code)
else:
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
crc = unicode(gi.country_code_by_addr(ipaddr), 'iso-8859-1')
if not crc:
return None
return crc
def get_geoip_crc(ipaddr):
import GeoIP
if os.path.exists(GEOIP_CITY_DB):
logger.debug('Opening geoip db in %s' % GEOIP_CITY_DB)
gi = GeoIP.open(GEOIP_CITY_DB, GeoIP.GEOIP_STANDARD)
else:
logger.debug('%s does not exist, using basic geoip db' % GEOIP_CITY_DB)
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
ginfo = gi.record_by_addr(ipaddr)
if ginfo is not None:
city = region_name = country_code = 'Unknown'
if ginfo['city'] is not None:
city = unicode(ginfo['city'], 'iso-8859-1')
if ginfo['region_name'] is not None:
region_name = unicode(ginfo['region_name'], 'iso-8859-1')
if ginfo['country_code'] is not None:
country_code = unicode(ginfo['country_code'], 'iso-8859-1')
crc = u'%s, %s, %s' % (city, region_name, country_code)
else:
# try just the country code, then
crc = gi.country_code_by_addr(ipaddr)
if not crc:
return None
crc = unicode(crc, 'iso-8859-1')
return crc
def data(self):
returnObject = {};
tempObject = {};
_ip = self._parent._ip.__str__();
if(self._loaded == False):
returnObject["GEOIP"] = {};
return returnObject;
if(self._parent._ipversion == 4):
if(self._hasGeoIPDat):
gi = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD);
gic = GeoIP.open("/usr/share/GeoIP/GeoLiteCity.dat", GeoIP.GEOIP_STANDARD);
gia = GeoIP.open("/usr/share/GeoIP/GeoIPASNum.dat", GeoIP.GEOIP_STANDARD);
else:
if(self._hasGeoIPv6Dat):
gi = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD);
tempObject["country_code"] = gi.country_code_by_addr(_ip);
t = gic.record_by_addr(_ip);
if(isinstance(t, dict)):
tempObject.update(t);
t = gi.range_by_ip(_ip);
if(isinstance(t, tuple)):
tempObject["range_start"] = t[0];
tempObject["range_end"] = t[1];
returnObject["GEOIP"] = tempObject;
return returnObject;
def __init__(self, maxmind_geo_file=None, maxmind_asn_file=None):
self.maxmind_geo = None
self.maxmind_asn = None
if maxmind_geo_file:
self.maxmind_geo = GeoIP.open(maxmind_geo_file, GeoIP.GEOIP_STANDARD)
if maxmind_asn_file:
self.maxmind_asn = GeoIP.open(maxmind_asn_file, GeoIP.GEOIP_STANDARD)
def start(self):
import socket
self.hpc = None
self.gi = {}
self.gi[socket.AF_INET] = GeoIP.open("GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
self.gi[socket.AF_INET6] = GeoIP.open("GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
try:
self.hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
logger.error('[hpfeed] feed exception: %s'% e)
def open_geoip_databases():
global gipv4
global gipv6
try:
gipv4 = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD)
except:
gipv4=None
try:
gipv6 = GeoIP.open("/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD)
except:
gipv6=None
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def connect_geoip(dbpath):
global gi_dbconn
if gi_dbconn is None:
# Open the GeoIP database and store globally
if os.path.exists(dbpath):
logger.debug('Opening geoip db in %s' % dbpath)
gi_dbconn = GeoIP.open(dbpath, GeoIP.GEOIP_STANDARD)
else:
logger.debug('%s does not exist, using basic geoip db' % dbpath)
gi_dbconn = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
return gi_dbconn
def main():
# local and web lookup results
results_local_lookup = {}
results_web_lookup = {}
# read command line options
(opts, args) = parseOptions()
# parse file
if opts.infile:
try:
f = open(opts.infile, 'r')
except:
print "Unable to open input file '%s'" % opts.infile
# search local database
if opts.dolocal:
__dir__ = os.path.dirname(os.path.abspath(__file__))
geoip_lib = os.path.join(__dir__, 'GeoLiteCity.dat')
geoip = GeoIP.open(geoip_lib, GeoIP.GEOIP_MEMORY_CACHE)
# parse command line ip's
for ip in args:
try:
r = geoip.record_by_addr(ip)
results_local_lookup[ip] = (r['city'],
r['region'],
r['country_name'])
except:
results_local_lookup[ip] = None
# search web
if opts.doweb:
geoip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
for ip in args:
results_web_lookup[ip] = geoip.country_code_by_addr(ip)
# print the local results
if results_local_lookup:
results_local_lookups = sorted(results_local_lookup.iteritems(), key=lambda(k,v): k)
print "*** local results ***"
for result in results_local_lookups:
print result
# print the web results
if results_web_lookup:
# note: web lookup always returns "None" if not found
results_web_lookups = sorted(results_web_lookup.iteritems(), key=lambda(k,v): k)
print "*** web results ***"
for result in results_web_lookups:
print result
def test_dbs(opts, args):
"""test reference.dat and test.dat against a list of IPs and print any differences"""
ref_file, tst_file = args[:2]
gi_ref = pygeoip.GeoIP(ref_file, pygeoip.MEMORY_CACHE)
gi_tst = pygeoip.GeoIP(tst_file, pygeoip.MEMORY_CACHE)
dbtype = gi_ref._databaseType
if gi_ref._databaseType != gi_tst._databaseType:
print "error: database types don't match"
exit(1)
if opts.geoip:
import GeoIP
logging.debug('using GeoIP module')
gi_ref = GeoIP.open(ref_file, pygeoip.MEMORY_CACHE)
gi_tst = GeoIP.open(test_file, pygeoip.MEMORY_CACHE)
else:
logging.debug('using pygeoip module')
isequal = lambda lhs, rhs: lhs == rhs
if dbtype in (pygeoip.const.ASNUM_EDITION, pygeoip.const.ASNUM_EDITION_V6,
pygeoip.const.ISP_EDITION, pygeoip.const.ORG_EDITION):
get_ref = gi_ref.org_by_addr
get_tst = gi_tst.org_by_addr
elif dbtype in (pygeoip.const.CITY_EDITION_REV1, pygeoip.const.CITY_EDITION_REV1_V6):
get_ref = gi_ref.record_by_addr
get_tst = gi_tst.record_by_addr
def isequal(lhs, rhs):
if lhs and rhs:
#Python's float rounding makes these unpredictable,
#so just stomp them to ints as a sanity check.
for k in ('latitude', 'longitude'):
lhs[k] = int(lhs[k])
rhs[k] = int(rhs[k])
return lhs == rhs
elif dbtype in (pygeoip.const.COUNTRY_EDITION, pygeoip.const.COUNTRY_EDITION_V6):
get_ref = gi_ref.country_code_by_addr
get_tst = gi_tst.country_code_by_addr
else:
print "error: unknown database type"
exit(1)
ok = bad = 0
for ip in fileinput.input(args[2:]):
ip = ip.strip()
ref = get_ref(ip)
tst = get_tst(ip)
if not isequal(ref, tst):
print ip, ref, tst
bad += 1
else:
ok += 1
print 'ok:', ok, 'bad:', bad
def geoip_lookup(ipaddr):
try:
if IP(ipaddr).version() == 6:
gip = GeoIP.open(settings.GEOIP_IPV6_DB, GeoIP.GEOIP_MEMORY_CACHE)
country_code = gip.country_code_by_addr_v6(ipaddr).lower()
country_name = gip.country_name_by_name_v6(ipaddr)
else:
gip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
country_code = gip.country_code_by_addr(ipaddr).lower()
country_name = gip.country_name_by_addr(ipaddr)
return country_name, country_code
except (GeoIP.error, AttributeError, ValueError):
return ('', '')
def open_geoip_databases():
info = {'gipv4': None,
'gipv6': None}
try:
info['gipv4'] = GeoIP.open(
"/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_STANDARD)
except:
pass
try:
info['gipv6'] = GeoIP.open(
"/usr/share/GeoIP/GeoIPv6.dat", GeoIP.GEOIP_STANDARD)
except:
pass
return info
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
with open(sys.argv[2], 'r') as f:
dl = dict(line.strip().split(None, 1) for line in f)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def finish_init(self):
import GeoIP
if hasattr(self, '_t'):
self._t.join()
delattr(self, '_t')
parent = self._parent
geo_db_path = self.get_geodb_path()
Geo = GeoIP.open(geo_db_path, GeoIP.GEOIP_STANDARD)
self.geo = Geo
self.set_has_tooltip(True)
self._max_points = 200
self._lasttime = 0.0
self.context = None
self.mapcontext = None
self._mappixbuf = None
self._selected = []
self._current_text = ["", 0.0]
self._stats = [0,0,0,0,0,0,0]
self.width = self._pixbuf.get_width()
self.height = self._pixbuf.get_height()
self._min_x = 0
self._max_x = self.width
self._drawn_points = []
self._lines = []
self._frozenlines = []
# self.set_size_request(self.width, self.height)
self._points = []
self._crosses = []
self.connect("expose_event", self.expose)
self.connect("query-tooltip", self.on_query_tooltip)
if self.window:
self.window.invalidate_rect(self.allocation, True)
if not self._onattack:
self.add_test_points()
def onConnect(client_socket, client_addr):
print "Accepted connection from: ", client_addr
conn = MySQLdb.connect(DB_HOST, DB_USERNAME, DB_PASSWORD)
cursor = conn.cursor()
gi = GeoIP.open(GEO_IP_DATA_FILE,GeoIP.GEOIP_STANDARD)
data = ""
while 1:
# wait for something to come from the socket
data += client_socket.recv(BUFFER_SIZE)
# socket is broken, need to quit
if data == "":
break
# we assume everything is terminated by a newline so if one isn't there
# we got a fragment (that needs to be prepended to the stream next time
# around the loop)
is_fragment = data[-1] != "\n"
lines = data.split("\n")
up_to = len(lines)
if is_fragment:
up_to -= 1
for i in xrange(up_to):
matches = re.split(r'\s+', lines[i])
if len(matches) >= 2:
flag = matches[0]
if flag == "money":
site = matches[1]
amount = matches[2]
cursor.execute(MONEY_SQL, (site, amount))
conn.commit()
else:
ip = flag
site = matches[-1].strip('"')
site = re.sub(r'^www\.', '', site)
gir = gi.record_by_name(ip)
if gir == None:
continue
lat = gir['latitude']
lon = gir['longitude']
cursor.execute(MAP_SQL, (ip, site, lat, lon))
conn.commit()
if is_fragment:
# save the last line for the next round
data = lines[-1]
else:
data = ""
# keep the connection alive
conn.ping()
client_socket.close()
cursor.close()
conn.close()
print "Closed connection from ", client_addr
def get_nearest_mirrors(mirrors=MIRRORS):
ip_addr = getip()
print "Current user IP:", ip_addr
gic = GeoIP.open(city_data_path, 1)
record = gic.record_by_addr(ip_addr)
latitude = record['latitude']
longitude = record['longitude']
if not (latitude and longitude):
return [official_mirror.hostname]
distance_list = []
for mirror in mirrors:
if mirror.latitude and mirror.longitude:
distance = get_distance(latitude, longitude, mirror.latitude, mirror.longitude)
distance_list.append((distance, mirror.hostname))
distance_list_sorted = sorted(distance_list, key=lambda distance: distance[0])[:5]
return_list = []
for (index, t) in enumerate(distance_list_sorted):
if index < 5:
return_list.append(t[1])
else:
break
return return_list
def geoip_init(request):
_ = request.getText
# Find GeoIP
GEO_IP_PATH = getattr(request.cfg, 'gwiki_geoip_path', None)
error = ''
GEO_IP = None
if not GeoIP:
error = _("ERROR: GeoIP Python extensions not installed.")
elif not GEO_IP_PATH:
error = _("ERROR: GeoIP data file not found.")
elif not os.path.isfile(GEO_IP_PATH):
error = _("ERROR: GeoIP data file not found.")
elif not os.access(GEO_IP_PATH, os.R_OK):
error = _("ERROR: GeoIP data file not found.")
else:
GEO_IP = GeoIP.open(GEO_IP_PATH, GeoIP.GEOIP_STANDARD)
return GEO_IP, error
def get_ad_to_serve(self, cookie):
size = self.get_argument('size')
marker = self.get_argument('marker', None)
marker_found = False
if not marker:
marker_found = True
tag_on_page = 1 # tag_on_page is a bad name, but it signifies that it has not been passed back as a default from a network
else:
tag_on_page = 0
for ad in ads.find({"size":size, "state":"active", "deleted": {'$ne': True}}).sort("price", pymongo.DESCENDING):
if not marker_found:
if marker == str(ad['_id']):
marker_found = True
continue
frequency_for_ad = ad.get('frequency')
frequency_for_user = None
if cookie.get('creative'):
frequency_for_user = cookie.get('creative').get('%s' % (ad['_id']), None)
if int(frequency_for_ad) == 0 or frequency_for_user == None or int(frequency_for_user) < int(frequency_for_ad):
cookies.update({'uuid':self.uuid}, {'$inc': {'creative.%s' % (ad['_id']): 1}, '$set': {'ip': self.request.headers.get('X-Real-Ip', self.request.remote_ip)}})
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
country = gi.country_code_by_addr(self.request.headers.get('X-Real-Ip', self.request.remote_ip))
db.impressions.update({'ad_id':str(ad['_id']), 'country': country, 'date': datetime.date.today().isoformat(), 'tag_on_page': tag_on_page},
{
'$inc': {'view': 1},
'$set': {'ad_id':str(ad['_id']), 'country': country, 'date': datetime.date.today().isoformat(), 'tag_on_page': tag_on_page}
}, upsert=True)
return ad
def coords(ip, db='/usr/share/GeoIP/GeoIPCity.dat'):
ret = None
try:
import GeoIP
except:
logging.warning("GeoIP python module not installed")
return ret
try:
socket.inet_aton(ip)
ip_type = 4
except socket.error:
try:
socket.inet_pton(socket.AF_INET6, ip)
ip_type = 6
except socket.error:
logging.warning("Ip %s is not parseable" % ip)
return None
r=None
if ip_type == 4:
if not hasattr(coords, "gi"):
coords.gi = GeoIP.open(db, GeoIP.GEOIP_STANDARD |GeoIP.GEOIP_MEMORY_CACHE | GeoIP.GEOIP_CHECK_CACHE)
r = coords.gi.record_by_addr(ip)
elif ip_type == 6:
pass
# For the moment there's no useable open ipv6 city database
if r is not None:
ret= "%f, %f" % (round(float(r['latitude']),4), round(float(r['longitude']),4))
return ret
def scan(self):
'''
Função para efectuar o portscan
Grava na base de dados o IP
com a respectiva georeferencia
e os resultados obtidos.
Simultâneamente os mesmos são impressos na consola
'''
nm = nmap.PortScanner()
nm.scan(self.ip, self.port)
now = datetime.now()
for host in nm.all_hosts():
try:
gi = GeoIP.open('GeoLiteCity.dat', GeoIP.GEOIP_STANDARD)
geo = gi.record_by_addr(host)
country_=geo['country_code']
country_name_=geo['country_name']
lon_=geo['longitude']
lat_=geo['latitude']
except:
country_ = None
country_name_ = None
lon_ = None
lat_ = None
finally:
new_ip = IP(ip=host,
country=country_,
country_name=country_name_,
lon=lon_,
lat=lat_)
#check if ip address is in database
ip_address = self.session.query(
IP).filter_by(
ip=host).first()
if ip_address == None:
self.session.add(new_ip)
ip_address = new_ip
print('----------------------------------------------------')
print('Host : %s (%s)' % (host, nm[host].hostname()))
print('State : %s' % nm[host].state())
for proto in nm[host].all_protocols():
if proto in ['tcp', 'udp']:
print('----------')
print('Protocol : %s' % proto)
lport = nm[host][proto].keys()
lport.sort()
print "Ports:"
for p in lport:
print p
nmp = NmapScanDB(port=p,
time=now,
protocol=proto,
ip=ip_address)
self.session.add(nmp)
self.session.commit()
def google_maps_js(vpns, loc_lat, loc_long):
gi = GeoIP.open("/usr/share/GeoIP/GeoIPCity.dat", GeoIP.GEOIP_STANDARD)
sessions = 0
print "<script type=\"text/javascript\" src=\"https://maps.google.com/maps/api/js?sensor=true\"></script>"
print "<script type=\"text/javascript\">"
print "function initialize() {"
print "var bounds = new google.maps.LatLngBounds();"
print "var markers = new Array();"
for vkey, vpn in vpns:
if 'sessions' in vpn:
for skey, session in vpn['sessions'].items():
gir = gi.record_by_addr(session['remote_ip'])
if gir is not None:
print "var latlng = new google.maps.LatLng(%s, %s);" % (gir['latitude'], gir['longitude'])
print "bounds.extend(latlng);"
print "markers.push(new google.maps.Marker({position: latlng, title: \"%s\\n%s\"}));" % (session['username'], session['remote_ip'])
sessions = sessions + 1
if sessions != 0:
if sessions == 1:
print "bounds.extend(new google.maps.LatLng(%s, %s));" % (loc_lat, loc_long)
print "var myOptions = { zoom: 8, mapTypeId: google.maps.MapTypeId.ROADMAP };"
print "var map = new google.maps.Map(document.getElementById(\"map_canvas\"), myOptions);"
print "map.fitBounds(bounds);"
print "for ( var i=markers.length-1; i>=0; --i ) { markers[i].setMap(map); }"
print "}"
print "</script>"
else:
print "var latlng = new google.maps.LatLng(%s, %s);" % (loc_lat, loc_long)
print "var myOptions = { zoom: 8, center: latlng, mapTypeId: google.maps.MapTypeId.ROADMAP };"
print "var map = new google.maps.Map(document.getElementById(\"map_canvas\"), myOptions);"
print "}"
print "</script>"
def geo_country(request):
try:
ip = request.META['REMOTE_ADDR']
except KeyError:
ip = "0.0.0.0"
gi = GeoIP.new(GeoIP.GEOIP_STANDARD)
return gi.country_code_by_addr(ip).lower()
def load_db(db_name, db_fpath):
'''
Load a geolocation service.
:param db_name: name of the geolocation service.
:param db_fpath: location of its data file.
'''
# validate database name
if db_name not in _dbs.keys(): raise ValueError("Database must be one of: {}".format(_dbs.keys()))
# load appropriate database
if db_name == "geoip": _dbs[db_name] = GeoIP.open(db_fpath, GeoIP.GEOIP_STANDARD)
elif db_name == "ip2location": _dbs[db_name] = IP2Location.IP2Location(db_fpath)
elif db_name == "known_networks":
_dbs['known_networks'] = _Radix()
with open(db_fpath, 'rb') as known_net_file:
known_networks = _json.load(known_net_file)
for p in known_networks:
# Only keep prefices whose country we know
if 'country' in p:
n = _dbs['known_networks'].add(p['net'])
n.data['cc'] = p['country']
elif db_name == "dbip": _dbs[db_name] = _loadDBIP(db_fpath)
elif db_name == "ipligence": _dbs[db_name] = IPligence(db_fpath)
else: raise ValueError("error loading db {}".format(db_name))
# update table of supported databases
global available_dbs
available_dbs = [k for k,v in _dbs.items() if v is not None]
def init():
global geoipdb, countries, ignorepaths, goodpaths, ignoremissing, filters
# Load the database once and store it globally in interpreter memory.
geoipdb = GeoIP.open('%s/data/GeoIP.dat' %
basepath,GeoIP.GEOIP_STANDARD)
csvfile = open('%s/data/countrylist.csv' % basepath,'r')
dialect = csv.Sniffer().sniff(csvfile.read(32768))
csvfile.seek(0)
headers = csv.reader(csvfile, dialect=dialect).next()
reader = UnicodeDictReader(csvfile,
fieldnames=headers,
dialect=dialect)
countries={}
for line in reader:
countries[line['ISO 3166-1 2 Letter Code']]=line
csvfile.close()
fp=open('%s/data/%s/ignorepaths' % (basepath, sys.argv[1]),'r')
ignorepaths=[re.compile(x.strip()) for x in fp]
fp.close()
fp=open('%s/data/%s/ignoremissing' % (basepath, sys.argv[1]),'r')
ignoremissing=[re.compile(x.strip()) for x in fp]
fp.close()
fp=open('%s/data/%s/goodpaths' % (basepath, sys.argv[1]),'r')
goodpaths=[re.compile(r"%s" % x.strip()) for x in fp]
fp.close()
filters, queries=init_plugins(sys.argv[1])
def getgeoip(remote_addr): gi = GeoIP.open(GEOIP_DAT_PATH,GeoIP.GEOIP_STANDARD) gir = gi.record_by_addr(remote_addr) if not gir: msg = '??' else: msg = gir['country_code'] # print gir['country_code'] # print gir['country_code3'] # print gir['country_name'] # print gir['city'] # print gir['region'] # print gir['region_name'] # print gir['postal_code'] # print gir['latitude'] # print gir['longitude'] # print gir['area_code'] # print gir['time_zone'] # print gir['metro_code'] return msg
def open_or_die(self):
''' Open the database or die '''
if not GEOIP:
logging.error("Missing dependency: GeoIP")
logging.info("Please install GeoIP python wrappers, e.g.")
logging.info(" sudo apt-get install python-geoip")
sys.exit(1)
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
# Detect the common error case, i.e. that the user has
# not downloaded the database. If something fancy is
# going on, let the GeoIP library stacktrace for us.
#
if not os.path.exists(path):
logging.error("Missing GeoLiteCountry database: %s", path)
logging.info("Please download it from "
"<http://www.maxmind.com/app/geolitecountry>.")
sys.exit(1)
self.countries = GEOIP.open(path, GEOIP.GEOIP_STANDARD)
def origin_country(item_list, fid):
"""
Looks up the origin country of the provided IP address.
"""
gip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
output = []
# Splits the string in item_list on blank spaces and assumes that index[2] will contain an IP
# address. The index[1] will be checked against the GeoIP database to find the country of origin
# that IP address belongs to. The string will be reassembled with the country name appended to
# the end of the string before the string is appended to the output list and returned from the
# function.
if fid == "access":
for item in item_list:
item = item.split(" ")
geo = gip.country_code_by_addr(item[2])
out = item[0], item[1], item[3], item[4], item[2], geo
output.append(out)
# The item_list, when fid == origin, will only have IP addreses in it. The origin country of
# these IP addresses are checked agains the GeoIP database and the country of origin is appended
# to the output list and returned from the function.
if fid == "origin":
for item in item_list:
geo = gip.country_name_by_addr(item)
output.append(geo)
return output
def get_mirrors(request):
mirrors = Mirror.objects.all().order_by('id')
set_mirror = False
try:
try:
mirror_id = request.GET['mirror']
except (MultiValueDictKeyError, KeyError):
mirror_id = request.COOKIES['mirror']
set_mirror = True
except (MultiValueDictKeyError, KeyError):
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
address = request.META.get('REMOTE_ADDR')
if address[:7] == '::ffff:':
address = address[7:]
country = gi.country_code_by_addr(address)
if country in ['CZ', 'SK', 'DE', 'PL', 'AT', 'HU', 'RU', 'UA', 'BL']:
mirror_id = 'cihar-com'
# elif country in ['GB', 'US', 'FR', 'NL', 'CA', 'DK', 'SE', 'FI']:
# mirror_id = 'clickcreations-com'
else:
mirror_id = 'sf-net'
try:
mirror = Mirror.objects.get(slug = mirror_id)
except Mirror.DoesNotExist:
mirror = Mirror.objects.get(slug = 'cihar-com')
return (mirror, mirrors, set_mirror, mirror_id)
def get_country_by_name(name):
gi = GeoIP.open("./GeoIP.dat", GeoIP.GEOIP_STANDARD)
return gi.country_code_by_name(name)
def run(self):
while not self.kill_received:
try:
domain = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain['domain-name'] = domain['domain-name'].encode('idna').decode()
if self.option_extdns:
if self.nameservers:
resolv = dns.resolver.Resolver(configure=False)
resolv.nameservers = self.nameservers
else:
resolv = dns.resolver.Resolver()
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
nxdomain = False
dns_ns = False
dns_a = False
dns_aaaa = False
dns_mx = False
try:
domain['dns-ns'] = self.__answer_to_list(resolv.query(domain['domain-name'], rdtype=dns.rdatatype.NS))
dns_ns = True
except dns.resolver.NXDOMAIN:
nxdomain = True
pass
except dns.resolver.NoNameservers:
domain['dns-ns'] = ['!ServFail']
pass
except DNSException:
pass
if nxdomain is False:
try:
domain['dns-a'] = self.__answer_to_list(resolv.query(domain['domain-name'], rdtype=dns.rdatatype.A))
dns_a = True
except dns.resolver.NoNameservers:
domain['dns-a'] = ['!ServFail']
pass
except DNSException:
pass
try:
domain['dns-aaaa'] = self.__answer_to_list(resolv.query(domain['domain-name'], rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except dns.resolver.NoNameservers:
domain['dns-aaaa'] = ['!ServFail']
pass
except DNSException:
pass
if nxdomain is False and dns_ns is True:
try:
domain['dns-mx'] = self.__answer_to_list(resolv.query(domain['domain-name'], rdtype=dns.rdatatype.MX))
dns_mx = True
except dns.resolver.NoNameservers:
domain['dns-mx'] = ['!ServFail']
pass
except DNSException:
pass
else:
try:
ip = socket.getaddrinfo(domain['domain-name'], 80)
except socket.gaierror as e:
if e.errno == -3:
domain['dns-a'] = ['!ServFail']
pass
except Exception:
pass
else:
domain['dns-a'] = list()
domain['dns-aaaa'] = list()
for j in ip:
if '.' in j[4][0]:
domain['dns-a'].append(j[4][0])
if ':' in j[4][0]:
domain['dns-aaaa'].append(j[4][0])
domain['dns-a'] = sorted(domain['dns-a'])
domain['dns-aaaa'] = sorted(domain['dns-aaaa'])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain['domain-name'] != self.domain_init:
if self.__mxcheck(domain['dns-mx'][0], self.domain_init, domain['domain-name']):
domain['mx-spy'] = True
if self.option_geoip:
if dns_a is True:
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
try:
country = gi.country_name_by_addr(domain['dns-a'][0])
except Exception:
pass
else:
if country:
domain['geoip-country'] = country.split(',')[0]
if self.option_banners:
if dns_a is True:
banner = self.__banner_http(domain['dns-a'][0], domain['domain-name'])
if banner:
domain['banner-http'] = banner
if dns_mx is True:
banner = self.__banner_smtp(domain['dns-mx'][0])
if banner:
domain['banner-smtp'] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(self.uri_scheme + '://' + domain['domain-name'] + self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP, headers={'User-Agent': self.useragent}, verify=False)
except Exception:
pass
else:
if req.status_code // 100 == 2 and req.url.split('?')[0] != self.ssdeep_effective_url:
ssdeep_curr = ssdeep.hash(''.join(req.text.split()).lower())
domain['ssdeep-score'] = ssdeep.compare(self.ssdeep_init, ssdeep_curr)
domain['domain-name'] = domain['domain-name'].encode().decode('idna')
self.jobs.task_done()
def run(self):
while not self.kill_received:
domain = self.jobs.get()
if self.option_extdns:
resolv = dns.resolver.Resolver()
resolv.lifetime = REQUEST_TIMEOUT_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
try:
ans = resolv.query(domain['domain-name'], 'SOA')
domain['dns-ns'] = str(
sorted(ans)[0]).split(' ')[0][:-1].lower()
except Exception:
pass
if 'dns-ns' in domain:
try:
ans = resolv.query(domain['domain-name'], 'A')
domain['dns-a'] = str(sorted(ans)[0])
except Exception:
pass
try:
ans = resolv.query(domain['domain-name'], 'AAAA')
domain['dns-aaaa'] = str(sorted(ans)[0])
except Exception:
pass
try:
ans = resolv.query(domain['domain-name'], 'MX')
mx = str(sorted(ans)[0].exchange)[:-1].lower()
if mx: domain['dns-mx'] = mx
except Exception:
pass
else:
try:
ip = socket.getaddrinfo(domain['domain-name'], 80)
except Exception:
pass
else:
for j in ip:
if '.' in j[4][0]:
domain['dns-a'] = j[4][0]
break
for j in ip:
if ':' in j[4][0]:
domain['dns-aaaa'] = j[4][0]
break
if self.option_mxcheck:
if 'dns-mx' in domain:
if domain['domain-name'] is not self.domain_orig:
if self.__mxcheck(domain['dns-mx'], self.domain_orig,
domain['domain-name']):
domain['mx-spy'] = True
if self.option_whois:
if 'dns-ns' in domain or 'dns-a' in domain:
try:
whoisdb = whois.query(domain['domain-name'])
domain['whois-created'] = str(
whoisdb.creation_date).replace(' ', 'T')
domain['whois-updated'] = str(
whoisdb.last_updated).replace(' ', 'T')
except Exception:
pass
if self.option_geoip:
if 'dns-a' in domain:
gi = GeoIP.open(
FILE_GEOIP,
GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
try:
country = gi.country_name_by_addr(domain['dns-a'])
except Exception:
pass
else:
if country:
domain['geoip-country'] = country.split(',')[0]
if self.option_banners:
if 'dns-a' in domain:
banner = self.__banner_http(domain['dns-a'],
domain['domain-name'])
if banner:
domain['banner-http'] = banner
if 'dns-mx' in domain:
banner = self.__banner_smtp(domain['dns-mx'])
if banner:
domain['banner-smtp'] = banner
if self.option_ssdeep:
if 'dns-a' in domain:
try:
req = requests.get(
self.uri_scheme + '://' + domain['domain-name'] +
self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP,
headers={'User-Agent': 'Mozilla/5.0 (dnstwist)'})
#ssdeep_fuzz = ssdeep.hash(req.text.replace(' ', '').replace('\n', ''))
ssdeep_fuzz = ssdeep.hash(req.text)
except Exception:
pass
else:
if req.status_code / 100 == 2:
domain['ssdeep-score'] = ssdeep.compare(
self.ssdeep_orig, ssdeep_fuzz)
self.jobs.task_done()
def run(self):
while not self.kill_received:
try:
domain = self.jobs.get(block=False)
except queue.Empty:
self.kill_received = True
return
domain["domain-name"] = domain["domain-name"].encode(
"idna").decode()
if self.option_extdns:
if args["--nameservers"]:
resolv = dns.resolver.Resolver(configure=False)
resolv.nameservers = args["--nameservers"].split(",")
if args["--port"]:
resolv.port = args["--port"]
else:
resolv = dns.resolver.Resolver()
resolv.lifetime = REQUEST_TIMEOUT_DNS * REQUEST_RETRIES_DNS
resolv.timeout = REQUEST_TIMEOUT_DNS
nxdomain = False
dns_ns = False
dns_a = False
dns_aaaa = False
dns_mx = False
try:
domain["dns-ns"] = self.answer_to_list(
resolv.query(domain["domain-name"],
rdtype=dns.rdatatype.NS))
dns_ns = True
except dns.resolver.NXDOMAIN:
nxdomain = True
pass
except dns.resolver.NoNameservers:
domain["dns-ns"] = ["!ServFail"]
pass
except DNSException:
pass
if nxdomain is False:
try:
domain["dns-a"] = self.answer_to_list(
resolv.query(domain["domain-name"],
rdtype=dns.rdatatype.A))
dns_a = True
except dns.resolver.NoNameservers:
domain["dns-a"] = ["!ServFail"]
pass
except DNSException:
pass
try:
domain["dns-aaaa"] = self.answer_to_list(
resolv.query(domain["domain-name"],
rdtype=dns.rdatatype.AAAA))
dns_aaaa = True
except dns.resolver.NoNameservers:
domain["dns-aaaa"] = ["!ServFail"]
pass
except DNSException:
pass
if nxdomain is False and dns_ns is True:
try:
domain["dns-mx"] = self.answer_to_list(
resolv.query(domain["domain-name"],
rdtype=dns.rdatatype.MX))
dns_mx = True
except dns.resolver.NoNameservers:
domain["dns-mx"] = ["!ServFail"]
pass
except DNSException:
pass
else:
try:
ip = socket.getaddrinfo(domain["domain-name"], 80)
except socket.gaierror as e:
if e.errno == -3:
domain["dns-a"] = ["!ServFail"]
pass
except Exception:
pass
else:
domain["dns-a"] = list()
domain["dns-aaaa"] = list()
for j in ip:
if "." in j[4][0]:
domain["dns-a"].append(j[4][0])
if ":" in j[4][0]:
domain["dns-aaaa"].append(j[4][0])
domain["dns-a"] = sorted(domain["dns-a"])
domain["dns-aaaa"] = sorted(domain["dns-aaaa"])
dns_a = True
dns_aaaa = True
if self.option_mxcheck:
if dns_mx is True:
if domain["domain-name"] is not self.domain_orig:
if self.__mxcheck(domain["dns-mx"][0],
self.domain_orig,
domain["domain-name"]):
domain["mx-spy"] = True
if self.option_whois:
if nxdomain is False and dns_ns is True:
try:
whoisdb = whois.query(domain["domain-name"])
domain["whois-created"] = str(
whoisdb.creation_date).split(" ")[0]
domain["whois-updated"] = str(
whoisdb.last_updated).split(" ")[0]
except Exception:
pass
if self.option_geoip:
if dns_a is True:
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
try:
country = gi.country_name_by_addr(domain["dns-a"][0])
except Exception:
pass
else:
if country:
domain["geoip-country"] = country.split(",")[0]
if self.option_banners:
if dns_a is True or dns_aaaa is True:
banner = self.__banner_http(domain["dns-a"][0],
domain["domain-name"])
if banner:
domain["banner-http"] = banner
if dns_mx is True:
banner = self.__banner_smtp(domain["dns-mx"][0])
if banner:
domain["banner-smtp"] = banner
if self.option_ssdeep:
if dns_a is True or dns_aaaa is True:
try:
req = requests.get(
self.uri_scheme + "://" + domain["domain-name"] +
self.uri_path + self.uri_query,
timeout=REQUEST_TIMEOUT_HTTP,
headers={"User-Agent": args["--useragent"]},
verify=False,
)
# ssdeep_fuzz = ssdeep.hash(req.text.replace(' ', '').replace('\n', ''))
ssdeep_fuzz = ssdeep.hash(req.text)
except Exception:
pass
else:
if req.status_code // 100 == 2:
domain["ssdeep-score"] = ssdeep.compare(
self.ssdeep_orig, ssdeep_fuzz)
domain["domain-name"] = domain["domain-name"].encode().decode(
"idna")
self.jobs.task_done()
def __init__(self, **kwargs):
self.gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
self.countries = {}
#!/usr/bin/python
from __future__ import print_function
import GeoIP
gi = GeoIP.open("/usr/local/share/GeoIP/GeoIPDomain.dat", GeoIP.GEOIP_STANDARD)
print(gi.org_by_addr("24.24.24.24"))
else:
paths = ("/usr/local/share/GeoIP", "/usr/share/GeoIP")
for path in paths:
for datafile in datafiles:
filename = os.path.join(path, datafile)
if os.path.isfile(filename):
return filename
# try importing both the C-library GeoIP and the pure-python pygeoip
geoip_dat = get_geoip_dat()
geoip = None
if geoip_dat:
try:
import GeoIP
geoip = GeoIP.open(geoip_dat, GeoIP.GEOIP_STANDARD)
geoip_error = GeoIP.error
except ImportError:
try:
import pygeoip
geoip = pygeoip.GeoIP(geoip_dat)
geoip_error = pygeoip.GeoIPError
except ImportError:
pass
if geoip_dat.endswith('GeoIPCity.dat'):
get_geoip_record = lambda host: geoip.record_by_name(host)
else:
get_geoip_record = lambda host: {
'country_name': geoip.country_name_by_name(host)
}
class Address:
"""Represents a single IP address."""
GI = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
__instances = {}
def __init__(self, address):
self.__address = address
self.__attempts = []
self.__country = Country.for_name(
Address.GI.country_name_by_addr(address))
self.__country.add_address(self)
def __repr__(self):
return self.__str__()
def __str__(self):
return self.__address
@property
def total_attempts(self):
return len(self.__attempts)
def total_attempts_for_date(self, date):
# return sum(filter(lambda a: a.when_as_date == date, self.__attempts))
result = 0
for attempt in self.__attempts:
if attempt.when_as_monthstamp == date:
result = result + 1
return result
@classmethod
def for_source(cls, address):
"""
Returns an instance for the specified class. If no such instance
exists then it creates a new one and returns it.
"""
if cls.__instances.has_key(address):
result = cls.__instances[address]
else:
result = Address(address)
cls.__instances[address] = result
return result
def add_attempt(self, date, port):
attempt = AccessAttempt(port, date)
if not attempt in self.__attempts:
self.__attempts.append(attempt)
return True
return False
@property
def number_of_attempts(self):
return len(self.__attempts)
@property
def attempts(self):
return self.__attempts
@property
def attempts_sorted_by_date(self):
return sorted(self.__attempts, key=lambda x: (x.port, x.when))
@property
def address(self):
return self.__address
@property
def country(self):
return self.__country
#!/usr/bin/python
# This example is only for Python 2. In Python 3, all returned strings are
# unicode.
import GeoIP
# open the citydatabase. All cities return in iso-8859-1 by default
gi = GeoIP.open("/usr/local/share/GeoIP/GeoIPCity.dat", GeoIP.GEOIP_STANDARD)
# lookup a record, where cityname contains chars > 127 ( eg != ascii )
gir = gi.record_by_name("www.osnabrueck.de")
# print the cityname with iso-8859-1 charset
print gir['city']
# print the cityname transformed to utf8
print unicode(gir['city'], 'iso-8859-1')
# however, at your option GeoIP can return the cityname in utf8
# just put GeoIP into utf8 mode
#
# from now on all records returns in UTF8 until you change the charset again
# Notice, that all previous records return in the previous charset
gi.set_charset(GeoIP.GEOIP_CHARSET_UTF8)
# get a new record, now in utf8
gir2 = gi.record_by_name("www.osnabrueck.de")
# and print it ( should be the same output as on line 2 )
print gir2['city']
def save_session(self,
cr,
uid,
tz,
sid,
unsuccessful_message='',
context=None):
now = fields.datetime.now()
session_obj = request.registry.get('ir.sessions')
cr = request.registry.cursor()
# Get IP, check if it's behind a proxy
ip = request.httprequest.headers.environ['REMOTE_ADDR']
forwarded_for = ''
if 'HTTP_X_FORWARDED_FOR' in request.httprequest.headers.environ and request.httprequest.headers.environ[
'HTTP_X_FORWARDED_FOR']:
forwarded_for = request.httprequest.headers.environ[
'HTTP_X_FORWARDED_FOR'].split(', ')
if forwarded_for and forwarded_for[0]:
ip = forwarded_for[0]
# for GeoIP
geo_ip_resolver = None
ip_location = ''
try:
import GeoIP
geo_ip_resolver = GeoIP.open('/usr/share/GeoIP/GeoIP.dat',
GeoIP.GEOIP_STANDARD)
except ImportError:
geo_ip_resolver = False
if geo_ip_resolver:
ip_location = (str(geo_ip_resolver.country_name_by_addr(ip)) or '')
# autocommit: our single update request will be performed atomically.
# (In this way, there is no opportunity to have two transactions
# interleaving their cr.execute()..cr.commit() calls and have one
# of them rolled back due to a concurrent access.)
cr.autocommit(True)
user = request.registry.get('res.users').browse(
cr, request.uid, uid, request.context)
logged_in = True
if unsuccessful_message:
uid = SUPERUSER_ID
logged_in = False
sessions = False
else:
sessions = session_obj.search(cr,
uid, [('session_id', '=', sid),
('ip', '=', ip),
('user_id', '=', uid),
('logged_in', '=', True)],
context=context)
if not sessions:
values = {
'user_id':
uid,
'logged_in':
logged_in,
'session_id':
sid,
'session_seconds':
user.session_default_seconds,
'multiple_sessions_block':
user.multiple_sessions_block,
'date_login':
now,
'expiration_date':
datetime.strftime(
(datetime.strptime(now, DEFAULT_SERVER_DATETIME_FORMAT) +
relativedelta(seconds=user.session_default_seconds)),
DEFAULT_SERVER_DATETIME_FORMAT),
'ip':
ip,
'ip_location':
ip_location,
'remote_tz':
tz or 'GMT',
'unsuccessful_message':
unsuccessful_message,
}
session_obj.create(cr, uid, values, context=context)
cr.commit()
cr.close()
return True
for train, test in kf.split(X):
X_train, X_test, Y_train, Y_test = data_set[train], data_set[
test], labels[train], labels[test]
print("Training...")
clf.fit(X_train, Y_train)
print("Training finished")
print("Starting the test...")
print("Score on test set: {0}\n".format(clf.score(X_test, Y_test)))
# ------------------------------ Main ------------------------------
start_time = time.time()
# gi = GeoIP.open("/usr/share/GeoIP/GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
gi_asn = GeoIP.open("/usr/share/GeoIP/GeoIPASNum.dat", GeoIP.GEOIP_STANDARD)
n_col = 2
dns_scan = sys.argv[1]
dns_ip = read_data(dns_scan)
not_dns_scan = sys.argv[2]
not_dns_ip = read_data(not_dns_scan)
n_row_regulars = int(sys.argv[3])
n_row_outliers = int(sys.argv[4])
check_params(n_row_regulars, n_row_outliers, len(dns_ip), len(not_dns_ip))
clf_name = sys.argv[5]
clf = set_classifier(clf_name)
def __init__(self): self.geo = GeoIP.open(Geo.GEO_PATH, GeoIP.GEOIP_STANDARD)
def __init__(self, epd):
self.epd = epd
self.menu_font = '/usr/share/fonts/truetype/freefont/FreeSans.ttf'
# TODO: Move these out into a separate file
self.possible_fonts = [{
'filename':
'/usr/share/fonts/truetype/fonts-georgewilliams/CaslonBold.ttf',
'title': 'Caslon Bold'
}, {
'filename':
'/usr/share/fonts/truetype/fonts-georgewilliams/Caslon-Black.ttf',
'title': 'Caslon Black'
}, {
'filename':
'/usr/share/fonts/truetype/fonts-georgewilliams/Caliban.ttf',
'title': 'Caliban'
}, {
'filename':
'/usr/share/fonts/truetype/fonts-georgewilliams/Cupola.ttf',
'title': 'Cupola'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeSerif.ttf',
'title': 'Serif'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeSerifBold.ttf',
'title': 'Serif Bold'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeMono.ttf',
'title': 'Monospace'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeMonoBold.ttf',
'title': 'Monospace Bold'
}, {
'filename':
'/usr/share/fonts/truetype/freefont/FreeMonoOblique.ttf',
'title': 'Monospace Oblique'
}, {
'filename':
'/usr/share/fonts/truetype/freefont/FreeMonoBoldOblique.ttf',
'title': 'Monospace Bold Oblique'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeSans.ttf',
'title': 'Sans-Serif'
}, {
'filename': '/usr/share/fonts/truetype/freefont/FreeSansBold.ttf',
'title': 'Sans-Serif Bold'
}, {
'filename':
'/usr/share/fonts/truetype/freefont/FreeSansOblique.ttf',
'title': 'Sans-Serif Oblique'
}, {
'filename':
'/usr/share/fonts/truetype/freefont/FreeSansBoldOblique.ttf',
'title': 'Sans-Serif Bold Oblique'
}, {
'filename': '/usr/share/fonts/truetype/humor-sans/Humor-Sans.ttf',
'title': 'Humor'
}, {
'filename': '/usr/share/fonts/truetype/dejavu/DejaVuSerif.ttf',
'title': 'DejaVu'
}, {
'filename':
'/usr/share/fonts/truetype/dejavu/DejaVuSerif-Bold.ttf',
'title': 'DejaVu Bold'
}, {
'filename': '/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf',
'title': 'DejaVu Monospace'
}, {
'filename':
'/usr/share/fonts/truetype/dejavu/DejaVuSansMono-Bold.ttf',
'title': 'DejaVu Monospace Bold'
}, {
'filename': '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf',
'title': 'DejaVu Sans'
}, {
'filename': '/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf',
'title': 'DejaVu Sans Bold'
}, {
'filename': '/usr/share/fonts/truetype/dustin/Dustismo.ttf',
'title': 'Dustismo'
}, {
'filename': '/usr/share/fonts/truetype/dustin/dustismo_bold.ttf',
'title': 'Dustismo Bold'
}, {
'filename': '/usr/share/fonts/truetype/dustin/dustismo_italic.ttf',
'title': 'Dustismo Italic'
}, {
'filename': '/usr/share/fonts/truetype/dustin/Dustismo_Roman.ttf',
'title': 'Dustismo Roman'
}, {
'filename':
'/usr/share/fonts/truetype/dustin/Dustismo_Roman_Bold.ttf',
'title': 'Dustismo Roman Bold'
}, {
'filename':
'/usr/share/fonts/truetype/dustin/Dustismo_Roman_Italic.ttf',
'title': 'Dustismo Roman Italic'
}, {
'filename':
'/usr/share/fonts/truetype/dustin/Domestic_Manners.ttf',
'title': 'Domestic Manners'
}, {
'filename': '/usr/share/fonts/truetype/dustin/Junkyard.ttf',
'title': 'Junkyard'
}, {
'filename': '/usr/share/fonts/truetype/dustin/Wargames.ttf',
'title': 'Wargames'
}, {
'filename': '/usr/share/fonts/truetype/dustin/PenguinAttack.ttf',
'title': 'Penguin Attack'
}, {
'filename': '/usr/share/fonts/truetype/dustin/It_wasn_t_me.ttf',
'title': 'It wasn\'t me!'
}]
self.possible_tones = [
{
'filename': '/root/tones/Blues.mp3',
'title': 'Blues'
},
{
'filename': '/root/tones/Piano Riff.mp3',
'title': 'Piano Riff'
},
{
'filename': '/root/tones/Sci-Fi.mp3',
'title': 'Sci-Fi'
},
{
'filename': '/root/tones/Pinball.mp3',
'title': 'Pinball'
},
{
'filename': '/root/tones/Crickets.mp3',
'title': 'Crickets'
},
{
'filename': '/root/tones/Motorcycle.mp3',
'title': 'Motorcycle'
},
{
'filename': '/root/tones/Timba.mp3',
'title': 'Timba'
},
{
'filename': '/root/tones/Bark.mp3',
'title': 'Bark'
},
{
'filename': '/root/tones/Trill.mp3',
'title': 'Trill'
},
{
'filename': '/root/tones/Robot.mp3',
'title': 'Robot'
},
{
'filename': '/root/tones/Old Phone.mp3',
'title': 'Old Phone'
},
{
'filename': '/root/tones/Marimba.mp3',
'title': 'Marimba'
},
{
'filename': '/root/tones/Boing.mp3',
'title': 'Boing'
},
{
'filename': '/root/tones/Strum.mp3',
'title': 'Strum'
},
{
'filename': '/root/tones/Xylophone.mp3',
'title': 'Xylophone'
},
{
'filename': '/root/tones/Digital.mp3',
'title': 'Digital'
},
{
'filename': '/root/tones/Time Passing.mp3',
'title': 'Time Passing'
},
{
'filename': '/root/tones/Harp.mp3',
'title': 'Harp'
},
{
'filename': '/root/tones/Bell Tower.mp3',
'title': 'Bell Tower'
},
{
'filename': '/root/tones/Alarm.mp3',
'title': 'Alarm'
},
{
'filename': '/root/tones/Old Car Horn.mp3',
'title': 'Old Car Horn'
},
{
'filename': '/root/tones/Doorbell.mp3',
'title': 'Doorbell'
},
{
'filename': '/root/tones/Sonar.mp3',
'title': 'Sonar'
},
{
'filename': '/root/tones/Ascending.mp3',
'title': 'Ascending'
},
{
'filename': '/root/tones/Duck.mp3',
'title': 'Duck'
},
]
try:
with open("/sys/class/gpio/unexport", "w") as unexport:
unexport.write("133\n")
except IOError:
pass
GPIO.setup("CSID1", GPIO.OUT)
GPIO.output("CSID1", GPIO.LOW)
self.mode = "weather"
self.timezones = country_timezones('US')
self.timezone_index = 0
self.stop_alarming()
atexit.register(self.stop_alarming)
self.snooze = 0
self.lock = threading.Lock() # create lock for rotary switch
self.pips = 0
self.presses = 0
self.re = RotaryEncoder([0x11], invert=False, factor=0.5)
self.re.register_callbacks(turn=self.turn_cb, press=self.press_cb)
try:
ip = json.load(urlopen('http://jsonip.com'))['ip']
except Exception:
try:
ip = json.load(
urlopen('https://api.ipify.org/?format=json'))['ip']
except Exception:
raise
g = GeoIP.open("/usr/share/GeoIP/GeoIPCity.dat", GeoIP.GEOIP_STANDARD)
try:
gr = g.record_by_addr(ip)
print gr
self.latitude = gr['latitude']
self.longitude = gr['longitude']
while self.timezones[self.timezone_index] != gr['time_zone']:
self.timezone_index += 1
except Exception:
raise
try:
with open('/root/alarmclock-settings.pickle', 'rb') as settings:
self.settings = pickle.load(settings)
except Exception as e:
print "Failed to load settings; using defaults"
print e
self.settings = {}
self.settings['twentyfour'] = False
self.settings['alarm'] = False
self.settings['font_index'] = 21
self.settings['tone_index'] = 0
self.settings['alarm_time'] = 360 # Minutes from midnight
try:
with open('/root/darksky.key', 'r') as f:
self.darksky_key = f.readline().strip()
self.weather = True
except Exception as e:
print "Couldn't get key from /root/darksky.key: " + str(e)
self.weather = False
if self.weather:
self._update_weather()
def _dispatch(self):
first_pass = not hasattr(request, 'website')
request.website = None
func = None
try:
func, arguments = self._find_handler()
request.website_enabled = func.routing.get('website', False)
except werkzeug.exceptions.NotFound:
# either we have a language prefixed route, either a real 404
# in all cases, website processes them
request.website_enabled = True
request.website_multilang = request.website_enabled and func and func.routing.get(
'multilang', True)
if 'geoip' not in request.session:
record = {}
if self.geo_ip_resolver is None:
try:
import GeoIP
# updated database can be downloaded on MaxMind website
# http://dev.maxmind.com/geoip/legacy/install/city/
geofile = config.get('geoip_database')
if os.path.exists(geofile):
self.geo_ip_resolver = GeoIP.open(
geofile, GeoIP.GEOIP_STANDARD)
else:
self.geo_ip_resolver = False
logger.warning(
'GeoIP database file %r does not exists', geofile)
except ImportError:
self.geo_ip_resolver = False
if self.geo_ip_resolver and request.httprequest.remote_addr:
record = self.geo_ip_resolver.record_by_addr(
request.httprequest.remote_addr) or {}
request.session['geoip'] = record
if request.website_enabled:
try:
if func:
self._authenticate(func.routing['auth'])
else:
self._auth_method_public()
except Exception as e:
return self._handle_exception(e)
request.redirect = lambda url, code=302: werkzeug.utils.redirect(
url_for(url), code)
request.website = request.registry['website'].get_current_website(
request.cr, request.uid, context=request.context)
langs = [lg[0] for lg in request.website.get_languages()]
path = request.httprequest.path.split('/')
if first_pass:
if request.website_multilang:
# If the url doesn't contains the lang and that it's the first connection, we to retreive the user preference if it exists.
if not path[
1] in langs and not request.httprequest.cookies.get(
'session_id'):
if request.lang not in langs:
# Try to find a similar lang. Eg: fr_BE and fr_FR
short = request.lang.split('_')[0]
langs_withshort = [
lg[0]
for lg in request.website.get_languages()
if lg[0].startswith(short)
]
if len(langs_withshort):
request.lang = langs_withshort[0]
else:
request.lang = request.website.default_lang_code
# We redirect with the right language in url
if request.lang != request.website.default_lang_code:
path.insert(1, request.lang)
path = '/'.join(path) or '/'
return request.redirect(
path + '?' + request.httprequest.query_string)
else:
request.lang = request.website.default_lang_code
request.context['lang'] = request.lang
if not func:
if path[1] in langs:
request.lang = request.context['lang'] = path.pop(1)
path = '/'.join(path) or '/'
if request.lang == request.website.default_lang_code:
# If language is in the url and it is the default language, redirect
# to url without language so google doesn't see duplicate content
return request.redirect(
path + '?' + request.httprequest.query_string,
code=301)
return self.reroute(path)
# bind modified context
request.website = request.website.with_context(request.context)
return super(ir_http, self)._dispatch()
help="Increase verbosity",
default="store_true")
args = parser.parse_args()
root = "/home/taavi/logs"
keywords = "Windows", "Linux", "OS X", "Ubuntu", "Googlebot", "bingbot", "Android", "YandexBot", "facebookexternalhit"
d = {}
urls = {}
users = {}
ip_addresses = {}
countries = {}
total = 0
usernr = 0
gi = GeoIP.open("GeoIP.dat", GeoIP.GEOIP_MEMORY_CACHE)
for filename in os.listdir(root):
if not filename.startswith("access.log"):
print "Skipping unknown file:", filename
continue
if filename.endswith(".gz"):
continue
fh = gzip.open(os.path.join(root, filename))
else:
fh = open(os.path.join(root, filename))
print "Going to process:", filename
for line in fh:
try:
source_timestamp, request, response, referrer, _, agent, _ = line.split(
"\"")
def geo_ip(ip):
gi = GeoIP.open("GeoLiteCity.dat",
GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
geo = gi.record_by_name(str(ip))
return geo
try:
from dns.resolver import Resolver, NXDOMAIN, NoNameservers
import dns.rdatatype
from dns.exception import DNSException
MODULE_DNSPYTHON = True
except ImportError:
MODULE_DNSPYTHON = False
GEOLITE2_MMDB = path.join(path.dirname(__file__), 'GeoLite2-Country.mmdb')
try:
import geoip2.database
_ = geoip2.database.Reader(GEOLITE2_MMDB)
except Exception:
try:
import GeoIP
_ = GeoIP.new(-1)
except Exception:
MODULE_GEOIP = False
else:
MODULE_GEOIP = True
class geoip:
def __init__(self):
self.reader = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
def country_by_addr(self, ipaddr):
return self.reader.country_name_by_addr(ipaddr)
else:
MODULE_GEOIP = True
class geoip:
def __init__(self):
self.reader = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
#!/usr/bin/python
from __future__ import print_function
import GeoIP
gi = GeoIP.open("GeoIP.dat", GeoIP.GEOIP_STANDARD)
print(gi.country_code_by_name("yahoo.com"))
print(gi.country_code_by_name("ifeng.com"))
def dump_to_json(config):
data_dir = "/home/birksworks/Dev/ptGlobe/www/data"
counts = {}
locations = {}
data = {"type": "FeatureCollection", "features": []}
city_data = {}
gi = GeoIP.open("/usr/share/GeoIP/GeoLiteCity.dat", GeoIP.GEOIP_STANDARD)
dbm = DataManager(config)
for project in dbm.get_project_data():
url = project['url']
if url:
try:
domain = url[7:].split('/')[0]
info = gi.record_by_addr(socket.gethostbyname(domain))
city = info['city']
region = info['region']
country = info['country_code']
key = "%s::%s::%s" % (city, region, country)
if key in counts:
counts[key] = counts[key] + 1
else:
counts[key] = 1
locations[key] = {
"type": "Point",
"coordinates": [info['longitude'], info['latitude']]
}
city_data[key] = {}
city_data[key][project['id']] = [
info['longitude'] +
(2.0 * round(random(), 2) - 1.0) / 10.0,
info['latitude'] + round(
(2.0 * random() - 1.0), 1) / 10.0, project['title']
]
except:
print url
for key in counts.keys():
city, region, country = key.split('::')
feature = {
"type": "Feature",
"properties": {
"city": city,
"region": region,
"country": country,
"mag": counts[key]
},
"geometry": locations[key],
'id': key
}
data['features'].append(feature)
longitude = locations[key]["coordinates"][0]
latitude = locations[key]["coordinates"][1]
if counts[key] <= 10:
n_sites = counts[key]
elif counts[key] <= 40:
n_sites = counts[key] / 2
else:
n_sites = 20
site_data = {}
for i in range(n_sites):
site_data["%d" % (i + 1)] = [
longitude + (2.0 * round(random(), 2) - 1.0) / 10.0,
latitude + round((2.0 * random() - 1.0), 1) / 10.0,
"%s Research Site %d" % (city, i + 1)
]
fp = open('%s/%s.json' % (data_dir, key.replace('::', '-')), 'w')
fp.write(json.dumps(site_data))
fp.close()
fp = open('%s/cities.json' % data_dir, 'w')
fp.write(json.dumps(data))
fp.close()
parser.add_argument('-c', type=str, help='config file')
args = parser.parse_args()
if args.c != None:
configfiles = args.c
try:
config = configparser.ConfigParser()
config.read(configfiles)
port = config['DEFAULT']['coreurl'].split(":")[-1]
databasef = config['core']['database']
geoloc = config['core']['geoloc']
if (geoloc == 'geoip'):
geoipdb = config['core']['geoipdb']
try:
geoip_loc = GeoIP.open(geoipdb + "/GeoIPCity.dat",
GeoIP.GEOIP_STANDARD)
geoip_as = GeoIP.open(geoipdb + "/GeoIPASNum.dat",
GeoIP.GEOIP_STANDARD)
except:
print(
'Cannot open GeoIP database. Did you install it ? (apt-get install geoip-database-contrib)'
)
exit(1)
except:
print('Cannot read config from ' + str(configfiles))
exit(1)
engine = create_engine(databasef, echo=False)
metadata.create_all(engine)
session_factory = sessionmaker(bind=engine)
Session = scoped_session(session_factory)
#!/usr/bin/python
import GeoIP
gi = GeoIP.open("/usr/local/share/GeoIP/GeoIPNetspeed.dat",
GeoIP.GEOIP_STANDARD)
# GEOIP_UNKNOWN_SPEED, GEOIP_DIALUP_SPEED, GEOIP_CABLEDSL_SPEED or
# GEOIP_CORPORATE_SPEED
print gi.id_by_name("yahoo.com")
print gi.id_by_name("www.maxmind.com") == GeoIP.GEOIP_UNKNOWN_SPEED
print gi.id_by_addr("203.195.93.0")
def __init__(self, filename):
self.__logger = logging.getLogger("seiscomp3.fdsnws.reqlog")
self.__logger.addHandler(MyFileHandler(filename))
self.__logger.setLevel(logging.INFO)
self.__geoip = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
def parse_status(data, geoip_data):
client_section = False
routes_section = False
status_version = 1
sessions = {}
client_session = {}
gi = GeoIP.open(geoip_data, GeoIP.GEOIP_STANDARD)
for line in data.splitlines():
if ',' in line:
parts = line.split(',')
else:
parts = line.split('\t')
if args.debug:
debug("=== begin split line\n{0!s}\n=== end split line".format(parts))
if parts[0].startswith('GLOBAL'):
break
if parts[0] == 'HEADER':
status_version = 3
if parts[1] == 'CLIENT_LIST':
client_section = True
routes_section = False
if parts[1] == 'ROUTING_TABLE':
client_section = False
routes_section = True
continue
if parts[0] == 'Updated':
continue
if parts[0] == 'Common Name':
status_version = 1
client_section = True
routes_section = False
continue
if parts[0] == 'ROUTING TABLE' or parts[0] == 'Virtual Address':
status_version = 1
client_section = False
routes_section = True
continue
if parts[0].startswith('>CLIENT'):
continue
session = {}
if parts[0] == 'TUN/TAP read bytes':
client_session['tuntap_read'] = int(parts[1])
continue
if parts[0] == 'TUN/TAP write bytes':
client_session['tuntap_write'] = int(parts[1])
continue
if parts[0] == 'TCP/UDP read bytes':
client_session['tcpudp_read'] = int(parts[1])
continue
if parts[0] == 'TCP/UDP write bytes':
client_session['tcpudp_write'] = int(parts[1])
continue
if parts[0] == 'Auth read bytes':
client_session['auth_read'] = int(parts[1])
sessions['Client'] = client_session
continue
if client_section and not routes_section:
if status_version == 1:
ident = parts[1]
sessions[ident] = session
session['username'] = parts[0]
remote_ip, port = parts[1].split(':')
session['bytes_recv'] = int(parts[2])
session['bytes_sent'] = int(parts[3])
session['connected_since'] = get_date(parts[4])
elif status_version == 3:
local_ip = parts[3]
if local_ip:
ident = local_ip
else:
ident = str(uuid4())
sessions[ident] = session
if parts[8] != 'UNDEF':
session['username'] = parts[8]
else:
session['username'] = parts[1]
if parts[2].count(':') == 1:
remote_ip, port = parts[2].split(':')
else:
remote_ip = parts[2]
port = None
remote_ip_address = ip_address(remote_ip)
if local_ip:
session['local_ip'] = ip_address(local_ip)
else:
session['local_ip'] = ''
session['bytes_recv'] = int(parts[4])
session['bytes_sent'] = int(parts[5])
session['connected_since'] = get_date(parts[7], uts=True)
session['last_seen'] = session['connected_since']
session['location'] = 'Unknown'
if isinstance(remote_ip_address, IPv6Address) and \
remote_ip_address.ipv4_mapped is not None:
session['remote_ip'] = remote_ip_address.ipv4_mapped
else:
session['remote_ip'] = remote_ip_address
if port:
session['port'] = int(port)
else:
session['port'] = ''
if session['remote_ip'].is_private:
session['location'] = 'RFC1918'
else:
try:
gir = gi.record_by_addr(str(session['remote_ip']))
except SystemError:
gir = None
if gir is not None:
session['location'] = gir['country_code']
session['city'] = get_str(gir['city'])
session['country_name'] = gir['country_name']
session['longitude'] = gir['longitude']
session['latitude'] = gir['latitude']
if routes_section and not client_section:
if status_version == 1:
ident = parts[2]
sessions[ident]['local_ip'] = ip_address(parts[0])
sessions[ident]['last_seen'] = get_date(parts[3])
elif status_version == 3:
local_ip = parts[1]
if local_ip in sessions:
sessions[local_ip]['last_seen'] = get_date(parts[5], uts=True)
if args.debug:
if sessions:
pretty_sessions = pformat(sessions)
debug("=== begin sessions\n{0!s}\n=== end sessions".format(pretty_sessions))
else:
debug("no sessions")
return sessions
#!/usr/bin/python
import GeoIP
#gi = GeoIP.new(GeoIP.GEOIP_STANDARD)
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
#gi = GeoIP.open("/usr/local/share/GeoIP/GeoIP.dat",GeoIP.GEOIP_STANDARD)
print gi.country_code_by_name("yahoo.com")
print gi.last_netmask()
print gi.country_name_by_name("www.bundestag.de")
print gi.country_code_by_addr("24.24.24.24")
print gi.country_name_by_addr("24.24.24.24")
print gi.range_by_ip("68.180.206.184")
print GeoIP.lib_version()
def GeoIP(database_path, *args, **kwargs):
return CGeoIP.open(database_path, CGeoIP.GEOIP_STANDARD)
import GeoIP
import cPickle as pickle
import re
import sys
import time
import os.path
#sResolution = "%W" # Split by week
sResolution = "%m" # Split by month
sPrefix = "country-months"
sGEOIPFile = "GeoIP.dat"
gi = GeoIP.open(sGEOIPFile,GeoIP.GEOIP_STANDARD)
def getCountryForIP(sIP):
return str(gi.country_name_by_addr(sIP))
dataLessDates = {
datetime.date(2013,1,3),
datetime.date(2013,2,28),
datetime.date(2013,3,6),
datetime.date(2013,3,7),
datetime.date(2013,4,11),
datetime.date(2013,4,12),
datetime.date(2013,4,13),
datetime.date(2013,4,14),
datetime.date(2013,8,28),
datetime.date(2013,8,29),
m.drawmapboundary(fill_color='white')
plt.show()
ringx = axes[1]
ringx.set_title("Node IDs")
ringx.set_autoscale_on(False)
ringx.set_aspect('equal', 'datalim')
ringx.set_xlim(-2., 2.)
ringx.set_ylim(-1.5, 1.5)
exitax = plt.axes([0.92, 0.95, 0.07, 0.04])
exitbtn = Button(exitax, 'Exit')
reloadax = plt.axes([0.92, 0.90, 0.07, 0.04])
button = Button(reloadax, 'Reload')
gi = GeoIP.open("GeoLiteCity.dat",
GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
gi6 = GeoIP.open("GeoLiteCityv6.dat",
GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
def gcb(v):
return True
r = DhtRunner()
i = Identity()
i.generate(bits=1024)
r.run(i, port=4112)
r.bootstrap("bootstrap.ring.cx", "4222")
template = "{0:64}{1:<10}{2:<14}{3:<16}"
c = ("IPs", "Bytes", "Transactions", "Country")
current_time = datetime.datetime.now().time()
print("Bitcoin Transaction Monitor (%s)" % current_time)
print(template.format(*c))
hs = sorted(ips.values(), key=operator.attrgetter("bytes"), reverse=True)
for v in hs:
items = (v.ip, v.bytes, v.transactions, v.country)
print(template.format(*items))
if __name__ == '__main__':
gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
# Load an instance of a Lan Stack
st = pyaiengine.StackLan()
st.tcp_flows = 327680
st.udp_flows = 0
""" Decrease the memory use for other protocols """
st.decrease_allocated_memory("ssl", 10000)
st.decrease_allocated_memory("http", 10000)
"""" Increase the capacity of the bitcoin protocol """
st.increase_allocated_memory("bitcoin", 10000)
with pyaiengine.PacketDispatcher("lo") as pd:
pd.add_timer(monitor_handler, 1)
pd.stack = st
def parseFile(filename):
g = GeoIP.open("GeoLiteCity.dat",
GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
f = open(filename, 'r')
for line in f:
templine = line.strip()
#print templine
#print(raw_input()) //step mode
if (templine[1:4] == "TCP" or templine[1:4] == "UDP"):
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', templine)
#print ip[0] //srcIP
#print ip[1] //dstIP
global srcIP
if srcIP.has_key(ip[0]):
srcIP[ip[0]] += 1
else:
srcIP[ip[0]] = 1
global dstIP
if dstIP.has_key(ip[1]):
dstIP[ip[1]] += 1
else:
dstIP[ip[1]] = 1
srcG = g.record_by_name(ip[0])
dstG = g.record_by_name(ip[1])
if srcG is None:
srcGStr = "N/A"
else:
if srcG["city"] is None:
srcG["city"] = "N/A"
if srcG["region_name"] is None:
srcG["region_name"] = "N/A"
if srcG["country_name"] is None:
srcG["country_name"] = "N/A"
srcGStr = srcG["city"] + ", " + srcG[
"region_name"] + ", " + srcG["country_name"]
if dstG is None:
dstGStr = "N/A"
else:
if dstG["city"] is None:
dstG["city"] = "N/A"
if dstG["region_name"] is None:
dstG["region_name"] = "N/A"
if dstG["country_name"] is None:
dstG["country_name"] = "N/A"
dstGStr = dstG["city"] + ", " + dstG[
"region_name"] + ", " + dstG["country_name"]
global srcGeoIP, dstGeoIP
if srcGeoIP.has_key(srcGStr):
srcGeoIP[srcGStr] += 1
else:
srcGeoIP[srcGStr] = 1
if dstGeoIP.has_key(dstGStr):
dstGeoIP[dstGStr] += 1
else:
dstGeoIP[dstGStr] = 1
srcP = templine.split(ip[0] + ':', 1)[1].split('(')[0]
dstP = templine.split(ip[1] + ':', 1)[1].split('(')[0]
global srcIPPort
if srcPort.has_key(srcP):
srcPort[srcP] += 1
else:
srcPort[srcP] = 1
global dstIPPort
if dstPort.has_key(dstP):
dstPort[dstP] += 1
else:
dstPort[dstP] = 1
global srcIPPort
if srcIPPort.has_key(ip[0] + ':' + srcP):
srcIPPort[ip[0] + ':' + srcP] += 1
else:
srcIPPort[ip[0] + ':' + srcP] = 1
global dstIPPort
if dstIPPort.has_key(ip[1] + ':' + dstP):
dstIPPort[ip[1] + ':' + dstP] += 1
else:
dstIPPort[ip[1] + ':' + dstP] = 1
if (templine[1:4] == "TCP"): # TCP
flags = [-1, -1, -1, -1, -1,
-1] # URG, ACK, PSH, RST, SYN, FIN
flagStr = templine.split("FLAGS=[")[1].split(']')[0]
if "URG" in flagStr:
flags[0] = 1
else:
flags[0] = 0
if "ACK" in flagStr:
flags[1] = 1
else:
flags[1] = 0
if "PSH" in flagStr:
flags[2] = 1
else:
flags[2] = 0
if "RST" in flagStr:
flags[3] = 1
else:
flags[3] = 0
if "SYN" in flagStr:
flags[4] = 1
else:
flags[4] = 0
if "FIN" in flagStr:
flags[5] = 1
else:
flags[5] = 0
if (flags[4] and not flags[1]): # SYN no ACK
tempTime = templine.split(
'-')[0][-4:] + "-" + templine.split('-',
1)[1].split(']')[0]
currentTime = datetime.strptime(tempTime,
"%Y-%m-%d %H:%M:%S")
global startTime
global endTime
if currentTime < startTime:
startTime = currentTime
if currentTime > endTime:
endTime = currentTime
global SYNFlagCount
SYNFlagCount += 1
# URG, ACK, PSH, RST, SYN, FIN
if (all(f == 0 for f in flags) or all(f == 1 for f in flags) or \
(flags[4] and flags[5]) or (flags[4] and flags[3]) or \
(flags[5] and flags[3]) or (flags[0] and flags[2] and flags[5]) or \
(flags[5] and not flags[0] and not flags[1] and not flags[2] and not flags[3]\
and not flags[4]) or\
(flags[0] and not flags[1] and not flags[2] and not flags[3] and not flags[4]\
and not flags[5]) or\
(flags[2] and not flags[0] and not flags[1] and not flags[3] and not flags[4]\
and not flags[5])) :
global abnormalFlags
abnormalFlags += 1
global TCPPckCount
TCPPckCount += 1
else:
token = templine.split("DATA_BINARY=", 1)[1]
if token[1] == '7' and token[
9:
11] == "2a" and srcP == "123": # NTP source port = 123
# moode is 7 and request MON_GETLIST_! (42)
global monListCount
monListCount += 1
elif srcP == "1900": # SSDP port = 1900
if "55 50 6e 50" in token and "53 54 3a" in token: # UPnP and ST:
global SSDPCount
SSDPCount += 1
elif srcP == "53":
global DNSCount
DNSCount += 1
global UDPPckCount
UDPPckCount += 1
f.close()
