def ip_to_location(ipaddr):
from ooni.settings import config
country_file = config.get_data_file_path(
'resources/maxmind-geoip/GeoIP.dat')
asn_file = config.get_data_file_path(
'resources/maxmind-geoip/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
if not asn_file or not country_file:
log.err("Could not find GeoIP data file in data directories."
"Try running ooniresources or"
" edit your ooniprobe.conf")
return location
country_dat = GeoIP(country_file)
asn_dat = GeoIP(asn_file)
country_code = country_dat.country_code_by_addr(ipaddr)
if country_code is not None:
location['countrycode'] = country_code
asn = asn_dat.org_by_addr(ipaddr)
if asn is not None:
location['asn'] = asn.split(' ')[0]
return location
def IPToLocation(ipaddr):
from ooni.settings import config
city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat')
country_file = config.get_data_file_path('GeoIP/GeoIP.dat')
asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat')
location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'}
def error():
log.err("Could not find GeoIP data file in %s."
"Try running ooniresources --update-geoip or"
" edit your ooniprobe.conf" % config.advanced.geoip_data_dir)
try:
country_dat = GeoIP(country_file)
location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
if not location['countrycode']:
location['countrycode'] = 'ZZ'
except IOError:
error()
try:
city_dat = GeoIP(city_file)
location['city'] = city_dat.record_by_addr(ipaddr)['city']
except:
error()
try:
asn_dat = GeoIP(asn_file)
location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
except:
error()
return location
def __init__(self):
# Load the GeoIP databases into class attributes since they each need 20+ MB in memory
if not self.__class__._geoip4:
self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE)
if not self.__class__._geoip6:
self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE)
self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'],
Config.REDIS['DB'])
def __init__(self, *args, **kwargs):
""" Initialiser le manager """
super(IPManager, self).__init__(*args, **kwargs)
# Initialiser l'outil GeoIP dans le manager
if not hasattr(self, 'geoip'):
try:
self.geoip = GeoIP(settings.GEOIP_PATH)
self.geoisp = GeoIP(settings.GEOISP_PATH)
except AttributeError:
pass
def getAutonomous_system_number(hostName):
try:
geo_ip = GeoIP('GeoIPASNum.dat')
autonomous_number = int(geo_ip.org_by_name(hostName).split()[0][2:])
return autonomous_number
except:
return notFound
def __init__(self,
packets,
layer=3,
geo_ip=os.path.expanduser('~/GeoIP.dat')):
self.graph = DiGraph()
self.layer = layer
self.geo_ip = None
self.data = {}
try:
self.geo_ip = GeoIP(geo_ip)
except:
logging.warning("could not load GeoIP data")
if self.layer == 2:
edges = map(self._layer_2_edge, packets)
elif self.layer == 3:
edges = map(self._layer_3_edge, packets)
elif self.layer == 4:
edges = map(self._layer_4_edge, packets)
else:
raise ValueError(
"Other layers than 2,3 and 4 are not supported yet!")
for src, dst, packet in filter(lambda x: not (x is None), edges):
if src in self.graph and dst in self.graph[src]:
self.graph[src][dst]['packets'].append(packet)
else:
self.graph.add_edge(src, dst, {'packets': [packet]})
for node in self.graph.nodes():
self._retrieve_node_info(node)
for src, dst in self.graph.edges():
self._retrieve_edge_info(src, dst)
def __init__(self):
"""
Constructor -
It starts with some basic initialisations and spawns a coordinator
thread which creates more threads to perform the master server query
and also the status updates for the servers.
"""
self.serverqueue = Queue()
self.messageque = Queue()
self.pulsemessageque = Queue()
self.threadcount = 0
self.servercount = 0
self.processedserver = 0
self.filterdcount = 0
self.gui_lock = None
self.geo_lock = None
coord = Thread(target=self.coordinator)
coord.daemon = True
coord.start()
dbname = Globals.geoip_dir+ '/GeoIP.dat'
self.pygeoip = GeoIP(dbname, pygeoip.const.MMAP_CACHE)
self.abort = False
def run(self):
log.debug("Updating mirror database")
geoip = GeoIP(Config.GEOIP_PATH_V4)
for status in mirror_statuses(
unofficial_mirrors=Config.UNOFFICIAL_MIRRORS):
name = status['mirror']
if name == "a.pypi.python.org":
# don't include 'a' in the list of mirrors - it's no mirror after all
continue
time_diff = status['time_diff']
if not isinstance(time_diff, timedelta):
continue
log.debug(" Processing mirror '%s'", name)
record = geoip.record_by_name(name)
lat = record['latitude']
lon = record['longitude']
log.debug(" Age: %d, Lat: %0.5f, Lon: %0.5f",
time_diff.total_seconds(), lat, lon)
try:
mirror = Mirror.objects.get(name=name)
except ObjectNotFound:
mirror = Mirror(name=name)
mirror.age = time_diff.total_seconds()
mirror.lat = lat
mirror.lon = lon
mirror.save()
self.redis.set(Config.KEY_LAST_UPDATE, time.time())
log.debug("Finished updating mirror database")
class AccountController(BaseController):
geoip = GeoIP(os.path.join(config['pylons.paths']['data'], 'geoip.dat'))
openid_store = FileOpenIDStore('/var/tmp')
@require('guest')
def login(self):
login = render('account/login.tpl', slacks=True)
if request.environ['REQUEST_METHOD'] != 'POST':
return login
try:
form = LoginForm().to_python(request.POST)
except validators.Invalid, e:
return h.htmlfill(e, form=login)
try:
cons = Consumer(session=session, store=self.openid_store)
auth_request = cons.begin(form['openid_identifier'])
auth_request.addExtension(
SRegRequest(optional=['nickname', 'email']))
except DiscoveryFailure:
h.flash(_('The specified URL is not a valid OpenID end-point.'),
'error')
redirect(url(controller='account', action='login'))
host = request.headers['host']
realm = '%s://%s' % (request_config().protocol, host)
return_url = url(host=host,
controller='account',
action='login_complete')
new_url = auth_request.redirectURL(return_to=return_url, realm=realm)
redirect(new_url)
def lookup_country_code(ip_address):
country_code = None
# Download geoip data file in background (non blocking, not wait for result)
gif = Globals.get_geoip_file(wait=False)
# If downloaded, use geoip API to get the country
if gif:
try:
country_code = GeoIP(gif).country_code_by_addr(ip_address)
except GeoIPError as e:
logging.error(e)
if not country_code:
# If geoip file not present (not yet downloaded) or it did not find the IP,
# use web API to get the country code
if inetcache.isonline:
try:
with closing(
urllib2.urlopen(
"http://getcitydetails.geobytes.com/GetCityDetails?fqcn=%s"
% ip_address, None, 5)) as resp:
meta = resp.read()
r = re.search(r"\"geobytesinternet\"\s*:\s*\"(.*?)\"", meta)
if r:
country_code = r.group(1)
except Exception, e:
logging.error(e)
def InitializeConnections(ConfigFile='IpLocator.ini'):
# Initialization file (default=IpLocator.ini) with two sections:
# [CONNECTION} section with connection string parameters and
# [GEODATABASE} section with the pathname of the GeoIP database file
# Returns: connection to MSSQL and gi handler to GeoIP database
ConnectionString = ''
config = ConfigParser()
try:
with open(ConfigFile) as f:
config.read_file(f)
Section = 'CONNECTION'
if config.has_section(Section):
#Driver = config[Section]['Driver'] # only used for odbc connections, not in pymssql or _mssql
Server = config[Section]['Server'] # server\instance_name
Database = config[Section]['Database']
Uid = config[Section]['Uid']
Pwd = config[Section]['Pwd']
m_writeBlocks = int(config[Section]['WriteBlocks'])
else:
print('Section: {} does not exist in config file.'.format(Section))
return None, None
connectionString = {}
connectionString['Server'] = Server
connectionString['User'] = Uid
connectionString['Password'] = Pwd
connectionString['Database'] = Database
connection = connect(connectionString['Server'],
connectionString['User'],
connectionString['Password'],
connectionString['Database'],
autocommit=False)
Section = 'GEODATABASE'
if config.has_section(Section):
GeoIPFile = config[Section]['GeoIPFile']
gi = GeoIP(GeoIPFile, flags=MEMORY_CACHE) #MEMORY_CACHE
else:
print('Section: {} does not exist in config file.'.format(Section))
return None, None
except IOError as e:
print('{}'.format(e))
return None, None
except GeoIPError as e:
print('{}'.format(e))
except KeyError as e:
print('Item {} does not exist in configuration file {}.'.format(
e, ConfigFile))
return None, None
except InterfaceError as e:
print('InterfaceError {}'.format(e))
return None, None
except DatabaseError as e:
print('DatabaseError {}'.format(e))
return None, None
except:
print('{}\n{}'.format(connectionString, exc_info()[0]))
return None, None
return connection, gi, m_writeBlocks
def IP_by_DataBase(ip):
"""
Retorna as iformações de geo posicionamento atraver da base de dados local
Disponivel no site http://appliedsec.github.com/pygeoip/
Data Base http://dev.maxmind.com/geoip/geolite
"""
gi = GeoIP(PATH_GEOIP_CITY)
return gi.record_by_addr(ip) or {}
def update_data(self, request, commit=True):
self.user_agent = request.META.get('HTTP_USER_AGENT', None)
geo = GeoIP(settings.GEOIP_DATABASE)
self.country_code = geo.country_code_by_addr(
request.META.get('REMOTE_ADDR', None))
self.visitor_ip = request.META.get('REMOTE_ADDR', None)
if hasattr(request, 'user') and request.user.is_authenticated():
self.visitor = request.user
if commit:
self.save()
def add_geo(nodes):
from pygeoip import GeoIP
gi = GeoIP(GEODB)
for k, v in nodes.items():
try:
nodes[k].update(gi.record_by_addr(v["external-ip"]))
except Exception as e:
sys.stderr.write(str(e))
sys.stderr.write("Cannot determine GeoData for %s\n" % k)
return nodes
def IPToLocation(ipaddr):
city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
location = {'city': None, 'countrycode': None, 'asn': None}
try:
city_dat = GeoIP(city_file)
location['city'] = city_dat.record_by_addr(ipaddr)['city']
country_dat = GeoIP(country_file)
location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
asn_dat = GeoIP(asn_file)
location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0]
except IOError:
log.err("Could not find GeoIP data files. Go into data/ "
"and run make geoip")
raise GeoIPDataFilesNotFound
return location
def calcDistance(self, ipAddr1, ipAddr2):
gi = GeoIP("c:\\temp\\GeoLiteCity.dat")
gir1 = gi.record_by_name(ipAddr1)
gir2 = gi.record_by_name(ipAddr2)
#Calculate distance
print "Calculating Distance between IP ", ipAddr1, " and IP ", ipAddr2
pt1 = geopy.Point(gir1['latitude'], gir1['longitude'])
pt2 = geopy.Point(gir2['latitude'], gir2['longitude'])
print "Distance:"
dist1 = geopy.distance.distance(pt1, pt2).km
print dist1
return dist1
def add_coords_to_edges(nodes):
from pygeoip import GeoIP
gi = GeoIP(GEODB)
for k, v in nodes.items():
for i, j in enumerate(v.get("to", [])):
data = gi.record_by_addr(j["addr"])
try:
j["latitude"] = data["latitude"]
j["longitude"] = data["longitude"]
except Exception as e:
pass
return nodes
def get_gic():
global gic
if gic == None:
if os.path.exists('/usr/share/GeoIP/GeoIP.dat'):
default = "/usr/share/GeoIP/GeoIP.dat"
elif os.path.exists("/usr/local/share/GeoIP/GeoLiteCity.dat"):
default = "/usr/local/share/GeoIP/GeoLiteCity.dat"
elif os.path.exists("/usr/local/var/lib/GeoLiteCity.dat"):
default = "/usr/local/var/lib/GeoLiteCity.dat"
else:
default = None
filename = BabeBase.get_config_with_env('geoip', 'GEOIP_FILE', {},
default)
from pygeoip import GeoIP
gic = GeoIP(filename)
return gic
def application(environ, start_response):
from pygeoip import GeoIP
from os import path
from urlparse import parse_qs
import json
status = '200 OK'
ip = environ.get('HTTP_X_REAL_IP',
environ.get('HTTP_X_FORWARDED_FOR',
environ.get('REMOTE_ADDR')
)
)
qs = parse_qs(environ.get('QUERY_STRING', ''))
callback = qs.get('callback', '')
if any(callback):
callback = callback[0]
else:
callback = None
if ip:
filename = path.join(path.dirname(__file__), 'data', 'GeoLiteCity.dat')
geo = GeoIP(filename)
record = geo.record_by_name(ip)
if record:
response_headers = [('Content-type', 'application/json')]
start_response(status, response_headers)
if callback:
yield '%s(' % callback
yield json.dumps(record)
if callback:
yield ')'
yield
class RawIPConsumer(Consumer):
""" Consumes dummy objects for testing like:
{
'ip': 'some_ip',
'tag': 'some_tag',
}
"""
topic = 'narcissus.hits'
jsonify = True
geoip_url = '/'.join(__file__.split('/')[:-1] +
["public/data/GeoLiteCity.dat"])
gi = GeoIP(geoip_url, GEOIP_MEMORY_CACHE)
def consume(self, message):
if not message:
#self.log.warn("%r got empty message." % self)
return
#self.log.info("%r got message '%r'" % (self, message))
message = simplejson.loads(message['body'])
# Get IP 2 LatLon info
rec = self.gi.record_by_addr(message['ip'])
if not(rec and rec['latitude'] and rec['longitude']):
self.log.warn("Failed to geo-encode %r" % message)
return
updates = {
'lat' : rec['latitude'],
'lon' : rec['longitude'],
'country' : rec.get('country_name', 'undefined'),
}
message.update(updates)
self.send_message('http_latlon', message)
def traceIP(target):
try:
base = GeoIP('GeoLiteCity.dat')
data = base.record_by_addr(target)
dnsName = socket.gethostbyaddr(target)[0]
formatedData = '''IP: {}
City: {}
State/Province: {}
Country: {}
Continent: {}
Zip/Postal code: {}
Timezone: {}
Latitude: {}
Longitude: {}
DNS name: {}'''.format(target, data['city'], data['region_code'], data['country_name'], data['continent'], data['postal_code'], data['time_zone'], str(data['latitude']), str(data['longitude']), dnsName)
print formatedData
# compares target to database and print results to console
askSave = raw_input('Save data? Y/n: ').lower()
if askSave == 'y':
ipFileName = raw_input('Filename: ')
with open(ipFileName, 'w') as fileName:
fileName.write(formatedData)
print 'Output saved as {}'.format(ipFileName)
else:
pass
# asks user if they want to save the output
pause()
main()
except socket.herror:
pass
def show():
g = GeoIP('pygeoip/GeoIP.dat')
user_country = g.country_code_by_addr(request.remote_addr);
return render_template('home.html', user_country=user_country)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from django.utils import timezone
from pygeoip import GeoIP, MEMORY_CACHE
from django_visitor_information import constants
from django_visitor_information import settings
__all__ = ['TimezoneMiddleware', 'VisitorInformationMiddleware']
logger = logging.getLogger('django_visitor_information.middleware')
gi4 = GeoIP(settings.VISITOR_INFO_GEOIP_DATABASE_PATH, MEMORY_CACHE)
class TimezoneMiddleware(object):
"""
This middleware activates a timezone for an authenticated user.
This middleware assumes that a User model references a UserProfile model
which has a "timezone" field.
"""
def process_request(self, request):
if request.user.is_authenticated():
profile = request.user.get_profile()
user_timezone = \
getattr(profile,
settings.VISITOR_INFO_PROFILE_TIMEZONE_FIELD,
def load_geodb(path, log=None):
def geoip(reader, ip):
try:
record = reader.city(ip)
except (AddressNotFoundError, ValueError):
return {}
if record is None:
return {}
result = {}
geoip_cc = record.country.iso_code
if geoip_cc:
result["geoip cc"] = [geoip_cc]
latitude = record.location.latitude
longitude = record.location.longitude
if latitude and longitude:
result["latitude"] = [unicode(latitude)]
result["longitude"] = [unicode(longitude)]
return result
def legacy_geoip(reader, ip):
if not is_ipv4(ip):
return {}
try:
record = reader.record_by_addr(ip)
except GeoIPError:
return {}
if record is None:
return {}
result = {}
geoip_cc = record.get("country_code", None)
if geoip_cc:
result["geoip cc"] = [geoip_cc]
latitude = record.get("latitude", None)
longitude = record.get("longitude", None)
if latitude and longitude:
result["latitude"] = [unicode(latitude)]
result["longitude"] = [unicode(longitude)]
return result
try:
from geoip2.database import Reader
from maxminddb.errors import InvalidDatabaseError
from geoip2.errors import AddressNotFoundError
try:
reader = Reader(path)
fun = geoip
except InvalidDatabaseError:
raise ImportError
if log:
log.info("GeoIP2 initiated")
except ImportError:
from pygeoip import GeoIP, GeoIPError
reader = GeoIP(path)
fun = legacy_geoip
if log:
log.info("Legacy GeoIP initiated")
def geoip_reader(ip):
return fun(reader, ip)
return geoip_reader
class Proxy(models.Model):
"""A proxy server"""
_geoip = GeoIP(defaults.PROXY_LIST_GEOIP_PATH)
anonymity_level_choices = (
# Anonymity can't be determined
(None, 'Unknown'),
# No anonymity; remote host knows your IP and knows you are using
# proxy.
(ANONYMITY_NONE, 'None'),
# Low anonymity; proxy sent our IP to remote host, but it was sent in
# non standard way (unknown header).
(ANONYMITY_LOW, 'Low'),
# Medium anonymity; remote host knows you are using proxy, but it does
# not know your IP
(ANONYMITY_MEDIUM, 'Medium'),
# High anonymity; remote host does not know your IP and has no direct
# proof of proxy usage (proxy-connection family header strings).
(ANONYMITY_HIGH, 'High'),
)
hostname = models.CharField(max_length=75, unique=True)
port = models.PositiveIntegerField()
user = models.CharField(blank=True, null=True, max_length=50)
password = models.CharField(blank=True, null=True, max_length=50)
country = CountryField(blank=True, editable=False)
proxy_type = models.CharField(default='http',
max_length=10,
choices=PROXY_TYPE_CHOICES)
anonymity_level = models.PositiveIntegerField(
null=True,
default=ANONYMITY_NONE,
choices=anonymity_level_choices,
editable=False)
last_check = models.DateTimeField(null=True, blank=True, editable=False)
next_check = models.DateTimeField(null=True, blank=True)
created = models.DateTimeField(auto_now=False,
auto_now_add=True,
db_index=True,
editable=False)
errors = models.PositiveIntegerField(default=0, editable=False)
elapsed_time = models.FloatField(blank=True, null=True, editable=False)
def _update_next_check(self):
""" Calculate and set next check time """
delay = randint(defaults.PROXY_LIST_MIN_CHECK_INTERVAL,
defaults.PROXY_LIST_MAX_CHECK_INTERVAL)
delay += defaults.PROXY_LIST_ERROR_DELAY * self.errors
if self.last_check:
self.next_check = self.last_check + timedelta(seconds=delay)
else:
self.next_check = now() + timedelta(seconds=delay)
def update_from_check(self, check, elapsed_time):
""" Update data from a ProxyCheckResult """
if check.check_start:
self.last_check = check.check_start
else:
self.last_check = now()
self.errors = 0
self.anonymity_level = check.anonymity()
self._update_next_check()
self.elapsed_time = elapsed_time
self.save()
def update_from_error(self):
""" Last check was an error """
self.last_check = now()
self.errors += 1
self._update_next_check()
self.save()
def save(self, *args, **kwargs):
if not self.country:
if self.hostname.count('.') == 3:
self.country = self._geoip.country_code_by_addr(
str(self.hostname))
else:
self.country = self._geoip.country_code_by_name(
str(self.hostname))
if not self.next_check:
self.next_check = (now() - timedelta(seconds=max_check))
super(Proxy, self).save(*args, **kwargs)
class Meta:
verbose_name = 'Proxy'
verbose_name_plural = 'Proxies'
ordering = ('-last_check', )
def __unicode__(self):
return "%s://%s:%s" % (self.proxy_type, self.hostname, self.port)
def geo_connection():
geo_conn = GeoIP("%s/%s" %
(settings.GEOIP_DB_LOCATION, settings.GEOIP_FILENAME))
return geo_conn
from trytond.modules.nereid_checkout.i18n import _
__metaclass__ = PoolMeta
__all__ = ['Address']
geoip = None
try:
from pygeoip import GeoIP
except ImportError:
logging.error("pygeoip is not installed")
else:
path = os.environ.get('GEOIP_DATA_PATH',
config.get('nereid_webshop', 'geoip_data_path'))
if path:
geoip = GeoIP(path)
class WebshopAddressForm(AddressForm):
"""Custom address form for webshop
"""
phone = TextField(_('Phone'), [
validators.DataRequired(),
])
def get_default_country(self):
"""Get the default country based on geoip data.
"""
if not geoip or not request.remote_addr:
return None
from models import *
from django.contrib.sessions.models import Session as django_session
if not settings.configured:
settings.configure()
geoip_city_dat = settings.GEOLITECITY_ABSOLUTE_PATH
geoip_org_dat = settings.GEOORGANIZATION_ABSOLUTE_PATH
# default amount of time which must pass before the same event from the
# same user is logged again
default_sleep_minutes = 30
gic = None
try:
gic = GeoIP(geoip_city_dat)
except IOError, err:
sys.stderr.write("""
ERROR: Could not find GeoIP database. Tried looking in "%s".
If this is not where you have your GeoIP .dat file stored, edit
GEOLITECITY_ABSOLUTE_PATH in live/local_settings.py\nIf you don't have the
GeoIP City database, you can get it from "http://dev.maxmind.com/geoip/geolite".
""" % (geoip_city_dat))
sys.exit(1)
gio = None
try:
gio = GeoIP(geoip_org_dat)
except IOError:
# we don't want to spam the log with warning messages, so don't do anything
# here. it's desgined to work without the GeoIP Organization database
]
__metaclass__ = PoolMeta
geoip = None
try:
from pygeoip import GeoIP
except ImportError:
logging.error("pygeoip is not installed")
else:
# Usual location in Ubuntu
path1 = '/usr/share/GeoIP/GeoIP.dat'
# this is where brew installs it
path2 = '/usr/local/Cellar/geoip/1.4.8/share/GeoIP/GeoIP.dat'
if os.path.isfile(path1):
geoip = GeoIP(path1)
elif os.path.isfile(path2):
geoip = GeoIP(path2)
class NereidUser:
"""
Add employee
"""
__name__ = "nereid.user"
#: Allow the nereid user to be connected to an internal employee. This
#: indicates that the user is an employee and not a regular participant
employee = fields.Many2One(
'company.employee',
'Employee',
import re
import sys
from pygeoip import GeoIP
g = GeoIP('GeoIP.dat')
r = re.compile(r'20. ([0-9]+) "')
d = {}
while True:
line = sys.stdin.readline()
if not line: break
match = r.search(line)
if match:
ip = line.split(' ')[0]
country = g.country_name_by_addr(ip)
if not country: country = 'Unknown'
d[country] = int(match.group(1)) + d.get(country, 0)
ds = d.items()
ds.sort(lambda x, y: cmp(y[1], x[1]))
for k, v in ds:
if v > (1024 * 1024 * 1024):
print k, '%0.2fGb' % (v / (1024 * 1024 * 1024.0))
