def ip_to_location(ipaddr): from ooni.settings import config country_file = config.get_data_file_path( 'resources/maxmind-geoip/GeoIP.dat') asn_file = config.get_data_file_path( 'resources/maxmind-geoip/GeoIPASNum.dat') location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} if not asn_file or not country_file: log.err("Could not find GeoIP data file in data directories." "Try running ooniresources or" " edit your ooniprobe.conf") return location country_dat = GeoIP(country_file) asn_dat = GeoIP(asn_file) country_code = country_dat.country_code_by_addr(ipaddr) if country_code is not None: location['countrycode'] = country_code asn = asn_dat.org_by_addr(ipaddr) if asn is not None: location['asn'] = asn.split(' ')[0] return location
def IPToLocation(ipaddr): from ooni.settings import config city_file = config.get_data_file_path('GeoIP/GeoLiteCity.dat') country_file = config.get_data_file_path('GeoIP/GeoIP.dat') asn_file = config.get_data_file_path('GeoIP/GeoIPASNum.dat') location = {'city': None, 'countrycode': 'ZZ', 'asn': 'AS0'} def error(): log.err("Could not find GeoIP data file in %s." "Try running ooniresources --update-geoip or" " edit your ooniprobe.conf" % config.advanced.geoip_data_dir) try: country_dat = GeoIP(country_file) location['countrycode'] = country_dat.country_code_by_addr(ipaddr) if not location['countrycode']: location['countrycode'] = 'ZZ' except IOError: error() try: city_dat = GeoIP(city_file) location['city'] = city_dat.record_by_addr(ipaddr)['city'] except: error() try: asn_dat = GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] except: error() return location
def __init__(self): # Load the GeoIP databases into class attributes since they each need 20+ MB in memory if not self.__class__._geoip4: self.__class__._geoip4 = GeoIP(Config.GEOIP_PATH_V4, MEMORY_CACHE) if not self.__class__._geoip6: self.__class__._geoip6 = GeoIP(Config.GEOIP_PATH_V6, MEMORY_CACHE) self.redis = StrictRedis(Config.REDIS['HOST'], Config.REDIS['PORT'], Config.REDIS['DB'])
def __init__(self, *args, **kwargs): """ Initialiser le manager """ super(IPManager, self).__init__(*args, **kwargs) # Initialiser l'outil GeoIP dans le manager if not hasattr(self, 'geoip'): try: self.geoip = GeoIP(settings.GEOIP_PATH) self.geoisp = GeoIP(settings.GEOISP_PATH) except AttributeError: pass
def getAutonomous_system_number(hostName): try: geo_ip = GeoIP('GeoIPASNum.dat') autonomous_number = int(geo_ip.org_by_name(hostName).split()[0][2:]) return autonomous_number except: return notFound
def __init__(self, packets, layer=3, geo_ip=os.path.expanduser('~/GeoIP.dat')): self.graph = DiGraph() self.layer = layer self.geo_ip = None self.data = {} try: self.geo_ip = GeoIP(geo_ip) except: logging.warning("could not load GeoIP data") if self.layer == 2: edges = map(self._layer_2_edge, packets) elif self.layer == 3: edges = map(self._layer_3_edge, packets) elif self.layer == 4: edges = map(self._layer_4_edge, packets) else: raise ValueError( "Other layers than 2,3 and 4 are not supported yet!") for src, dst, packet in filter(lambda x: not (x is None), edges): if src in self.graph and dst in self.graph[src]: self.graph[src][dst]['packets'].append(packet) else: self.graph.add_edge(src, dst, {'packets': [packet]}) for node in self.graph.nodes(): self._retrieve_node_info(node) for src, dst in self.graph.edges(): self._retrieve_edge_info(src, dst)
def __init__(self): """ Constructor - It starts with some basic initialisations and spawns a coordinator thread which creates more threads to perform the master server query and also the status updates for the servers. """ self.serverqueue = Queue() self.messageque = Queue() self.pulsemessageque = Queue() self.threadcount = 0 self.servercount = 0 self.processedserver = 0 self.filterdcount = 0 self.gui_lock = None self.geo_lock = None coord = Thread(target=self.coordinator) coord.daemon = True coord.start() dbname = Globals.geoip_dir+ '/GeoIP.dat' self.pygeoip = GeoIP(dbname, pygeoip.const.MMAP_CACHE) self.abort = False
def run(self): log.debug("Updating mirror database") geoip = GeoIP(Config.GEOIP_PATH_V4) for status in mirror_statuses( unofficial_mirrors=Config.UNOFFICIAL_MIRRORS): name = status['mirror'] if name == "a.pypi.python.org": # don't include 'a' in the list of mirrors - it's no mirror after all continue time_diff = status['time_diff'] if not isinstance(time_diff, timedelta): continue log.debug(" Processing mirror '%s'", name) record = geoip.record_by_name(name) lat = record['latitude'] lon = record['longitude'] log.debug(" Age: %d, Lat: %0.5f, Lon: %0.5f", time_diff.total_seconds(), lat, lon) try: mirror = Mirror.objects.get(name=name) except ObjectNotFound: mirror = Mirror(name=name) mirror.age = time_diff.total_seconds() mirror.lat = lat mirror.lon = lon mirror.save() self.redis.set(Config.KEY_LAST_UPDATE, time.time()) log.debug("Finished updating mirror database")
class AccountController(BaseController): geoip = GeoIP(os.path.join(config['pylons.paths']['data'], 'geoip.dat')) openid_store = FileOpenIDStore('/var/tmp') @require('guest') def login(self): login = render('account/login.tpl', slacks=True) if request.environ['REQUEST_METHOD'] != 'POST': return login try: form = LoginForm().to_python(request.POST) except validators.Invalid, e: return h.htmlfill(e, form=login) try: cons = Consumer(session=session, store=self.openid_store) auth_request = cons.begin(form['openid_identifier']) auth_request.addExtension( SRegRequest(optional=['nickname', 'email'])) except DiscoveryFailure: h.flash(_('The specified URL is not a valid OpenID end-point.'), 'error') redirect(url(controller='account', action='login')) host = request.headers['host'] realm = '%s://%s' % (request_config().protocol, host) return_url = url(host=host, controller='account', action='login_complete') new_url = auth_request.redirectURL(return_to=return_url, realm=realm) redirect(new_url)
def lookup_country_code(ip_address): country_code = None # Download geoip data file in background (non blocking, not wait for result) gif = Globals.get_geoip_file(wait=False) # If downloaded, use geoip API to get the country if gif: try: country_code = GeoIP(gif).country_code_by_addr(ip_address) except GeoIPError as e: logging.error(e) if not country_code: # If geoip file not present (not yet downloaded) or it did not find the IP, # use web API to get the country code if inetcache.isonline: try: with closing( urllib2.urlopen( "http://getcitydetails.geobytes.com/GetCityDetails?fqcn=%s" % ip_address, None, 5)) as resp: meta = resp.read() r = re.search(r"\"geobytesinternet\"\s*:\s*\"(.*?)\"", meta) if r: country_code = r.group(1) except Exception, e: logging.error(e)
def InitializeConnections(ConfigFile='IpLocator.ini'): # Initialization file (default=IpLocator.ini) with two sections: # [CONNECTION} section with connection string parameters and # [GEODATABASE} section with the pathname of the GeoIP database file # Returns: connection to MSSQL and gi handler to GeoIP database ConnectionString = '' config = ConfigParser() try: with open(ConfigFile) as f: config.read_file(f) Section = 'CONNECTION' if config.has_section(Section): #Driver = config[Section]['Driver'] # only used for odbc connections, not in pymssql or _mssql Server = config[Section]['Server'] # server\instance_name Database = config[Section]['Database'] Uid = config[Section]['Uid'] Pwd = config[Section]['Pwd'] m_writeBlocks = int(config[Section]['WriteBlocks']) else: print('Section: {} does not exist in config file.'.format(Section)) return None, None connectionString = {} connectionString['Server'] = Server connectionString['User'] = Uid connectionString['Password'] = Pwd connectionString['Database'] = Database connection = connect(connectionString['Server'], connectionString['User'], connectionString['Password'], connectionString['Database'], autocommit=False) Section = 'GEODATABASE' if config.has_section(Section): GeoIPFile = config[Section]['GeoIPFile'] gi = GeoIP(GeoIPFile, flags=MEMORY_CACHE) #MEMORY_CACHE else: print('Section: {} does not exist in config file.'.format(Section)) return None, None except IOError as e: print('{}'.format(e)) return None, None except GeoIPError as e: print('{}'.format(e)) except KeyError as e: print('Item {} does not exist in configuration file {}.'.format( e, ConfigFile)) return None, None except InterfaceError as e: print('InterfaceError {}'.format(e)) return None, None except DatabaseError as e: print('DatabaseError {}'.format(e)) return None, None except: print('{}\n{}'.format(connectionString, exc_info()[0])) return None, None return connection, gi, m_writeBlocks
def IP_by_DataBase(ip): """ Retorna as iformações de geo posicionamento atraver da base de dados local Disponivel no site http://appliedsec.github.com/pygeoip/ Data Base http://dev.maxmind.com/geoip/geolite """ gi = GeoIP(PATH_GEOIP_CITY) return gi.record_by_addr(ip) or {}
def update_data(self, request, commit=True): self.user_agent = request.META.get('HTTP_USER_AGENT', None) geo = GeoIP(settings.GEOIP_DATABASE) self.country_code = geo.country_code_by_addr( request.META.get('REMOTE_ADDR', None)) self.visitor_ip = request.META.get('REMOTE_ADDR', None) if hasattr(request, 'user') and request.user.is_authenticated(): self.visitor = request.user if commit: self.save()
def add_geo(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) for k, v in nodes.items(): try: nodes[k].update(gi.record_by_addr(v["external-ip"])) except Exception as e: sys.stderr.write(str(e)) sys.stderr.write("Cannot determine GeoData for %s\n" % k) return nodes
def IPToLocation(ipaddr): city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat') country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat') asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat') location = {'city': None, 'countrycode': None, 'asn': None} try: city_dat = GeoIP(city_file) location['city'] = city_dat.record_by_addr(ipaddr)['city'] country_dat = GeoIP(country_file) location['countrycode'] = country_dat.country_code_by_addr(ipaddr) asn_dat = GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr).split(' ')[0] except IOError: log.err("Could not find GeoIP data files. Go into data/ " "and run make geoip") raise GeoIPDataFilesNotFound return location
def calcDistance(self, ipAddr1, ipAddr2): gi = GeoIP("c:\\temp\\GeoLiteCity.dat") gir1 = gi.record_by_name(ipAddr1) gir2 = gi.record_by_name(ipAddr2) #Calculate distance print "Calculating Distance between IP ", ipAddr1, " and IP ", ipAddr2 pt1 = geopy.Point(gir1['latitude'], gir1['longitude']) pt2 = geopy.Point(gir2['latitude'], gir2['longitude']) print "Distance:" dist1 = geopy.distance.distance(pt1, pt2).km print dist1 return dist1
def add_coords_to_edges(nodes): from pygeoip import GeoIP gi = GeoIP(GEODB) for k, v in nodes.items(): for i, j in enumerate(v.get("to", [])): data = gi.record_by_addr(j["addr"]) try: j["latitude"] = data["latitude"] j["longitude"] = data["longitude"] except Exception as e: pass return nodes
def get_gic(): global gic if gic == None: if os.path.exists('/usr/share/GeoIP/GeoIP.dat'): default = "/usr/share/GeoIP/GeoIP.dat" elif os.path.exists("/usr/local/share/GeoIP/GeoLiteCity.dat"): default = "/usr/local/share/GeoIP/GeoLiteCity.dat" elif os.path.exists("/usr/local/var/lib/GeoLiteCity.dat"): default = "/usr/local/var/lib/GeoLiteCity.dat" else: default = None filename = BabeBase.get_config_with_env('geoip', 'GEOIP_FILE', {}, default) from pygeoip import GeoIP gic = GeoIP(filename) return gic
def application(environ, start_response): from pygeoip import GeoIP from os import path from urlparse import parse_qs import json status = '200 OK' ip = environ.get('HTTP_X_REAL_IP', environ.get('HTTP_X_FORWARDED_FOR', environ.get('REMOTE_ADDR') ) ) qs = parse_qs(environ.get('QUERY_STRING', '')) callback = qs.get('callback', '') if any(callback): callback = callback[0] else: callback = None if ip: filename = path.join(path.dirname(__file__), 'data', 'GeoLiteCity.dat') geo = GeoIP(filename) record = geo.record_by_name(ip) if record: response_headers = [('Content-type', 'application/json')] start_response(status, response_headers) if callback: yield '%s(' % callback yield json.dumps(record) if callback: yield ')' yield
class RawIPConsumer(Consumer): """ Consumes dummy objects for testing like: { 'ip': 'some_ip', 'tag': 'some_tag', } """ topic = 'narcissus.hits' jsonify = True geoip_url = '/'.join(__file__.split('/')[:-1] + ["public/data/GeoLiteCity.dat"]) gi = GeoIP(geoip_url, GEOIP_MEMORY_CACHE) def consume(self, message): if not message: #self.log.warn("%r got empty message." % self) return #self.log.info("%r got message '%r'" % (self, message)) message = simplejson.loads(message['body']) # Get IP 2 LatLon info rec = self.gi.record_by_addr(message['ip']) if not(rec and rec['latitude'] and rec['longitude']): self.log.warn("Failed to geo-encode %r" % message) return updates = { 'lat' : rec['latitude'], 'lon' : rec['longitude'], 'country' : rec.get('country_name', 'undefined'), } message.update(updates) self.send_message('http_latlon', message)
def traceIP(target): try: base = GeoIP('GeoLiteCity.dat') data = base.record_by_addr(target) dnsName = socket.gethostbyaddr(target)[0] formatedData = '''IP: {} City: {} State/Province: {} Country: {} Continent: {} Zip/Postal code: {} Timezone: {} Latitude: {} Longitude: {} DNS name: {}'''.format(target, data['city'], data['region_code'], data['country_name'], data['continent'], data['postal_code'], data['time_zone'], str(data['latitude']), str(data['longitude']), dnsName) print formatedData # compares target to database and print results to console askSave = raw_input('Save data? Y/n: ').lower() if askSave == 'y': ipFileName = raw_input('Filename: ') with open(ipFileName, 'w') as fileName: fileName.write(formatedData) print 'Output saved as {}'.format(ipFileName) else: pass # asks user if they want to save the output pause() main() except socket.herror: pass
def show(): g = GeoIP('pygeoip/GeoIP.dat') user_country = g.country_code_by_addr(request.remote_addr); return render_template('home.html', user_country=user_country)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging from django.utils import timezone from pygeoip import GeoIP, MEMORY_CACHE from django_visitor_information import constants from django_visitor_information import settings __all__ = ['TimezoneMiddleware', 'VisitorInformationMiddleware'] logger = logging.getLogger('django_visitor_information.middleware') gi4 = GeoIP(settings.VISITOR_INFO_GEOIP_DATABASE_PATH, MEMORY_CACHE) class TimezoneMiddleware(object): """ This middleware activates a timezone for an authenticated user. This middleware assumes that a User model references a UserProfile model which has a "timezone" field. """ def process_request(self, request): if request.user.is_authenticated(): profile = request.user.get_profile() user_timezone = \ getattr(profile, settings.VISITOR_INFO_PROFILE_TIMEZONE_FIELD,
def load_geodb(path, log=None): def geoip(reader, ip): try: record = reader.city(ip) except (AddressNotFoundError, ValueError): return {} if record is None: return {} result = {} geoip_cc = record.country.iso_code if geoip_cc: result["geoip cc"] = [geoip_cc] latitude = record.location.latitude longitude = record.location.longitude if latitude and longitude: result["latitude"] = [unicode(latitude)] result["longitude"] = [unicode(longitude)] return result def legacy_geoip(reader, ip): if not is_ipv4(ip): return {} try: record = reader.record_by_addr(ip) except GeoIPError: return {} if record is None: return {} result = {} geoip_cc = record.get("country_code", None) if geoip_cc: result["geoip cc"] = [geoip_cc] latitude = record.get("latitude", None) longitude = record.get("longitude", None) if latitude and longitude: result["latitude"] = [unicode(latitude)] result["longitude"] = [unicode(longitude)] return result try: from geoip2.database import Reader from maxminddb.errors import InvalidDatabaseError from geoip2.errors import AddressNotFoundError try: reader = Reader(path) fun = geoip except InvalidDatabaseError: raise ImportError if log: log.info("GeoIP2 initiated") except ImportError: from pygeoip import GeoIP, GeoIPError reader = GeoIP(path) fun = legacy_geoip if log: log.info("Legacy GeoIP initiated") def geoip_reader(ip): return fun(reader, ip) return geoip_reader
class Proxy(models.Model): """A proxy server""" _geoip = GeoIP(defaults.PROXY_LIST_GEOIP_PATH) anonymity_level_choices = ( # Anonymity can't be determined (None, 'Unknown'), # No anonymity; remote host knows your IP and knows you are using # proxy. (ANONYMITY_NONE, 'None'), # Low anonymity; proxy sent our IP to remote host, but it was sent in # non standard way (unknown header). (ANONYMITY_LOW, 'Low'), # Medium anonymity; remote host knows you are using proxy, but it does # not know your IP (ANONYMITY_MEDIUM, 'Medium'), # High anonymity; remote host does not know your IP and has no direct # proof of proxy usage (proxy-connection family header strings). (ANONYMITY_HIGH, 'High'), ) hostname = models.CharField(max_length=75, unique=True) port = models.PositiveIntegerField() user = models.CharField(blank=True, null=True, max_length=50) password = models.CharField(blank=True, null=True, max_length=50) country = CountryField(blank=True, editable=False) proxy_type = models.CharField(default='http', max_length=10, choices=PROXY_TYPE_CHOICES) anonymity_level = models.PositiveIntegerField( null=True, default=ANONYMITY_NONE, choices=anonymity_level_choices, editable=False) last_check = models.DateTimeField(null=True, blank=True, editable=False) next_check = models.DateTimeField(null=True, blank=True) created = models.DateTimeField(auto_now=False, auto_now_add=True, db_index=True, editable=False) errors = models.PositiveIntegerField(default=0, editable=False) elapsed_time = models.FloatField(blank=True, null=True, editable=False) def _update_next_check(self): """ Calculate and set next check time """ delay = randint(defaults.PROXY_LIST_MIN_CHECK_INTERVAL, defaults.PROXY_LIST_MAX_CHECK_INTERVAL) delay += defaults.PROXY_LIST_ERROR_DELAY * self.errors if self.last_check: self.next_check = self.last_check + timedelta(seconds=delay) else: self.next_check = now() + timedelta(seconds=delay) def update_from_check(self, check, elapsed_time): """ Update data from a ProxyCheckResult """ if check.check_start: self.last_check = check.check_start else: self.last_check = now() self.errors = 0 self.anonymity_level = check.anonymity() self._update_next_check() self.elapsed_time = elapsed_time self.save() def update_from_error(self): """ Last check was an error """ self.last_check = now() self.errors += 1 self._update_next_check() self.save() def save(self, *args, **kwargs): if not self.country: if self.hostname.count('.') == 3: self.country = self._geoip.country_code_by_addr( str(self.hostname)) else: self.country = self._geoip.country_code_by_name( str(self.hostname)) if not self.next_check: self.next_check = (now() - timedelta(seconds=max_check)) super(Proxy, self).save(*args, **kwargs) class Meta: verbose_name = 'Proxy' verbose_name_plural = 'Proxies' ordering = ('-last_check', ) def __unicode__(self): return "%s://%s:%s" % (self.proxy_type, self.hostname, self.port)
def geo_connection(): geo_conn = GeoIP("%s/%s" % (settings.GEOIP_DB_LOCATION, settings.GEOIP_FILENAME)) return geo_conn
from trytond.modules.nereid_checkout.i18n import _ __metaclass__ = PoolMeta __all__ = ['Address'] geoip = None try: from pygeoip import GeoIP except ImportError: logging.error("pygeoip is not installed") else: path = os.environ.get('GEOIP_DATA_PATH', config.get('nereid_webshop', 'geoip_data_path')) if path: geoip = GeoIP(path) class WebshopAddressForm(AddressForm): """Custom address form for webshop """ phone = TextField(_('Phone'), [ validators.DataRequired(), ]) def get_default_country(self): """Get the default country based on geoip data. """ if not geoip or not request.remote_addr: return None
from models import * from django.contrib.sessions.models import Session as django_session if not settings.configured: settings.configure() geoip_city_dat = settings.GEOLITECITY_ABSOLUTE_PATH geoip_org_dat = settings.GEOORGANIZATION_ABSOLUTE_PATH # default amount of time which must pass before the same event from the # same user is logged again default_sleep_minutes = 30 gic = None try: gic = GeoIP(geoip_city_dat) except IOError, err: sys.stderr.write(""" ERROR: Could not find GeoIP database. Tried looking in "%s". If this is not where you have your GeoIP .dat file stored, edit GEOLITECITY_ABSOLUTE_PATH in live/local_settings.py\nIf you don't have the GeoIP City database, you can get it from "http://dev.maxmind.com/geoip/geolite". """ % (geoip_city_dat)) sys.exit(1) gio = None try: gio = GeoIP(geoip_org_dat) except IOError: # we don't want to spam the log with warning messages, so don't do anything # here. it's desgined to work without the GeoIP Organization database
] __metaclass__ = PoolMeta geoip = None try: from pygeoip import GeoIP except ImportError: logging.error("pygeoip is not installed") else: # Usual location in Ubuntu path1 = '/usr/share/GeoIP/GeoIP.dat' # this is where brew installs it path2 = '/usr/local/Cellar/geoip/1.4.8/share/GeoIP/GeoIP.dat' if os.path.isfile(path1): geoip = GeoIP(path1) elif os.path.isfile(path2): geoip = GeoIP(path2) class NereidUser: """ Add employee """ __name__ = "nereid.user" #: Allow the nereid user to be connected to an internal employee. This #: indicates that the user is an employee and not a regular participant employee = fields.Many2One( 'company.employee', 'Employee',
import re import sys from pygeoip import GeoIP g = GeoIP('GeoIP.dat') r = re.compile(r'20. ([0-9]+) "') d = {} while True: line = sys.stdin.readline() if not line: break match = r.search(line) if match: ip = line.split(' ')[0] country = g.country_name_by_addr(ip) if not country: country = 'Unknown' d[country] = int(match.group(1)) + d.get(country, 0) ds = d.items() ds.sort(lambda x, y: cmp(y[1], x[1])) for k, v in ds: if v > (1024 * 1024 * 1024): print k, '%0.2fGb' % (v / (1024 * 1024 * 1024.0))