def test_no_get_args(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) self.assertRaises(TypeError, reader.get) reader.close()
def _check_concurrency(self, worker_class): reader = open_database( 'tests/data/test-data/GeoIP2-Domain-Test.mmdb', self.mode ) def lookup(pipe): try: for i in range(32): reader.get('65.115.240.{i}'.format(i=i)) pipe.send(1) except: pipe.send(0) finally: if worker_class is Process: reader.close() pipe.close() pipes = [Pipe() for _ in range(32)] procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes] for proc in procs: proc.start() for proc in procs: proc.join() reader.close() count = sum([p.recv() for (p, c) in pipes]) self.assertEqual(count, 32, 'expected number of successful lookups')
def test_metadata_args(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) self.assertRaises(TypeError, reader.metadata, ('blah')) reader.close()
def mirror(): """Get the encrypted anti-censorship data from Mothership""" import urllib v = request.values.get ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr)) reader = maxminddb.open_database('GeoLite2-City.mmdb') result = reader.get(ip) reader.close() if v("_ac_force_locale") or (result and result.get('country')): locale = v("_ac_force_locale") if v("_ac_force_locale") else \ result.get("country").get('iso_code') link = "%s/%s?api_key=%s&domain=%s&user_key=%s" % (MOTHERSHIP_URL, locale, os.environ.get('API_KEY'), os.environ.get('DOMAIN'), v("key")) f = urllib.urlopen(link) encrypted_data = f.read() return encrypted_data return ""
def test_decoder(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode) record = reader.get('::1.1.1.0') self.assertEqual(record['array'], [1, 2, 3]) self.assertEqual(record['boolean'], True) self.assertEqual(record['bytes'], bytearray(b'\x00\x00\x00*')) self.assertEqual(record['double'], 42.123456) self.assertAlmostEqual(record['float'], 1.1) self.assertEqual(record['int32'], -268435456) self.assertEqual( { 'mapX': { 'arrayX': [7, 8, 9], 'utf8_stringX': 'hello' }, }, record['map'] ) self.assertEqual(record['uint16'], 100) self.assertEqual(record['uint32'], 268435456) self.assertEqual(record['uint64'], 1152921504606846976) self.assertEqual(record['utf8_string'], 'unicode! ☯ - ♫') self.assertEqual( 1329227995784915872903807060280344576, record['uint128'] ) reader.close()
def report(): """Report something sketchy back to Mothership""" import urllib import requests v = request.values.get ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr)) reader = maxminddb.open_database('GeoLite2-City.mmdb') result = reader.get(ip) reader.close() payload = { 'ip': ip, 'host': v('host'), 'response': v('response') } if result and result.get('country'): payload['locale'] = result.get("country").get('iso_code') r = requests.post(MOTHERSHIP_URL + "/report", data=payload) print r.text return "kthx"
def test_no_ipv4_search_tree(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb', self.mode) self.assertEqual(reader.get('1.1.1.1'), '::0/64') self.assertEqual(reader.get('192.1.1.1'), '::0/64') reader.close()
def _get_db(cls): if cls._db: return cls._db try: import maxminddb except ImportError: return None path = cls._try_all() if not path: return None cls._db = maxminddb.open_database(path) return cls._db
def test_ip_validation(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode) self.assertRaisesRegex(ValueError, "'not_ip' does not appear to be an IPv4 or " "IPv6 address", reader.get, ('not_ip')) reader.close()
def test_double_close(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) reader.close() self.assertIsNone( reader.close(), 'Double close does not throw an exception')
def test_ipv6_address_in_ipv4_database(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb', self.mode) with self.assertRaisesRegex(ValueError, 'Error looking up 2001::. ' 'You attempted to look up an IPv6 address ' 'in an IPv4-only database'): reader.get('2001::') reader.close()
def _prepare_factor_pipeline(self, cfg): pipeline = [] if cfg.factors.asn_movement is not None: pipeline.append( factors.asn_movement( mmdb.open_database( cfg.factors.asn_movement.maxmind_db_path), cfg.asn_movement_severity)) return pipeline
def test_broken_database(self): reader = open_database('tests/data/test-data/' 'GeoIP2-City-Test-Broken-Double-Format.mmdb', self.mode) with self.assertRaisesRegex(InvalidDatabaseError, "The MaxMind DB file's data " "section contains bad data \(unknown data " "type or corrupt data\)" ): reader.get('2001:220::') reader.close()
def test_metadata_unknown_attribute(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) metadata = reader.metadata() with self.assertRaisesRegex(AttributeError, "'Metadata' object has no " "attribute 'blah'"): metadata.blah reader.close()
def test_broken_database(self): reader = open_database('tests/data/test-data/' 'GeoIP2-City-Test-Broken-Double-Format.mmdb', self.mode) with self.assertRaisesRegex(InvalidDatabaseError, "The MaxMind DB file's data " "section contains bad data \(unknown data " "type or corrupt data\)" ): reader.get('2001:220::') reader.close()
def test_metadata_unknown_attribute(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) metadata = reader.metadata() with self.assertRaisesRegex(AttributeError, "'Metadata' object has no " "attribute 'blah'"): metadata.blah reader.close()
def test_with_statement_close(self): filename = 'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb' reader = open_database(filename, self.mode) reader.close() def use_with(reader): with reader: pass self.assertRaisesRegex(ValueError, 'Attempt to reopen a closed MaxMind DB', use_with, reader)
def test_closed_get(self): if self.mode == MODE_MEMORY: return reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) reader.close() self.assertRaisesRegex(ValueError, 'Attempt to read from a closed MaxMind DB.' '|closed', reader.get, ('1.1.1.1'))
def consult_geo(self): geodb = maxminddb.open_database('GeoLite2-Country.mmdb') s = geodb.get(self.host) try: self.iso_code = s['country']['iso_code'] except: self.iso_code = "Null" try: self.country = s['country']['names']['en'] except: self.country = "Null"
def parse_maxminddb(db_path, ip): try: reader = maxminddb.open_database(db_path) response = reader.get(ip) reader.close() return response except FileNotFoundError: print('DB not found') print('SHUTTING DOWN') exit() except ValueError: return False
def parse_maxminddb(db_path, ip): try: reader = maxminddb.open_database(db_path) response = reader.get(ip) reader.close() return response except FileNotFoundError: print('DB not found') print('SHUTTING DOWN') exit() except ValueError: return False
def test_closed_get(self): if self.mode == MODE_MEMORY: return reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) reader.close() self.assertRaisesRegex(ValueError, 'Attempt to read from a closed MaxMind DB.' '|closed', reader.get, ('1.1.1.1'))
def open_mmdb_database(mmdb_content): """ Test MaxMindDB database format with only its content as binary :return True if the database is correct, False otherwise """ # Check if the response content is MMDB database tmpfile = BytesIO() tmpfile.write(mmdb_content) tmpfile.seek(0) setattr(tmpfile, "name", "test") try: return open_database(tmpfile, mode=MODE_FD) except Exception: return None
def nmap_scan(self, file, fk): progress_recorder = ProgressRecorder(self) result = 0 print(os.getcwd() + file) search = Search.objects.get(id=fk) max_reader = maxminddb.open_database('GeoLite2-City.mmdb') nmap_report = NmapParser.parse_fromfile(os.getcwd() + file) total = len(nmap_report.hosts) for c, i in enumerate(nmap_report.hosts): result += c nmap_host_worker(host_arg=i, max_reader=max_reader, search=search) progress_recorder.set_progress(c + 1, total=total) return result
def getCityInformation(ipdaddress): """ Retrieve city information about the provided IP address. :param ipdaddress: address to retrieve information from. """ dbName = "GeoLite2-City.mmdb" if not os.path.exists(dbName): utils.getCityDatabase() conx = maxminddb.open_database(dbName) utils.CityLocation(conx.get(ipdaddress))
def getPeerLocations(self, peers): import maxminddb db_path = self.getGeoipDb() if not db_path: self.log.debug("Not showing peer locations: no GeoIP database") return False self.log.info("Loading GeoIP database from: %s" % db_path) geodb = maxminddb.open_database(db_path) peers = list(peers.values()) # Place bars peer_locations = [] placed = {} # Already placed bars here for peer in peers: # Height of bar if peer.connection and peer.connection.last_ping_delay: ping = round(peer.connection.last_ping_delay * 1000) else: ping = None loc = self.getLoc(geodb, peer.ip) if not loc: continue # Create position array lat, lon = loc["lat"], loc["lon"] latlon = "%s,%s" % (lat, lon) if latlon in placed and helper.getIpType( peer.ip ) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) placed[latlon] = True peer_location = {} peer_location.update(loc) peer_location["lat"] = lat peer_location["lon"] = lon peer_location["ping"] = ping peer_locations.append(peer_location) # Append myself for ip in self.site.connection_server.ip_external_list: my_loc = self.getLoc(geodb, ip) if my_loc: my_loc["ping"] = 0 peer_locations.append(my_loc) return peer_locations
def get_ip_geoinfo(self, ip: str) -> dict: """ 根据ip去查询所在的地址 :param ip: :return: """ res = {} try: # 1、先检测下东西在不在 # 这样当前dbname就一定会有 # with DbipMmdb.__scouter_ip_locker: # 一次只能有一个玩意在这检测版本,虽然慢但是多线程没办法 # self.__check_db_version() reader = maxminddb.open_database( './_ip_location_dbs/dbfile/2020-04.mmdb') res = reader.get(ip) if res is None: return res reader.close() # data_city = res.get('city', {}).get('names', {}).get('en', None) # data_state = None # subdivisions = res.get('subdivisions', []) # if len(subdivisions) == 0: # data_state = None # else: # data_state = subdivisions[0].get('names', {}).get('en', None) # data_country = res.get('country', {}).get('iso_code', None) # location = res.get('location', {}) # lng = location.get('longitude', None) # lat = location.get('latitude', None) # 这里表示国家不属于主权国家,所以直接返回None,写注释是一个非常好的习惯 # if not ALL_COUNTRIES.__contains__( # data_country) or lat is None or lng is None: # return res # geoinfo: GeoInfo = GeoInfo(level, lng, lat) # exrtinfo: CountryCode = ALL_COUNTRIES.get(data_country) # geoinfo.set_country( # data_country, { # 'zh-CN': exrtinfo.country_names.get('CN'), # 'en': exrtinfo.country_names.get('EN') # }) # geoinfo.set_province({'zh-CN': None, 'en': data_state}) # geoinfo.set_city({'zh-CN': None, 'en': data_city}) # geoinfo.globaltelcode = exrtinfo.countrycode # geoinfo.timezone = exrtinfo.timediffer # res = geoinfo except Exception as error: print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error)) finally: return res
def test_broken_database(self): reader = open_database( "tests/data/test-data/" "GeoIP2-City-Test-Broken-Double-Format.mmdb", self.mode, ) with self.assertRaisesRegex( InvalidDatabaseError, r"The MaxMind DB file's data " r"section contains bad data \(unknown data " r"type or corrupt data\)", ): reader.get(self.ipf("2001:220::")) reader.close()
def query(self): import maxminddb db = maxminddb.open_database(self._res.get_file_path()) result = db.get(self._query.get("ip", "127.0.0.1")[0]) prop = self._query.get("prop", "all")[0] if result: if prop == "countrycode": return "200 OK", [("Content-Type", "text/plain") ], result["country"]["iso_code"] else: return "200 OK", [("Content-Type", "text/json")], dumps(result) else: return "404 Not Found", [], ""
def run( analyzer_name, job_id, observable_name, observable_classification, additional_config_params, ): logger.info("started analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) report = general.get_basic_report_template(analyzer_name) try: try: if not os.path.isfile(database_location): updater(additional_config_params) reader = maxminddb.open_database(database_location) maxmind_result = reader.get(observable_name) reader.close() except maxminddb.InvalidDatabaseError as e: error_message = "invalid database error: {}".format(e) logger.exception(error_message) maxmind_result = {"error": error_message} if not maxmind_result: maxmind_result = {} # pprint.pprint(maxmind_result) report["report"] = maxmind_result except AnalyzerRunException as e: error_message = ( "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.error(error_message) report["errors"].append(error_message) report["success"] = False except Exception as e: traceback.print_exc() error_message = ( "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" "".format(job_id, analyzer_name, observable_name, e)) logger.exception(error_message) report["errors"].append(str(e)) report["success"] = False else: report["success"] = True general.set_report_and_cleanup(job_id, report) logger.info("finished analyzer {} job_id {} observable {}" "".format(analyzer_name, job_id, observable_name)) return report
def load_from(base): for root, directories, filenames in os.walk(base): for directory in directories: match = re.match('^Geo(?:IP|Lite)2-(?:.+)_(?:.+)$', directory) if match: path = os.path.join(root, directory) load_from(path) for filename in filenames: match = re.match('^Geo(?:IP|Lite)2-(.+)\.mmdb$', filename) if match: typ = match.group(1) path = os.path.join(root, filename) reader = maxminddb.open_database(path) readers.append(reader)
def __init__(self, fileish: str, locales: Optional[List[str]] = None, mode: int = MODE_AUTO) -> None: """Create GeoIP2 Reader. :param fileish: The string path to the GeoIP2 database, or an existing file descriptor pointing to the database. Note that this latter usage is only valid when mode is MODE_FD. :param locales: This is list of locale codes. This argument will be passed on to record classes to use when their name properties are called. The default value is ['en']. The order of the locales is significant. When a record class has multiple names (country, city, etc.), its name property will return the name in the first locale that has one. Note that the only locale which is always present in the GeoIP2 data is "en". If you do not include this locale, the name property may end up returning None even when the record has an English name. Currently, the valid locale codes are: * de -- German * en -- English names may still include accented characters if that is the accepted spelling in English. In other words, English does not mean ASCII. * es -- Spanish * fr -- French * ja -- Japanese * pt-BR -- Brazilian Portuguese * ru -- Russian * zh-CN -- Simplified Chinese. :param mode: The mode to open the database with. Valid mode are: * MODE_MMAP_EXT - use the C extension with memory map. * MODE_MMAP - read from memory map. Pure Python. * MODE_FILE - read database as standard file. Pure Python. * MODE_MEMORY - load database into memory. Pure Python. * MODE_FD - the param passed via fileish is a file descriptor, not a path. This mode implies MODE_MEMORY. Pure Python. * MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order. Default. """ if locales is None: locales = ["en"] self._db_reader = maxminddb.open_database(fileish, mode) self._db_type = self._db_reader.metadata().database_type self._locales = locales
def test_reader(self): for record_size in [24, 28, 32]: for ip_version in [4, 6]: file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' + str(ip_version) + '-' + str(record_size) + '.mmdb') reader = open_database(file_name, self.mode) self._check_metadata(reader, ip_version, record_size) if ip_version == 4: self._check_ip_v4(reader, file_name) else: self._check_ip_v6(reader, file_name) reader.close()
async def parse_mmdb(self, mmdb): ''' create and return mmdb reader of the mmdb :param:mmdb: the byte array of the .mmdb file ''' # maminddb expects a file or file descriptor, # use Tempfile to createa a file pointer ''' tfile = TemporaryFile() tfile.write(mmdb) # after writing must seek back to the begggining tfile.seek(0) reader = maxminddb.open_database(tfile, mode=maxminddb.MODE_MMAP_EXT) tfile.close() ''' temp_file = '/tmp/maxmin.db' with open(temp_file, 'wb+') as tf: tf.write(mmdb) reader = None try: reader = maxminddb.open_database(temp_file, mode=maxminddb.MODE_MMAP_EXT) except: reader = maxminddb.open_database(temp_file) return reader
def getASNInformation(ipaddress): """ Get ASN information of the provided IP address. :param ipaddress: """ dbName = "GeoLite2-ASN.mmdb" if not os.path.exists(dbName): utils.getASNDatabase() conx = maxminddb.open_database(dbName) info = conx.get(ipaddress) print("\n\033[0;36m[*]\033[0;0m ASN: ", info["autonomous_system_number"]) print("\033[0;36m[*]\033[0;0m Organization: ", info["autonomous_system_organization"])
def main(): """The main entry point.""" metrics = get_prometheus_metrics() prometheus_client.start_http_server(addr=ADDRESS, port=PORT) log.info(f'listing on {ADDRESS}:{PORT}') if MAXMIND_LICENSE_KEY and MAXMIND_DATABASE_EDITION: log.info('geoip lookup enabled') database_path = prepare_maxmind_database(MAXMIND_LICENSE_KEY, MAXMIND_DATABASE_EDITION, MAXMIND_CACHE_DIRECTORY) with maxminddb.open_database(database_path.as_posix()) as database: collect_metrics(*metrics, geoip_db=database) else: log.info('geoip lookup disabled') collect_metrics(*metrics)
def test_reader(self): for record_size in [24, 28, 32]: for ip_version in [4, 6]: file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' + str(ip_version) + '-' + str(record_size) + '.mmdb') reader = open_database(file_name, self.mode) self._check_metadata(reader, ip_version, record_size) if ip_version == 4: self._check_ip_v4(reader, file_name) else: self._check_ip_v6(reader, file_name) reader.close()
async def loader(): try: if not os.path.isfile(settings.db_file): raise FileNotFoundError("db file not found") state.db = maxminddb.open_database(settings.db_file) logger.info('DB loaded') for num in count(): info = await cache_info() logger.info('cache stat', loop=num, info=info) await asyncio.sleep(60 * 5) except asyncio.CancelledError: pass except Exception: logger.exception('error while opening database file')
def test_closed_metadata(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode) reader.close() # The primary purpose of this is to ensure the extension doesn't # segfault try: metadata = reader.metadata() except IOError as ex: self.assertEqual('Attempt to read from a closed MaxMind DB.', str(ex), 'extension throws exception') else: self.assertIsNotNone(metadata, 'pure Python implementation returns value')
def _validate_cachefile(self, filepath): try: db = maxminddb.open_database(filepath) except Exception as e: log.debug('Exception raised when trying to open %r: %r', filepath, e) return False test_ip = '1.1.1.1' try: db.get(test_ip) except Exception as e: log.debug('Exception raised when looking up %r: %r', test_ip, e) return False else: return True
def update_db(db_path, license_key, edition_id): db_dir_path = os.path.abspath(os.path.dirname(db_path)) db_archive_path = '%s.tar.gz' % db_path def maxmind_download(suffix, **kwargs): return requests.get('https://download.maxmind.com/app/geoip_download', params={ 'license_key': license_key, 'edition_id': edition_id, 'suffix': suffix, }, **kwargs) expected_md5 = maxmind_download('tar.gz.md5').content curr_md5 = _hash_file(db_archive_path) if expected_md5 == curr_md5 and os.path.exists(db_path): return with open(db_archive_path, 'wb') as local_zip: for chunk in maxmind_download( 'tar.gz', stream=True).iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks local_zip.write(chunk) with tarfile.open(db_archive_path) as tar_file: # We only want the mmdb file. Maxmind kindly includes things # we don't want. extract_members = [ member for member in tar_file.getmembers() if member.name.endswith('.mmdb') ] assert len(extract_members) == 1 tar_file.extractall(path=db_dir_path, members=extract_members) # extractall keeps the subfolder structure. Account for this by # appending the path to the db_dir_path where it was extracted. new_db_path = os.path.join(db_dir_path, extract_members[0].path) try: db = maxminddb.open_database(new_db_path) db.get('8.8.8.8') db.get('2001:420::') except Exception: sys.stderr.write('Retrieved invalid GeoIP database - ' 'check MaxMind account details.\n') raise if not os.path.exists(db_dir_path): os.makedirs(db_dir_path) shutil.move(new_db_path, db_path) os.rmdir(os.path.dirname(new_db_path))
def getPeerLocations(self, peers): import maxminddb db_path = config.data_dir + '/GeoLite2-City.mmdb' if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: if not self.downloadGeoLiteDb(db_path): return False geodb = maxminddb.open_database(db_path) peers = peers.values() # Place bars peer_locations = [] placed = {} # Already placed bars here for peer in peers: # Height of bar if peer.connection and peer.connection.last_ping_delay: ping = round(peer.connection.last_ping_delay * 1000) else: ping = None loc = self.getLoc(geodb, peer.ip) if not loc: continue # Create position array lat, lon = loc["lat"], loc["lon"] latlon = "%s,%s" % (lat, lon) if latlon in placed and helper.getIpType( peer.ip ) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) placed[latlon] = True peer_location = {} peer_location.update(loc) peer_location["lat"] = lat peer_location["lon"] = lon peer_location["ping"] = ping peer_locations.append(peer_location) # Append myself for ip in config.ip_external: my_loc = self.getLoc(geodb, ip) if my_loc: my_loc["ping"] = 0 peer_locations.append(my_loc) return peer_locations
def _swap_database(self, new_database_path): """ Swap the current database with a new database :param new_database_path: The path to the new database """ # Open the new database self._reader = maxminddb.open_database(new_database_path, maxminddb.const.MODE_MEMORY) # Delete the temporary new database file try: os.remove(new_database_path) except Exception: logger.exception("Failed to remove old database file.") self._database_checksum = self._response_checksum
def morph(self, data): mmreader = maxminddb.open_database(self.config['dbpath']['value']) if data['name'] in ['connections','connscan']: loccol = data['columns'].index('LocalAddress') remcol = data['columns'].index('RemoteAddress') for idx,row in enumerate(data['data']): row = list(row) if self.IsInternal(row[loccol]): row[loccol] = {'value':row[loccol],'style':'background-color:lightyellow;'} else: row[loccol] = self.PrepareExtData(row[loccol], mmreader) if self.IsInternal(row[remcol]): row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'} else: row[remcol] = self.PrepareExtData(row[remcol], mmreader) data['data'][idx] = row
def main(): with open(a,"r") as f: for line in f.readlines(): try: reader = maxminddb.open_database('/home/caifeng/ip.mndb') #strip() 方法用于移除字符串头尾指定的字符(默认为空格或换行符)或字符序列 result1 = reader.get(line.strip('\n')) result = json.dumps(result1).decode('unicode_escape') reader.close() except Exception as e: print(e) else: with open(a+'-result.txt','a+') as r: r.write(line.strip('\n') + ' ') r.write(result + '\n')
def run(self): try: if not os.path.isfile(database_location): self.updater(self.additional_config_params) reader = maxminddb.open_database(database_location) maxmind_result = reader.get(self.observable_name) reader.close() except maxminddb.InvalidDatabaseError as e: error_message = f"Invalid database error: {e}" logger.exception(error_message) maxmind_result = {"error": error_message} if not maxmind_result: maxmind_result = {} return maxmind_result
def get_ip_geoinfo(self, ip: str) -> dict: """ 根据ip去查询所在的地址 :param ip: :return: """ res = None try: reader = maxminddb.open_database( './_ip_location_dbs/dbfile/geoinfo.mmdb') res = reader.get(ip) reader.close() except Exception as error: print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error)) finally: return res
def test_closed_metadata(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) reader.close() # The primary purpose of this is to ensure the extension doesn't # segfault try: metadata = reader.metadata() except IOError as ex: self.assertEqual('Attempt to read from a closed MaxMind DB.', str(ex), 'extension throws exception') else: self.assertIsNotNone( metadata, 'pure Python implementation returns value')
def getPeerLocations(self, peers): import maxminddb db_path = config.data_dir + '/GeoLite2-City.mmdb' if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: if not self.downloadGeoLiteDb(db_path): return False geodb = maxminddb.open_database(db_path) peers = peers.values() # Place bars peer_locations = [] placed = {} # Already placed bars here for peer in peers: # Height of bar if peer.connection and peer.connection.last_ping_delay: ping = round(peer.connection.last_ping_delay * 1000) else: ping = None loc = self.getLoc(geodb, peer.ip) if not loc: continue # Create position array lat, lon = loc["lat"], loc["lon"] latlon = "%s,%s" % (lat, lon) if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) placed[latlon] = True peer_location = {} peer_location.update(loc) peer_location["lat"] = lat peer_location["lon"] = lon peer_location["ping"] = ping peer_locations.append(peer_location) # Append myself my_loc = self.getLoc(geodb, config.ip_external) if my_loc: my_loc["ping"] = 0 peer_locations.append(my_loc) return peer_locations
def __init__(self, fileish, locales=None, mode=MODE_AUTO): """Create GeoIP2 Reader. :param fileish: The string path to the GeoIP2 database, or an existing file descriptor pointing to the database. Note that this latter usage is only valid when mode is MODE_FD. :param locales: This is list of locale codes. This argument will be passed on to record classes to use when their name properties are called. The default value is ['en']. The order of the locales is significant. When a record class has multiple names (country, city, etc.), its name property will return the name in the first locale that has one. Note that the only locale which is always present in the GeoIP2 data is "en". If you do not include this locale, the name property may end up returning None even when the record has an English name. Currently, the valid locale codes are: * de -- German * en -- English names may still include accented characters if that is the accepted spelling in English. In other words, English does not mean ASCII. * es -- Spanish * fr -- French * ja -- Japanese * pt-BR -- Brazilian Portuguese * ru -- Russian * zh-CN -- Simplified Chinese. :param mode: The mode to open the database with. Valid mode are: * MODE_MMAP_EXT - use the C extension with memory map. * MODE_MMAP - read from memory map. Pure Python. * MODE_FILE - read database as standard file. Pure Python. * MODE_MEMORY - load database into memory. Pure Python. * MODE_FD - the param passed via fileish is a file descriptor, not a path. This mode implies MODE_MEMORY. Pure Python. * MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order. Default. """ if locales is None: locales = ['en'] self._db_reader = maxminddb.open_database(fileish, mode) self._locales = locales
def update_db(db_path, license_key, edition_id): db_dir_path = os.path.abspath(os.path.dirname(db_path)) db_archive_path = '%s.tar.gz' % db_path def maxmind_download(suffix, **kwargs): return requests.get('https://download.maxmind.com/app/geoip_download', params={'license_key': license_key, 'edition_id': edition_id, 'suffix': suffix, }, **kwargs) expected_md5 = maxmind_download('tar.gz.md5').content curr_md5 = _hash_file(db_archive_path) if expected_md5 == curr_md5 and os.path.exists(db_path): return with open(db_archive_path, 'wb') as local_zip: for chunk in maxmind_download('tar.gz', stream=True ).iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks local_zip.write(chunk) with tarfile.open(db_archive_path) as tar_file: # We only want the mmdb file. Maxmind kindly includes things # we don't want. extract_members = [member for member in tar_file.getmembers() if member.name.endswith('.mmdb')] assert len(extract_members) == 1 tar_file.extractall(path=db_dir_path, members=extract_members) # extractall keeps the subfolder structure. Account for this by # appending the path to the db_dir_path where it was extracted. new_db_path = os.path.join(db_dir_path, extract_members[0].path) try: db = maxminddb.open_database(new_db_path) db.get('8.8.8.8') db.get('2001:420::') except Exception: sys.stderr.write('Retrieved invalid GeoIP database - ' 'check MaxMind account details.\n') raise if not os.path.exists(db_dir_path): os.makedirs(db_dir_path) shutil.move(new_db_path, db_path) os.rmdir(os.path.dirname(new_db_path))
def morph(self, data): mmreader = maxminddb.open_database(self.config['dbpath']['value']) if data['name'] in ['connections','connscan','netscan','sockets','sockscan','linux_route_cache','linux_netstat']: for colname in ['LocalAddress','RemoteAddress','LocalAddr','ForeignAddr','Address','Destination']: if colname in data['columns']: col = data['columns'].index(colname) #remcol = data['columns'].index('RemoteAddress') for idx,row in enumerate(data['data']): row = list(row) if self.IsInternal(row[col]): row[col] = {'value':row[col],'style':'background-color:lightyellow;'} else: row[col] = self.PrepareExtData(row[col], mmreader) #if self.IsInternal(row[remcol]): # row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'} #else: # row[remcol] = self.PrepareExtData(row[remcol], mmreader) data['data'][idx] = row
def lookup(*args): ipstring = args[0] result = "" try: reader = maxminddb.open_database('GeoLite2-Country.mmdb') ipdata = reader.get(ipstring) reader.close() result = ipdata['country']['names']['en'] except maxminddb.InvalidDatabaseError: result = "InvalidDatabaseError" except FileNotFoundError: result = "FileNotFoundError" except ValueError: result = "ValueError" return result
import argparse import maxminddb import random import socket import struct import timeit parser = argparse.ArgumentParser(description='Benchmark maxminddb.') parser.add_argument( '--count', default=250000, type=int, help='number of lookups') parser.add_argument('--mode', default=0, type=int, help='reader mode to use') parser.add_argument( '--file', default='GeoIP2-City.mmdb', help='path to mmdb file') args = parser.parse_args() reader = maxminddb.open_database(args.file, args.mode) def lookup_ip_address(): ip = socket.inet_ntoa(struct.pack('!L', random.getrandbits(32))) record = reader.get(str(ip)) elapsed = timeit.timeit( 'lookup_ip_address()', setup='from __main__ import lookup_ip_address', number=args.count) print(args.count / elapsed, 'lookups per second')
def actionSidebarGetPeers(self, to): permissions = self.getPermissions(to) if "ADMIN" not in permissions: return self.response(to, "You don't have permission to run this command") try: import maxminddb db_path = config.data_dir + '/GeoLite2-City.mmdb' if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0: if not self.downloadGeoLiteDb(db_path): return False geodb = maxminddb.open_database(db_path) peers = self.site.peers.values() # Find avg ping ping_times = [ peer.connection.last_ping_delay for peer in peers if peer.connection and peer.connection.last_ping_delay and peer.connection.last_ping_delay ] if ping_times: ping_avg = sum(ping_times) / float(len(ping_times)) else: ping_avg = 0 # Place bars globe_data = [] placed = {} # Already placed bars here for peer in peers: # Height of bar if peer.connection and peer.connection.last_ping_delay: ping = min(0.20, math.log(1 + peer.connection.last_ping_delay / ping_avg, 300)) else: ping = -0.03 # Query and cache location if peer.ip in loc_cache: loc = loc_cache[peer.ip] else: try: loc = geodb.get(peer.ip) except: loc = None loc_cache[peer.ip] = loc if not loc or "location" not in loc: continue # Create position array lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"]) latlon = "%s,%s" % (lat, lon) if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part lat += float(128 - int(peer.ip.split(".")[-2])) / 50 lon += float(128 - int(peer.ip.split(".")[-1])) / 50 latlon = "%s,%s" % (lat, lon) placed[latlon] = True globe_data += (lat, lon, ping) # Append myself loc = geodb.get(config.ip_external) if loc: lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"]) globe_data += (lat, lon, -0.135) self.response(to, globe_data) except Exception, err: self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err)) self.response(to, {"error": err})
def test_close(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) reader.close()
r'(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)') IPPortPatternLine = re.compile( r'^.*?(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)).*?(?P<port>\d{2,5}).*$', flags=re.MULTILINE) IPPortPatternGlobal = re.compile( r'(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))' r'(?=.*?(?:(?:(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?P<port>\d{2,5})))', flags=re.DOTALL) # IsIpPattern = re.compile( # r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$') mmdbReader = maxminddb.open_database( os.path.join(BASE_DIR, 'data', 'GeoLite2-Country.mmdb')) async def resolve_host(host, timeout, loop): global RESOLVER if not RESOLVER: RESOLVER = aiodns.DNSResolver(loop=loop) try: ip = await asyncio.wait_for(RESOLVER.query(host, 'A'), timeout=timeout) ip = ip[0].host except (aiodns.error.DNSError, asyncio.TimeoutError): return False else: return ip def get_base_dir():
import maxminddb from PIL import Image from PIL.ExifTags import TAGS from pprint import pprint import sys imageName = sys.argv[1] reader = maxminddb.open_database('GeoLite2-City.mmdb') data = reader.get('1.1.1.1') # print(data) reader.close() exifData = {} imgFile = Image.open(imageName) # imgFile = Image.open('image_test2.jpg') info = imgFile._getexif() if info: for (tag, value) in info.items(): decoded = TAGS.get(tag, tag) exifData[decoded] = value exifGPS = exifData.get('GPSInfo', None) if exifGPS: print(imgFile, ' contains GPS MetaData', exifGPS) pprint(exifData)
# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from django.conf import settings try: import maxminddb except ImportError: maxminddb = None if maxminddb is not None: try: geo = maxminddb.open_database(settings.MAXMIND_DB_PATH) except (IOError, maxminddb.InvalidDatabaseError): geo = None else: geo = None def get_country_from_ip(ip_addr): """Return country info for the given IP Address.""" if geo is not None: try: data = geo.get(ip_addr) except ValueError: data = None if data: country = data.get('country', data.get('registered_country'))
def test_too_many_get_args(self): reader = open_database( 'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode ) self.assertRaises(TypeError, reader.get, ('1.1.1.1', 'blah')) reader.close()