def test_no_get_args(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
self.assertRaises(TypeError, reader.get)
reader.close()
def _check_concurrency(self, worker_class):
reader = open_database(
'tests/data/test-data/GeoIP2-Domain-Test.mmdb',
self.mode
)
def lookup(pipe):
try:
for i in range(32):
reader.get('65.115.240.{i}'.format(i=i))
pipe.send(1)
except:
pipe.send(0)
finally:
if worker_class is Process:
reader.close()
pipe.close()
pipes = [Pipe() for _ in range(32)]
procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
for proc in procs:
proc.start()
for proc in procs:
proc.join()
reader.close()
count = sum([p.recv() for (p, c) in pipes])
self.assertEqual(count, 32, 'expected number of successful lookups')
def test_metadata_args(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
self.assertRaises(TypeError, reader.metadata, ('blah'))
reader.close()
def mirror():
"""Get the encrypted anti-censorship data from Mothership"""
import urllib
v = request.values.get
ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr))
reader = maxminddb.open_database('GeoLite2-City.mmdb')
result = reader.get(ip)
reader.close()
if v("_ac_force_locale") or (result and result.get('country')):
locale = v("_ac_force_locale") if v("_ac_force_locale") else \
result.get("country").get('iso_code')
link = "%s/%s?api_key=%s&domain=%s&user_key=%s" % (MOTHERSHIP_URL,
locale, os.environ.get('API_KEY'), os.environ.get('DOMAIN'),
v("key"))
f = urllib.urlopen(link)
encrypted_data = f.read()
return encrypted_data
return ""
def test_decoder(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
record = reader.get('::1.1.1.0')
self.assertEqual(record['array'], [1, 2, 3])
self.assertEqual(record['boolean'], True)
self.assertEqual(record['bytes'], bytearray(b'\x00\x00\x00*'))
self.assertEqual(record['double'], 42.123456)
self.assertAlmostEqual(record['float'], 1.1)
self.assertEqual(record['int32'], -268435456)
self.assertEqual(
{
'mapX': {
'arrayX': [7, 8, 9],
'utf8_stringX': 'hello'
},
},
record['map']
)
self.assertEqual(record['uint16'], 100)
self.assertEqual(record['uint32'], 268435456)
self.assertEqual(record['uint64'], 1152921504606846976)
self.assertEqual(record['utf8_string'], 'unicode! ☯ - ♫')
self.assertEqual(
1329227995784915872903807060280344576,
record['uint128']
)
reader.close()
def report():
"""Report something sketchy back to Mothership"""
import urllib
import requests
v = request.values.get
ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr))
reader = maxminddb.open_database('GeoLite2-City.mmdb')
result = reader.get(ip)
reader.close()
payload = {
'ip': ip,
'host': v('host'),
'response': v('response')
}
if result and result.get('country'):
payload['locale'] = result.get("country").get('iso_code')
r = requests.post(MOTHERSHIP_URL + "/report", data=payload)
print r.text
return "kthx"
def test_no_ipv4_search_tree(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb',
self.mode)
self.assertEqual(reader.get('1.1.1.1'), '::0/64')
self.assertEqual(reader.get('192.1.1.1'), '::0/64')
reader.close()
def _get_db(cls):
if cls._db: return cls._db
try: import maxminddb
except ImportError: return None
path = cls._try_all()
if not path: return None
cls._db = maxminddb.open_database(path)
return cls._db
def test_ip_validation(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
self.assertRaisesRegex(ValueError,
"'not_ip' does not appear to be an IPv4 or "
"IPv6 address",
reader.get, ('not_ip'))
reader.close()
def test_double_close(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
reader.close()
self.assertIsNone(
reader.close(), 'Double close does not throw an exception')
def test_ipv6_address_in_ipv4_database(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb', self.mode)
with self.assertRaisesRegex(ValueError,
'Error looking up 2001::. '
'You attempted to look up an IPv6 address '
'in an IPv4-only database'):
reader.get('2001::')
reader.close()
def _prepare_factor_pipeline(self, cfg):
pipeline = []
if cfg.factors.asn_movement is not None:
pipeline.append(
factors.asn_movement(
mmdb.open_database(
cfg.factors.asn_movement.maxmind_db_path),
cfg.asn_movement_severity))
return pipeline
def test_broken_database(self):
reader = open_database('tests/data/test-data/'
'GeoIP2-City-Test-Broken-Double-Format.mmdb',
self.mode)
with self.assertRaisesRegex(InvalidDatabaseError,
"The MaxMind DB file's data "
"section contains bad data \(unknown data "
"type or corrupt data\)"
):
reader.get('2001:220::')
reader.close()
def test_metadata_unknown_attribute(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
metadata = reader.metadata()
with self.assertRaisesRegex(AttributeError,
"'Metadata' object has no "
"attribute 'blah'"):
metadata.blah
reader.close()
def test_broken_database(self):
reader = open_database('tests/data/test-data/'
'GeoIP2-City-Test-Broken-Double-Format.mmdb',
self.mode)
with self.assertRaisesRegex(InvalidDatabaseError,
"The MaxMind DB file's data "
"section contains bad data \(unknown data "
"type or corrupt data\)"
):
reader.get('2001:220::')
reader.close()
def test_metadata_unknown_attribute(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
metadata = reader.metadata()
with self.assertRaisesRegex(AttributeError,
"'Metadata' object has no "
"attribute 'blah'"):
metadata.blah
reader.close()
def test_with_statement_close(self):
filename = 'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb'
reader = open_database(filename, self.mode)
reader.close()
def use_with(reader):
with reader:
pass
self.assertRaisesRegex(ValueError, 'Attempt to reopen a closed MaxMind DB',
use_with, reader)
def test_closed_get(self):
if self.mode == MODE_MEMORY:
return
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
reader.close()
self.assertRaisesRegex(ValueError,
'Attempt to read from a closed MaxMind DB.'
'|closed',
reader.get, ('1.1.1.1'))
def consult_geo(self):
geodb = maxminddb.open_database('GeoLite2-Country.mmdb')
s = geodb.get(self.host)
try:
self.iso_code = s['country']['iso_code']
except:
self.iso_code = "Null"
try:
self.country = s['country']['names']['en']
except:
self.country = "Null"
def parse_maxminddb(db_path, ip):
try:
reader = maxminddb.open_database(db_path)
response = reader.get(ip)
reader.close()
return response
except FileNotFoundError:
print('DB not found')
print('SHUTTING DOWN')
exit()
except ValueError:
return False
def parse_maxminddb(db_path, ip):
try:
reader = maxminddb.open_database(db_path)
response = reader.get(ip)
reader.close()
return response
except FileNotFoundError:
print('DB not found')
print('SHUTTING DOWN')
exit()
except ValueError:
return False
def test_closed_get(self):
if self.mode == MODE_MEMORY:
return
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
reader.close()
self.assertRaisesRegex(ValueError,
'Attempt to read from a closed MaxMind DB.'
'|closed',
reader.get, ('1.1.1.1'))
def open_mmdb_database(mmdb_content):
""" Test MaxMindDB database format with only its content as binary
:return True if the database is correct, False otherwise
"""
# Check if the response content is MMDB database
tmpfile = BytesIO()
tmpfile.write(mmdb_content)
tmpfile.seek(0)
setattr(tmpfile, "name", "test")
try:
return open_database(tmpfile, mode=MODE_FD)
except Exception:
return None
def nmap_scan(self, file, fk):
progress_recorder = ProgressRecorder(self)
result = 0
print(os.getcwd() + file)
search = Search.objects.get(id=fk)
max_reader = maxminddb.open_database('GeoLite2-City.mmdb')
nmap_report = NmapParser.parse_fromfile(os.getcwd() + file)
total = len(nmap_report.hosts)
for c, i in enumerate(nmap_report.hosts):
result += c
nmap_host_worker(host_arg=i, max_reader=max_reader, search=search)
progress_recorder.set_progress(c + 1, total=total)
return result
def getCityInformation(ipdaddress):
"""
Retrieve city information about the provided IP address.
:param ipdaddress: address to retrieve information from.
"""
dbName = "GeoLite2-City.mmdb"
if not os.path.exists(dbName):
utils.getCityDatabase()
conx = maxminddb.open_database(dbName)
utils.CityLocation(conx.get(ipdaddress))
def getPeerLocations(self, peers):
import maxminddb
db_path = self.getGeoipDb()
if not db_path:
self.log.debug("Not showing peer locations: no GeoIP database")
return False
self.log.info("Loading GeoIP database from: %s" % db_path)
geodb = maxminddb.open_database(db_path)
peers = list(peers.values())
# Place bars
peer_locations = []
placed = {} # Already placed bars here
for peer in peers:
# Height of bar
if peer.connection and peer.connection.last_ping_delay:
ping = round(peer.connection.last_ping_delay * 1000)
else:
ping = None
loc = self.getLoc(geodb, peer.ip)
if not loc:
continue
# Create position array
lat, lon = loc["lat"], loc["lon"]
latlon = "%s,%s" % (lat, lon)
if latlon in placed and helper.getIpType(
peer.ip
) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part
lat += float(128 - int(peer.ip.split(".")[-2])) / 50
lon += float(128 - int(peer.ip.split(".")[-1])) / 50
latlon = "%s,%s" % (lat, lon)
placed[latlon] = True
peer_location = {}
peer_location.update(loc)
peer_location["lat"] = lat
peer_location["lon"] = lon
peer_location["ping"] = ping
peer_locations.append(peer_location)
# Append myself
for ip in self.site.connection_server.ip_external_list:
my_loc = self.getLoc(geodb, ip)
if my_loc:
my_loc["ping"] = 0
peer_locations.append(my_loc)
return peer_locations
def get_ip_geoinfo(self, ip: str) -> dict:
"""
根据ip去查询所在的地址
:param ip:
:return:
"""
res = {}
try:
# 1、先检测下东西在不在
# 这样当前dbname就一定会有
# with DbipMmdb.__scouter_ip_locker:
# 一次只能有一个玩意在这检测版本，虽然慢但是多线程没办法
# self.__check_db_version()
reader = maxminddb.open_database(
'./_ip_location_dbs/dbfile/2020-04.mmdb')
res = reader.get(ip)
if res is None:
return res
reader.close()
# data_city = res.get('city', {}).get('names', {}).get('en', None)
# data_state = None
# subdivisions = res.get('subdivisions', [])
# if len(subdivisions) == 0:
# data_state = None
# else:
# data_state = subdivisions[0].get('names', {}).get('en', None)
# data_country = res.get('country', {}).get('iso_code', None)
# location = res.get('location', {})
# lng = location.get('longitude', None)
# lat = location.get('latitude', None)
# 这里表示国家不属于主权国家，所以直接返回None，写注释是一个非常好的习惯
# if not ALL_COUNTRIES.__contains__(
# data_country) or lat is None or lng is None:
# return res
# geoinfo: GeoInfo = GeoInfo(level, lng, lat)
# exrtinfo: CountryCode = ALL_COUNTRIES.get(data_country)
# geoinfo.set_country(
# data_country, {
# 'zh-CN': exrtinfo.country_names.get('CN'),
# 'en': exrtinfo.country_names.get('EN')
# })
# geoinfo.set_province({'zh-CN': None, 'en': data_state})
# geoinfo.set_city({'zh-CN': None, 'en': data_city})
# geoinfo.globaltelcode = exrtinfo.countrycode
# geoinfo.timezone = exrtinfo.timediffer
# res = geoinfo
except Exception as error:
print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error))
finally:
return res
def test_broken_database(self):
reader = open_database(
"tests/data/test-data/"
"GeoIP2-City-Test-Broken-Double-Format.mmdb",
self.mode,
)
with self.assertRaisesRegex(
InvalidDatabaseError,
r"The MaxMind DB file's data "
r"section contains bad data \(unknown data "
r"type or corrupt data\)",
):
reader.get(self.ipf("2001:220::"))
reader.close()
def query(self):
import maxminddb
db = maxminddb.open_database(self._res.get_file_path())
result = db.get(self._query.get("ip", "127.0.0.1")[0])
prop = self._query.get("prop", "all")[0]
if result:
if prop == "countrycode":
return "200 OK", [("Content-Type", "text/plain")
], result["country"]["iso_code"]
else:
return "200 OK", [("Content-Type", "text/json")], dumps(result)
else:
return "404 Not Found", [], ""
def run(
analyzer_name,
job_id,
observable_name,
observable_classification,
additional_config_params,
):
logger.info("started analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
report = general.get_basic_report_template(analyzer_name)
try:
try:
if not os.path.isfile(database_location):
updater(additional_config_params)
reader = maxminddb.open_database(database_location)
maxmind_result = reader.get(observable_name)
reader.close()
except maxminddb.InvalidDatabaseError as e:
error_message = "invalid database error: {}".format(e)
logger.exception(error_message)
maxmind_result = {"error": error_message}
if not maxmind_result:
maxmind_result = {}
# pprint.pprint(maxmind_result)
report["report"] = maxmind_result
except AnalyzerRunException as e:
error_message = (
"job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.error(error_message)
report["errors"].append(error_message)
report["success"] = False
except Exception as e:
traceback.print_exc()
error_message = (
"job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
"".format(job_id, analyzer_name, observable_name, e))
logger.exception(error_message)
report["errors"].append(str(e))
report["success"] = False
else:
report["success"] = True
general.set_report_and_cleanup(job_id, report)
logger.info("finished analyzer {} job_id {} observable {}"
"".format(analyzer_name, job_id, observable_name))
return report
def load_from(base):
for root, directories, filenames in os.walk(base):
for directory in directories:
match = re.match('^Geo(?:IP|Lite)2-(?:.+)_(?:.+)$', directory)
if match:
path = os.path.join(root, directory)
load_from(path)
for filename in filenames:
match = re.match('^Geo(?:IP|Lite)2-(.+)\.mmdb$', filename)
if match:
typ = match.group(1)
path = os.path.join(root, filename)
reader = maxminddb.open_database(path)
readers.append(reader)
def __init__(self,
fileish: str,
locales: Optional[List[str]] = None,
mode: int = MODE_AUTO) -> None:
"""Create GeoIP2 Reader.
:param fileish: The string path to the GeoIP2 database, or an existing
file descriptor pointing to the database. Note that this latter
usage is only valid when mode is MODE_FD.
:param locales: This is list of locale codes. This argument will be
passed on to record classes to use when their name properties are
called. The default value is ['en'].
The order of the locales is significant. When a record class has
multiple names (country, city, etc.), its name property will return
the name in the first locale that has one.
Note that the only locale which is always present in the GeoIP2
data is "en". If you do not include this locale, the name property
may end up returning None even when the record has an English name.
Currently, the valid locale codes are:
* de -- German
* en -- English names may still include accented characters if that
is the accepted spelling in English. In other words, English does
not mean ASCII.
* es -- Spanish
* fr -- French
* ja -- Japanese
* pt-BR -- Brazilian Portuguese
* ru -- Russian
* zh-CN -- Simplified Chinese.
:param mode: The mode to open the database with. Valid mode are:
* MODE_MMAP_EXT - use the C extension with memory map.
* MODE_MMAP - read from memory map. Pure Python.
* MODE_FILE - read database as standard file. Pure Python.
* MODE_MEMORY - load database into memory. Pure Python.
* MODE_FD - the param passed via fileish is a file descriptor, not a
path. This mode implies MODE_MEMORY. Pure Python.
* MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
Default.
"""
if locales is None:
locales = ["en"]
self._db_reader = maxminddb.open_database(fileish, mode)
self._db_type = self._db_reader.metadata().database_type
self._locales = locales
def test_reader(self):
for record_size in [24, 28, 32]:
for ip_version in [4, 6]:
file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
str(ip_version) + '-' + str(record_size) +
'.mmdb')
reader = open_database(file_name, self.mode)
self._check_metadata(reader, ip_version, record_size)
if ip_version == 4:
self._check_ip_v4(reader, file_name)
else:
self._check_ip_v6(reader, file_name)
reader.close()
async def parse_mmdb(self, mmdb):
'''
create and return mmdb reader of the mmdb
:param:mmdb: the byte array of the .mmdb file
'''
# maminddb expects a file or file descriptor,
# use Tempfile to createa a file pointer
'''
tfile = TemporaryFile()
tfile.write(mmdb)
# after writing must seek back to the begggining
tfile.seek(0)
reader = maxminddb.open_database(tfile, mode=maxminddb.MODE_MMAP_EXT)
tfile.close()
'''
temp_file = '/tmp/maxmin.db'
with open(temp_file, 'wb+') as tf:
tf.write(mmdb)
reader = None
try:
reader = maxminddb.open_database(temp_file, mode=maxminddb.MODE_MMAP_EXT)
except:
reader = maxminddb.open_database(temp_file)
return reader
def getASNInformation(ipaddress):
"""
Get ASN information of the provided IP address.
:param ipaddress:
"""
dbName = "GeoLite2-ASN.mmdb"
if not os.path.exists(dbName):
utils.getASNDatabase()
conx = maxminddb.open_database(dbName)
info = conx.get(ipaddress)
print("\n\033[0;36m[*]\033[0;0m ASN: ", info["autonomous_system_number"])
print("\033[0;36m[*]\033[0;0m Organization: ", info["autonomous_system_organization"])
def main():
"""The main entry point."""
metrics = get_prometheus_metrics()
prometheus_client.start_http_server(addr=ADDRESS, port=PORT)
log.info(f'listing on {ADDRESS}:{PORT}')
if MAXMIND_LICENSE_KEY and MAXMIND_DATABASE_EDITION:
log.info('geoip lookup enabled')
database_path = prepare_maxmind_database(MAXMIND_LICENSE_KEY,
MAXMIND_DATABASE_EDITION,
MAXMIND_CACHE_DIRECTORY)
with maxminddb.open_database(database_path.as_posix()) as database:
collect_metrics(*metrics, geoip_db=database)
else:
log.info('geoip lookup disabled')
collect_metrics(*metrics)
def test_reader(self):
for record_size in [24, 28, 32]:
for ip_version in [4, 6]:
file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
str(ip_version) + '-' + str(record_size) +
'.mmdb')
reader = open_database(file_name, self.mode)
self._check_metadata(reader, ip_version, record_size)
if ip_version == 4:
self._check_ip_v4(reader, file_name)
else:
self._check_ip_v6(reader, file_name)
reader.close()
async def loader():
try:
if not os.path.isfile(settings.db_file):
raise FileNotFoundError("db file not found")
state.db = maxminddb.open_database(settings.db_file)
logger.info('DB loaded')
for num in count():
info = await cache_info()
logger.info('cache stat', loop=num, info=info)
await asyncio.sleep(60 * 5)
except asyncio.CancelledError:
pass
except Exception:
logger.exception('error while opening database file')
def test_closed_metadata(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
reader.close()
# The primary purpose of this is to ensure the extension doesn't
# segfault
try:
metadata = reader.metadata()
except IOError as ex:
self.assertEqual('Attempt to read from a closed MaxMind DB.',
str(ex), 'extension throws exception')
else:
self.assertIsNotNone(metadata,
'pure Python implementation returns value')
def _validate_cachefile(self, filepath):
try:
db = maxminddb.open_database(filepath)
except Exception as e:
log.debug('Exception raised when trying to open %r: %r', filepath, e)
return False
test_ip = '1.1.1.1'
try:
db.get(test_ip)
except Exception as e:
log.debug('Exception raised when looking up %r: %r', test_ip, e)
return False
else:
return True
def update_db(db_path, license_key, edition_id):
db_dir_path = os.path.abspath(os.path.dirname(db_path))
db_archive_path = '%s.tar.gz' % db_path
def maxmind_download(suffix, **kwargs):
return requests.get('https://download.maxmind.com/app/geoip_download',
params={
'license_key': license_key,
'edition_id': edition_id,
'suffix': suffix,
},
**kwargs)
expected_md5 = maxmind_download('tar.gz.md5').content
curr_md5 = _hash_file(db_archive_path)
if expected_md5 == curr_md5 and os.path.exists(db_path):
return
with open(db_archive_path, 'wb') as local_zip:
for chunk in maxmind_download(
'tar.gz', stream=True).iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
local_zip.write(chunk)
with tarfile.open(db_archive_path) as tar_file:
# We only want the mmdb file. Maxmind kindly includes things
# we don't want.
extract_members = [
member for member in tar_file.getmembers()
if member.name.endswith('.mmdb')
]
assert len(extract_members) == 1
tar_file.extractall(path=db_dir_path, members=extract_members)
# extractall keeps the subfolder structure. Account for this by
# appending the path to the db_dir_path where it was extracted.
new_db_path = os.path.join(db_dir_path, extract_members[0].path)
try:
db = maxminddb.open_database(new_db_path)
db.get('8.8.8.8')
db.get('2001:420::')
except Exception:
sys.stderr.write('Retrieved invalid GeoIP database - '
'check MaxMind account details.\n')
raise
if not os.path.exists(db_dir_path):
os.makedirs(db_dir_path)
shutil.move(new_db_path, db_path)
os.rmdir(os.path.dirname(new_db_path))
def getPeerLocations(self, peers):
import maxminddb
db_path = config.data_dir + '/GeoLite2-City.mmdb'
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
if not self.downloadGeoLiteDb(db_path):
return False
geodb = maxminddb.open_database(db_path)
peers = peers.values()
# Place bars
peer_locations = []
placed = {} # Already placed bars here
for peer in peers:
# Height of bar
if peer.connection and peer.connection.last_ping_delay:
ping = round(peer.connection.last_ping_delay * 1000)
else:
ping = None
loc = self.getLoc(geodb, peer.ip)
if not loc:
continue
# Create position array
lat, lon = loc["lat"], loc["lon"]
latlon = "%s,%s" % (lat, lon)
if latlon in placed and helper.getIpType(
peer.ip
) == "ipv4": # Dont place more than 1 bar to same place, fake repos using ip address last two part
lat += float(128 - int(peer.ip.split(".")[-2])) / 50
lon += float(128 - int(peer.ip.split(".")[-1])) / 50
latlon = "%s,%s" % (lat, lon)
placed[latlon] = True
peer_location = {}
peer_location.update(loc)
peer_location["lat"] = lat
peer_location["lon"] = lon
peer_location["ping"] = ping
peer_locations.append(peer_location)
# Append myself
for ip in config.ip_external:
my_loc = self.getLoc(geodb, ip)
if my_loc:
my_loc["ping"] = 0
peer_locations.append(my_loc)
return peer_locations
def _swap_database(self, new_database_path):
"""
Swap the current database with a new database
:param new_database_path: The path to the new database
"""
# Open the new database
self._reader = maxminddb.open_database(new_database_path,
maxminddb.const.MODE_MEMORY)
# Delete the temporary new database file
try:
os.remove(new_database_path)
except Exception:
logger.exception("Failed to remove old database file.")
self._database_checksum = self._response_checksum
def morph(self, data):
mmreader = maxminddb.open_database(self.config['dbpath']['value'])
if data['name'] in ['connections','connscan']:
loccol = data['columns'].index('LocalAddress')
remcol = data['columns'].index('RemoteAddress')
for idx,row in enumerate(data['data']):
row = list(row)
if self.IsInternal(row[loccol]):
row[loccol] = {'value':row[loccol],'style':'background-color:lightyellow;'}
else:
row[loccol] = self.PrepareExtData(row[loccol], mmreader)
if self.IsInternal(row[remcol]):
row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'}
else:
row[remcol] = self.PrepareExtData(row[remcol], mmreader)
data['data'][idx] = row
def main():
with open(a,"r") as f:
for line in f.readlines():
try:
reader = maxminddb.open_database('/home/caifeng/ip.mndb')
#strip() 方法用于移除字符串头尾指定的字符（默认为空格或换行符）或字符序列
result1 = reader.get(line.strip('\n'))
result = json.dumps(result1).decode('unicode_escape')
reader.close()
except Exception as e:
print(e)
else:
with open(a+'-result.txt','a+') as r:
r.write(line.strip('\n') + ' ')
r.write(result + '\n')
def run(self):
try:
if not os.path.isfile(database_location):
self.updater(self.additional_config_params)
reader = maxminddb.open_database(database_location)
maxmind_result = reader.get(self.observable_name)
reader.close()
except maxminddb.InvalidDatabaseError as e:
error_message = f"Invalid database error: {e}"
logger.exception(error_message)
maxmind_result = {"error": error_message}
if not maxmind_result:
maxmind_result = {}
return maxmind_result
def get_ip_geoinfo(self, ip: str) -> dict:
"""
根据ip去查询所在的地址
:param ip:
:return:
"""
res = None
try:
reader = maxminddb.open_database(
'./_ip_location_dbs/dbfile/geoinfo.mmdb')
res = reader.get(ip)
reader.close()
except Exception as error:
print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error))
finally:
return res
def test_closed_metadata(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
reader.close()
# The primary purpose of this is to ensure the extension doesn't
# segfault
try:
metadata = reader.metadata()
except IOError as ex:
self.assertEqual('Attempt to read from a closed MaxMind DB.',
str(ex), 'extension throws exception')
else:
self.assertIsNotNone(
metadata, 'pure Python implementation returns value')
def getPeerLocations(self, peers):
import maxminddb
db_path = config.data_dir + '/GeoLite2-City.mmdb'
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
if not self.downloadGeoLiteDb(db_path):
return False
geodb = maxminddb.open_database(db_path)
peers = peers.values()
# Place bars
peer_locations = []
placed = {} # Already placed bars here
for peer in peers:
# Height of bar
if peer.connection and peer.connection.last_ping_delay:
ping = round(peer.connection.last_ping_delay * 1000)
else:
ping = None
loc = self.getLoc(geodb, peer.ip)
if not loc:
continue
# Create position array
lat, lon = loc["lat"], loc["lon"]
latlon = "%s,%s" % (lat, lon)
if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part
lat += float(128 - int(peer.ip.split(".")[-2])) / 50
lon += float(128 - int(peer.ip.split(".")[-1])) / 50
latlon = "%s,%s" % (lat, lon)
placed[latlon] = True
peer_location = {}
peer_location.update(loc)
peer_location["lat"] = lat
peer_location["lon"] = lon
peer_location["ping"] = ping
peer_locations.append(peer_location)
# Append myself
my_loc = self.getLoc(geodb, config.ip_external)
if my_loc:
my_loc["ping"] = 0
peer_locations.append(my_loc)
return peer_locations
def __init__(self, fileish, locales=None, mode=MODE_AUTO):
"""Create GeoIP2 Reader.
:param fileish: The string path to the GeoIP2 database, or an existing
file descriptor pointing to the database. Note that this latter
usage is only valid when mode is MODE_FD.
:param locales: This is list of locale codes. This argument will be
passed on to record classes to use when their name properties are
called. The default value is ['en'].
The order of the locales is significant. When a record class has
multiple names (country, city, etc.), its name property will return
the name in the first locale that has one.
Note that the only locale which is always present in the GeoIP2
data is "en". If you do not include this locale, the name property
may end up returning None even when the record has an English name.
Currently, the valid locale codes are:
* de -- German
* en -- English names may still include accented characters if that
is the accepted spelling in English. In other words, English does
not mean ASCII.
* es -- Spanish
* fr -- French
* ja -- Japanese
* pt-BR -- Brazilian Portuguese
* ru -- Russian
* zh-CN -- Simplified Chinese.
:param mode: The mode to open the database with. Valid mode are:
* MODE_MMAP_EXT - use the C extension with memory map.
* MODE_MMAP - read from memory map. Pure Python.
* MODE_FILE - read database as standard file. Pure Python.
* MODE_MEMORY - load database into memory. Pure Python.
* MODE_FD - the param passed via fileish is a file descriptor, not a
path. This mode implies MODE_MEMORY. Pure Python.
* MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
Default.
"""
if locales is None:
locales = ['en']
self._db_reader = maxminddb.open_database(fileish, mode)
self._locales = locales
def update_db(db_path, license_key, edition_id):
db_dir_path = os.path.abspath(os.path.dirname(db_path))
db_archive_path = '%s.tar.gz' % db_path
def maxmind_download(suffix, **kwargs):
return requests.get('https://download.maxmind.com/app/geoip_download',
params={'license_key': license_key,
'edition_id': edition_id,
'suffix': suffix,
},
**kwargs)
expected_md5 = maxmind_download('tar.gz.md5').content
curr_md5 = _hash_file(db_archive_path)
if expected_md5 == curr_md5 and os.path.exists(db_path):
return
with open(db_archive_path, 'wb') as local_zip:
for chunk in maxmind_download('tar.gz', stream=True
).iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
local_zip.write(chunk)
with tarfile.open(db_archive_path) as tar_file:
# We only want the mmdb file. Maxmind kindly includes things
# we don't want.
extract_members = [member for member in tar_file.getmembers()
if member.name.endswith('.mmdb')]
assert len(extract_members) == 1
tar_file.extractall(path=db_dir_path, members=extract_members)
# extractall keeps the subfolder structure. Account for this by
# appending the path to the db_dir_path where it was extracted.
new_db_path = os.path.join(db_dir_path, extract_members[0].path)
try:
db = maxminddb.open_database(new_db_path)
db.get('8.8.8.8')
db.get('2001:420::')
except Exception:
sys.stderr.write('Retrieved invalid GeoIP database - '
'check MaxMind account details.\n')
raise
if not os.path.exists(db_dir_path):
os.makedirs(db_dir_path)
shutil.move(new_db_path, db_path)
os.rmdir(os.path.dirname(new_db_path))
def morph(self, data):
mmreader = maxminddb.open_database(self.config['dbpath']['value'])
if data['name'] in ['connections','connscan','netscan','sockets','sockscan','linux_route_cache','linux_netstat']:
for colname in ['LocalAddress','RemoteAddress','LocalAddr','ForeignAddr','Address','Destination']:
if colname in data['columns']:
col = data['columns'].index(colname)
#remcol = data['columns'].index('RemoteAddress')
for idx,row in enumerate(data['data']):
row = list(row)
if self.IsInternal(row[col]):
row[col] = {'value':row[col],'style':'background-color:lightyellow;'}
else:
row[col] = self.PrepareExtData(row[col], mmreader)
#if self.IsInternal(row[remcol]):
# row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'}
#else:
# row[remcol] = self.PrepareExtData(row[remcol], mmreader)
data['data'][idx] = row
def lookup(*args):
ipstring = args[0]
result = ""
try:
reader = maxminddb.open_database('GeoLite2-Country.mmdb')
ipdata = reader.get(ipstring)
reader.close()
result = ipdata['country']['names']['en']
except maxminddb.InvalidDatabaseError:
result = "InvalidDatabaseError"
except FileNotFoundError:
result = "FileNotFoundError"
except ValueError:
result = "ValueError"
return result
import argparse
import maxminddb
import random
import socket
import struct
import timeit
parser = argparse.ArgumentParser(description='Benchmark maxminddb.')
parser.add_argument(
'--count', default=250000, type=int, help='number of lookups')
parser.add_argument('--mode', default=0, type=int, help='reader mode to use')
parser.add_argument(
'--file', default='GeoIP2-City.mmdb', help='path to mmdb file')
args = parser.parse_args()
reader = maxminddb.open_database(args.file, args.mode)
def lookup_ip_address():
ip = socket.inet_ntoa(struct.pack('!L', random.getrandbits(32)))
record = reader.get(str(ip))
elapsed = timeit.timeit(
'lookup_ip_address()',
setup='from __main__ import lookup_ip_address',
number=args.count)
print(args.count / elapsed, 'lookups per second')
def actionSidebarGetPeers(self, to):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
return self.response(to, "You don't have permission to run this command")
try:
import maxminddb
db_path = config.data_dir + '/GeoLite2-City.mmdb'
if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
if not self.downloadGeoLiteDb(db_path):
return False
geodb = maxminddb.open_database(db_path)
peers = self.site.peers.values()
# Find avg ping
ping_times = [
peer.connection.last_ping_delay
for peer in peers
if peer.connection and peer.connection.last_ping_delay and peer.connection.last_ping_delay
]
if ping_times:
ping_avg = sum(ping_times) / float(len(ping_times))
else:
ping_avg = 0
# Place bars
globe_data = []
placed = {} # Already placed bars here
for peer in peers:
# Height of bar
if peer.connection and peer.connection.last_ping_delay:
ping = min(0.20, math.log(1 + peer.connection.last_ping_delay / ping_avg, 300))
else:
ping = -0.03
# Query and cache location
if peer.ip in loc_cache:
loc = loc_cache[peer.ip]
else:
try:
loc = geodb.get(peer.ip)
except:
loc = None
loc_cache[peer.ip] = loc
if not loc or "location" not in loc:
continue
# Create position array
lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"])
latlon = "%s,%s" % (lat, lon)
if latlon in placed: # Dont place more than 1 bar to same place, fake repos using ip address last two part
lat += float(128 - int(peer.ip.split(".")[-2])) / 50
lon += float(128 - int(peer.ip.split(".")[-1])) / 50
latlon = "%s,%s" % (lat, lon)
placed[latlon] = True
globe_data += (lat, lon, ping)
# Append myself
loc = geodb.get(config.ip_external)
if loc:
lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"])
globe_data += (lat, lon, -0.135)
self.response(to, globe_data)
except Exception, err:
self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
self.response(to, {"error": err})
def test_close(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
self.mode
)
reader.close()
r'(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)')
IPPortPatternLine = re.compile(
r'^.*?(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)).*?(?P<port>\d{2,5}).*$',
flags=re.MULTILINE)
IPPortPatternGlobal = re.compile(
r'(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))'
r'(?=.*?(?:(?:(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?P<port>\d{2,5})))',
flags=re.DOTALL)
# IsIpPattern = re.compile(
# r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$')
mmdbReader = maxminddb.open_database(
os.path.join(BASE_DIR, 'data', 'GeoLite2-Country.mmdb'))
async def resolve_host(host, timeout, loop):
global RESOLVER
if not RESOLVER:
RESOLVER = aiodns.DNSResolver(loop=loop)
try:
ip = await asyncio.wait_for(RESOLVER.query(host, 'A'),
timeout=timeout)
ip = ip[0].host
except (aiodns.error.DNSError, asyncio.TimeoutError):
return False
else:
return ip
def get_base_dir():
import maxminddb
from PIL import Image
from PIL.ExifTags import TAGS
from pprint import pprint
import sys
imageName = sys.argv[1]
reader = maxminddb.open_database('GeoLite2-City.mmdb')
data = reader.get('1.1.1.1')
# print(data)
reader.close()
exifData = {}
imgFile = Image.open(imageName)
# imgFile = Image.open('image_test2.jpg')
info = imgFile._getexif()
if info:
for (tag, value) in info.items():
decoded = TAGS.get(tag, tag)
exifData[decoded] = value
exifGPS = exifData.get('GPSInfo', None)
if exifGPS:
print(imgFile, ' contains GPS MetaData', exifGPS)
pprint(exifData)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from django.conf import settings
try:
import maxminddb
except ImportError:
maxminddb = None
if maxminddb is not None:
try:
geo = maxminddb.open_database(settings.MAXMIND_DB_PATH)
except (IOError, maxminddb.InvalidDatabaseError):
geo = None
else:
geo = None
def get_country_from_ip(ip_addr):
"""Return country info for the given IP Address."""
if geo is not None:
try:
data = geo.get(ip_addr)
except ValueError:
data = None
if data:
country = data.get('country', data.get('registered_country'))
def test_too_many_get_args(self):
reader = open_database(
'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode
)
self.assertRaises(TypeError, reader.get, ('1.1.1.1', 'blah'))
reader.close()