def test_no_get_args(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     self.assertRaises(TypeError, reader.get)
     reader.close()
    def _check_concurrency(self, worker_class):
        reader = open_database(
            'tests/data/test-data/GeoIP2-Domain-Test.mmdb',
            self.mode
        )

        def lookup(pipe):
            try:
                for i in range(32):
                    reader.get('65.115.240.{i}'.format(i=i))
                pipe.send(1)
            except:
                pipe.send(0)
            finally:
                if worker_class is Process:
                    reader.close()
                pipe.close()

        pipes = [Pipe() for _ in range(32)]
        procs = [worker_class(target=lookup, args=(c,)) for (p, c) in pipes]
        for proc in procs:
            proc.start()
        for proc in procs:
            proc.join()

        reader.close()

        count = sum([p.recv() for (p, c) in pipes])

        self.assertEqual(count, 32, 'expected number of successful lookups')
 def test_metadata_args(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     self.assertRaises(TypeError, reader.metadata, ('blah'))
     reader.close()
Exemplo n.º 4
0
def mirror():
    """Get the encrypted anti-censorship data from Mothership"""

    import urllib

    v = request.values.get
    ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr))

    reader = maxminddb.open_database('GeoLite2-City.mmdb')
    result = reader.get(ip)
    reader.close()

    if v("_ac_force_locale") or (result and result.get('country')):

        locale = v("_ac_force_locale") if v("_ac_force_locale") else \
                 result.get("country").get('iso_code')

        link = "%s/%s?api_key=%s&domain=%s&user_key=%s" % (MOTHERSHIP_URL,
               locale, os.environ.get('API_KEY'), os.environ.get('DOMAIN'),
               v("key"))

        f = urllib.urlopen(link)
        encrypted_data = f.read()
        return encrypted_data

    return ""
    def test_decoder(self):
        reader = open_database(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
        record = reader.get('::1.1.1.0')

        self.assertEqual(record['array'], [1, 2, 3])
        self.assertEqual(record['boolean'], True)
        self.assertEqual(record['bytes'], bytearray(b'\x00\x00\x00*'))
        self.assertEqual(record['double'], 42.123456)
        self.assertAlmostEqual(record['float'], 1.1)
        self.assertEqual(record['int32'], -268435456)
        self.assertEqual(
            {
                'mapX': {
                    'arrayX': [7, 8, 9],
                    'utf8_stringX': 'hello'
                },
            },
            record['map']
        )

        self.assertEqual(record['uint16'], 100)
        self.assertEqual(record['uint32'], 268435456)
        self.assertEqual(record['uint64'], 1152921504606846976)
        self.assertEqual(record['utf8_string'], 'unicode! ☯ - ♫')

        self.assertEqual(
            1329227995784915872903807060280344576,
            record['uint128']
        )
        reader.close()
Exemplo n.º 6
0
def report():
    """Report something sketchy back to Mothership"""

    import urllib
    import requests

    v = request.values.get
    ip = fix_ip(request.headers.get('x-forwarded-for', request.remote_addr))

    reader = maxminddb.open_database('GeoLite2-City.mmdb')
    result = reader.get(ip)
    reader.close()

    payload = {
        'ip': ip,
        'host': v('host'),
        'response': v('response')
    }
    if result and result.get('country'):
        payload['locale'] = result.get("country").get('iso_code')

    r = requests.post(MOTHERSHIP_URL + "/report", data=payload)
    print r.text

    return "kthx"
    def test_no_ipv4_search_tree(self):
        reader = open_database(
            'tests/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb',
            self.mode)

        self.assertEqual(reader.get('1.1.1.1'), '::0/64')
        self.assertEqual(reader.get('192.1.1.1'), '::0/64')
        reader.close()
Exemplo n.º 8
0
 def _get_db(cls):
     if cls._db: return cls._db
     try: import maxminddb
     except ImportError: return None
     path = cls._try_all()
     if not path: return None
     cls._db = maxminddb.open_database(path)
     return cls._db
 def test_ip_validation(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
     self.assertRaisesRegex(ValueError,
                            "'not_ip' does not appear to be an IPv4 or "
                            "IPv6 address",
                            reader.get, ('not_ip'))
     reader.close()
Exemplo n.º 10
0
 def test_double_close(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     reader.close()
     self.assertIsNone(
         reader.close(), 'Double close does not throw an exception')
Exemplo n.º 11
0
 def test_ipv6_address_in_ipv4_database(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb', self.mode)
     with self.assertRaisesRegex(ValueError,
                                 'Error looking up 2001::. '
                                 'You attempted to look up an IPv6 address '
                                 'in an IPv4-only database'):
         reader.get('2001::')
     reader.close()
Exemplo n.º 12
0
    def _prepare_factor_pipeline(self, cfg):
        pipeline = []

        if cfg.factors.asn_movement is not None:
            pipeline.append(
                factors.asn_movement(
                    mmdb.open_database(
                        cfg.factors.asn_movement.maxmind_db_path),
                    cfg.asn_movement_severity))

        return pipeline
 def test_broken_database(self):
     reader = open_database('tests/data/test-data/'
                            'GeoIP2-City-Test-Broken-Double-Format.mmdb',
                            self.mode)
     with self.assertRaisesRegex(InvalidDatabaseError,
                                 "The MaxMind DB file's data "
                                 "section contains bad data \(unknown data "
                                 "type or corrupt data\)"
                                 ):
         reader.get('2001:220::')
     reader.close()
 def test_metadata_unknown_attribute(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     metadata = reader.metadata()
     with self.assertRaisesRegex(AttributeError,
                                 "'Metadata' object has no "
                                 "attribute 'blah'"):
         metadata.blah
     reader.close()
Exemplo n.º 15
0
 def test_broken_database(self):
     reader = open_database('tests/data/test-data/'
                            'GeoIP2-City-Test-Broken-Double-Format.mmdb',
                            self.mode)
     with self.assertRaisesRegex(InvalidDatabaseError,
                                 "The MaxMind DB file's data "
                                 "section contains bad data \(unknown data "
                                 "type or corrupt data\)"
                                 ):
         reader.get('2001:220::')
     reader.close()
Exemplo n.º 16
0
 def test_metadata_unknown_attribute(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     metadata = reader.metadata()
     with self.assertRaisesRegex(AttributeError,
                                 "'Metadata' object has no "
                                 "attribute 'blah'"):
         metadata.blah
     reader.close()
    def test_with_statement_close(self):
        filename = 'tests/data/test-data/MaxMind-DB-test-ipv4-24.mmdb'
        reader = open_database(filename, self.mode)
        reader.close()

        def use_with(reader):
            with reader:
                pass

        self.assertRaisesRegex(ValueError, 'Attempt to reopen a closed MaxMind DB',
                               use_with, reader)
 def test_closed_get(self):
     if self.mode == MODE_MEMORY:
         return
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     reader.close()
     self.assertRaisesRegex(ValueError,
                            'Attempt to read from a closed MaxMind DB.'
                            '|closed',
                            reader.get, ('1.1.1.1'))
Exemplo n.º 19
0
 def consult_geo(self):
    geodb = maxminddb.open_database('GeoLite2-Country.mmdb')
    s = geodb.get(self.host)
    try:
        self.iso_code = s['country']['iso_code']
    except:
        self.iso_code = "Null"
        
    try: 
        self.country = s['country']['names']['en']
    except:
        self.country = "Null"
Exemplo n.º 20
0
def parse_maxminddb(db_path, ip):
    try:
        reader = maxminddb.open_database(db_path)
        response = reader.get(ip)
        reader.close()
        return response
    except FileNotFoundError:
        print('DB not found')
        print('SHUTTING DOWN')
        exit()
    except ValueError:
        return False
Exemplo n.º 21
0
def parse_maxminddb(db_path, ip):
    try:
        reader = maxminddb.open_database(db_path)
        response = reader.get(ip)
        reader.close()
        return response
    except FileNotFoundError:
        print('DB not found')
        print('SHUTTING DOWN')
        exit()
    except ValueError:
        return False
Exemplo n.º 22
0
 def test_closed_get(self):
     if self.mode == MODE_MEMORY:
         return
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     reader.close()
     self.assertRaisesRegex(ValueError,
                            'Attempt to read from a closed MaxMind DB.'
                            '|closed',
                            reader.get, ('1.1.1.1'))
Exemplo n.º 23
0
def open_mmdb_database(mmdb_content):
    """ Test MaxMindDB database format with only its content as binary
    :return True if the database is correct, False otherwise
    """
    # Check if the response content is MMDB database
    tmpfile = BytesIO()
    tmpfile.write(mmdb_content)
    tmpfile.seek(0)
    setattr(tmpfile, "name", "test")
    try:
        return open_database(tmpfile, mode=MODE_FD)
    except Exception:
        return None
Exemplo n.º 24
0
def nmap_scan(self, file, fk):
    progress_recorder = ProgressRecorder(self)
    result = 0
    print(os.getcwd() + file)
    search = Search.objects.get(id=fk)
    max_reader = maxminddb.open_database('GeoLite2-City.mmdb')
    nmap_report = NmapParser.parse_fromfile(os.getcwd() + file)
    total = len(nmap_report.hosts)
    for c, i in enumerate(nmap_report.hosts):
        result += c
        nmap_host_worker(host_arg=i, max_reader=max_reader, search=search)
        progress_recorder.set_progress(c + 1, total=total)
    return result
Exemplo n.º 25
0
def getCityInformation(ipdaddress):
    """
    Retrieve city information about the provided IP address.
    :param ipdaddress: address to retrieve information from.
    """
    dbName = "GeoLite2-City.mmdb"

    if not os.path.exists(dbName):
        utils.getCityDatabase()

    conx = maxminddb.open_database(dbName)

    utils.CityLocation(conx.get(ipdaddress))
Exemplo n.º 26
0
    def getPeerLocations(self, peers):
        import maxminddb

        db_path = self.getGeoipDb()
        if not db_path:
            self.log.debug("Not showing peer locations: no GeoIP database")
            return False

        self.log.info("Loading GeoIP database from: %s" % db_path)
        geodb = maxminddb.open_database(db_path)

        peers = list(peers.values())
        # Place bars
        peer_locations = []
        placed = {}  # Already placed bars here
        for peer in peers:
            # Height of bar
            if peer.connection and peer.connection.last_ping_delay:
                ping = round(peer.connection.last_ping_delay * 1000)
            else:
                ping = None
            loc = self.getLoc(geodb, peer.ip)

            if not loc:
                continue
            # Create position array
            lat, lon = loc["lat"], loc["lon"]
            latlon = "%s,%s" % (lat, lon)
            if latlon in placed and helper.getIpType(
                    peer.ip
            ) == "ipv4":  # Dont place more than 1 bar to same place, fake repos using ip address last two part
                lat += float(128 - int(peer.ip.split(".")[-2])) / 50
                lon += float(128 - int(peer.ip.split(".")[-1])) / 50
                latlon = "%s,%s" % (lat, lon)
            placed[latlon] = True
            peer_location = {}
            peer_location.update(loc)
            peer_location["lat"] = lat
            peer_location["lon"] = lon
            peer_location["ping"] = ping

            peer_locations.append(peer_location)

        # Append myself
        for ip in self.site.connection_server.ip_external_list:
            my_loc = self.getLoc(geodb, ip)
            if my_loc:
                my_loc["ping"] = 0
                peer_locations.append(my_loc)

        return peer_locations
Exemplo n.º 27
0
    def get_ip_geoinfo(self, ip: str) -> dict:
        """
        根据ip去查询所在的地址
        :param ip:
        :return:
        """
        res = {}
        try:
            # 1、先检测下东西在不在
            # 这样当前dbname就一定会有
            # with DbipMmdb.__scouter_ip_locker:
            # 一次只能有一个玩意在这检测版本,虽然慢但是多线程没办法
            # self.__check_db_version()
            reader = maxminddb.open_database(
                './_ip_location_dbs/dbfile/2020-04.mmdb')
            res = reader.get(ip)
            if res is None:
                return res
            reader.close()

            # data_city = res.get('city', {}).get('names', {}).get('en', None)
            # data_state = None
            # subdivisions = res.get('subdivisions', [])
            # if len(subdivisions) == 0:
            #     data_state = None
            # else:
            #     data_state = subdivisions[0].get('names', {}).get('en', None)
            # data_country = res.get('country', {}).get('iso_code', None)
            # location = res.get('location', {})
            # lng = location.get('longitude', None)
            # lat = location.get('latitude', None)
            # 这里表示国家不属于主权国家,所以直接返回None,写注释是一个非常好的习惯
            # if not ALL_COUNTRIES.__contains__(
            #         data_country) or lat is None or lng is None:
            #     return res
            # geoinfo: GeoInfo = GeoInfo(level, lng, lat)
            # exrtinfo: CountryCode = ALL_COUNTRIES.get(data_country)
            # geoinfo.set_country(
            #     data_country, {
            #         'zh-CN': exrtinfo.country_names.get('CN'),
            #         'en': exrtinfo.country_names.get('EN')
            #     })
            # geoinfo.set_province({'zh-CN': None, 'en': data_state})
            # geoinfo.set_city({'zh-CN': None, 'en': data_city})
            # geoinfo.globaltelcode = exrtinfo.countrycode
            # geoinfo.timezone = exrtinfo.timediffer
            # res = geoinfo
        except Exception as error:
            print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error))
        finally:
            return res
Exemplo n.º 28
0
 def test_broken_database(self):
     reader = open_database(
         "tests/data/test-data/"
         "GeoIP2-City-Test-Broken-Double-Format.mmdb",
         self.mode,
     )
     with self.assertRaisesRegex(
             InvalidDatabaseError,
             r"The MaxMind DB file's data "
             r"section contains bad data \(unknown data "
             r"type or corrupt data\)",
     ):
         reader.get(self.ipf("2001:220::"))
     reader.close()
Exemplo n.º 29
0
    def query(self):
        import maxminddb
        db = maxminddb.open_database(self._res.get_file_path())
        result = db.get(self._query.get("ip", "127.0.0.1")[0])
        prop = self._query.get("prop", "all")[0]

        if result:
            if prop == "countrycode":
                return "200 OK", [("Content-Type", "text/plain")
                                  ], result["country"]["iso_code"]
            else:
                return "200 OK", [("Content-Type", "text/json")], dumps(result)
        else:
            return "404 Not Found", [], ""
Exemplo n.º 30
0
def run(
    analyzer_name,
    job_id,
    observable_name,
    observable_classification,
    additional_config_params,
):
    logger.info("started analyzer {} job_id {} observable {}"
                "".format(analyzer_name, job_id, observable_name))
    report = general.get_basic_report_template(analyzer_name)
    try:
        try:
            if not os.path.isfile(database_location):
                updater(additional_config_params)
            reader = maxminddb.open_database(database_location)
            maxmind_result = reader.get(observable_name)
            reader.close()
        except maxminddb.InvalidDatabaseError as e:
            error_message = "invalid database error: {}".format(e)
            logger.exception(error_message)
            maxmind_result = {"error": error_message}

        if not maxmind_result:
            maxmind_result = {}
        # pprint.pprint(maxmind_result)
        report["report"] = maxmind_result
    except AnalyzerRunException as e:
        error_message = (
            "job_id:{} analyzer:{} observable_name:{} Analyzer error {}"
            "".format(job_id, analyzer_name, observable_name, e))
        logger.error(error_message)
        report["errors"].append(error_message)
        report["success"] = False
    except Exception as e:
        traceback.print_exc()
        error_message = (
            "job_id:{} analyzer:{} observable_name:{} Unexpected error {}"
            "".format(job_id, analyzer_name, observable_name, e))
        logger.exception(error_message)
        report["errors"].append(str(e))
        report["success"] = False
    else:
        report["success"] = True

    general.set_report_and_cleanup(job_id, report)

    logger.info("finished analyzer {} job_id {} observable {}"
                "".format(analyzer_name, job_id, observable_name))

    return report
Exemplo n.º 31
0
Arquivo: mmdb.py Projeto: telnet23/bin
def load_from(base):
    for root, directories, filenames in os.walk(base):
        for directory in directories:
            match = re.match('^Geo(?:IP|Lite)2-(?:.+)_(?:.+)$', directory)
            if match:
                path = os.path.join(root, directory)
                load_from(path)
        for filename in filenames:
            match = re.match('^Geo(?:IP|Lite)2-(.+)\.mmdb$', filename)
            if match:
                typ = match.group(1)
                path = os.path.join(root, filename)
                reader = maxminddb.open_database(path)
                readers.append(reader)
Exemplo n.º 32
0
    def __init__(self,
                 fileish: str,
                 locales: Optional[List[str]] = None,
                 mode: int = MODE_AUTO) -> None:
        """Create GeoIP2 Reader.

        :param fileish: The string path to the GeoIP2 database, or an existing
          file descriptor pointing to the database. Note that this latter
          usage is only valid when mode is MODE_FD.
        :param locales: This is list of locale codes. This argument will be
          passed on to record classes to use when their name properties are
          called. The default value is ['en'].

          The order of the locales is significant. When a record class has
          multiple names (country, city, etc.), its name property will return
          the name in the first locale that has one.

          Note that the only locale which is always present in the GeoIP2
          data is "en". If you do not include this locale, the name property
          may end up returning None even when the record has an English name.

          Currently, the valid locale codes are:

          * de -- German
          * en -- English names may still include accented characters if that
            is the accepted spelling in English. In other words, English does
            not mean ASCII.
          * es -- Spanish
          * fr -- French
          * ja -- Japanese
          * pt-BR -- Brazilian Portuguese
          * ru -- Russian
          * zh-CN -- Simplified Chinese.
        :param mode: The mode to open the database with. Valid mode are:
          * MODE_MMAP_EXT - use the C extension with memory map.
          * MODE_MMAP - read from memory map. Pure Python.
          * MODE_FILE - read database as standard file. Pure Python.
          * MODE_MEMORY - load database into memory. Pure Python.
          * MODE_FD - the param passed via fileish is a file descriptor, not a
             path. This mode implies MODE_MEMORY. Pure Python.
          * MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
             Default.

        """
        if locales is None:
            locales = ["en"]
        self._db_reader = maxminddb.open_database(fileish, mode)
        self._db_type = self._db_reader.metadata().database_type
        self._locales = locales
Exemplo n.º 33
0
    def test_reader(self):
        for record_size in [24, 28, 32]:
            for ip_version in [4, 6]:
                file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
                             str(ip_version) + '-' + str(record_size) +
                             '.mmdb')
                reader = open_database(file_name, self.mode)

                self._check_metadata(reader, ip_version, record_size)

                if ip_version == 4:
                    self._check_ip_v4(reader, file_name)
                else:
                    self._check_ip_v6(reader, file_name)
                reader.close()
Exemplo n.º 34
0
 async def parse_mmdb(self, mmdb):
     '''
     create and return mmdb reader of the mmdb
     :param:mmdb: the byte array of the .mmdb file
     '''
     # maminddb expects a file or file descriptor, 
     # use Tempfile to createa a file pointer
     '''
     tfile = TemporaryFile()
     tfile.write(mmdb)
     # after writing must seek back to the begggining
     tfile.seek(0)
     reader = maxminddb.open_database(tfile, mode=maxminddb.MODE_MMAP_EXT)
     tfile.close()
     '''
     temp_file = '/tmp/maxmin.db'
     with open(temp_file, 'wb+') as tf:
         tf.write(mmdb)
     reader = None
     try:
         reader = maxminddb.open_database(temp_file, mode=maxminddb.MODE_MMAP_EXT)
     except:
         reader = maxminddb.open_database(temp_file)
     return reader
Exemplo n.º 35
0
def getASNInformation(ipaddress):
    """
    Get ASN information of the provided IP address.
    :param ipaddress:
    """
    dbName = "GeoLite2-ASN.mmdb"

    if not os.path.exists(dbName):
        utils.getASNDatabase()

    conx = maxminddb.open_database(dbName)
    info = conx.get(ipaddress)

    print("\n\033[0;36m[*]\033[0;0m ASN: ", info["autonomous_system_number"])
    print("\033[0;36m[*]\033[0;0m Organization: ", info["autonomous_system_organization"])
Exemplo n.º 36
0
def main():
    """The main entry point."""
    metrics = get_prometheus_metrics()
    prometheus_client.start_http_server(addr=ADDRESS, port=PORT)
    log.info(f'listing on {ADDRESS}:{PORT}')
    if MAXMIND_LICENSE_KEY and MAXMIND_DATABASE_EDITION:
        log.info('geoip lookup enabled')
        database_path = prepare_maxmind_database(MAXMIND_LICENSE_KEY,
                                                 MAXMIND_DATABASE_EDITION,
                                                 MAXMIND_CACHE_DIRECTORY)
        with maxminddb.open_database(database_path.as_posix()) as database:
            collect_metrics(*metrics, geoip_db=database)
    else:
        log.info('geoip lookup disabled')
        collect_metrics(*metrics)
    def test_reader(self):
        for record_size in [24, 28, 32]:
            for ip_version in [4, 6]:
                file_name = ('tests/data/test-data/MaxMind-DB-test-ipv' +
                             str(ip_version) + '-' + str(record_size) +
                             '.mmdb')
                reader = open_database(file_name, self.mode)

                self._check_metadata(reader, ip_version, record_size)

                if ip_version == 4:
                    self._check_ip_v4(reader, file_name)
                else:
                    self._check_ip_v6(reader, file_name)
                reader.close()
Exemplo n.º 38
0
async def loader():
    try:
        if not os.path.isfile(settings.db_file):
            raise FileNotFoundError("db file not found")
        state.db = maxminddb.open_database(settings.db_file)
        logger.info('DB loaded')

        for num in count():
            info = await cache_info()
            logger.info('cache stat', loop=num, info=info)
            await asyncio.sleep(60 * 5)
    except asyncio.CancelledError:
        pass
    except Exception:
        logger.exception('error while opening database file')
    def test_closed_metadata(self):
        reader = open_database(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode)
        reader.close()

        # The primary purpose of this is to ensure the extension doesn't
        # segfault
        try:
            metadata = reader.metadata()
        except IOError as ex:
            self.assertEqual('Attempt to read from a closed MaxMind DB.',
                             str(ex), 'extension throws exception')
        else:
            self.assertIsNotNone(metadata,
                                 'pure Python implementation returns value')
Exemplo n.º 40
0
        def _validate_cachefile(self, filepath):
            try:
                db = maxminddb.open_database(filepath)
            except Exception as e:
                log.debug('Exception raised when trying to open %r: %r', filepath, e)
                return False

            test_ip = '1.1.1.1'
            try:
                db.get(test_ip)
            except Exception as e:
                log.debug('Exception raised when looking up %r: %r', test_ip, e)
                return False
            else:
                return True
Exemplo n.º 41
0
def update_db(db_path, license_key, edition_id):
    db_dir_path = os.path.abspath(os.path.dirname(db_path))
    db_archive_path = '%s.tar.gz' % db_path

    def maxmind_download(suffix, **kwargs):
        return requests.get('https://download.maxmind.com/app/geoip_download',
                            params={
                                'license_key': license_key,
                                'edition_id': edition_id,
                                'suffix': suffix,
                            },
                            **kwargs)

    expected_md5 = maxmind_download('tar.gz.md5').content
    curr_md5 = _hash_file(db_archive_path)
    if expected_md5 == curr_md5 and os.path.exists(db_path):
        return

    with open(db_archive_path, 'wb') as local_zip:
        for chunk in maxmind_download(
                'tar.gz', stream=True).iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                local_zip.write(chunk)

    with tarfile.open(db_archive_path) as tar_file:
        # We only want the mmdb file. Maxmind kindly includes things
        # we don't want.
        extract_members = [
            member for member in tar_file.getmembers()
            if member.name.endswith('.mmdb')
        ]
        assert len(extract_members) == 1
        tar_file.extractall(path=db_dir_path, members=extract_members)
        # extractall keeps the subfolder structure. Account for this by
        # appending the path to the db_dir_path where it was extracted.
        new_db_path = os.path.join(db_dir_path, extract_members[0].path)
    try:
        db = maxminddb.open_database(new_db_path)
        db.get('8.8.8.8')
        db.get('2001:420::')
    except Exception:
        sys.stderr.write('Retrieved invalid GeoIP database - '
                         'check MaxMind account details.\n')
        raise
    if not os.path.exists(db_dir_path):
        os.makedirs(db_dir_path)
    shutil.move(new_db_path, db_path)
    os.rmdir(os.path.dirname(new_db_path))
Exemplo n.º 42
0
    def getPeerLocations(self, peers):
        import maxminddb
        db_path = config.data_dir + '/GeoLite2-City.mmdb'
        if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
            if not self.downloadGeoLiteDb(db_path):
                return False
        geodb = maxminddb.open_database(db_path)

        peers = peers.values()
        # Place bars
        peer_locations = []
        placed = {}  # Already placed bars here
        for peer in peers:
            # Height of bar
            if peer.connection and peer.connection.last_ping_delay:
                ping = round(peer.connection.last_ping_delay * 1000)
            else:
                ping = None
            loc = self.getLoc(geodb, peer.ip)

            if not loc:
                continue
            # Create position array
            lat, lon = loc["lat"], loc["lon"]
            latlon = "%s,%s" % (lat, lon)
            if latlon in placed and helper.getIpType(
                    peer.ip
            ) == "ipv4":  # Dont place more than 1 bar to same place, fake repos using ip address last two part
                lat += float(128 - int(peer.ip.split(".")[-2])) / 50
                lon += float(128 - int(peer.ip.split(".")[-1])) / 50
                latlon = "%s,%s" % (lat, lon)
            placed[latlon] = True
            peer_location = {}
            peer_location.update(loc)
            peer_location["lat"] = lat
            peer_location["lon"] = lon
            peer_location["ping"] = ping

            peer_locations.append(peer_location)

        # Append myself
        for ip in config.ip_external:
            my_loc = self.getLoc(geodb, ip)
            if my_loc:
                my_loc["ping"] = 0
                peer_locations.append(my_loc)

        return peer_locations
    def _swap_database(self, new_database_path):
        """
        Swap the current database with a new database

        :param new_database_path: The path to the new database
        """
        # Open the new database
        self._reader = maxminddb.open_database(new_database_path,
                                               maxminddb.const.MODE_MEMORY)

        # Delete the temporary new database file
        try:
            os.remove(new_database_path)
        except Exception:
            logger.exception("Failed to remove old database file.")
        self._database_checksum = self._response_checksum
Exemplo n.º 44
0
 def morph(self, data):
     mmreader = maxminddb.open_database(self.config['dbpath']['value'])
     if data['name'] in ['connections','connscan']:
         loccol = data['columns'].index('LocalAddress')
         remcol = data['columns'].index('RemoteAddress')
         for idx,row in enumerate(data['data']):
             row = list(row)
             if self.IsInternal(row[loccol]):
                 row[loccol] = {'value':row[loccol],'style':'background-color:lightyellow;'}
             else:
                 row[loccol] = self.PrepareExtData(row[loccol], mmreader)
             if self.IsInternal(row[remcol]):
                 row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'}
             else:
                 row[remcol] = self.PrepareExtData(row[remcol], mmreader)
             data['data'][idx] = row
Exemplo n.º 45
0
def main():

    with open(a,"r") as f:
        for line in f.readlines():
            try:
                reader = maxminddb.open_database('/home/caifeng/ip.mndb')
                #strip() 方法用于移除字符串头尾指定的字符(默认为空格或换行符)或字符序列
                result1 = reader.get(line.strip('\n'))
                result = json.dumps(result1).decode('unicode_escape')
                reader.close()
            except Exception as e:
                print(e)
            else:
                with open(a+'-result.txt','a+') as r:
                    r.write(line.strip('\n') + ' ')
                    r.write(result + '\n')
Exemplo n.º 46
0
    def run(self):
        try:
            if not os.path.isfile(database_location):
                self.updater(self.additional_config_params)
            reader = maxminddb.open_database(database_location)
            maxmind_result = reader.get(self.observable_name)
            reader.close()
        except maxminddb.InvalidDatabaseError as e:
            error_message = f"Invalid database error: {e}"
            logger.exception(error_message)
            maxmind_result = {"error": error_message}

        if not maxmind_result:
            maxmind_result = {}

        return maxmind_result
Exemplo n.º 47
0
 def get_ip_geoinfo(self, ip: str) -> dict:
     """
     根据ip去查询所在的地址
     :param ip:
     :return:
     """
     res = None
     try:
         reader = maxminddb.open_database(
             './_ip_location_dbs/dbfile/geoinfo.mmdb')
         res = reader.get(ip)
         reader.close()
     except Exception as error:
         print("Get geoinfo error:\nip={}\nerror:{}".format(ip, error))
     finally:
         return res
Exemplo n.º 48
0
    def test_closed_metadata(self):
        reader = open_database(
            'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
            self.mode
        )
        reader.close()

        # The primary purpose of this is to ensure the extension doesn't
        # segfault
        try:
            metadata = reader.metadata()
        except IOError as ex:
            self.assertEqual('Attempt to read from a closed MaxMind DB.',
                             str(ex), 'extension throws exception')
        else:
            self.assertIsNotNone(
                metadata, 'pure Python implementation returns value')
Exemplo n.º 49
0
    def getPeerLocations(self, peers):
        import maxminddb
        db_path = config.data_dir + '/GeoLite2-City.mmdb'
        if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
            if not self.downloadGeoLiteDb(db_path):
                return False
        geodb = maxminddb.open_database(db_path)

        peers = peers.values()
        # Place bars
        peer_locations = []
        placed = {}  # Already placed bars here
        for peer in peers:
            # Height of bar
            if peer.connection and peer.connection.last_ping_delay:
                ping = round(peer.connection.last_ping_delay * 1000)
            else:
                ping = None
            loc = self.getLoc(geodb, peer.ip)

            if not loc:
                continue
            # Create position array
            lat, lon = loc["lat"], loc["lon"]
            latlon = "%s,%s" % (lat, lon)
            if latlon in placed:  # Dont place more than 1 bar to same place, fake repos using ip address last two part
                lat += float(128 - int(peer.ip.split(".")[-2])) / 50
                lon += float(128 - int(peer.ip.split(".")[-1])) / 50
                latlon = "%s,%s" % (lat, lon)
            placed[latlon] = True
            peer_location = {}
            peer_location.update(loc)
            peer_location["lat"] = lat
            peer_location["lon"] = lon
            peer_location["ping"] = ping

            peer_locations.append(peer_location)

        # Append myself
        my_loc = self.getLoc(geodb, config.ip_external)
        if my_loc:
            my_loc["ping"] = 0
            peer_locations.append(my_loc)

        return peer_locations
Exemplo n.º 50
0
    def __init__(self, fileish, locales=None, mode=MODE_AUTO):
        """Create GeoIP2 Reader.

        :param fileish: The string path to the GeoIP2 database, or an existing
          file descriptor pointing to the database. Note that this latter
          usage is only valid when mode is MODE_FD.
        :param locales: This is list of locale codes. This argument will be
          passed on to record classes to use when their name properties are
          called. The default value is ['en'].

          The order of the locales is significant. When a record class has
          multiple names (country, city, etc.), its name property will return
          the name in the first locale that has one.

          Note that the only locale which is always present in the GeoIP2
          data is "en". If you do not include this locale, the name property
          may end up returning None even when the record has an English name.

          Currently, the valid locale codes are:

          * de -- German
          * en -- English names may still include accented characters if that
            is the accepted spelling in English. In other words, English does
            not mean ASCII.
          * es -- Spanish
          * fr -- French
          * ja -- Japanese
          * pt-BR -- Brazilian Portuguese
          * ru -- Russian
          * zh-CN -- Simplified Chinese.
        :param mode: The mode to open the database with. Valid mode are:
          * MODE_MMAP_EXT - use the C extension with memory map.
          * MODE_MMAP - read from memory map. Pure Python.
          * MODE_FILE - read database as standard file. Pure Python.
          * MODE_MEMORY - load database into memory. Pure Python.
          * MODE_FD - the param passed via fileish is a file descriptor, not a
             path. This mode implies MODE_MEMORY. Pure Python.
          * MODE_AUTO - try MODE_MMAP_EXT, MODE_MMAP, MODE_FILE in that order.
             Default.

        """
        if locales is None:
            locales = ['en']
        self._db_reader = maxminddb.open_database(fileish, mode)
        self._locales = locales
Exemplo n.º 51
0
def update_db(db_path, license_key, edition_id):
    db_dir_path = os.path.abspath(os.path.dirname(db_path))
    db_archive_path = '%s.tar.gz' % db_path

    def maxmind_download(suffix, **kwargs):
        return requests.get('https://download.maxmind.com/app/geoip_download',
                            params={'license_key': license_key,
                                    'edition_id': edition_id,
                                    'suffix': suffix,
                                    },
                            **kwargs)

    expected_md5 = maxmind_download('tar.gz.md5').content
    curr_md5 = _hash_file(db_archive_path)
    if expected_md5 == curr_md5 and os.path.exists(db_path):
        return

    with open(db_archive_path, 'wb') as local_zip:
        for chunk in maxmind_download('tar.gz', stream=True
                                      ).iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                local_zip.write(chunk)

    with tarfile.open(db_archive_path) as tar_file:
        # We only want the mmdb file. Maxmind kindly includes things
        # we don't want.
        extract_members = [member for member in tar_file.getmembers()
                           if member.name.endswith('.mmdb')]
        assert len(extract_members) == 1
        tar_file.extractall(path=db_dir_path, members=extract_members)
        # extractall keeps the subfolder structure. Account for this by
        # appending the path to the db_dir_path where it was extracted.
        new_db_path = os.path.join(db_dir_path, extract_members[0].path)
    try:
        db = maxminddb.open_database(new_db_path)
        db.get('8.8.8.8')
        db.get('2001:420::')
    except Exception:
        sys.stderr.write('Retrieved invalid GeoIP database - '
                         'check MaxMind account details.\n')
        raise
    if not os.path.exists(db_dir_path):
        os.makedirs(db_dir_path)
    shutil.move(new_db_path, db_path)
    os.rmdir(os.path.dirname(new_db_path))
Exemplo n.º 52
0
 def morph(self, data):
     mmreader = maxminddb.open_database(self.config['dbpath']['value'])
     if data['name'] in ['connections','connscan','netscan','sockets','sockscan','linux_route_cache','linux_netstat']:
         for colname in ['LocalAddress','RemoteAddress','LocalAddr','ForeignAddr','Address','Destination']:
             if colname in data['columns']:
                 col = data['columns'].index(colname)
                 #remcol = data['columns'].index('RemoteAddress')
                 for idx,row in enumerate(data['data']):
                     row = list(row)
                     if self.IsInternal(row[col]):
                         row[col] = {'value':row[col],'style':'background-color:lightyellow;'}
                     else:
                         row[col] = self.PrepareExtData(row[col], mmreader)
                     #if self.IsInternal(row[remcol]):
                     #    row[remcol] = {'value':row[remcol],'style':'background-color:lightyellow;'}
                     #else:
                     #    row[remcol] = self.PrepareExtData(row[remcol], mmreader)
                     data['data'][idx] = row
Exemplo n.º 53
0
def lookup(*args):
	ipstring = args[0]

	result = ""

	try:
		reader = maxminddb.open_database('GeoLite2-Country.mmdb')
		ipdata = reader.get(ipstring)
		reader.close()
		result = ipdata['country']['names']['en']

	except maxminddb.InvalidDatabaseError:
		result = "InvalidDatabaseError"

	except FileNotFoundError:
		result = "FileNotFoundError"

	except ValueError:
		result = "ValueError"

	return result
Exemplo n.º 54
0
import argparse
import maxminddb
import random
import socket
import struct
import timeit

parser = argparse.ArgumentParser(description='Benchmark maxminddb.')
parser.add_argument(
    '--count', default=250000, type=int, help='number of lookups')
parser.add_argument('--mode', default=0, type=int, help='reader mode to use')
parser.add_argument(
    '--file', default='GeoIP2-City.mmdb', help='path to mmdb file')

args = parser.parse_args()

reader = maxminddb.open_database(args.file, args.mode)


def lookup_ip_address():
    ip = socket.inet_ntoa(struct.pack('!L', random.getrandbits(32)))
    record = reader.get(str(ip))


elapsed = timeit.timeit(
    'lookup_ip_address()',
    setup='from __main__ import lookup_ip_address',
    number=args.count)

print(args.count / elapsed, 'lookups per second')
Exemplo n.º 55
0
    def actionSidebarGetPeers(self, to):
        permissions = self.getPermissions(to)
        if "ADMIN" not in permissions:
            return self.response(to, "You don't have permission to run this command")
        try:
            import maxminddb
            db_path = config.data_dir + '/GeoLite2-City.mmdb'
            if not os.path.isfile(db_path) or os.path.getsize(db_path) == 0:
                if not self.downloadGeoLiteDb(db_path):
                    return False
            geodb = maxminddb.open_database(db_path)

            peers = self.site.peers.values()
            # Find avg ping
            ping_times = [
                peer.connection.last_ping_delay
                for peer in peers
                if peer.connection and peer.connection.last_ping_delay and peer.connection.last_ping_delay
            ]
            if ping_times:
                ping_avg = sum(ping_times) / float(len(ping_times))
            else:
                ping_avg = 0
            # Place bars
            globe_data = []
            placed = {}  # Already placed bars here
            for peer in peers:
                # Height of bar
                if peer.connection and peer.connection.last_ping_delay:
                    ping = min(0.20, math.log(1 + peer.connection.last_ping_delay / ping_avg, 300))
                else:
                    ping = -0.03

                # Query and cache location
                if peer.ip in loc_cache:
                    loc = loc_cache[peer.ip]
                else:
                    try:
                        loc = geodb.get(peer.ip)
                    except:
                        loc = None
                    loc_cache[peer.ip] = loc
                if not loc or "location" not in loc:
                    continue

                # Create position array
                lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"])
                latlon = "%s,%s" % (lat, lon)
                if latlon in placed:  # Dont place more than 1 bar to same place, fake repos using ip address last two part
                    lat += float(128 - int(peer.ip.split(".")[-2])) / 50
                    lon += float(128 - int(peer.ip.split(".")[-1])) / 50
                    latlon = "%s,%s" % (lat, lon)
                placed[latlon] = True

                globe_data += (lat, lon, ping)
            # Append myself
            loc = geodb.get(config.ip_external)
            if loc:
                lat, lon = (loc["location"]["latitude"], loc["location"]["longitude"])
                globe_data += (lat, lon, -0.135)

            self.response(to, globe_data)
        except Exception, err:
            self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
            self.response(to, {"error": err})
Exemplo n.º 56
0
 def test_close(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb',
         self.mode
     )
     reader.close()
Exemplo n.º 57
0
    r'(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)')

IPPortPatternLine = re.compile(
    r'^.*?(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)).*?(?P<port>\d{2,5}).*$',
    flags=re.MULTILINE)

IPPortPatternGlobal = re.compile(
    r'(?P<ip>(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))'
    r'(?=.*?(?:(?:(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?))|(?P<port>\d{2,5})))',
    flags=re.DOTALL)

# IsIpPattern = re.compile(
#     r'^(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)$')


mmdbReader = maxminddb.open_database(
                os.path.join(BASE_DIR, 'data', 'GeoLite2-Country.mmdb'))

async def resolve_host(host, timeout, loop):
    global RESOLVER
    if not RESOLVER:
        RESOLVER = aiodns.DNSResolver(loop=loop)
    try:
        ip = await asyncio.wait_for(RESOLVER.query(host, 'A'),
                                    timeout=timeout)
        ip = ip[0].host
    except (aiodns.error.DNSError, asyncio.TimeoutError):
        return False
    else:
        return ip

def get_base_dir():
Exemplo n.º 58
0
import maxminddb
from PIL import Image
from PIL.ExifTags import TAGS
from pprint import pprint 
import sys

imageName = sys.argv[1]

reader = maxminddb.open_database('GeoLite2-City.mmdb')

data = reader.get('1.1.1.1')
# print(data)

reader.close()

exifData = {}
imgFile = Image.open(imageName)
# imgFile = Image.open('image_test2.jpg')
info = imgFile._getexif()
if info:
    for (tag, value) in info.items():
        decoded = TAGS.get(tag, tag)
        exifData[decoded] = value
    exifGPS = exifData.get('GPSInfo', None)
    if exifGPS:
        print(imgFile, ' contains GPS MetaData', exifGPS)

pprint(exifData)        
Exemplo n.º 59
0
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from django.conf import settings

try:
    import maxminddb
except ImportError:
    maxminddb = None


if maxminddb is not None:
    try:
        geo = maxminddb.open_database(settings.MAXMIND_DB_PATH)
    except (IOError, maxminddb.InvalidDatabaseError):
        geo = None
else:
    geo = None


def get_country_from_ip(ip_addr):
    """Return country info for the given IP Address."""
    if geo is not None:
        try:
            data = geo.get(ip_addr)
        except ValueError:
            data = None

        if data:
            country = data.get('country', data.get('registered_country'))
Exemplo n.º 60
0
 def test_too_many_get_args(self):
     reader = open_database(
         'tests/data/test-data/MaxMind-DB-test-decoder.mmdb', self.mode
     )
     self.assertRaises(TypeError, reader.get, ('1.1.1.1', 'blah'))
     reader.close()