def download_geolite2_city_db(dest):
"""
Download the GeoLite2 database, decompress it, and save it to disk.
:param str dest: The file path to save the database to.
"""
try:
response = requests.get(DB_DOWNLOAD_URL, stream=True)
except requests.ConnectionError:
logger.error(
'geoip database download failed (could not connect to the server)')
raise errors.KingPhisherResourceError(
'could not download the geoip database') from None
except requests.RequestException:
logger.error('geoip database download failed', exc_info=True)
raise errors.KingPhisherResourceError(
'could not download the geoip database') from None
tfile = tempfile.mkstemp()
os.close(tfile[0])
tfile = tfile[1]
try:
with open(tfile, 'wb') as file_h:
for chunk in response.iter_content(chunk_size=1024):
file_h.write(chunk)
file_h.flush()
with open(dest, 'wb') as file_h:
shutil.copyfileobj(gzip.GzipFile(tfile, mode='rb'), file_h)
finally:
os.remove(tfile)
os.chmod(dest, 0o644)
return os.stat(dest).st_size
def download_geolite2_city_db(dest, license, date=None):
"""
Download the GeoLite2 database, decompress it, and save it to disk.
.. versionchanged:: 1.16.0
Added the *license* and *date* parameters.
:param str dest: The file path to save the database to.
:param str license: The MaxMind license key to use to download the database.
:param date: The date for which to download the database.
:type date: :py:class:`datetime.date`
"""
params = {
'edition_id': 'GeoLite2-City',
'license_key': license,
'suffix': 'tar.gz'
}
if date is not None:
params['date'] = date.strftime('%Y%m%d')
try:
response = requests.get(
'https://download.maxmind.com/app/geoip_download',
params=params,
stream=True)
except requests.ConnectionError:
logger.error(
'geoip database download failed (could not connect to the server)')
raise errors.KingPhisherResourceError(
'could not download the geoip database') from None
except requests.RequestException:
logger.error('geoip database download failed', exc_info=True)
raise errors.KingPhisherResourceError(
'could not download the geoip database') from None
tmp_file = tempfile.mkstemp()
os.close(tmp_file[0])
tmp_file = tmp_file[1]
try:
with open(tmp_file, 'wb') as file_h:
for chunk in response.iter_content(chunk_size=4096):
file_h.write(chunk)
file_h.flush()
tar_file = tarfile.open(tmp_file, mode='r:gz')
member = next((name for name in tar_file.getnames()
if name.endswith('GeoLite2-City.mmdb')), None)
if member is None:
raise errors.KingPhisherResourceError(
'could not find the GeoLite2-City.mmdb file in the archive')
with open(dest, 'wb') as file_h:
shutil.copyfileobj(tar_file.extractfile(member), file_h)
finally:
os.remove(tmp_file)
os.chmod(dest, 0o644)
return os.stat(dest).st_size
def init_database(database_file):
"""
Create and initialize the GeoLite2 database engine. This must be done before
classes and functions in this module attempt to look up results. If the
specified database file does not exist, a new copy will be downloaded.
:param str database_file: The GeoLite2 database file to use.
:return: The initialized GeoLite2 database object.
:rtype: :py:class:`geoip2.database.Reader`
"""
# pylint: disable=global-statement
global _geoip_db
if not os.path.isfile(database_file):
db_path = find.data_file('GeoLite2-City.mmdb')
if db_path is None:
raise errors.KingPhisherResourceError(
'the default geoip database file is unavailable')
logger.info('initializing the default geoip database')
shutil.copyfile(db_path, database_file)
_geoip_db = geoip2.database.Reader(database_file)
metadata = _geoip_db.metadata()
if not metadata.database_type == 'GeoLite2-City':
raise ValueError(
'the connected database is not a GeoLite2-City database')
build_date = datetime.datetime.fromtimestamp(metadata.build_epoch)
if build_date < datetime.datetime.utcnow() - datetime.timedelta(days=90):
logger.warning('the geoip database is older than 90 days')
return _geoip_db
def install_packages(self, packages):
"""
This function will take a list of Python packages and attempt to install
them through pip to the :py:attr:`.library_path`.
.. versionadded:: 1.14.0
:param list packages: list of python packages to install using pip.
:return: The process results from the command execution.
:rtype: :py:class:`~.ProcessResults`
"""
pip_options = []
if self.library_path is None:
raise errors.KingPhisherResourceError('missing plugin-specific library path')
pip_options.extend(['--target', self.library_path])
if its.frozen:
args = [os.path.join(os.path.dirname(sys.executable), 'python.exe')]
else:
args = [sys.executable] + pip_options + packages
args += ['-m', 'pip', 'install'] + pip_options + packages
if len(packages) > 1:
info_string = "installing packages: {}"
else:
info_string = "installing package: {}"
self.logger.info(info_string.format(', '.join(packages)))
return startup.run_process(args)
def load(self, name, reload_module=False):
"""
Load a plugin into memory, this is effectively the Python equivalent of
importing it. A reference to the plugin class is kept in
:py:attr:`.loaded_plugins`. If the plugin is already loaded, no changes
are made.
:param str name: The name of the plugin to load.
:param bool reload_module: Reload the module to allow changes to take affect.
:return:
"""
self._lock.acquire()
if not reload_module and name in self.loaded_plugins:
self._lock.release()
return
try:
module = self.plugin_source.load_plugin(name)
except Exception as error:
self._lock.release()
raise error
if reload_module:
_reload(module)
klass = getattr(module, 'Plugin', None)
if klass is None:
self._lock.release()
self.logger.warning(
"failed to load plugin '{0}', Plugin class not found".format(
name))
raise errors.KingPhisherResourceError(
'the Plugin class is missing')
if not issubclass(klass, self._plugin_klass):
self._lock.release()
self.logger.warning(
"failed to load plugin '{0}', Plugin class is invalid".format(
name))
raise errors.KingPhisherResourceError(
'the Plugin class is invalid')
self.loaded_plugins[name] = klass
self.logger.debug("plugin '{0}' has been {1}loaded".format(
name, 're' if reload_module else ''))
self._lock.release()
return klass
def graphql_find_file(self, query_file, **query_vars):
"""
This method is similar to :py:meth:`~.graphql_file`. The first argument
(*query_file*) is the name of a query file that will be located using
:py:func:`find.data_file`. Additional keyword arguments are passed as
the variables to the query.
:param str query_file: The name of the query file to locate.
:param query_vars: These keyword arguments are passed as the variables to the query.
:return: The query results.
:rtype: dict
"""
path = find.data_file(os.path.join('queries', query_file))
if path is None:
raise errors.KingPhisherResourceError(
'could not find GraphQL query file: ' + query_file)
return self.graphql_file(path, query_vars=query_vars)
def load_campaigns(self, cursor=None):
"""Load campaigns from the remote server and populate the :py:class:`Gtk.TreeView`."""
if cursor is None:
self._tv_model.clear()
self.gobjects['revealer_loading'].set_reveal_child(True)
self.gobjects['progressbar_loading'].set_fraction(0.0)
path = find.data_file(os.path.join('queries', 'get_campaigns.graphql'))
if path is None:
raise errors.KingPhisherResourceError(
'could not find GraphQL query file: get_campaigns.graphql')
self.application.rpc.async_graphql_file(
path,
query_vars={
'cursor': cursor,
'page': _QUERY_PAGE_SIZE
},
on_success=self.__async_rpc_cb_load_campaigns,
when_idle=True)
def _init_tables_api(self):
# initialize the tables api dataset, this is to effectively pin the schema exposed allowing new columns to be
# added without breaking rpc compatibility
file_path = find.data_file('table-api.json')
if file_path is None:
raise errors.KingPhisherResourceError('missing the table-api.json data file')
with open(file_path, 'r') as file_h:
tables_api_data = serializers.JSON.load(file_h)
if tables_api_data['schema'] > db_models.SCHEMA_VERSION:
raise errors.KingPhisherInputValidationError('the table-api.json data file\'s schema version is incompatible')
for table_name, columns in tables_api_data['tables'].items():
model = db_models.database_tables[table_name].model
self.tables_api[table_name] = db_models.MetaTable(
column_names=columns,
model=model,
name=table_name,
table=model.__table__
)
self.logger.debug("initialized the table api dataset (schema version: {0})".format(tables_api_data['schema']))
def _graphql_find_file(query_file):
path = find.data_file(os.path.join('queries', query_file))
if path is None:
raise errors.KingPhisherResourceError(
'could not find GraphQL query file: ' + query_file)
return _graphql_file(path)