def __update_codec(self):
# character encoding detect
with open(self.path, 'r+') as f:
overwrite = False
content = f.read()
result = chardet.detect(content)
if not result:
raise ImproperlyConfigured("Unknown encoding for '%s'." %
self.path)
# coding with utf-N-BOM, removes the BOM signs '\xff\xfe' or '\xfe\xff'
if content.startswith(codecs.BOM_LE) or content.startswith(
codecs.BOM_BE):
content = content[2:]
overwrite = True # flag to rewrite the config file
file_encoding = result['encoding']
# coding with other codecs except for ascii or utf-8
if file_encoding != 'ascii' and file_encoding != settings.FILE_CHARSET:
try:
content = content.decode(file_encoding).encode(
settings.FILE_CHARSET)
overwrite = True
except UnicodeDecodeError as e:
raise ImproperlyConfigured("Unknown encoding for '%s'." %
self.path)
except UnicodeEncodeError as e:
raise ImproperlyConfigured(
"Unrecognized symbols in '%s'." % self.path)
# erases the content and rewrites with 'utf-8' encoding
if overwrite:
f.truncate(0) # truncate the config size to 0
f.seek(0) # rewind
f.write(content)
def _path_from_module(self, module): """Attempt to determine app's filesystem path from its module.""" # See #21874 for extended discussion of the behavior of this method in # various cases. # Convert paths to list because Python 3's _NamespacePath does not # support indexing. paths = list(getattr(module, '__path__', [])) if len(paths) != 1: filename = getattr(module, '__file__', None) if filename is not None: paths = [os.path.dirname(filename)] else: # For unknown reasons, sometimes the list returned by __path__ # contains duplicates that must be removed (#25246). paths = list(set(paths)) if len(paths) > 1: raise ImproperlyConfigured( "The app module %r has multiple filesystem locations (%r); " "you must configure this app with an AppConfig subclass " "with a 'path' class attribute." % (module, paths)) elif not paths: raise ImproperlyConfigured( "The app module %r has no filesystem location, " "you must configure this app with an AppConfig subclass " "with a 'path' class attribute." % (module,)) return upath(paths[0])
def get_connection(self, settings_dict):
if settings_dict.get('HOST') and settings_dict.get('PORT'):
stdout.debug(
"Trying to connect to {} servered on '{}:{}'...".format(
self.db_name, settings_dict['HOST'],
settings_dict['PORT']))
conn_cnt = 0
while conn_cnt < settings.DB_CONN_MAX_TIMES:
try:
conn = self._connect(settings_dict)
stdout.success("Connection to {} established.".format(
self.db_name))
return conn
except ConnectionTimeout as e:
conn_cnt += 1
interval = random.randint(0, settings.DB_CONN_MAX_INTERVAL)
stdout.warn(
"Connection timeout to {}: {}\nWill retry to connect in {} \
seconds.".format(self.db_name, str(e), interval))
time.sleep(interval)
except KeyError as e:
# May raise when resolving the settings dict
stdout.error("Fields missing, check '{}' was set.".format(
str(e)))
raise ImproperlyConfigured("Fields missing: {}".format(str(e)))
stdout.error("Unable to establish connection to '{}'".format(
self.db_name))
raise ImproperlyConfigured()
def __init__(self, template_path):
if not os.path.exists(template_path):
raise ImproperlyConfigured("Config path '%s' does not exist." %
npath(template_path))
self.template_path = template_path
self._config = self.__load_template()
def __connect(self):
mongo_url = self.__get_mongo_url(self.settings_dict)
logger.debug("Connecting to '%s'..." % self.displayed_mongo_url)
conn_cnt = 0
while conn_cnt < settings.DB_CONN_MAX_TIMES:
try:
# More details about `MongoClient` API, see:
# https://api.mongodb.com/python/2.8/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
client = MongoClient(mongo_url)
# the constructor returns immediately and launches the
# connection process on background threads.
# Checks if the server is available like this:
client.admin.command('ismaster')
logger.debug("Connection to MongoDB is established.")
return client
# If auto-reconnection will be performed, AutoReconnect will be raised.
# Application code should handle this exception (recognizing that the
# operation failed) and then continue to execute.
except errors.AutoReconnect as e:
conn_cnt += 1
msg = ("Connection to MongoDB is lost and an attempt to "
"auto-connect will be made ...")
stdout.warn(msg)
logger.warning(msg)
logger.error("Unable to establish the connection to MongoDB.")
raise ImproperlyConfigured(
"Unable to establish the connection to MongoDB.")
def write(self, conf_path):
if os.path.exists(conf_path):
raise ImproperlyConfigured("Config path '%s' exists already." %
conf_path)
with open(conf_path, 'wb') as configfile:
self._config.write(configfile)
def query(self, **context):
try:
operation = self.operation_template.format(**context)
except KeyError as e:
raise ImproperlyConfigured("Key missing: {}".format(str(e)))
# execute the query with context provided
return self.handler.exec_query(operation)
def copy_blobs(self,
blob_names,
container_name,
src_container=None,
pattern=None):
"""
Copy blobs listed in `blob_names` to the dest container.
`src_container`
if src_container was given, blob_names are OK to be relative path
to the container, and will be extended to `http://self.host/src_container/blob_name`
`pattern`
if src_container and pattern was given and blob_names was None,
copies blobs in the src_container meanwhile matches the pattern to
dest container.
"""
if blob_names == None:
if src_container:
blobs_in_container = self.list_blobs(src_container)
matchfn = get_matchfn(pattern, True)
# gets blobs from the src_container which matches the pattern(with ignorecase)
blob_names = filter(lambda x: matchfn(x), blobs_in_container)
else:
raise ImproperlyConfigured(
"Method `copy_blobs` is ought to be called with "
"`src_container` given if blob_names was set to None.")
if src_container:
urls = []
for blob_name in blob_names:
# not absolute url path
if not blob_name.startswith('http'):
# extends with the account and container
blob_name = "http://{}/{}/{}".format(
self.host, src_container, blob_name)
urls.append(escape_uri_path(blob_name))
blob_names = urls
blobs = []
logger.info("Will copy {} blobs to [{}].".format(
len(blob_names), container_name))
for copy_source in progressbar.progressbar(blob_names,
widgets=self.widgets):
r = re.match(self.blob_pattern, copy_source)
if r:
blob_name = r.group(3)
else:
logger.error("Blob name specified must be a url: '{}'.".format(
copy_source))
continue
self.block_blob_service.copy_blob(container_name, \
blob_name, copy_source)
logger.debug("Copied '{}' to '{}'.".format(copy_source, blob_name))
blobs.append(blob_name)
return blobs
def __init__(self, settings_dict):
if not isinstance(settings_dict, dict):
raise ImproperlyConfigured(
"Argument `settings_dict` is the configure for sql drivers, "
"which must be an instance of `dict`.")
self.settings_dict = settings_dict
self._conn = self.get_connection(settings_dict)
if settings_dict.get(self.table_alias_name):
self._table_alias = settings_dict[self.table_alias_name]
else:
raise ImproperlyConfigured(\
"Key missing: `{}` is required to provide a map of "
"alias to table names in the database.".format(self.table_alias_name))
self._cursor = None
def set_option_code(self, option_code):
for opt_type_klass in self.option_types:
if option_code in opt_type_klass.opt_code:
self._option = opt_type_klass(self.value)
return
# TODO: lists all supported option code
raise ImproperlyConfigured("Does not support option type %s." %
option_code)
def query(self, **context):
try:
context.update(self.table_alias)
sql_statement = self.statement_template.format(**context)
except KeyError as e:
raise ImproperlyConfigured("Keys missing: {}".format(str(e)))
# execute the query with context provided
return self.handler.exec_query(sql_statement)
def __load_or_import(self, handler_cls, base_cls, context):
if isinstance(handler_cls, six.string_types):
handler_cls = import_string(handler_cls)
if issubclass(handler_cls, base_cls):
return handler_cls(context)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(base_cls))
def get_default_template(self):
config_template_path = self.app_config.config_template
if not os.path.exists(config_template_path):
raise ImproperlyConfigured(
"Config template path '%s' does not exist." %
config_template_path)
return ConfigWriter(config_template_path)
def db(self):
if self._db != None:
return self._db
else:
logger.error(
"Database is not specified, call `set_database()` first.")
raise ImproperlyConfigured(
"Database is not specified, call `set_database()` first.")
def coll(self):
if self._coll != None:
return self._coll
else:
logger.error(
"Collection is not specified, call `set_collection()` first.")
raise ImproperlyConfigured(
"Collection is not specified, call `set_collection()` first.")
def query(self, **context):
try:
context.update(self.base_context)
sql_statement = self.statement_template.format(**context)
except KeyError as e:
raise ImproperlyConfigured("Keys missing: %s" % e.message)
# execute the query with context provided
return self.handler.exec_query(sql_statement)
def __get_mongo_url(self, settings_dict):
try:
return "mongodb://{USER}:{PASSWORD}@{HOST}:{PORT}".format(
**settings_dict)
except KeyError as e:
logger.error("Key missing, check '{}' was set.".format(str(e)))
raise ImproperlyConfigured(
"Key missing, check '{}' was set.".format(str(e)))
def set_database(self, db_name):
logger.debug("Set database to '{}'".format(db_name))
try:
self._db = self._client[db_name]
self._db_name = db_name
except errors.InvalidName as e:
logger.warning("Unknown database specified: '{}'".format(db_name))
raise ImproperlyConfigured(
"Unknown database specified: '{}'".format(db_name))
def verbose(self, verbose):
if verbose in [0, 1, 2, 3]:
self._verbose = verbose
elif verbose == None:
self._verbose = 1
else:
raise ImproperlyConfigured(
"Verbose level was ought to be set from "
"0 to 3, but got %s." % verbose)
def create_from_context(cls, query_context):
handler_cls = query_context['sql_handler']
handler_settings = query_context['sql_context']
if isinstance(handler_cls, six.string_types):
try:
handler_cls = import_string(handler_cls)
except ImportError as e:
raise ImproperlyConfigured(\
"SQL handler specified is not an importable module: {}.".format(handler_cls))
if issubclass(handler_cls, cls.base_handler_cls):
handler = handler_cls(handler_settings)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(cls.base_handler_cls))
return cls(handler)
def append(self, config_name):
if not isinstance(self.template_conf, ConfigWriter):
raise ImproperlyConfigured(
"Member of the class 'template_conf' must "
"be an instance of 'ConfigWriter'")
config_path = os.path.join(self.app_config.configs_dirname,
config_name)
self.template_conf.write(config_path)
def __init__(self, query_cls, context):
self.sql_handler = self.__load_or_import(context['sql_handler'], \
database.BaseSQLHandler, context['sql_context'])
sql_table_alias = context['sql_context']['TABLE_ALIAS']
if not issubclass(query_cls, query.BaseGuidQuery):
raise ImproperlyConfigured(\
"Query handler must be a subclass of query.BaseGuidQuery.")
self.querier = query_cls(self.sql_handler, sql_table_alias)
self.mongodb = self.__load_or_import(context['mongo_handler'], \
database.MongoDBHandler, context['mongo_context'])
def register(self, action_klass, alias, default=False, entry=None):
"""
To store the map between actions and names, which would be called by
the sub-command 'run' and option '-a'
"""
if self.alias_action_table.get(alias):
raise ImproperlyConfigured("Action alias '%s' was registered." % alias)
# TODO: take into account argument default and entry
self.alias_action_table[alias] = getattr(self.actions_module, action_klass)
def choice(exclude=None):
if isinstance(exclude, list) or isinstance(exclude, tuple):
colors = tuple(set(_colors) - set(exclude))
else:
colors = _colors
if len(colors) == 0:
raise ImproperlyConfigured("Colors were used up.")
return random.choice(_colors)
def set_collection(self, coll_name):
logger.debug("Set collection to '{}'".format(coll_name))
if self._coll_name != coll_name:
try:
self._coll = self.db[coll_name]
self._coll_name = coll_name
except errors.InvalidName as e:
logger.warning(
"Unknown collection specified: '{}'".format(coll_name))
raise ImproperlyConfigured(
"Unknown collection specified: '{}'".format(coll_name))
def _update_codec(self, conf_path):
# character encoding detect
with open(conf_path, 'rb+') as f:
raw_content = f.read()
if raw_content.strip() == "":
# do nothing if it was empty
return
try:
content = self.__update_with_locales(raw_content)
except UnicodeError as e:
result = chardet.detect(raw_content)
if not result or result['encoding'] in [
'ascii', settings.FILE_CHARSET
]:
# Tried, but failed
raise ImproperlyConfigured("Unknown encoding for '%s'." %
self.path)
if result['confidence'] < settings.CONF_CHARDET_CONFIDENCE:
logger.warning(
"Confidence for file encoding is too low: '%s'" %
self.path)
raise ImproperlyConfigured(
"Ambigious encoding for '%s', make sure it was "
"encoded with 'UTF-8'." % self.path)
file_encoding = result['encoding']
try:
content = raw_content.decode(file_encoding).encode(
settings.FILE_CHARSET)
except UnicodeError as e:
raise ImproperlyConfigured("Unknown encoding for '%s'." %
self.path)
# erases the content and rewrites with 'utf-8' encoding
if content != raw_content:
f.truncate(0) # truncate the config size to 0
f.seek(0) # rewind
f.write(content)
def __init__(self, settings_dict):
if not isinstance(settings_dict, dict):
raise ImproperlyConfigured(
"Argument `settings_dict` is the configure for mongodb drivers, "
"which must be an instance of `dict`.")
self.settings_dict = settings_dict
self.displayed_mongo_url = self.__get_displayed_url(settings_dict)
self._client = self.__connect()
self._db_name = None
self._db = None
self._coll_name = None
self._coll = None
def get_color(self, label):
if self._use_default:
return None
if self._pallet.get(label) == None:
if self._autofill:
# dict.values() returns a iterator, uses list() to force convert
color = colors.choice(exclude=list(self._pallet.values()))
self.add_color(label, color)
else:
raise ImproperlyConfigured(
"Color for lable '{}' was not set.".format(label))
return self._pallet[label]
def create_from_context(cls, query_context):
handler_cls = query_context['sql_handler']
handler_settings = query_context['sql_context']
if isinstance(handler_cls, six.string_types):
handler_cls = import_string(handler_cls)
if issubclass(handler_cls, cls.base_handler_cls):
handler = handler_cls(handler_settings)
else:
raise ImproperlyConfigured(\
"Database handlers must be a subclass of {}.".format(base_cls))
table_alias = handler_settings['TABLE_ALIAS']
return cls(handler, table_alias)
def get_connection(self, settings_dict):
import _mssql
logger.debug(
"Trying to connect to SQLServer servered on '%s:%s'..." % (settings_dict['HOST'], settings_dict['PORT']))
conn = None
try:
conn = _mssql.connect(
server=settings_dict['HOST'],
port=settings_dict['PORT'],
user=settings_dict['USER'],
password=settings_dict['PASSWORD'],
database=settings_dict['DATABASE'],
charset=settings_dict['CHARSET']
)
except _mssql.MssqlDatabaseException as e:
logger.error(str(e))
raise ImproperlyConfigured("Failed to connect to SQL database '%s'." % settings_dict['HOST'])
except KeyError as e:
logger.error(
"Fields missing, please check {} was in settings.".format(str(e)))
raise ImproperlyConfigured("Fields missing: {}".format(str(e)))
return conn