def __update_codec(self): # character encoding detect with open(self.path, 'r+') as f: overwrite = False content = f.read() result = chardet.detect(content) if not result: raise ImproperlyConfigured("Unknown encoding for '%s'." % self.path) # coding with utf-N-BOM, removes the BOM signs '\xff\xfe' or '\xfe\xff' if content.startswith(codecs.BOM_LE) or content.startswith( codecs.BOM_BE): content = content[2:] overwrite = True # flag to rewrite the config file file_encoding = result['encoding'] # coding with other codecs except for ascii or utf-8 if file_encoding != 'ascii' and file_encoding != settings.FILE_CHARSET: try: content = content.decode(file_encoding).encode( settings.FILE_CHARSET) overwrite = True except UnicodeDecodeError as e: raise ImproperlyConfigured("Unknown encoding for '%s'." % self.path) except UnicodeEncodeError as e: raise ImproperlyConfigured( "Unrecognized symbols in '%s'." % self.path) # erases the content and rewrites with 'utf-8' encoding if overwrite: f.truncate(0) # truncate the config size to 0 f.seek(0) # rewind f.write(content)
def _path_from_module(self, module): """Attempt to determine app's filesystem path from its module.""" # See #21874 for extended discussion of the behavior of this method in # various cases. # Convert paths to list because Python 3's _NamespacePath does not # support indexing. paths = list(getattr(module, '__path__', [])) if len(paths) != 1: filename = getattr(module, '__file__', None) if filename is not None: paths = [os.path.dirname(filename)] else: # For unknown reasons, sometimes the list returned by __path__ # contains duplicates that must be removed (#25246). paths = list(set(paths)) if len(paths) > 1: raise ImproperlyConfigured( "The app module %r has multiple filesystem locations (%r); " "you must configure this app with an AppConfig subclass " "with a 'path' class attribute." % (module, paths)) elif not paths: raise ImproperlyConfigured( "The app module %r has no filesystem location, " "you must configure this app with an AppConfig subclass " "with a 'path' class attribute." % (module,)) return upath(paths[0])
def get_connection(self, settings_dict): if settings_dict.get('HOST') and settings_dict.get('PORT'): stdout.debug( "Trying to connect to {} servered on '{}:{}'...".format( self.db_name, settings_dict['HOST'], settings_dict['PORT'])) conn_cnt = 0 while conn_cnt < settings.DB_CONN_MAX_TIMES: try: conn = self._connect(settings_dict) stdout.success("Connection to {} established.".format( self.db_name)) return conn except ConnectionTimeout as e: conn_cnt += 1 interval = random.randint(0, settings.DB_CONN_MAX_INTERVAL) stdout.warn( "Connection timeout to {}: {}\nWill retry to connect in {} \ seconds.".format(self.db_name, str(e), interval)) time.sleep(interval) except KeyError as e: # May raise when resolving the settings dict stdout.error("Fields missing, check '{}' was set.".format( str(e))) raise ImproperlyConfigured("Fields missing: {}".format(str(e))) stdout.error("Unable to establish connection to '{}'".format( self.db_name)) raise ImproperlyConfigured()
def __init__(self, template_path): if not os.path.exists(template_path): raise ImproperlyConfigured("Config path '%s' does not exist." % npath(template_path)) self.template_path = template_path self._config = self.__load_template()
def __connect(self): mongo_url = self.__get_mongo_url(self.settings_dict) logger.debug("Connecting to '%s'..." % self.displayed_mongo_url) conn_cnt = 0 while conn_cnt < settings.DB_CONN_MAX_TIMES: try: # More details about `MongoClient` API, see: # https://api.mongodb.com/python/2.8/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient client = MongoClient(mongo_url) # the constructor returns immediately and launches the # connection process on background threads. # Checks if the server is available like this: client.admin.command('ismaster') logger.debug("Connection to MongoDB is established.") return client # If auto-reconnection will be performed, AutoReconnect will be raised. # Application code should handle this exception (recognizing that the # operation failed) and then continue to execute. except errors.AutoReconnect as e: conn_cnt += 1 msg = ("Connection to MongoDB is lost and an attempt to " "auto-connect will be made ...") stdout.warn(msg) logger.warning(msg) logger.error("Unable to establish the connection to MongoDB.") raise ImproperlyConfigured( "Unable to establish the connection to MongoDB.")
def write(self, conf_path): if os.path.exists(conf_path): raise ImproperlyConfigured("Config path '%s' exists already." % conf_path) with open(conf_path, 'wb') as configfile: self._config.write(configfile)
def query(self, **context): try: operation = self.operation_template.format(**context) except KeyError as e: raise ImproperlyConfigured("Key missing: {}".format(str(e))) # execute the query with context provided return self.handler.exec_query(operation)
def copy_blobs(self, blob_names, container_name, src_container=None, pattern=None): """ Copy blobs listed in `blob_names` to the dest container. `src_container` if src_container was given, blob_names are OK to be relative path to the container, and will be extended to `http://self.host/src_container/blob_name` `pattern` if src_container and pattern was given and blob_names was None, copies blobs in the src_container meanwhile matches the pattern to dest container. """ if blob_names == None: if src_container: blobs_in_container = self.list_blobs(src_container) matchfn = get_matchfn(pattern, True) # gets blobs from the src_container which matches the pattern(with ignorecase) blob_names = filter(lambda x: matchfn(x), blobs_in_container) else: raise ImproperlyConfigured( "Method `copy_blobs` is ought to be called with " "`src_container` given if blob_names was set to None.") if src_container: urls = [] for blob_name in blob_names: # not absolute url path if not blob_name.startswith('http'): # extends with the account and container blob_name = "http://{}/{}/{}".format( self.host, src_container, blob_name) urls.append(escape_uri_path(blob_name)) blob_names = urls blobs = [] logger.info("Will copy {} blobs to [{}].".format( len(blob_names), container_name)) for copy_source in progressbar.progressbar(blob_names, widgets=self.widgets): r = re.match(self.blob_pattern, copy_source) if r: blob_name = r.group(3) else: logger.error("Blob name specified must be a url: '{}'.".format( copy_source)) continue self.block_blob_service.copy_blob(container_name, \ blob_name, copy_source) logger.debug("Copied '{}' to '{}'.".format(copy_source, blob_name)) blobs.append(blob_name) return blobs
def __init__(self, settings_dict): if not isinstance(settings_dict, dict): raise ImproperlyConfigured( "Argument `settings_dict` is the configure for sql drivers, " "which must be an instance of `dict`.") self.settings_dict = settings_dict self._conn = self.get_connection(settings_dict) if settings_dict.get(self.table_alias_name): self._table_alias = settings_dict[self.table_alias_name] else: raise ImproperlyConfigured(\ "Key missing: `{}` is required to provide a map of " "alias to table names in the database.".format(self.table_alias_name)) self._cursor = None
def set_option_code(self, option_code): for opt_type_klass in self.option_types: if option_code in opt_type_klass.opt_code: self._option = opt_type_klass(self.value) return # TODO: lists all supported option code raise ImproperlyConfigured("Does not support option type %s." % option_code)
def query(self, **context): try: context.update(self.table_alias) sql_statement = self.statement_template.format(**context) except KeyError as e: raise ImproperlyConfigured("Keys missing: {}".format(str(e))) # execute the query with context provided return self.handler.exec_query(sql_statement)
def __load_or_import(self, handler_cls, base_cls, context): if isinstance(handler_cls, six.string_types): handler_cls = import_string(handler_cls) if issubclass(handler_cls, base_cls): return handler_cls(context) else: raise ImproperlyConfigured(\ "Database handlers must be a subclass of {}.".format(base_cls))
def get_default_template(self): config_template_path = self.app_config.config_template if not os.path.exists(config_template_path): raise ImproperlyConfigured( "Config template path '%s' does not exist." % config_template_path) return ConfigWriter(config_template_path)
def db(self): if self._db != None: return self._db else: logger.error( "Database is not specified, call `set_database()` first.") raise ImproperlyConfigured( "Database is not specified, call `set_database()` first.")
def coll(self): if self._coll != None: return self._coll else: logger.error( "Collection is not specified, call `set_collection()` first.") raise ImproperlyConfigured( "Collection is not specified, call `set_collection()` first.")
def query(self, **context): try: context.update(self.base_context) sql_statement = self.statement_template.format(**context) except KeyError as e: raise ImproperlyConfigured("Keys missing: %s" % e.message) # execute the query with context provided return self.handler.exec_query(sql_statement)
def __get_mongo_url(self, settings_dict): try: return "mongodb://{USER}:{PASSWORD}@{HOST}:{PORT}".format( **settings_dict) except KeyError as e: logger.error("Key missing, check '{}' was set.".format(str(e))) raise ImproperlyConfigured( "Key missing, check '{}' was set.".format(str(e)))
def set_database(self, db_name): logger.debug("Set database to '{}'".format(db_name)) try: self._db = self._client[db_name] self._db_name = db_name except errors.InvalidName as e: logger.warning("Unknown database specified: '{}'".format(db_name)) raise ImproperlyConfigured( "Unknown database specified: '{}'".format(db_name))
def verbose(self, verbose): if verbose in [0, 1, 2, 3]: self._verbose = verbose elif verbose == None: self._verbose = 1 else: raise ImproperlyConfigured( "Verbose level was ought to be set from " "0 to 3, but got %s." % verbose)
def create_from_context(cls, query_context): handler_cls = query_context['sql_handler'] handler_settings = query_context['sql_context'] if isinstance(handler_cls, six.string_types): try: handler_cls = import_string(handler_cls) except ImportError as e: raise ImproperlyConfigured(\ "SQL handler specified is not an importable module: {}.".format(handler_cls)) if issubclass(handler_cls, cls.base_handler_cls): handler = handler_cls(handler_settings) else: raise ImproperlyConfigured(\ "Database handlers must be a subclass of {}.".format(cls.base_handler_cls)) return cls(handler)
def append(self, config_name): if not isinstance(self.template_conf, ConfigWriter): raise ImproperlyConfigured( "Member of the class 'template_conf' must " "be an instance of 'ConfigWriter'") config_path = os.path.join(self.app_config.configs_dirname, config_name) self.template_conf.write(config_path)
def __init__(self, query_cls, context): self.sql_handler = self.__load_or_import(context['sql_handler'], \ database.BaseSQLHandler, context['sql_context']) sql_table_alias = context['sql_context']['TABLE_ALIAS'] if not issubclass(query_cls, query.BaseGuidQuery): raise ImproperlyConfigured(\ "Query handler must be a subclass of query.BaseGuidQuery.") self.querier = query_cls(self.sql_handler, sql_table_alias) self.mongodb = self.__load_or_import(context['mongo_handler'], \ database.MongoDBHandler, context['mongo_context'])
def register(self, action_klass, alias, default=False, entry=None): """ To store the map between actions and names, which would be called by the sub-command 'run' and option '-a' """ if self.alias_action_table.get(alias): raise ImproperlyConfigured("Action alias '%s' was registered." % alias) # TODO: take into account argument default and entry self.alias_action_table[alias] = getattr(self.actions_module, action_klass)
def choice(exclude=None): if isinstance(exclude, list) or isinstance(exclude, tuple): colors = tuple(set(_colors) - set(exclude)) else: colors = _colors if len(colors) == 0: raise ImproperlyConfigured("Colors were used up.") return random.choice(_colors)
def set_collection(self, coll_name): logger.debug("Set collection to '{}'".format(coll_name)) if self._coll_name != coll_name: try: self._coll = self.db[coll_name] self._coll_name = coll_name except errors.InvalidName as e: logger.warning( "Unknown collection specified: '{}'".format(coll_name)) raise ImproperlyConfigured( "Unknown collection specified: '{}'".format(coll_name))
def _update_codec(self, conf_path): # character encoding detect with open(conf_path, 'rb+') as f: raw_content = f.read() if raw_content.strip() == "": # do nothing if it was empty return try: content = self.__update_with_locales(raw_content) except UnicodeError as e: result = chardet.detect(raw_content) if not result or result['encoding'] in [ 'ascii', settings.FILE_CHARSET ]: # Tried, but failed raise ImproperlyConfigured("Unknown encoding for '%s'." % self.path) if result['confidence'] < settings.CONF_CHARDET_CONFIDENCE: logger.warning( "Confidence for file encoding is too low: '%s'" % self.path) raise ImproperlyConfigured( "Ambigious encoding for '%s', make sure it was " "encoded with 'UTF-8'." % self.path) file_encoding = result['encoding'] try: content = raw_content.decode(file_encoding).encode( settings.FILE_CHARSET) except UnicodeError as e: raise ImproperlyConfigured("Unknown encoding for '%s'." % self.path) # erases the content and rewrites with 'utf-8' encoding if content != raw_content: f.truncate(0) # truncate the config size to 0 f.seek(0) # rewind f.write(content)
def __init__(self, settings_dict): if not isinstance(settings_dict, dict): raise ImproperlyConfigured( "Argument `settings_dict` is the configure for mongodb drivers, " "which must be an instance of `dict`.") self.settings_dict = settings_dict self.displayed_mongo_url = self.__get_displayed_url(settings_dict) self._client = self.__connect() self._db_name = None self._db = None self._coll_name = None self._coll = None
def get_color(self, label): if self._use_default: return None if self._pallet.get(label) == None: if self._autofill: # dict.values() returns a iterator, uses list() to force convert color = colors.choice(exclude=list(self._pallet.values())) self.add_color(label, color) else: raise ImproperlyConfigured( "Color for lable '{}' was not set.".format(label)) return self._pallet[label]
def create_from_context(cls, query_context): handler_cls = query_context['sql_handler'] handler_settings = query_context['sql_context'] if isinstance(handler_cls, six.string_types): handler_cls = import_string(handler_cls) if issubclass(handler_cls, cls.base_handler_cls): handler = handler_cls(handler_settings) else: raise ImproperlyConfigured(\ "Database handlers must be a subclass of {}.".format(base_cls)) table_alias = handler_settings['TABLE_ALIAS'] return cls(handler, table_alias)
def get_connection(self, settings_dict): import _mssql logger.debug( "Trying to connect to SQLServer servered on '%s:%s'..." % (settings_dict['HOST'], settings_dict['PORT'])) conn = None try: conn = _mssql.connect( server=settings_dict['HOST'], port=settings_dict['PORT'], user=settings_dict['USER'], password=settings_dict['PASSWORD'], database=settings_dict['DATABASE'], charset=settings_dict['CHARSET'] ) except _mssql.MssqlDatabaseException as e: logger.error(str(e)) raise ImproperlyConfigured("Failed to connect to SQL database '%s'." % settings_dict['HOST']) except KeyError as e: logger.error( "Fields missing, please check {} was in settings.".format(str(e))) raise ImproperlyConfigured("Fields missing: {}".format(str(e))) return conn