Beispiel #1
0
 def clean_cache_backend(self):
     """Validates that the specified cache backend is parseable by Django."""
     backend = self.cleaned_data['cache_backend'].strip()
     if backend:
         try:
             parse_backend_uri(backend)
         except InvalidCacheBackendError, e:
             raise forms.ValidationError(e)
Beispiel #2
0
 def clean_cache_backend(self):
     """Validates that the specified cache backend is parseable by Django."""
     backend = self.cleaned_data['cache_backend'].strip()
     if backend:
         try:
             parse_backend_uri(backend)
         except InvalidCacheBackendError, e:
             raise forms.ValidationError(e)
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Copied from Django 1.2.4 django.db.backends.creation.BaseDatabaseCreation

        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name
        can_rollback = self.connection.creation._rollback_works()
        self.connection.settings_dict["SUPPORTS_TRANSACTIONS"] = can_rollback

        call_command('syncdb', verbosity=verbosity, interactive=False, database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri, cache
            from django.db import router
            if router.allow_syncdb(self.connection.alias, cache.cache_model_class):
                _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
                call_command('createcachetable', cache_name, database=self.connection.alias)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #4
0
    def on_task_init(self, task_id, task):
        """This method is called before a task is executed.

        Does everything necessary for Django to work in a long-living,
        multiprocessing environment.

        """
        # See: http://groups.google.com/group/django-users/browse_thread/
        #       thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst&
        #       q=multiprocessing#38402e76cf3233e8
        from django.db import connection
        connection.close()

        # Reset cache connection only if using memcached/libmemcached
        from django.core import cache
        # XXX At Opera we use a custom memcached backend that uses libmemcached
        # instead of libmemcache (cmemcache). Should find a better solution for
        # this, but for now "memcached" should probably be unique enough of a
        # string to not make problems.
        cache_backend = cache.settings.CACHE_BACKEND
        if hasattr(cache, "parse_backend_uri"):
            cache_scheme = cache.parse_backend_uri(cache_backend)[0]
        else:
            # Django <= 1.0.2
            cache_scheme = cache_backend.split(":", 1)[0]
        if "memcached" in cache_scheme:
            cache.cache.close()
Beispiel #5
0
def normalize_cache_backend(cache_backend, cache_name=DEFAULT_CACHE_ALIAS):
    """Return a new-style ``CACHES`` dictionary from any given cache_backend.

    Over time, Django has had support for two formats for a cache backend. The
    old-style :django:setting:`CACHE_BACKEND` string, and the new-style
    :django:setting:`CACHES` dictionary.

    This function will accept either as input and return a dictionary
    containing a single normalized entry for the given cache backend. This can
    be merged into the existing :django:setting:`CACHES` setting.

    If a :django:setting:`CACHES` dictionary is passed, the "default" cache
    will be the result.

    Args:
        cache_backend (dict or str):
            The new-style or old-style cache backend dictionary or str to
            normalize.

        cache_name (str):
            The name of the cache backend to look up in ``cache_backend``, if
            a new-style dictionary is provided.

    Returns:
        A new-style cache backend dictionary containing the single cache
        backend referenced. If there were any parsing issues, an empty
        dictionary will be returned.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        backend_info = cache_backend.get(cache_name, {})
        backend_name = backend_info.get('BACKEND')

        if backend_name in RENAMED_BACKENDS:
            backend_info['BACKEND'] = RENAMED_BACKENDS[backend_name]

        return backend_info

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError as e:
        logger.error(
            'Invalid cache backend (%s) found while loading '
            'siteconfig: %s', cache_backend, e)
        return {}

    if engine in BACKEND_CLASSES:
        engine = 'django.core.cache.backends.%s' % BACKEND_CLASSES[engine]
    else:
        engine = '%s.CacheClass' % engine

    defaults = {
        'BACKEND': engine,
        'LOCATION': host,
    }
    defaults.update(params)

    return defaults
Beispiel #6
0
def get_redis_backend(backend_uri):
    # From django-redis-cache
    # This is temporary https://github.com/washort/nuggets/pull/1
    _, server, params = parse_backend_uri(backend_uri)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 0
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    return redislib.Redis(host=host, port=port, db=db, password=password,
                          socket_timeout=socket_timeout)
def get_redis_backend():
    """Connect to redis from a string like CACHE_BACKEND."""
    # From django-redis-cache.
    _, server, params = parse_backend_uri(settings.REDIS_BACKEND)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 1
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = 'localhost'
        port = 6379
    return redislib.Redis(host=host, port=port, db=db, password=password,
                          socket_timeout=socket_timeout)
Beispiel #8
0
def normalize_cache_backend(cache_backend):
    """Returns a new-style CACHES dictionary from any given cache_backend.

    Django has supported two formats for a cache backend. The old-style
    CACHE_BACKEND string, and the new-style CACHES dictionary.

    This function will accept either as input and return a cahe backend in the
    form of a CACHES dictionary as a result. The result won't be a full-on
    CACHES, with named cache entries inside. Rather, it will be a cache entry.

    If a CACHES dictionary is passed, the "default" cache will be the result.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        if DEFAULT_CACHE_ALIAS in cache_backend:
            return cache_backend[DEFAULT_CACHE_ALIAS]

        return {}

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError, e:
        logging.error('Invalid cache backend (%s) found while loading '
                      'siteconfig: %s' % (cache_backend, e))
        return {}
Beispiel #9
0
def normalize_cache_backend(cache_backend, cache_name=DEFAULT_CACHE_ALIAS):
    """Return a new-style ``CACHES`` dictionary from any given cache_backend.

    Over time, Django has had support for two formats for a cache backend. The
    old-style :django:setting:`CACHE_BACKEND` string, and the new-style
    :django:setting:`CACHES` dictionary.

    This function will accept either as input and return a dictionary
    containing a single normalized entry for the given cache backend. This can
    be merged into the existing :django:setting:`CACHES` setting.

    If a :django:setting:`CACHES` dictionary is passed, the "default" cache
    will be the result.

    Args:
        cache_backend (dict or str):
            The new-style or old-style cache backend dictionary or str to
            normalize.

        cache_name (str):
            The name of the cache backend to look up in ``cache_backend``, if
            a new-style dictionary is provided.

    Returns:
        A new-style cache backend dictionary containing the single cache
        backend referenced. If there were any parsing issues, an empty
        dictionary will be returned.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        backend_info = cache_backend.get(cache_name, {})
        backend_name = backend_info.get('BACKEND')

        if backend_name in RENAMED_BACKENDS:
            backend_info['BACKEND'] = RENAMED_BACKENDS[backend_name]

        return backend_info

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError as e:
        logger.error('Invalid cache backend (%s) found while loading '
                     'siteconfig: %s',
                     cache_backend, e)
        return {}

    if engine in BACKEND_CLASSES:
        engine = 'django.core.cache.backends.%s' % BACKEND_CLASSES[engine]
    else:
        engine = '%s.CacheClass' % engine

    defaults = {
        'BACKEND': engine,
        'LOCATION': host,
    }
    defaults.update(params)

    return defaults
Beispiel #10
0
def get_cache_server_list():
    """Returns configured memcached servers.

    Works with both old-style (CACHE_BACKEND) and new-style (CACHES)
    cache configurations.

    """
    engine = ''

    # Django >= 1.3
    #
    # If somebody uses CACHE_BACKEND instead of CACHES in 1.3, it
    # automatically converts their CACHE_BACKEND configuration to the
    # appropriate CACHES configuration.  So we can safely use
    # parse_backend_conf here and it'll work with both old and new styles.
    try:
        from django.core.cache import parse_backend_conf, DEFAULT_CACHE_ALIAS
        engine, hosts, params = parse_backend_conf(DEFAULT_CACHE_ALIAS)

    # Django < 1.3
    #
    # No parse_backend_conf and DEFAULT_CACHE_ALIAS.
    except ImportError:
        from django.core.cache import parse_backend_uri
        engine, hosts, params = parse_backend_uri(settings.CACHE_BACKEND)

    if 'memcached' not in engine:
        raise ImproperlyConfigured(
            "django-memcached2 only works with memcached.  Currently using '%s'"
            % engine)

    return hosts if isinstance(hosts, list) else hosts.split(';')
Beispiel #11
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name
        can_rollback = self._rollback_works()
        self.connection.settings_dict["SUPPORTS_TRANSACTIONS"] = can_rollback

        call_command('syncdb',
                     verbosity=verbosity,
                     interactive=False,
                     database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri
            _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
            call_command('createcachetable', cache_name)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #12
0
def get_cache_server_list():
    """Returns configured memcached servers.

    Works with both old-style (CACHE_BACKEND) and new-style (CACHES)
    cache configurations.

    """
    engine = ''

    # Django >= 1.3
    #
    # If somebody uses CACHE_BACKEND instead of CACHES in 1.3, it
    # automatically converts their CACHE_BACKEND configuration to the
    # appropriate CACHES configuration.  So we can safely use
    # parse_backend_conf here and it'll work with both old and new styles.
    try:
        from django.core.cache import parse_backend_conf, DEFAULT_CACHE_ALIAS
        engine, hosts, params = parse_backend_conf(DEFAULT_CACHE_ALIAS)

    # Django < 1.3
    #
    # No parse_backend_conf and DEFAULT_CACHE_ALIAS.
    except ImportError:
        from django.core.cache import parse_backend_uri
        engine, hosts, params = parse_backend_uri(settings.CACHE_BACKEND)

    if 'memcached' not in engine:
        raise ImproperlyConfigured("django-memcached2 only works with memcached.  Currently using '%s'" % engine)

    return hosts if isinstance(hosts, list) else hosts.split(';')
Beispiel #13
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name

        # Confirm the feature set of the test database
        self.connection.features.confirm()

        # Report syncdb messages at one level lower than that requested.
        # This ensures we don't get flooded with messages during testing
        # (unless you really ask to be flooded)
        call_command("syncdb", verbosity=max(verbosity - 1, 0), interactive=False, database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith("db://"):
            from django.core.cache import parse_backend_uri, cache
            from django.db import router

            if router.allow_syncdb(self.connection.alias, cache.cache_model_class):
                _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
                call_command("createcachetable", cache_name, database=self.connection.alias)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #14
0
def get_caches():
    caches = []
    schema, hosts, params = parse_backend_uri(conf.BACKEND)
    for host in hosts.split(';'):
        caches.append(
            (host, get_cache('%s://%s?%s' % (schema, host, params))._cache))
    return caches
Beispiel #15
0
def monitor(request, format=None):

    # For each check, a boolean pass/fail status to show in the template
    status_summary = {}
    status = 200

    # Check all memcached servers
    scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND)
    memcache_results = []
    status_summary['memcache'] = True
    if 'memcached' in scheme:
        hosts = servers.split(';')
        for host in hosts:
            ip, port = host.split(':')
            try:
                s = socket.socket()
                s.connect((ip, int(port)))
            except Exception, e:
                result = False
                status_summary['memcache'] = False
                monitor_log.critical('Failed to connect to memcached (%s): %s'
                                     % (host, e))
            else:
                result = True
            finally:
Beispiel #16
0
def get_redis_backend(backend_uri):
    # From django-redis-cache
    # This is temporary https://github.com/washort/nuggets/pull/1
    _, server, params = parse_backend_uri(backend_uri)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 0
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    return redislib.Redis(host=host,
                          port=port,
                          db=db,
                          password=password,
                          socket_timeout=socket_timeout)
Beispiel #17
0
def normalize_cache_backend(cache_backend):
    """Returns a new-style CACHES dictionary from any given cache_backend.

    Django has supported two formats for a cache backend. The old-style
    CACHE_BACKEND string, and the new-style CACHES dictionary.

    This function will accept either as input and return a cahe backend in the
    form of a CACHES dictionary as a result. The result won't be a full-on
    CACHES, with named cache entries inside. Rather, it will be a cache entry.

    If a CACHES dictionary is passed, the "default" cache will be the result.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        if DEFAULT_CACHE_ALIAS in cache_backend:
            return cache_backend[DEFAULT_CACHE_ALIAS]

        return {}

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError, e:
        logging.error('Invalid cache backend (%s) found while loading '
                      'siteconfig: %s' % (cache_backend, e))
        return {}
Beispiel #18
0
def monitor(request):

    # For each check, a boolean pass/fail to show in the template.
    status_summary = {}
    status = 200

    # Check all memcached servers.
    scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND)
    memcache_results = []
    status_summary['memcache'] = True
    if 'memcached' in scheme:
        hosts = servers.split(';')
        for host in hosts:
            ip, port = host.split(':')
            try:
                s = socket.socket()
                s.connect((ip, int(port)))
            except Exception, e:
                result = False
                status_summary['memcache'] = False
                log.critical('Failed to connect to memcached (%s): %s' %
                             (host, e))
            else:
                result = True
            finally:
Beispiel #19
0
def get_redis_backend(backend_uri):
    _, server, params = parse_backend_uri(backend_uri)
    db = params.pop('db', 0)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 0
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    return redislib.Redis(host=host,
                          port=port,
                          db=db,
                          password=password,
                          socket_timeout=socket_timeout)
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.

        This method is overloaded to load up the SpatiaLite initialization
        SQL prior to calling the `syncdb` command.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()

        self.connection.settings_dict["NAME"] = test_database_name
        # Confirm the feature set of the test database
        self.connection.features.confirm()
        # Need to load the SpatiaLite initialization SQL before running `syncdb`.
        self.load_spatialite_sql()
        call_command('syncdb', verbosity=verbosity, interactive=False, database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri
            _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
            call_command('createcachetable', cache_name)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #21
0
def redis_client(name):
    """Get a Redis client.

    Uses the name argument to lookup the connection string in the
    settings.REDIS_BACKEND dict.
    """
    uri = settings.REDIS_BACKENDS[name]
    _, server, params = parse_backend_uri(uri)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 1
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    return Redis(host=host, port=port, db=db, password=password,
                 socket_timeout=socket_timeout)
Beispiel #22
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database..."

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        settings.DATABASE_NAME = test_database_name
        self.connection.settings_dict["DATABASE_NAME"] = test_database_name
        can_rollback = self._rollback_works()
        settings.DATABASE_SUPPORTS_TRANSACTIONS = can_rollback
        self.connection.settings_dict["DATABASE_SUPPORTS_TRANSACTIONS"] = can_rollback

        call_command('syncdb', verbosity=verbosity, interactive=False)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri
            _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
            call_command('createcachetable', cache_name)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #23
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.

        This method is overloaded to load up the SpatiaLite initialization
        SQL prior to calling the `syncdb` command.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()

        self.connection.settings_dict["NAME"] = test_database_name
        can_rollback = self._rollback_works()
        self.connection.settings_dict["SUPPORTS_TRANSACTIONS"] = can_rollback
        # Need to load the SpatiaLite initialization SQL before running `syncdb`.
        self.load_spatialite_sql()
        call_command('syncdb', verbosity=verbosity, interactive=False, database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri

            _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
            call_command('createcachetable', cache_name)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #24
0
def get_redis_backend():
    """Connect to redis from a string like CACHE_BACKEND."""
    # From django-redis-cache.
    _, server, params = parse_backend_uri(settings.REDIS_BACKEND)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 1
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = 'localhost'
        port = 6379
    return redislib.Redis(host=host, port=port, db=db, password=password,
                          socket_timeout=socket_timeout)
Beispiel #25
0
 def get_caches(self):
     caches = []
     schema, hosts, params = parse_backend_uri(conf.BACKEND)
     for host in hosts.split(';'):
         try:
             caches.append((host, get_cache('%s://%s?%s' % (schema, host, params))._cache))
         except Exception, e:
             self.logger.exception(e)
Beispiel #26
0
def openMongoDB(new = False):
    global _mongo

    if _mongo and new == False:
        return _mongo

    (type, servers, params) = parse_backend_uri(SESSION_BACKEND_MEMCACHE)

    if type != "memcached":
        raise InvalidCacheBackendError("Unsupported CACHE_BACKEND type")
    (host, port) = parse_backend_uri(SESSION_BACKEND_MONGO)[1].split(";")[0].split(":")

    mongo = pymongo.Connection(host, int(port))

    if new == False:
        _mongo = mongo

    return mongo
Beispiel #27
0
def jail(task_id, task_name, func, args, kwargs):
    """Wraps the task in a jail, which catches all exceptions, and
    saves the status and result of the task execution to the task
    meta backend.

    If the call was successful, it saves the result to the task result
    backend, and sets the task status to ``"DONE"``.

    If the call results in an exception, it saves the exception as the task
    result, and sets the task status to ``"FAILURE"``.

    :param task_id: The id of the task.
    :param task_name: The name of the task.
    :param func: Callable object to execute.
    :param args: List of positional args to pass on to the function.
    :param kwargs: Keyword arguments mapping to pass on to the function.

    :returns: the function return value on success, or
        the exception instance on failure.

    """
    ignore_result = getattr(func, "ignore_result", False)
    timer_stat = TaskTimerStats.start(task_id, task_name, args, kwargs)

    # See: http://groups.google.com/group/django-users/browse_thread/
    #       thread/78200863d0c07c6d/38402e76cf3233e8?hl=en&lnk=gst&
    #       q=multiprocessing#38402e76cf3233e8
    from django.db import connection
    connection.close()

    # Reset cache connection only if using memcached/libmemcached
    from django.core import cache
    # XXX At Opera we use a custom memcached backend that uses libmemcached
    # instead of libmemcache (cmemcache). Should find a better solution for
    # this, but for now "memcached" should probably be unique enough of a
    # string to not make problems.
    cache_backend = cache.settings.CACHE_BACKEND
    if hasattr(cache, "parse_backend_uri"):
        cache_scheme = cache.parse_backend_uri(cache_backend)[0]
    else:
        # Django <= 1.0.2
        cache_scheme = cache_backend.split(":", 1)[0]
    if "memcached" in cache_scheme:
        cache.cache.close()

    # Backend process cleanup
    default_backend.process_cleanup()

    try:
        result = func(*args, **kwargs)
    except (SystemExit, KeyboardInterrupt):
        raise
    except Exception, exc:
        stored_exc = default_backend.mark_as_failure(task_id, exc)
        type_, _, tb = sys.exc_info()
        retval = ExceptionInfo((type_, stored_exc, tb))
Beispiel #28
0
def get_memcached_hosts():
    """
    Returns the hosts currently configured for memcached.
    """
    if not memcache:
        return None

    scheme, host, params = parse_backend_uri(settings.CACHE_BACKEND)

    if scheme == "memcached":
        return host.split(";")

    return None
Beispiel #29
0
def openMemcache(new = False):
    global _mc

    if _mc and new == False:
        return _mc

    (type, servers, params) = parse_backend_uri(SESSION_BACKEND_MEMCACHE)
    if type != "memcached":
        raise InvalidCacheBackendError("Unsupported CACHE_BACKEND type")

    mc = memcache.Client(servers.split(";"))
    if new == False:
        _mc = mc
    return mc
Beispiel #30
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name
        can_rollback = self._rollback_works()
        self.connection.settings_dict["SUPPORTS_TRANSACTIONS"] = can_rollback

        call_command('syncdb',
                     verbosity=verbosity,
                     interactive=False,
                     database=self.connection.alias,
                     load_initial_data=False)

        # We need to then do a flush to ensure that any data installed by
        # custom SQL has been removed. The only test data should come from
        # test fixtures, or autogenerated from post_syncdb triggers.
        # This has the side effect of loading initial data (which was
        # intentionally skipped in the syncdb).
        call_command('flush',
                     verbosity=verbosity,
                     interactive=False,
                     database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri, cache
            from django.db import router
            if router.allow_syncdb(self.connection.alias,
                                   cache.cache_model_class):
                _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
                call_command('createcachetable',
                             cache_name,
                             database=self.connection.alias)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #31
0
def redis_client(name):
    """Get a Redis client.

    Uses the name argument to lookup the connection string in the
    settings.REDIS_BACKEND dict.
    """
    if name not in settings.REDIS_BACKENDS:
        raise RedisError(
            '{k} is not defined in settings.REDIS_BACKENDS'.format(k=name))

    uri = settings.REDIS_BACKENDS[name]
    _, server, params = parse_backend_uri(uri)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 1
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    redis = Redis(host=host,
                  port=port,
                  db=db,
                  password=password,
                  socket_timeout=socket_timeout)
    try:
        # Make a cheap call to verify we can connect.
        redis.exists('dummy-key')
    except ConnectionError:
        raise RedisError(
            'Unable to connect to redis backend: {k}'.format(k=name))
    return RetryForeverRedis(redis)
Beispiel #32
0
def normalize_cache_backend(cache_backend, cache_name=DEFAULT_CACHE_ALIAS):
    """Returns a new-style CACHES dictionary from any given cache_backend.

    Django has supported two formats for a cache backend. The old-style
    CACHE_BACKEND string, and the new-style CACHES dictionary.

    This function will accept either as input and return a cahe backend in the
    form of a CACHES dictionary as a result. The result won't be a full-on
    CACHES, with named cache entries inside. Rather, it will be a cache entry.

    If a CACHES dictionary is passed, the "default" cache will be the result.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        backend_info = cache_backend.get(cache_name, {})
        backend_name = backend_info.get('BACKEND')

        if backend_name in RENAMED_BACKENDS:
            backend_info['BACKEND'] = RENAMED_BACKENDS[backend_name]

        return backend_info

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError as e:
        logging.error('Invalid cache backend (%s) found while loading '
                      'siteconfig: %s' % (cache_backend, e))
        return {}

    if engine in BACKEND_CLASSES:
        engine = 'django.core.cache.backends.%s' % BACKEND_CLASSES[engine]
    else:
        engine = '%s.CacheClass' % engine

    defaults = {
        'BACKEND': engine,
        'LOCATION': host,
    }
    defaults.update(params)

    return defaults
Beispiel #33
0
def normalize_cache_backend(cache_backend, cache_name=DEFAULT_CACHE_ALIAS):
    """Returns a new-style CACHES dictionary from any given cache_backend.

    Django has supported two formats for a cache backend. The old-style
    CACHE_BACKEND string, and the new-style CACHES dictionary.

    This function will accept either as input and return a cahe backend in the
    form of a CACHES dictionary as a result. The result won't be a full-on
    CACHES, with named cache entries inside. Rather, it will be a cache entry.

    If a CACHES dictionary is passed, the "default" cache will be the result.
    """
    if not cache_backend:
        return {}

    if isinstance(cache_backend, dict):
        backend_info = cache_backend.get(cache_name, {})
        backend_name = backend_info.get('BACKEND')

        if backend_name in RENAMED_BACKENDS:
            backend_info['BACKEND'] = RENAMED_BACKENDS[backend_name]

        return backend_info

    try:
        engine, host, params = parse_backend_uri(cache_backend)
    except InvalidCacheBackendError as e:
        logging.error('Invalid cache backend (%s) found while loading '
                      'siteconfig: %s' % (cache_backend, e))
        return {}

    if engine in BACKEND_CLASSES:
        engine = 'django.core.cache.backends.%s' % BACKEND_CLASSES[engine]
    else:
        engine = '%s.CacheClass' % engine

    defaults = {
        'BACKEND': engine,
        'LOCATION': host,
    }
    defaults.update(params)

    return defaults
Beispiel #34
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print ("Creating test database '%s'..." % self.connection.alias)

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name
        can_rollback = self._rollback_works()
        self.connection.settings_dict["SUPPORTS_TRANSACTIONS"] = can_rollback

        call_command('syncdb',
            verbosity=verbosity,
            interactive=False,
            database=self.connection.alias,
            load_initial_data=False)

        # We need to then do a flush to ensure that any data installed by
        # custom SQL has been removed. The only test data should come from
        # test fixtures, or autogenerated from post_syncdb triggers.
        # This has the side effect of loading initial data (which was
        # intentionally skipped in the syncdb).
        call_command('flush',
            verbosity=verbosity,
            interactive=False,
            database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri, cache
            from django.db import router
            if router.allow_syncdb(self.connection.alias, cache.cache_model_class):
                _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
                call_command('createcachetable', cache_name, database=self.connection.alias)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #35
0
def redis_client(name):
    """Get a Redis client.

    Uses the name argument to lookup the connection string in the
    settings.REDIS_BACKEND dict.
    """
    if name not in settings.REDIS_BACKENDS:
        raise RedisError(
            '{k} is not defined in settings.REDIS_BACKENDS'.format(k=name))

    uri = settings.REDIS_BACKENDS[name]
    _, server, params = parse_backend_uri(uri)
    db = params.pop('db', 1)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 1
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    redis = Redis(host=host, port=port, db=db, password=password,
                  socket_timeout=socket_timeout)
    try:
        # Make a cheap call to verify we can connect.
        redis.exists('dummy-key')
    except ConnectionError:
        raise RedisError(
            'Unable to connect to redis backend: {k}'.format(k=name))
    return redis
Beispiel #36
0
    def test_mysterious_contribution(self, urlopen):
        scheme, servers, _ = parse_backend_uri(settings.CACHE_BACKEND)
        if 'dummy' in scheme:
            raise SkipTest()
        urlopen.return_value = self.urlopener('VERIFIED')

        key = "%s%s:%s" % (settings.CACHE_PREFIX, 'contrib', self.item)

        data = {'txn_id': 100,
                'payer_email': '*****@*****.**',
                'receiver_email': '*****@*****.**',
                'mc_gross': '99.99',
                'item_number': self.item,
                'payment_status': 'Completed'}
        response = self.client.post(self.url, data)
        assert isinstance(response, http.HttpResponseServerError)
        eq_(cache.get(key), 1)

        cache.set(key, 10, 1209600)
        response = self.client.post(self.url, data)
        assert isinstance(response, http.HttpResponse)
        eq_(cache.get(key), None)
Beispiel #37
0
    def create_test_db(self, verbosity=1, autoclobber=False):
        """
        Creates a test database, prompting the user for confirmation if the
        database already exists. Returns the name of the test database created.
        """
        if verbosity >= 1:
            print "Creating test database '%s'..." % self.connection.alias

        test_database_name = self._create_test_db(verbosity, autoclobber)

        self.connection.close()
        self.connection.settings_dict["NAME"] = test_database_name

        # Confirm the feature set of the test database
        self.connection.features.confirm()

        # Report syncdb messages at one level lower than that requested.
        # This ensures we don't get flooded with messages during testing
        # (unless you really ask to be flooded)
        call_command('syncdb',
                     verbosity=max(verbosity - 1, 0),
                     interactive=False,
                     database=self.connection.alias)

        if settings.CACHE_BACKEND.startswith('db://'):
            from django.core.cache import parse_backend_uri, cache
            from django.db import router
            if router.allow_syncdb(self.connection.alias,
                                   cache.cache_model_class):
                _, cache_name, _ = parse_backend_uri(settings.CACHE_BACKEND)
                call_command('createcachetable',
                             cache_name,
                             database=self.connection.alias)

        # Get a cursor (even though we don't need one yet). This has
        # the side effect of initializing the test database.
        cursor = self.connection.cursor()

        return test_database_name
Beispiel #38
0
def get_redis_backend(backend_uri):
    _, server, params = parse_backend_uri(backend_uri)
    db = params.pop('db', 0)
    try:
        db = int(db)
    except (ValueError, TypeError):
        db = 0
    try:
        socket_timeout = float(params.pop('socket_timeout'))
    except (KeyError, ValueError):
        socket_timeout = None
    password = params.pop('password', None)
    if ':' in server:
        host, port = server.split(':')
        try:
            port = int(port)
        except (ValueError, TypeError):
            port = 6379
    else:
        host = server
        port = 6379
    return redislib.Redis(host=host, port=port, db=db, password=password,
                          socket_timeout=socket_timeout)
Beispiel #39
0
def get_cache():
    """Get a Jinja2 bytecode cache which uses the configured Django cache."""
    
    from django.conf import settings
    from django.core import cache
    
    cache_backend = cache.parse_backend_uri(settings.CACHE_BACKEND)[0]
    memcached_client = None
    
    if cache_backend == 'memcached':
        # We can get the actual memcached client object itself. This will
        # avoid the Django problem of storing binary data (as Django tries to
        # coerce everything to Unicode).
        
        # Here, we look for either `cache.cache._cache` or
        # `cache.cache._client`; I believe there is some discrepancy between
        # different versions of Django and where they put this.
        memcached_client = getattr(
            cache.cache, '_cache', getattr(cache.cache, '_client', None))
    
    memcached_client = memcached_client or B64CacheClient(cache.cache)
    
    return jinja2.MemcachedBytecodeCache(memcached_client)
Beispiel #40
0
 def __getitem__(self, name):
     try:
         return self.connections[name]
     except KeyError:
         for alias, backend in settings.REDIS_BACKENDS.items():
             if alias != name:
                 continue
             _, server, params = parse_backend_uri(backend)
             try:
                 socket_timeout = float(params.pop('socket_timeout'))
             except (KeyError, ValueError):
                 socket_timeout = None
             try:
                 db = int(params.pop('db'))
             except (KeyError, ValueError):
                 db = 0
             password = params.pop('password', None)
             if ':' in server:
                 host, port = server.split(':')
                 try:
                     port = int(port)
                 except (ValueError, TypeError):
                     port = 6379
             else:
                 host = 'localhost'
                 port = 6379
             self.connections[alias] = redislib.Redis(
                 host=host,
                 port=port,
                 db=db,
                 password=password,
                 socket_timeout=socket_timeout
             )
             break
         else:
             raise ImproperlyConfigured('No backend called %s' % name)
     return self.connections[name]
Beispiel #41
0
from django.db import models
from django.db.models import signals
from django.db.models.sql import query
from django.utils import encoding, translation

log = logging.getLogger('caching')

FOREVER = 0
NO_CACHE = -1
CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
FLUSH = CACHE_PREFIX + ':flush:'
CACHE_COUNT_TIMEOUT = getattr(settings, 'CACHE_COUNT_TIMEOUT', None)

CACHE_BACKEND = getattr(settings, 'CACHE_BACKEND', '')
if CACHE_BACKEND:
    scheme, _, _ = parse_backend_uri(CACHE_BACKEND)
else:
    scheme, _, args = parse_backend_conf('default')
    CACHE_COUNT_TIMEOUT = args.get('TIMEOUT', None)
cache.scheme = scheme


class CachingManager(models.Manager):

    # Tell Django to use this manager when resolving foreign keys.
    use_for_related_fields = True

    def get_query_set(self):
        return CachingQuerySet(self.model)

    def contribute_to_class(self, cls, name):
def is_dict(obj):
    return isinstance(obj, collections.Mapping)


if not JINJA2_CACHE_ENABLED or JINJA2_CACHE_BACKEND is None:
    cache = None
    bccache = None
else:
    cache_kwargs = {}
    if is_dict(JINJA2_CACHE_BACKEND) and 'BACKEND' in JINJA2_CACHE_BACKEND:
        cache_kwargs = copy.deepcopy(JINJA2_CACHE_BACKEND)
        cache_backend = cache_kwargs.pop('BACKEND')
        cache = get_cache(cache_backend, **cache_kwargs)
    elif isinstance(JINJA2_CACHE_BACKEND, basestring):
        scheme, host, params = parse_backend_uri(JINJA2_CACHE_BACKEND)
        if scheme == 'locmem':
            from .backends.locmem import LocMemCache
            cache = LocMemCache('templates', {'timeout': None})
            bccache = MemcachedBytecodeCache(cache)
        else:
            cache = get_cache(JINJA2_CACHE_BACKEND)
            if hasattr(cache, '_cache'):
                bccache = MemcachedBytecodeCache(cache._cache)
            else:
                bccache = MemcachedBytecodeCache(cache)


if not JINJA2_CACHE_ENABLED or cache is None:
    cache_loader = None
else:
Beispiel #43
0
from django.conf import settings

try:
    memcache_settings = settings.memcache_settings
except:
    mdefault = "memcached://127.0.0.1:11211/"
    memcache_settings = {
        "CACHE_BACKEND": mdefault,
        "PAGE_CACHE_BACKEND": mdefault
    }
    memcache_settings.update({"NINGX_CACHE_BACKEND": mdefault})

try:
    import pylibmc

    scheme, host, params = parse_backend_uri(
        memcache_settings.get("CACHE_BACKEND"))
    model_cache = pylibmc.Client([host])
except Exception, e:
    import memcache

    scheme, host, params = parse_backend_uri(
        memcache_settings.get("CACHE_BACKEND"))
    model_cache = memcache.Client([host])

try:
    import pylibmc

    scheme, host, params = parse_backend_uri(
        memcache_setting.get("NINGX_CACHE_BACKEND"))
    nginx_cache = pylibmc.Client([host])
except Exception, e:
Beispiel #44
0
    except ZeroDivisionError:
        stats['hit_rate'] = stats['get_hits']
    
    return stats
########NEW FILE########
__FILENAME__ = views
from django.http import Http404
from django.shortcuts import render_to_response
from django.conf import settings
from django.template import RequestContext
from django.core.cache import parse_backend_uri

from django_memcached.util import get_memcached_stats
from django.contrib.auth.decorators import user_passes_test

_, hosts, _ = parse_backend_uri(settings.CACHE_BACKEND)
SERVERS = hosts.split(';')

def server_list(request):
    statuses = zip(range(len(SERVERS)), SERVERS, map(get_memcached_stats, SERVERS))
    context = {
        'statuses': statuses,
    }
    return render_to_response(
        'memcached/server_list.html',
        context,
        context_instance=RequestContext(request)
    )

def server_status(request, index):
    try:
Beispiel #45
0
    def emit(self, record):
        pass


log = logging.getLogger('caching')
log.setLevel(logging.INFO)
log.addHandler(NullHandler())

FOREVER = 0
NO_CACHE = -1
CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
FLUSH = CACHE_PREFIX + ':flush:'
CACHE_SET_TIMEOUT = getattr(settings, 'CACHE_SET_TIMEOUT', 1)

scheme, _, _ = parse_backend_uri(settings.CACHE_BACKEND)
cache.scheme = scheme


class CachingManager(models.Manager):

    # Tell Django to use this manager when resolving foreign keys.
    use_for_related_fields = True

    def get_query_set(self):
        return CachingQuerySet(self.model)

    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(CachingManager, self).contribute_to_class(cls, name)
Beispiel #46
0
class NullHandler(logging.Handler):

    def emit(self, record):
        pass


log = logging.getLogger('caching')
log.setLevel(logging.INFO)
log.addHandler(NullHandler())

FOREVER = 0
NO_CACHE = -1
CACHE_PREFIX = getattr(settings, 'CACHE_PREFIX', '')
FLUSH = CACHE_PREFIX + ':flush:'

scheme, _, _ = parse_backend_uri(settings.CACHE_BACKEND)
cache.scheme = scheme


class CachingManager(models.Manager):

    # Tell Django to use this manager when resolving foreign keys.
    use_for_related_fields = True

    def get_query_set(self):
        return CachingQuerySet(self.model)

    def contribute_to_class(self, cls, name):
        signals.post_save.connect(self.post_save, sender=cls)
        signals.post_delete.connect(self.post_delete, sender=cls)
        return super(CachingManager, self).contribute_to_class(cls, name)
def get_caches():
    caches = []
    schema, hosts, params = parse_backend_uri(conf.BACKEND)
    for host in hosts.split(';'):
        caches.append((host, get_cache('%s://%s?%s' % (schema, host, params))._cache))
    return caches
Beispiel #48
0
from django.conf import settings
from django.core.cache import parse_backend_uri

try:
    import redis as redislib
except:
    redislib = None

connections = {}

if not connections:  # don't set this repeatedly
    for alias, backend in settings.REDIS_BACKENDS.items():
        _, server, params = parse_backend_uri(backend)

        try:
            socket_timeout = float(params.pop('socket_timeout'))
        except (KeyError, ValueError):
            socket_timeout = None
        password = params.pop('password', None)
        if ':' in server:
            host, port = server.split(':')
            try:
                port = int(port)
            except (ValueError, TypeError):
                port = 6379
        else:
            host = 'localhost'
            port = 6379

        connections[alias] = redislib.Redis(host=host, port=port, db=0,
                                            password=password,
Beispiel #49
0
from django.conf import settings
from django.core.cache import parse_backend_uri

try:
    import redis as redislib
except:
    redislib = None

connections = {}

if not connections:  # don't set this repeatedly
    for alias, backend in settings.REDIS_BACKENDS.items():
        _, server, params = parse_backend_uri(backend)

        try:
            socket_timeout = float(params.pop('socket_timeout'))
        except (KeyError, ValueError):
            socket_timeout = None
        password = params.pop('password', None)
        if ':' in server:
            host, port = server.split(':')
            try:
                port = int(port)
            except (ValueError, TypeError):
                port = 6379
        else:
            host = 'localhost'
            port = 6379

        connections[alias] = redislib.Redis(host=host,
                                            port=port,