Пример #1
0
def submission_listing_cache_key(*args, **kw):
    ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
    if ns_key is None:
        ns_key = random.randint(1, 10000)
        memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
    full_path = args[0].get_full_path()
    username = args[0].user.username
    return "demos_%s:%s" % (ns_key, hashlib.md5(full_path + username).hexdigest())
Пример #2
0
def submission_listing_cache_key(*args, **kw):
    ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
    if ns_key is None:
        ns_key = random.randint(1, 10000)
        memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
    full_path = args[0].get_full_path()
    username = args[0].user.username
    return 'demos_%s:%s' % (ns_key,
                            hashlib.md5(full_path + username).hexdigest())
Пример #3
0
def _invalidate_submission_listing_helper_cache():
    """Invalidate the cache for submission_listing helper used in templates"""
    # TODO: Does this belong in helpers.py? Better done with a model save event
    # subscription?
    ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
    if ns_key is None:
        ns_key = random.randint(1, 10000)
        memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
    else:
        memcache.incr(DEMOS_CACHE_NS_KEY)
Пример #4
0
def _invalidate_submission_listing_helper_cache():
    """Invalidate the cache for submission_listing helper used in templates"""
    # TODO: Does this belong in helpers.py? Better done with a model save event
    # subscription?
    ns_key = memcache.get(DEMOS_CACHE_NS_KEY)
    if ns_key is None:
        ns_key = random.randint(1, 10000)
        memcache.set(DEMOS_CACHE_NS_KEY, ns_key)
    else:
        memcache.incr(DEMOS_CACHE_NS_KEY)
Пример #5
0
def test_home_community_stats(client, db, cleared_memcache):
    stats = {
        'contributors': 'so many, like more than 10,000',
        'locales': 'lots, maybe fifty'
    }
    memcache.set('community_stats', stats)
    response = client.get(reverse('home'), follow=True)
    assert response.status_code == 200
    assert_shared_cache_header(response)
    assert stats['contributors'] in response.content
    assert stats['locales'] in response.content
Пример #6
0
        def wrapper(*args, **kw):
            if type(key_fn) is str:
                cache_key = key_fn
            else:
                cache_key = key_fn(*args, **kw)

            out = memcache.get(cache_key)
            if out is None:
                context = f(*args, **kw)
                t = get_template(template).render(context)
                out = jinja2.Markup(t)
                memcache.set(cache_key, out, expires)
            return out
Пример #7
0
def bitly_shorten(url):
    """Attempt to shorten a given URL through bit.ly / mzl.la"""
    cache_key = "bitly:%s" % hashlib.md5(smart_str(url)).hexdigest()
    short_url = memcache.get(cache_key)
    if short_url is None:
        try:
            short_url = bitly.shorten(url)["url"]
            memcache.set(cache_key, short_url, 60 * 60 * 24 * 30 * 12)
        except (bitly_api.BitlyError, KeyError):
            # Just in case the bit.ly service fails or the API key isn't
            # configured, fall back to using the original URL.
            return url
    return short_url
Пример #8
0
        def wrapper(*args, **kw):
            if type(key_fn) is str:
                cache_key = key_fn
            else:
                cache_key = key_fn(*args, **kw)

            out = memcache.get(cache_key)
            if out is None:
                context = f(*args, **kw)
                t = jingo.env.get_template(template).render(context)
                out = jinja2.Markup(t)
                memcache.set(cache_key, out, expires)
            return out
Пример #9
0
def bitly_shorten(url):
    """Attempt to shorten a given URL through bit.ly / mzl.la"""
    cache_key = 'bitly:%s' % hashlib.md5(smart_str(url)).hexdigest()
    short_url = memcache.get(cache_key)
    if short_url is None:
        try:
            short_url = bitly.shorten(url)['url']
            memcache.set(cache_key, short_url, 60 * 60 * 24 * 30 * 12)
        except (bitly_api.BitlyError, KeyError):
            # Just in case the bit.ly service fails or the API key isn't
            # configured, fall back to using the original URL.
            return url
    return short_url
Пример #10
0
    def get_url_remaps(self, locale):
        cache_key = URL_REMAPS_CACHE_KEY_TMPL % locale
        remaps = memcache.get(cache_key)

        if not remaps:
            qs = (self.filter(document__locale=locale,
                              url_root__isnull=False).exclude(url_root=''))
            remaps = [{
                'original_path': '/docs/%s' % zone.document.slug,
                'new_path': '/%s' % zone.url_root
            } for zone in qs]
            memcache.set(cache_key, remaps)

        return remaps
Пример #11
0
def update_community_stats():
    cursor = connection.cursor()
    try:
        cursor.execute(
            """
            SELECT count(creator_id)
            FROM
              (SELECT DISTINCT creator_id
               FROM wiki_revision
               WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
            """
        )
        contributors = cursor.fetchone()

        cursor.execute(
            """
            SELECT count(locale)
            FROM
              (SELECT DISTINCT wd.locale
               FROM wiki_document wd,
                                  wiki_revision wr
               WHERE wd.id = wr.document_id
                 AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
            """
        )
        locales = cursor.fetchone()
    finally:
        cursor.close()

    community_stats = {}

    try:
        community_stats["contributors"] = contributors[0]
        community_stats["locales"] = locales[0]
    except IndexError:
        community_stats = None

    # storing a None value in cache allows a better check for
    # emptiness in the view
    if 0 in community_stats.values():
        community_stats = None

    memcache.set("community_stats", community_stats, 86400)
def update_community_stats():
    cursor = connection.cursor()
    try:
        cursor.execute("""
            SELECT count(creator_id)
            FROM
              (SELECT DISTINCT creator_id
               FROM wiki_revision
               WHERE created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS contributors
            """)
        contributors = cursor.fetchone()

        cursor.execute("""
            SELECT count(locale)
            FROM
              (SELECT DISTINCT wd.locale
               FROM wiki_document wd,
                                  wiki_revision wr
               WHERE wd.id = wr.document_id
                 AND wr.created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)) AS locales
            """)
        locales = cursor.fetchone()
    finally:
        cursor.close()

    community_stats = {}

    try:
        community_stats['contributors'] = contributors[0]
        community_stats['locales'] = locales[0]
    except IndexError:
        community_stats = None

    # storing a None value in cache allows a better check for
    # emptiness in the view
    if 0 in community_stats.values():
        community_stats = None

    memcache.set('community_stats', community_stats, 86400)
Пример #13
0
def get(document, cache_control, base_url, timeout=None):
    """Perform a kumascript GET request for a document locale and slug."""
    if not cache_control:
        # Default to the configured max-age for cache control.
        max_age = constance.config.KUMASCRIPT_MAX_AGE
        cache_control = 'max-age=%s' % max_age

    if not base_url:
        site = Site.objects.get_current()
        base_url = 'http://%s' % site.domain

    if not timeout:
        timeout = constance.config.KUMASCRIPT_TIMEOUT

    document_locale = document.locale
    document_slug = document.slug
    max_age = constance.config.KUMASCRIPT_MAX_AGE

    # 1063580 - Kumascript converts template name calls to lower case and bases
    # caching keys off of that.
    document_slug_for_kumascript = document_slug
    if document.is_template:
        document_slug_for_kumascript = _format_slug_for_request(document_slug)

    body, errors = None, None

    try:
        url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
        url = unicode(url_tmpl).format(path=u'%s/%s' %
                                       (document_locale,
                                        document_slug_for_kumascript))

        cache_keys = build_cache_keys(document_slug, document_locale)
        etag_key, modified_key, body_key, errors_key = cache_keys

        headers = {
            'X-FireLogger': '1.2',
            'Cache-Control': cache_control,
        }

        # Create the file interface
        files = []
        for attachment in document.attachments.all():
            files.append(_get_attachment_metadata_dict(attachment))

        # Assemble some KumaScript env vars
        # TODO: See dekiscript vars for future inspiration
        # http://developer.mindtouch.com/en/docs/DekiScript/Reference/
        #   Wiki_Functions_and_Variables
        path = document.get_absolute_url()
        # TODO: Someday merge with _get_document_for_json in views.py
        # where most of this is duplicated code.
        env_vars = dict(
            path=path,
            url=urljoin(base_url, path),
            id=document.pk,
            revision_id=document.current_revision.pk,
            locale=document.locale,
            title=document.title,
            files=files,
            attachments=files,  # Just for sake of verbiage?
            slug=document.slug,
            tags=list(document.tags.values_list('name', flat=True)),
            review_tags=list(document.current_revision
                                     .review_tags
                                     .values_list('name', flat=True)),
            modified=time.mktime(document.modified.timetuple()),
            cache_control=cache_control,
        )
        add_env_headers(headers, env_vars)

        # Set up for conditional GET, if we have the details cached.
        cached_meta = memcache.get_many([etag_key, modified_key])
        if etag_key in cached_meta:
            headers['If-None-Match'] = cached_meta[etag_key]
        if modified_key in cached_meta:
            headers['If-Modified-Since'] = cached_meta[modified_key]

        # Finally, fire off the request.
        response = requests.get(url, headers=headers, timeout=timeout)

        if response.status_code == 304:
            # Conditional GET was a pass, so use the cached content.
            result = memcache.get_many([body_key, errors_key])
            body = result.get(body_key, '').decode('utf-8')
            errors = result.get(errors_key, None)

        elif response.status_code == 200:
            body = process_body(response)
            errors = process_errors(response)

            # Cache the request for conditional GET, but use the max_age for
            # the cache timeout here too.
            headers = response.headers
            memcache.set(etag_key, headers.get('etag'), timeout=max_age)
            memcache.set(modified_key, headers.get('last-modified'), timeout=max_age)
            memcache.set(body_key, body.encode('utf-8'), timeout=max_age)
            if errors:
                memcache.set(errors_key, errors, timeout=max_age)

        elif response.status_code is None:
            errors = KUMASCRIPT_TIMEOUT_ERROR

        else:
            errors = [
                {
                    "level": "error",
                    "message": "Unexpected response from Kumascript service: %s" %
                               response.status_code,
                    "args": ["UnknownError"],
                },
            ]

    except Exception, exc:
        # Last resort: Something went really haywire. Kumascript server died
        # mid-request, or something. Try to report at least some hint.
        errors = [
            {
                "level": "error",
                "message": "Kumascript service failed unexpectedly: %s" % exc,
                "args": ["UnknownError"],
            },
        ]
Пример #14
0
def get(document, cache_control, base_url, timeout=None):
    """Perform a kumascript GET request for a document locale and slug."""
    if not cache_control:
        # Default to the configured max-age for cache control.
        max_age = config.KUMASCRIPT_MAX_AGE
        cache_control = 'max-age=%s' % max_age

    if not base_url:
        site = Site.objects.get_current()
        base_url = 'http://%s' % site.domain

    if not timeout:
        timeout = config.KUMASCRIPT_TIMEOUT

    document_locale = document.locale
    document_slug = document.slug
    max_age = config.KUMASCRIPT_MAX_AGE

    # 1063580 - Kumascript converts template name calls to lower case and bases
    # caching keys off of that.
    document_slug_for_kumascript = document_slug
    body, errors = None, None

    try:
        url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
        url = unicode(url_tmpl).format(path=u'%s/%s' %
                                       (document_locale,
                                        document_slug_for_kumascript))

        cache_keys = build_cache_keys(document_slug, document_locale)
        etag_key, modified_key, body_key, errors_key = cache_keys

        headers = {
            'X-FireLogger': '1.2',
            'Cache-Control': cache_control,
        }

        # Create the file interface
        files = []
        for attachment in document.files.select_related('current_revision'):
            files.append(_get_attachment_metadata_dict(attachment))

        # Assemble some KumaScript env vars
        # TODO: See dekiscript vars for future inspiration
        # http://developer.mindtouch.com/en/docs/DekiScript/Reference/
        #   Wiki_Functions_and_Variables
        path = document.get_absolute_url()
        # TODO: Someday merge with _get_document_for_json in views.py
        # where most of this is duplicated code.
        env_vars = dict(
            path=path,
            url=urljoin(base_url, path),
            id=document.pk,
            revision_id=document.current_revision.pk,
            locale=document.locale,
            title=document.title,
            files=files,
            attachments=files,  # Just for sake of verbiage?
            slug=document.slug,
            tags=list(document.tags.names()),
            review_tags=list(document.current_revision.review_tags.names()),
            modified=time.mktime(document.modified.timetuple()),
            cache_control=cache_control,
        )
        add_env_headers(headers, env_vars)

        # Set up for conditional GET, if we have the details cached.
        cached_meta = memcache.get_many([etag_key, modified_key])
        if etag_key in cached_meta:
            headers['If-None-Match'] = cached_meta[etag_key]
        if modified_key in cached_meta:
            headers['If-Modified-Since'] = cached_meta[modified_key]

        # Finally, fire off the request.
        response = requests.get(url, headers=headers, timeout=timeout)

        if response.status_code == 304:
            # Conditional GET was a pass, so use the cached content.
            result = memcache.get_many([body_key, errors_key])
            body = result.get(body_key, '').decode('utf-8')
            errors = result.get(errors_key, None)

        elif response.status_code == 200:
            body = process_body(response)
            errors = process_errors(response)

            # Cache the request for conditional GET, but use the max_age for
            # the cache timeout here too.
            headers = response.headers
            memcache.set(etag_key, headers.get('etag'), timeout=max_age)
            memcache.set(modified_key, headers.get('last-modified'), timeout=max_age)
            memcache.set(body_key, body.encode('utf-8'), timeout=max_age)
            if errors:
                memcache.set(errors_key, errors, timeout=max_age)

        elif response.status_code is None:
            errors = KUMASCRIPT_TIMEOUT_ERROR

        else:
            errors = [
                {
                    "level": "error",
                    "message": "Unexpected response from Kumascript service: %s" %
                               response.status_code,
                    "args": ["UnknownError"],
                },
            ]

    except Exception as exc:
        # Last resort: Something went really haywire. Kumascript server died
        # mid-request, or something. Try to report at least some hint.
        errors = [
            {
                "level": "error",
                "message": "Kumascript service failed unexpectedly: %s" % exc,
                "args": ["UnknownError"],
            },
        ]
    return (body, errors)
Пример #15
0
 def reset_url_remaps(self, locale, cache_key=None):
     if cache_key is None:
         cache_key = self.model.cache_key(locale)
     remaps = self.build_url_remaps(locale)
     memcache.set(cache_key, remaps, timeout=60 * 60 * 24)
     return remaps