Example #1
0
def get_many_for_update(namespaces, keys):
    """Get cached data, full keys and create namespace counter where necessary.

    Arguments namespaces and keys should be a lists of the same size.

    Returns a pair (cached_data, full_keys),
        where cached_data is a dictionary {full_keys: data}
        and full_keys is a list of full keys, in the same order as
        given namespaces and keys.

    Use cache.set_many({full_key: value}) to finish updating.
    It is assumed namespace counters won't change in between. (if they do,
    maybe it is even better to immediately 'mark' data as invalid...)
    """
    counters = cache.get_many(namespaces)

    # Create new counters
    new_counters = {x: 1 for x in namespaces if x not in counters}
    cache.set_many(new_counters)

    # Generate list of full_keys and list of keys to read from cache.
    full_keys = []
    to_get = []
    for namespace, key in zip(namespaces, keys):
        counter = counters.get(namespace, None)
        if counter:
            full_key = make_full_key(namespace, counter, key)
            full_keys.append(full_key)
            to_get.append(full_key)
        else:
            full_keys.append(make_full_key(namespace, 1, key))

    return cache.get_many(to_get), full_keys
Example #2
0
def get_programs_by_uuids(uuids):
    """
    Gets a list of programs for the provided uuids
    """
    # a list of UUID objects would be a perfectly reasonable parameter to provide
    uuid_strings = [six.text_type(handle) for handle in uuids]

    programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=handle) for handle in uuid_strings])
    programs = list(programs.values())

    # The get_many above sometimes fails to bring back details cached on one or
    # more Memcached nodes. It doesn't look like these keys are being evicted.
    # 99% of the time all keys come back, but 1% of the time all the keys stored
    # on one or more nodes are missing from the result of the get_many. One
    # get_many may fail to bring these keys back, but a get_many occurring
    # immediately afterwards will succeed in bringing back all the keys. This
    # behavior can be mitigated by trying again for the missing keys, which is
    # what we do here. Splitting the get_many into smaller chunks may also help.
    missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
    if missing_uuids:
        logger.info(
            u'Failed to get details for {count} programs. Retrying.'.format(count=len(missing_uuids))
        )

        retried_programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids])
        programs += list(retried_programs.values())

        still_missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
        for uuid in still_missing_uuids:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

    return programs
 def test_get_many(self):
     # Multiple cache keys can be returned using get_many
     cache.set("a", "a")
     cache.set("b", "b")
     cache.set("c", "c")
     cache.set("d", "d")
     self.assertEqual(cache.get_many(["a", "c", "d"]), {"a": "a", "c": "c", "d": "d"})
     self.assertEqual(cache.get_many(["a", "b", "e"]), {"a": "a", "b": "b"})
Example #4
0
 def test_get_many(self):
     # get_many
     cache.set('a', 'a')
     cache.set('b', 'b')
     cache.set('c', 'c')
     cache.set('d', 'd')
     self.assertEqual(cache.get_many(['a', 'c', 'd']), {'a' : 'a', 'c' : 'c', 'd' : 'd'})
     self.assertEqual(cache.get_many(['a', 'b', 'e']), {'a' : 'a', 'b' : 'b'})
 def test_get_many(self):
     # Multiple cache keys can be returned using get_many
     cache.set('a', 'a')
     cache.set('b', 'b')
     cache.set('c', 'c')
     cache.set('d', 'd')
     self.assertEqual(cache.get_many(['a', 'c', 'd']), {'a': 'a', 'c': 'c', 'd': 'd'})
     self.assertEqual(cache.get_many(['a', 'b', 'e']), {'a': 'a', 'b': 'b'})
Example #6
0
    def iterator(self):
        cache_key = self.model._generate_cache_key("QUERY:%s" %
                                                   self._get_query_hash())
        on_cache_query_attr = self.model.value_to_list_on_cache_query()

        to_return = None
        to_cache = {}

        with_aggregates = len(self.query.aggregates) > 0
        key_list = self._fetch_from_query_cache(cache_key)

        if key_list is None:
            if not with_aggregates:
                values_list = [on_cache_query_attr]

                if len(self.query.extra):
                    values_list += self.query.extra.keys()

                key_list = [v[0] for v in self.values_list(*values_list)]
                to_cache[cache_key] = (datetime.datetime.now(), key_list)
            else:
                to_return = list(super(CachedQuerySet, self).iterator())
                to_cache[cache_key] = (datetime.datetime.now(), [
                    (row.__dict__[on_cache_query_attr],
                     dict([(k, row.__dict__[k])
                           for k in self.query.aggregates.keys()]))
                    for row in to_return
                ])
        elif with_aggregates:
            tmp = key_list
            key_list = [k[0] for k in tmp]
            with_aggregates = [k[1] for k in tmp]
            del tmp

        if (not to_return) and key_list:
            row_keys = [
                self.model.infer_cache_key({on_cache_query_attr: attr})
                for attr in key_list
            ]
            cached = cache.get_many(row_keys)

            to_return = [(ck in cached) and self.obj_from_datadict(cached[ck])
                         or ToFetch(force_unicode(key_list[i]))
                         for i, ck in enumerate(row_keys)]

            if len(cached) != len(row_keys):
                to_fetch = [
                    unicode(tr) for tr in to_return if isinstance(tr, ToFetch)
                ]

                fetched = dict([
                    (force_unicode(r.__dict__[on_cache_query_attr]), r)
                    for r in models.query.QuerySet(self.model).filter(
                        **{"%s__in" % on_cache_query_attr: to_fetch})
                ])

                to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)]
                              or tr) for tr in to_return]
                to_cache.update(
                    dict([(self.model.infer_cache_key(
                        {on_cache_query_attr: attr}), r._as_dict())
                          for attr, r in fetched.items()]))

            if with_aggregates:
                for i, r in enumerate(to_return):
                    r.__dict__.update(with_aggregates[i])

        if len(to_cache):
            cache.set_many(to_cache, 60 * 60)

        if to_return:
            for row in to_return:
                if hasattr(row, 'leaf'):
                    row = row.leaf

                row.reset_original_state()
                yield row
Example #7
0
    def bulk_get_rule_status(
            self, rules: Sequence[Rule]) -> Mapping[int, GroupRuleStatus]:
        keys = [self._build_rule_status_cache_key(rule.id) for rule in rules]
        cache_results: Mapping[str, GroupRuleStatus] = cache.get_many(keys)
        missing_rule_ids: Set[int] = set()
        rule_statuses: Mapping[int, GroupRuleStatus] = {}
        for key, rule in zip(keys, rules):
            rule_status = cache_results.get(key)
            if not rule_status:
                missing_rule_ids.add(rule.id)
            else:
                rule_statuses[rule.id] = rule_status

        if missing_rule_ids:
            # If not cached, attempt to fetch status from the database
            statuses = GroupRuleStatus.objects.filter(
                group=self.group, rule_id__in=missing_rule_ids)
            to_cache: Sequence[GroupRuleStatus] = list()
            for status in statuses:
                rule_statuses[status.rule_id] = status
                missing_rule_ids.remove(status.rule_id)
                to_cache.append(status)

            # We might need to create some statuses if they don't already exist
            if missing_rule_ids:
                # We use `ignore_conflicts=True` here to avoid race conditions where the statuses
                # might be created between when we queried above and attempt to create the rows now.
                GroupRuleStatus.objects.bulk_create(
                    [
                        GroupRuleStatus(rule_id=rule_id,
                                        group=self.group,
                                        project=self.project)
                        for rule_id in missing_rule_ids
                    ],
                    ignore_conflicts=True,
                )
                # Using `ignore_conflicts=True` prevents the pk from being set on the model
                # instances. Re-query the database to fetch the rows, they should all exist at this
                # point.
                statuses = GroupRuleStatus.objects.filter(
                    group=self.group, rule_id__in=missing_rule_ids)
                for status in statuses:
                    rule_statuses[status.rule_id] = status
                    missing_rule_ids.remove(status.rule_id)
                    to_cache.append(status)

                if missing_rule_ids:
                    # Shouldn't happen, but log just in case
                    self.logger.error(
                        "Failed to fetch some GroupRuleStatuses in RuleProcessor",
                        extra={
                            "missing_rule_ids": missing_rule_ids,
                            "group_id": self.group.id
                        },
                    )
            if to_cache:
                cache.set_many({
                    self._build_rule_status_cache_key(item.rule_id): item
                    for item in to_cache
                })

        return rule_statuses
Example #8
0
 def list(self, request):
     stats_keys = utils_stats.DEFAULTS.keys()
     cached_stats = cache.get_many(stats_keys)
     stats = utils_stats.DEFAULTS.copy()
     stats.update(cached_stats)
     return Response(stats)
Example #9
0
    def post(self, request, request_id):
        try:
            data = json.loads(request.body)
            validate(data, ANALYSIS_ID_SCHEMA)

            ids: Optional[Ids] = cache.get(f'{request_id}/analysis_ids')

            if ids is None:
                return HttpResponseNotFound('Query Analysis Ids not available')

            data = {(tuple(idx[0]), idx[1]): opt for idx, opt in data}

            if not any(v['show'] for v in data.values()):
                return HttpResponseBadRequest("cannot hide all analyses")

            for key in ids:
                try:
                    if ids[key]['name'] != data[key]['name']:
                        ids[key] = {
                            **data[key], 'version': ids[key]['version'] + 1
                        }
                    else:
                        ids[key] = {
                            **data[key], 'version': ids[key]['version']
                        }
                except KeyError:
                    pass

            cache.set(f'{request_id}/analysis_ids', ids)

            cached_result = cache.get_many([
                f'{request_id}/target_genes',
                f'{request_id}/tabular_output_unfiltered'
            ])

            result = cached_result[f'{request_id}/tabular_output_unfiltered']
            result = filter_df_by_ids(result, ids)

            try:
                user_lists = cached_result[f'{request_id}/target_genes']
                result = result[result.index.str.upper().isin(
                    user_lists[0].index.str.upper())].dropna(axis=1, how='all')

                if result.empty:
                    raise QueryError(
                        "Empty result (user list too restrictive).")

                result = reorder_data(result)
            except KeyError:
                pass

            cache.set(f'{request_id}/tabular_output',
                      result)  # refresh filtered tabular output

            # delete cache keys and refresh cache here.
            cache.delete_many([
                # formatted output
                f'{request_id}/formatted_tabular_output',
                # network
                f'{request_id}/network',
                # AUPR curve
                f'{request_id}/figure',
                f'{request_id}/figure_data',
                # network stats
                f'{request_id}/stats',
                # Gene list enrichment
                f'{request_id}/list_enrichment',
                f'{request_id}/list_enrichment_legend',
                f'{request_id}/list_enrichment_data',
                # motif enrichment
                *(f'{request_id}/{r}_enrich' for r in MOTIFS.regions),
                # analysis enrichment
                f'{request_id}/analysis_enrichment',
                # summary
                f'{request_id}/summary'
            ])

            return JsonResponse(list(ids.items()),
                                status=201,
                                safe=False,
                                encoder=PandasJSONEncoder)
        except (json.JSONDecodeError, ValidationError, QueryError) as e:
            return HttpResponseBadRequest(e)
        except KeyError:
            return HttpResponseNotFound(
                'Query does not exist. Please start a new query.')
Example #10
0
def get_cached_objects(pks, model=None, timeout=CACHE_TIMEOUT, missing=RAISE):
    """
    Return a list of objects with given PKs using cache.

    Params:
        pks - list of Primary Key values to look up or list of content_type_id, pk tuples
        model - ContentType instance representing the model's class or the model class itself
        timeout - TTL for the items in cache, defaults to CACHE_TIMEOUT

    Throws:
        model.DoesNotExist is propagated from content_type.get_object_for_this_type
    """
    if model is not None:
        if not isinstance(model, ContentType):
            model = ContentType.objects.get_for_model(model)
        pks = [(model, pk) for pk in pks]
    else:
        pks = [(ContentType.objects.get_for_id(ct_id), pk) for (ct_id, pk) in pks]

    keys = [_get_key(KEY_PREFIX, model, pk=pk) for (model, pk) in pks]

    cached = cache.get_many(keys)

    # keys not in cache
    keys_to_set = set(keys) - set(cached.keys())
    if keys_to_set:
        # build lookup to get model and pks from the key
        lookup = dict(zip(keys, pks))

        to_get = {}
        # group lookups by CT so we can do in_bulk
        for k in keys_to_set:
            ct, pk = lookup[k]
            to_get.setdefault(ct, {})[int(pk)] = k

        # take out all the publishables
        publishable_ct = ContentType.objects.get_for_model(get_model('core', 'publishable'))
        if publishable_ct in to_get:
            publishable_keys = to_get.pop(publishable_ct)
            models = publishable_ct.model_class()._default_manager.values('content_type_id', 'id').filter(id__in=publishable_keys.keys())
            for m in models:
                ct = ContentType.objects.get_for_id(m['content_type_id'])
                pk = m['id']
                # and put them back as their native content_type
                to_get.setdefault(ct, {})[pk] = publishable_keys[pk]

        to_set = {}
        # retrieve all the models from DB
        for ct, vals in to_get.items():
            models = ct.model_class()._default_manager.in_bulk(vals.keys())
            for pk, m in models.items():
                k = vals[pk]
                cached[k] = to_set[k] = m

        if not isinstance(cache, DummyCache):
            # write them into cache
            cache.set_many(to_set, timeout=timeout)

    out = []
    for k in keys:
        try:
            out.append(cached[k])
        except KeyError:
            if missing == NONE:
                out.append(None)
            elif missing == SKIP:
                pass
            elif missing == RAISE:
                ct = ContentType.objects.get_for_id(int(k.split(':')[1]))
                raise ct.model_class().DoesNotExist(
                    '%s matching query does not exist.' % ct.model_class()._meta.object_name)
    return out
Example #11
0
def query_request(kb_ids, issue, kb_vers_map):
    """
    requirement: (1)cache中查找知识库的地址,cache中没有的,则为其在cache中没有被其他kb占用的box中选取BOX,并异步写入其TARGET
                 (2)遍历/B/vm/下的子节点,找到空闲节点,为其赋一个知识库,单独起进程发送请求,请求结果放入queue

    param: kb_ids 列表 
    return: 
    """

    # 取cache中已有的kb
    logger.debug('start get add')
    start_get_add = time()

    kb_add_dict = cache.get_many(kb_ids)

    no_kbs = set(kb_ids) - set(kb_add_dict.keys())

    logger.debug('no_kbs:%s' % no_kbs)
    # 为cache中没有的kb赋予box
    boxs = [('/B/83c4ee846cf2/B70/', '192.168.30.187:8000/70'),
            ('/B/83c4ee846cf2/B74/', '192.168.30.187:8000/74'),
            ('/B/83c4ee846cf2/B73/', '192.168.30.187:8000/73'),
            ('/B/83c4ee846cf2/B72/', '192.168.30.187:8000/72'),
            ('/B/83c4ee846cf2/B71/', '192.168.30.187:8000/71'),
            ('/B/83c4ee846cf2/B30/', '192.168.30.187:8000/30'),
            ('/B/83c4ee846cf2/B23/', '192.168.30.187:8000/23'),
            ('/B/83c4ee846cf2/B22/', '192.168.30.187:8000/22'),
            ('/B/83c4ee846cf2/B21/', '192.168.30.187:8000/21'),
            ('/B/83c4ee846cf2/B20/', '192.168.30.187:8000/20'),
            ('/B/83c4ee846cf2/B27/', '192.168.30.187:8000/27'),
            ('/B/83c4ee846cf2/B26/', '192.168.30.187:8000/26'),
            ('/B/83c4ee846cf2/B25/', '192.168.30.187:8000/25'),
            ('/B/83c4ee846cf2/B24/', '192.168.30.187:8000/24'),
            ('/B/83c4ee846cf2/B66/', '192.168.30.187:8000/66'),
            ('/B/83c4ee846cf2/B67/', '192.168.30.187:8000/67'),
            ('/B/83c4ee846cf2/B64/', '192.168.30.187:8000/64'),
            ('/B/83c4ee846cf2/B29/', '192.168.30.187:8000/29'),
            ('/B/83c4ee846cf2/B65/', '192.168.30.187:8000/65'),
            ('/B/83c4ee846cf2/B28/', '192.168.30.187:8000/28'),
            ('/B/83c4ee846cf2/B68/', '192.168.30.187:8000/68'),
            ('/B/83c4ee846cf2/B69/', '192.168.30.187:8000/69'),
            ('/B/83c4ee846cf2/B5/', '192.168.30.187:8000/5'),
            ('/B/83c4ee846cf2/B4/', '192.168.30.187:8000/4'),
            ('/B/83c4ee846cf2/B81/', '192.168.30.187:8000/81'),
            ('/B/83c4ee846cf2/B3/', '192.168.30.187:8000/3'),
            ('/B/83c4ee846cf2/B80/', '192.168.30.187:8000/80'),
            ('/B/83c4ee846cf2/B2/', '192.168.30.187:8000/2'),
            ('/B/83c4ee846cf2/B83/', '192.168.30.187:8000/83'),
            ('/B/83c4ee846cf2/B9/', '192.168.30.187:8000/9'),
            ('/B/83c4ee846cf2/B82/', '192.168.30.187:8000/82'),
            ('/B/83c4ee846cf2/B8/', '192.168.30.187:8000/8'),
            ('/B/83c4ee846cf2/B85/', '192.168.30.187:8000/85'),
            ('/B/83c4ee846cf2/B7/', '192.168.30.187:8000/7'),
            ('/B/83c4ee846cf2/B84/', '192.168.30.187:8000/84'),
            ('/B/83c4ee846cf2/B6/', '192.168.30.187:8000/6'),
            ('/B/83c4ee846cf2/B40/', '192.168.30.187:8000/40'),
            ('/B/83c4ee846cf2/B41/', '192.168.30.187:8000/41'),
            ('/B/83c4ee846cf2/B32/', '192.168.30.187:8000/32'),
            ('/B/83c4ee846cf2/B31/', '192.168.30.187:8000/31'),
            ('/B/83c4ee846cf2/B34/', '192.168.30.187:8000/34'),
            ('/B/83c4ee846cf2/B33/', '192.168.30.187:8000/33'),
            ('/B/83c4ee846cf2/B36/', '192.168.30.187:8000/36'),
            ('/B/83c4ee846cf2/B35/', '192.168.30.187:8000/35'),
            ('/B/83c4ee846cf2/B38/', '192.168.30.187:8000/38'),
            ('/B/83c4ee846cf2/B37/', '192.168.30.187:8000/37'),
            ('/B/83c4ee846cf2/B75/', '192.168.30.187:8000/75'),
            ('/B/83c4ee846cf2/B76/', '192.168.30.187:8000/76'),
            ('/B/83c4ee846cf2/B39/', '192.168.30.187:8000/39'),
            ('/B/83c4ee846cf2/B77/', '192.168.30.187:8000/77'),
            ('/B/83c4ee846cf2/B78/', '192.168.30.187:8000/78'),
            ('/B/83c4ee846cf2/B79/', '192.168.30.187:8000/79'),
            ('/B/83c4ee846cf2/B1/', '192.168.30.187:8000/1'),
            ('/B/83c4ee846cf2/B19/', '192.168.30.187:8000/19'),
            ('/B/83c4ee846cf2/B17/', '192.168.30.187:8000/17'),
            ('/B/83c4ee846cf2/B18/', '192.168.30.187:8000/18'),
            ('/B/83c4ee846cf2/B90/', '192.168.30.187:8000/90'),
            ('/B/83c4ee846cf2/B51/', '192.168.30.187:8000/51'),
            ('/B/83c4ee846cf2/B11/', '192.168.30.187:8000/11'),
            ('/B/83c4ee846cf2/B52/', '192.168.30.187:8000/52'),
            ('/B/83c4ee846cf2/B12/', '192.168.30.187:8000/12'),
            ('/B/83c4ee846cf2/B50/', '192.168.30.187:8000/50'),
            ('/B/83c4ee846cf2/B10/', '192.168.30.187:8000/10'),
            ('/B/83c4ee846cf2/B15/', '192.168.30.187:8000/15'),
            ('/B/83c4ee846cf2/B16/', '192.168.30.187:8000/16'),
            ('/B/83c4ee846cf2/B13/', '192.168.30.187:8000/13'),
            ('/B/83c4ee846cf2/B14/', '192.168.30.187:8000/14'),
            ('/B/83c4ee846cf2/B49/', '192.168.30.187:8000/49'),
            ('/B/83c4ee846cf2/B48/', '192.168.30.187:8000/48'),
            ('/B/83c4ee846cf2/B47/', '192.168.30.187:8000/47'),
            ('/B/83c4ee846cf2/B46/', '192.168.30.187:8000/46'),
            ('/B/83c4ee846cf2/B45/', '192.168.30.187:8000/45'),
            ('/B/83c4ee846cf2/B44/', '192.168.30.187:8000/44'),
            ('/B/83c4ee846cf2/B43/', '192.168.30.187:8000/43'),
            ('/B/83c4ee846cf2/B42/', '192.168.30.187:8000/42'),
            ('/B/83c4ee846cf2/B88/', '192.168.30.187:8000/88'),
            ('/B/83c4ee846cf2/B89/', '192.168.30.187:8000/89'),
            ('/B/83c4ee846cf2/B86/', '192.168.30.187:8000/86'),
            ('/B/83c4ee846cf2/B87/', '192.168.30.187:8000/87'),
            ('/B/83c4ee846cf2/B60/', '192.168.30.187:8000/60'),
            ('/B/83c4ee846cf2/B61/', '192.168.30.187:8000/61'),
            ('/B/83c4ee846cf2/B62/', '192.168.30.187:8000/62'),
            ('/B/83c4ee846cf2/B63/', '192.168.30.187:8000/63'),
            ('/B/83c4ee846cf2/B58/', '192.168.30.187:8000/58'),
            ('/B/83c4ee846cf2/B57/', '192.168.30.187:8000/57'),
            ('/B/83c4ee846cf2/B59/', '192.168.30.187:8000/59'),
            ('/B/83c4ee846cf2/B54/', '192.168.30.187:8000/54'),
            ('/B/83c4ee846cf2/B53/', '192.168.30.187:8000/53'),
            ('/B/83c4ee846cf2/B56/', '192.168.30.187:8000/56'),
            ('/B/83c4ee846cf2/B55/', '192.168.30.187:8000/55'),
            ('/B/d204c1d12b8a/B70/', '192.168.30.186:8000/70'),
            ('/B/d204c1d12b8a/B74/', '192.168.30.186:8000/74'),
            ('/B/d204c1d12b8a/B73/', '192.168.30.186:8000/73'),
            ('/B/d204c1d12b8a/B72/', '192.168.30.186:8000/72'),
            ('/B/d204c1d12b8a/B71/', '192.168.30.186:8000/71'),
            ('/B/d204c1d12b8a/B30/', '192.168.30.186:8000/30'),
            ('/B/d204c1d12b8a/B23/', '192.168.30.186:8000/23'),
            ('/B/d204c1d12b8a/B22/', '192.168.30.186:8000/22'),
            ('/B/d204c1d12b8a/B21/', '192.168.30.186:8000/21'),
            ('/B/d204c1d12b8a/B20/', '192.168.30.186:8000/20'),
            ('/B/d204c1d12b8a/B27/', '192.168.30.186:8000/27'),
            ('/B/d204c1d12b8a/B26/', '192.168.30.186:8000/26'),
            ('/B/d204c1d12b8a/B25/', '192.168.30.186:8000/25'),
            ('/B/d204c1d12b8a/B24/', '192.168.30.186:8000/24'),
            ('/B/d204c1d12b8a/B66/', '192.168.30.186:8000/66'),
            ('/B/d204c1d12b8a/B67/', '192.168.30.186:8000/67'),
            ('/B/d204c1d12b8a/B64/', '192.168.30.186:8000/64'),
            ('/B/d204c1d12b8a/B29/', '192.168.30.186:8000/29'),
            ('/B/d204c1d12b8a/B65/', '192.168.30.186:8000/65'),
            ('/B/d204c1d12b8a/B28/', '192.168.30.186:8000/28'),
            ('/B/d204c1d12b8a/B68/', '192.168.30.186:8000/68'),
            ('/B/d204c1d12b8a/B69/', '192.168.30.186:8000/69'),
            ('/B/d204c1d12b8a/B5/', '192.168.30.186:8000/5'),
            ('/B/d204c1d12b8a/B4/', '192.168.30.186:8000/4'),
            ('/B/d204c1d12b8a/B81/', '192.168.30.186:8000/81'),
            ('/B/d204c1d12b8a/B3/', '192.168.30.186:8000/3'),
            ('/B/d204c1d12b8a/B80/', '192.168.30.186:8000/80'),
            ('/B/d204c1d12b8a/B2/', '192.168.30.186:8000/2'),
            ('/B/d204c1d12b8a/B83/', '192.168.30.186:8000/83'),
            ('/B/d204c1d12b8a/B9/', '192.168.30.186:8000/9'),
            ('/B/d204c1d12b8a/B82/', '192.168.30.186:8000/82'),
            ('/B/d204c1d12b8a/B8/', '192.168.30.186:8000/8'),
            ('/B/d204c1d12b8a/B85/', '192.168.30.186:8000/85'),
            ('/B/d204c1d12b8a/B7/', '192.168.30.186:8000/7'),
            ('/B/d204c1d12b8a/B84/', '192.168.30.186:8000/84'),
            ('/B/d204c1d12b8a/B6/', '192.168.30.186:8000/6'),
            ('/B/d204c1d12b8a/B40/', '192.168.30.186:8000/40'),
            ('/B/d204c1d12b8a/B41/', '192.168.30.186:8000/41'),
            ('/B/d204c1d12b8a/B32/', '192.168.30.186:8000/32'),
            ('/B/d204c1d12b8a/B31/', '192.168.30.186:8000/31'),
            ('/B/d204c1d12b8a/B34/', '192.168.30.186:8000/34'),
            ('/B/d204c1d12b8a/B33/', '192.168.30.186:8000/33'),
            ('/B/d204c1d12b8a/B36/', '192.168.30.186:8000/36'),
            ('/B/d204c1d12b8a/B35/', '192.168.30.186:8000/35'),
            ('/B/d204c1d12b8a/B38/', '192.168.30.186:8000/38'),
            ('/B/d204c1d12b8a/B37/', '192.168.30.186:8000/37'),
            ('/B/d204c1d12b8a/B75/', '192.168.30.186:8000/75'),
            ('/B/d204c1d12b8a/B76/', '192.168.30.186:8000/76'),
            ('/B/d204c1d12b8a/B39/', '192.168.30.186:8000/39'),
            ('/B/d204c1d12b8a/B77/', '192.168.30.186:8000/77'),
            ('/B/d204c1d12b8a/B78/', '192.168.30.186:8000/78'),
            ('/B/d204c1d12b8a/B79/', '192.168.30.186:8000/79'),
            ('/B/d204c1d12b8a/B1/', '192.168.30.186:8000/1'),
            ('/B/d204c1d12b8a/B19/', '192.168.30.186:8000/19'),
            ('/B/d204c1d12b8a/B17/', '192.168.30.186:8000/17'),
            ('/B/d204c1d12b8a/B18/', '192.168.30.186:8000/18'),
            ('/B/d204c1d12b8a/B90/', '192.168.30.186:8000/90'),
            ('/B/d204c1d12b8a/B51/', '192.168.30.186:8000/51'),
            ('/B/d204c1d12b8a/B11/', '192.168.30.186:8000/11'),
            ('/B/d204c1d12b8a/B52/', '192.168.30.186:8000/52'),
            ('/B/d204c1d12b8a/B12/', '192.168.30.186:8000/12'),
            ('/B/d204c1d12b8a/B50/', '192.168.30.186:8000/50'),
            ('/B/d204c1d12b8a/B10/', '192.168.30.186:8000/10'),
            ('/B/d204c1d12b8a/B15/', '192.168.30.186:8000/15'),
            ('/B/d204c1d12b8a/B16/', '192.168.30.186:8000/16'),
            ('/B/d204c1d12b8a/B13/', '192.168.30.186:8000/13'),
            ('/B/d204c1d12b8a/B14/', '192.168.30.186:8000/14'),
            ('/B/d204c1d12b8a/B49/', '192.168.30.186:8000/49'),
            ('/B/d204c1d12b8a/B48/', '192.168.30.186:8000/48'),
            ('/B/d204c1d12b8a/B47/', '192.168.30.186:8000/47'),
            ('/B/d204c1d12b8a/B46/', '192.168.30.186:8000/46'),
            ('/B/d204c1d12b8a/B45/', '192.168.30.186:8000/45'),
            ('/B/d204c1d12b8a/B44/', '192.168.30.186:8000/44'),
            ('/B/d204c1d12b8a/B43/', '192.168.30.186:8000/43'),
            ('/B/d204c1d12b8a/B42/', '192.168.30.186:8000/42'),
            ('/B/d204c1d12b8a/B88/', '192.168.30.186:8000/88'),
            ('/B/d204c1d12b8a/B89/', '192.168.30.186:8000/89'),
            ('/B/d204c1d12b8a/B86/', '192.168.30.186:8000/86'),
            ('/B/d204c1d12b8a/B87/', '192.168.30.186:8000/87'),
            ('/B/d204c1d12b8a/B60/', '192.168.30.186:8000/60'),
            ('/B/d204c1d12b8a/B61/', '192.168.30.186:8000/61'),
            ('/B/d204c1d12b8a/B62/', '192.168.30.186:8000/62'),
            ('/B/d204c1d12b8a/B63/', '192.168.30.186:8000/63'),
            ('/B/d204c1d12b8a/B58/', '192.168.30.186:8000/58'),
            ('/B/d204c1d12b8a/B57/', '192.168.30.186:8000/57'),
            ('/B/d204c1d12b8a/B59/', '192.168.30.186:8000/59'),
            ('/B/d204c1d12b8a/B54/', '192.168.30.186:8000/54'),
            ('/B/d204c1d12b8a/B53/', '192.168.30.186:8000/53'),
            ('/B/d204c1d12b8a/B56/', '192.168.30.186:8000/56'),
            ('/B/d204c1d12b8a/B55/', '192.168.30.186:8000/55')]
    boxs_free = []
    if kb_add_dict:
        boxs_free = set(dict(boxs).keys()) - set(
            dict(kb_add_dict.values()).keys())
    else:
        boxs_free = set(dict(boxs).keys())

    if len(boxs_free) < len(no_kbs):
        rest_kbs = no_kbs[len(boxs_free):]
        kb_ids = set(kb_ids) - set(rest_kbs)

    # 写入cache
    boxs_free_info = filter(lambda x: x[0] in boxs_free, boxs)
    temp_kb_box_list = list(zip(no_kbs, boxs_free_info))
    cache_ret = map(lambda x: cache.set(x[0], x[1], 30 * 60), temp_kb_box_list)
    logger.debug('cache_ret:%s' % list(cache_ret))

    kb_add_dict = cache.get_many(kb_ids)
    logger.debug('kb_add_dict:%s' % kb_add_dict)
    logger.debug('------get address time:%.5f' % (time() - start_get_add))
    logger.debug('start box-request ')
    start_request = time()
    num = len(kb_ids)
    q = Manager().Queue()
    p_list = []
    for i in range(0, num):
        kb = kb_ids[i]
        version = kb_vers_map[kb]
        add = kb_add_dict[kb][1]
        logger.debug('Target:%s Add:%s' % (kb, add))
        temp_p = Process(target=_writer, args=(q, kb, version, add, issue))
        p_list.append(temp_p)
        temp_p.start()

    for pr in p_list:
        pr.join()
    logger.debug('------box-request time:%.5f' % (time() - start_request))

    start_get_msg = time()
    i = 0
    ret = {'no_box': [], 'ans': [], 'not_match': [], 'fail': []}
    while not q.empty():
        msg = q.get()
        if 'not_match' in msg.keys():
            ret['not_match'].append(msg['not_match'])
        elif 'fail' in msg.keys():
            ret['fail'].append(msg['fail'])
        else:
            ret['ans'].append(msg)
        logger.debug('------%d msg:%s' % (i, msg))
        i += 1
    logger.debug('------get answers time:%.5f' % (time() - start_get_msg))

    # 异步写入zk
    # set_box_zk.delay(temp_kb_box_list)

    return ret
Example #12
0
def get(document, cache_control, base_url, timeout=None):
    """Perform a kumascript GET request for a document locale and slug."""
    if not cache_control:
        # Default to the configured max-age for cache control.
        max_age = config.KUMASCRIPT_MAX_AGE
        cache_control = 'max-age=%s' % max_age

    if not base_url:
        site = Site.objects.get_current()
        base_url = 'http://%s' % site.domain

    if not timeout:
        timeout = config.KUMASCRIPT_TIMEOUT

    document_locale = document.locale
    document_slug = document.slug
    max_age = config.KUMASCRIPT_MAX_AGE

    # 1063580 - Kumascript converts template name calls to lower case and bases
    # caching keys off of that.
    document_slug_for_kumascript = document_slug
    body, errors = None, None

    try:
        url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
        url = unicode(url_tmpl).format(path=u'%s/%s' %
                                       (document_locale,
                                        document_slug_for_kumascript))

        cache_keys = build_cache_keys(document_slug, document_locale)
        etag_key, modified_key, body_key, errors_key = cache_keys

        headers = {
            'X-FireLogger': '1.2',
            'Cache-Control': cache_control,
        }

        # Create the file interface
        files = []
        for attachment in document.files.select_related('current_revision'):
            files.append(_get_attachment_metadata_dict(attachment))

        # Assemble some KumaScript env vars
        # TODO: See dekiscript vars for future inspiration
        # http://developer.mindtouch.com/en/docs/DekiScript/Reference/
        #   Wiki_Functions_and_Variables
        path = document.get_absolute_url()
        # TODO: Someday merge with _get_document_for_json in views.py
        # where most of this is duplicated code.
        env_vars = dict(
            path=path,
            url=urljoin(base_url, path),
            id=document.pk,
            revision_id=document.current_revision.pk,
            locale=document.locale,
            title=document.title,
            files=files,
            slug=document.slug,
            tags=list(document.tags.names()),
            review_tags=list(document.current_revision.review_tags.names()),
            modified=time.mktime(document.modified.timetuple()),
            cache_control=cache_control,
        )
        add_env_headers(headers, env_vars)

        # Set up for conditional GET, if we have the details cached.
        cached_meta = cache.get_many([etag_key, modified_key])
        if etag_key in cached_meta:
            headers['If-None-Match'] = cached_meta[etag_key]
        if modified_key in cached_meta:
            headers['If-Modified-Since'] = cached_meta[modified_key]

        # Finally, fire off the request.
        response = requests.get(url, headers=headers, timeout=timeout)

        if response.status_code == 304:
            # Conditional GET was a pass, so use the cached content.
            result = cache.get_many([body_key, errors_key])
            body = result.get(body_key, '').decode('utf-8')
            errors = result.get(errors_key, None)

        elif response.status_code == 200:
            body = process_body(response)
            errors = process_errors(response)

            # Cache the request for conditional GET, but use the max_age for
            # the cache timeout here too.
            headers = response.headers
            cache.set(etag_key, headers.get('etag'), timeout=max_age)
            cache.set(modified_key, headers.get('last-modified'), timeout=max_age)
            cache.set(body_key, body.encode('utf-8'), timeout=max_age)
            if errors:
                cache.set(errors_key, errors, timeout=max_age)

        elif response.status_code is None:
            errors = KUMASCRIPT_TIMEOUT_ERROR

        else:
            errors = [
                {
                    "level": "error",
                    "message": "Unexpected response from Kumascript service: %s" %
                               response.status_code,
                    "args": ["UnknownError"],
                },
            ]

    except Exception as exc:
        # Last resort: Something went really haywire. Kumascript server died
        # mid-request, or something. Try to report at least some hint.
        errors = [
            {
                "level": "error",
                "message": "Kumascript service failed unexpectedly: %s" % exc,
                "args": ["UnknownError"],
            },
        ]
    return (body, errors)
Example #13
0
    def get(self, request, project):
        """
        Get list with release setup progress for a project
        1. tag an error
        2. link a repo
        3. associate commits
        4. tell sentry about a deploy
        """

        tag_key = "onboard_tag:1:%s" % (project.id)
        repo_key = "onboard_repo:1:%s" % (project.organization_id)
        commit_key = "onboard_commit:1:%s" % hash_values(
            [project.organization_id, project.id])
        deploy_key = "onboard_deploy:1:%s" % hash_values(
            [project.organization_id, project.id])
        onboard_cache = cache.get_many(
            [tag_key, repo_key, commit_key, deploy_key])

        tag = onboard_cache.get(tag_key)
        if tag is None:
            tag = Group.objects.filter(project=project.id,
                                       first_release__isnull=False).exists()
            cache.set(tag_key, tag, 3600 if tag else 60)

        repo = onboard_cache.get(repo_key)
        if repo is None:
            repo = Repository.objects.filter(
                organization_id=project.organization_id).exists()
            cache.set(repo_key, repo, 3600 if repo else 60)

        commit = onboard_cache.get(commit_key)
        if commit is None:
            commit = ReleaseCommit.objects.filter(
                organization_id=project.organization_id,
                release__projects=project.id).exists()
            cache.set(commit_key, commit, 3600 if commit else 60)

        deploy = onboard_cache.get(deploy_key)
        if deploy is None:
            deploy = Deploy.objects.filter(
                organization_id=project.organization_id,
                release__projects=project.id).exists()
            cache.set(deploy_key, deploy, 3600 if deploy else 60)

        return Response([
            {
                "step": "tag",
                "complete": bool(tag)
            },
            {
                "step": "repo",
                "complete": bool(repo)
            },
            {
                "step": "commit",
                "complete": bool(commit)
            },
            {
                "step": "deploy",
                "complete": bool(deploy)
            },
        ])
Example #14
0
def query_request_new(kb_ids, issue, kb_vers_map):
    """
    tips: cache形式:cache.set(box ,(kb, 0/1), 过期时间)
    requirement: (1)查看boxs列表, 对比cache中的boxs, 若有空闲的box,则为其赋一个知识库;
                    若没有空闲box,则在cache中寻找该KB已有的box。
                 (2)取出请求中所有知识库对应的BOX的地址,单独起进程发送请求,请求结果放入queue

    param: kb_ids 列表 
    return: 
    """

    # 取cache中已有的kb
    logger.debug('start get add')
    start_get_add = time()

    kb_add_dict = cache.get_many(kb_ids)

    no_kbs = set(kb_ids) - set(kb_add_dict.keys())
    boxs = cache.get('boxs')
    box_addr_dict = cache.get('box_infos')
    box_kb_dict = cache.get_many(boxs)

    box_kb_rest = list(
        filter(lambda x: (x[1][0] in boxs) and (not x[1][1]),
               box_kb_dict.items()))
    boxs_idle = list(filter(lambda x: not cache.get(x), boxs))
    logger.debug('boxs_idle:%s' % boxs_idle)

    # 为cache中没有的kb赋予box
    boxs_free = []
    if kb_add_dict:
        boxs_free = set(dict(boxs).keys()) - set(
            dict(kb_add_dict.values()).keys())
    else:
        boxs_free = set(dict(boxs).keys())

    if len(boxs_free) < len(no_kbs):
        rest_kbs = no_kbs[len(boxs_free):]
        kb_ids = set(kb_ids) - set(rest_kbs)

    # 写入cache
    boxs_free_info = filter(lambda x: x[0] in boxs_free, boxs)
    temp_kb_box_list = list(zip(no_kbs, boxs_free_info))
    cache_ret = map(lambda x: cache.set(x[0], x[1], 30 * 60), temp_kb_box_list)
    logger.debug('cache_ret:%s' % list(cache_ret))

    kb_add_dict = cache.get_many(kb_ids)
    logger.debug('kb_add_dict:%s' % kb_add_dict)
    logger.debug('------get address time:%.5f' % (time() - start_get_add))
    logger.debug('start box-request ')
    start_request = time()
    num = len(kb_ids)
    q = Manager().Queue()
    p_list = []
    for i in range(0, num):
        kb = kb_ids[i]
        version = kb_vers_map[kb]
        add = kb_add_dict[kb][1]
        logger.debug('Target:%s Add:%s' % (kb, add))
        temp_p = Process(target=_writer, args=(q, kb, version, add, issue))
        p_list.append(temp_p)
        temp_p.start()

    for pr in p_list:
        pr.join()
    logger.debug('------box-request time:%.5f' % (time() - start_request))

    start_get_msg = time()
    i = 0
    ret = {'no_box': [], 'ans': [], 'not_match': [], 'fail': []}
    while not q.empty():
        msg = q.get()
        if 'not_match' in msg.keys():
            ret['not_match'].append(msg['not_match'])
        elif 'fail' in msg.keys():
            ret['fail'].append(msg['fail'])
        else:
            ret['ans'].append(msg)
        logger.debug('------%d msg:%s' % (i, msg))
        i += 1
    logger.debug('------get answers time:%.5f' % (time() - start_get_msg))

    # 异步写入zk
    # set_box_zk.delay(temp_kb_box_list)

    return ret
Example #15
0
 def _get_cached_long_urls(self, long_urls):
     return cache.get_many(long_urls)
Example #16
0
def get(document, cache_control, base_url, timeout=None):
    """Perform a kumascript GET request for a document locale and slug."""
    if not cache_control:
        # Default to the configured max-age for cache control.
        max_age = constance.config.KUMASCRIPT_MAX_AGE
        cache_control = 'max-age=%s' % max_age

    if not base_url:
        site = Site.objects.get_current()
        base_url = 'http://%s' % site.domain

    if not timeout:
        timeout = constance.config.KUMASCRIPT_TIMEOUT

    document_locale = document.locale
    document_slug = document.slug
    max_age = constance.config.KUMASCRIPT_MAX_AGE

    resp_body, resp_errors = None, None

    try:
        url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
        url = unicode(url_tmpl).format(path=u'%s/%s' %
                                       (document_locale, document_slug))

        ck_etag, ck_modified, ck_body, ck_errors = (build_cache_keys(
            document_slug, document_locale))

        headers = {'X-FireLogger': '1.2', 'Cache-Control': cache_control}

        # Create the file interface
        files = []
        for attachment in document.attachments.all():
            files.append({
                'title': attachment.title,
                'description': attachment.current_revision.description,
                'filename': attachment.current_revision.filename(),
                'size': attachment.current_revision.file.size,
                'author': attachment.current_revision.creator.username,
                'mime': attachment.current_revision.mime_type,
                'url': attachment.get_file_url(),
            })

        # Assemble some KumaScript env vars
        # TODO: See dekiscript vars for future inspiration
        # http://developer.mindtouch.com/en/docs/DekiScript/Reference/
        #   Wiki_Functions_and_Variables
        path = document.get_absolute_url()
        env_vars = dict(
            path=path,
            url=urljoin(base_url, path),
            id=document.pk,
            locale=document.locale,
            title=document.title,
            files=files,
            attachments=files,  # Just for sake of verbiage?
            slug=document.slug,
            tags=[x.name for x in document.tags.all()],
            modified=time.mktime(document.modified.timetuple()),
            cache_control=cache_control,
        )
        add_env_headers(headers, env_vars)

        # Set up for conditional GET, if we have the details cached.
        c_meta = cache.get_many([ck_etag, ck_modified])
        if ck_etag in c_meta:
            headers['If-None-Match'] = c_meta[ck_etag]
        if ck_modified in c_meta:
            headers['If-Modified-Since'] = c_meta[ck_modified]

        # Finally, fire off the request.
        statsd.incr('wiki.ks_get')
        with statsd.timer('wiki.ks_get'):
            resp = requests.get(url, headers=headers, timeout=timeout)

        if resp.status_code == 304:
            # Conditional GET was a pass, so use the cached content.
            c_result = cache.get_many([ck_body, ck_errors])
            resp_body = c_result.get(ck_body, '').decode('utf-8')
            resp_errors = c_result.get(ck_errors, None)

        elif resp.status_code == 200:
            resp_body = process_body(resp)
            resp_errors = process_errors(resp)

            # Cache the request for conditional GET, but use the max_age for
            # the cache timeout here too.
            cache.set(ck_etag, resp.headers.get('etag'), timeout=max_age)
            cache.set(ck_modified,
                      resp.headers.get('last-modified'),
                      timeout=max_age)
            cache.set(ck_body, resp_body.encode('utf-8'), timeout=max_age)
            if resp_errors:
                cache.set(ck_errors, resp_errors, timeout=max_age)

        elif resp.status_code == None:
            resp_errors = KUMASCRIPT_TIMEOUT_ERROR

        else:
            resp_errors = [{
                "level":
                "error",
                "message":
                "Unexpected response from Kumascript service: %s" %
                resp.status_code,
                "args": ["UnknownError"]
            }]

    except Exception, e:
        # Last resort: Something went really haywire. Kumascript server died
        # mid-request, or something. Try to report at least some hint.
        resp_errors = [{
            "level":
            "error",
            "message":
            "Kumascript service failed unexpectedly: %s" % type(e),
            "args": ["UnknownError"]
        }]
Example #17
0
from django.core.cache import cache  # 导入的cache就是配置的默认缓存
from django.core.cache import caches  # caches相当于全部缓存集
from django.views.decorators.cache import never_cache, cache_page, cache_control

def_cache = caches['default']
rds_cache = caches['redis']

assert def_cache == cache
assert rds_cache != cache
print('def_cache', def_cache)
print('rds_cache', rds_cache)
# 2)设置,如果将exp过期时间设置0或负值,就是删除缓存
# cache.set('key', 'value', exp=1000)
cache.set('key', 'value')
print(cache.set_many({'a': 1, 'b': 2, 'c': 3}))
print(cache.get_many(['a', 'b', 'c']))
# 3)获取
cache.get('key')
cache.set('num', 1)
cache.incr('num')
cache.incr('num', 10)
cache.decr('num')
cache.decr('num', 5)
cache.clear()


@cache_page(60 * 15, cache="redis")  # 可以选用缓存方式
@cache_page(60 * 15, key_prefix="site1")
@cache_control(max_age=3600)
@never_cache
def myview(request):
Example #18
0
    def test_cache_versioning_get_set_many(self):
        # set, using default version = 1
        cache.set_many({'ford1': 37, 'arthur1': 42})
        assert (
            cache.get_many(['ford1', 'arthur1']) ==
            {'ford1': 37, 'arthur1': 42}
        )
        assert (
            cache.get_many(['ford1', 'arthur1'], version=1) ==
            {'ford1': 37, 'arthur1': 42}
        )
        assert cache.get_many(['ford1', 'arthur1'], version=2) == {}

        assert caches['v2'].get_many(['ford1', 'arthur1']) == {}
        assert (
            caches['v2'].get_many(['ford1', 'arthur1'], version=1) ==
            {'ford1': 37, 'arthur1': 42}
        )
        assert caches['v2'].get_many(['ford1', 'arthur1'], version=2) == {}

        # set, default version = 1, but manually override version = 2
        cache.set_many({'ford2': 37, 'arthur2': 42}, version=2)
        assert cache.get_many(['ford2', 'arthur2']) == {}
        assert cache.get_many(['ford2', 'arthur2'], version=1) == {}
        assert (
            cache.get_many(['ford2', 'arthur2'], version=2) ==
            {'ford2': 37, 'arthur2': 42}
        )

        assert (
            caches['v2'].get_many(['ford2', 'arthur2']) ==
            {'ford2': 37, 'arthur2': 42}
        )
        assert caches['v2'].get_many(['ford2', 'arthur2'], version=1) == {}
        assert (
            caches['v2'].get_many(['ford2', 'arthur2'], version=2) ==
            {'ford2': 37, 'arthur2': 42}
        )

        # v2 set, using default version = 2
        caches['v2'].set_many({'ford3': 37, 'arthur3': 42})
        assert cache.get_many(['ford3', 'arthur3']) == {}
        assert cache.get_many(['ford3', 'arthur3'], version=1) == {}
        assert (
            cache.get_many(['ford3', 'arthur3'], version=2) ==
            {'ford3': 37, 'arthur3': 42}
        )

        assert (
            caches['v2'].get_many(['ford3', 'arthur3']) ==
            {'ford3': 37, 'arthur3': 42}
        )
        assert (
            caches['v2'].get_many(['ford3', 'arthur3'], version=1) ==
            {}
        )
        assert (
            caches['v2'].get_many(['ford3', 'arthur3'], version=2) ==
            {'ford3': 37, 'arthur3': 42}
        )

        # v2 set, default version = 2, but manually override version = 1
        caches['v2'].set_many({'ford4': 37, 'arthur4': 42}, version=1)
        assert (
            cache.get_many(['ford4', 'arthur4']) ==
            {'ford4': 37, 'arthur4': 42}
        )
        assert (
            cache.get_many(['ford4', 'arthur4'], version=1) ==
            {'ford4': 37, 'arthur4': 42}
        )
        assert cache.get_many(['ford4', 'arthur4'], version=2) == {}

        assert caches['v2'].get_many(['ford4', 'arthur4']) == {}
        assert (
            caches['v2'].get_many(['ford4', 'arthur4'], version=1) ==
            {'ford4': 37, 'arthur4': 42}
        )
        assert caches['v2'].get_many(['ford4', 'arthur4'], version=2) == {}
Example #19
0
    def get_cached(self,
                   slugs=None,
                   pks=None,
                   select_related_media_tag=True,
                   portal_id=None):
        """
        Gets all ideas defined by either `slugs` or `pks`.

        `slugs` and `pks` may be a list or tuple of identifiers to use for
        request where the elements are of type string / unicode or int,
        respectively. You may provide a single string / unicode or int directly
        to query only one object.

        :returns: An instance or a list of instances of :class:`CosinnusGroup`.
        :raises: If a single object is defined a `CosinnusGroup.DoesNotExist`
            will be raised in case the requested object does not exist.
        """
        if portal_id is None:
            portal_id = CosinnusPortal.get_current().id

        # Check that at most one of slugs and pks is set
        assert not (slugs and pks)
        assert not (slugs or pks)

        if slugs is not None:
            if isinstance(slugs, six.string_types):
                # We request a single idea
                slugs = [slugs]

            # We request multiple ideas by slugs
            keys = [self._IDEAS_SLUG_CACHE_KEY % (portal_id, s) for s in slugs]
            ideas = cache.get_many(keys)
            missing = [key.split('/')[-1] for key in keys if key not in ideas]
            if missing:
                # we can only find ideas via this function that are in the same portal we run in
                query = self.get_queryset().filter(portal__id=portal_id,
                                                   is_active=True,
                                                   slug__in=missing)
                if select_related_media_tag:
                    query = query.select_related('media_tag')

                for idea in query:
                    ideas[self._IDEAS_SLUG_CACHE_KEY %
                          (portal_id, idea.slug)] = idea
                cache.set_many(ideas, settings.COSINNUS_IDEA_CACHE_TIMEOUT)

            # sort by a good sorting function that acknowldges umlauts, etc, case insensitive
            idea_list = list(ideas.values())
            idea_list = sorted(idea_list, key=sort_key_strcoll_attr('name'))
            return idea_list

        elif pks is not None:
            if isinstance(pks, int):
                pks = [pks]
            else:
                # We request multiple ideas
                cached_pks = self.get_pks(portal_id=portal_id)
                slugs = [
                    _f for _f in (cached_pks.get(pk, []) for pk in pks) if _f
                ]
                if slugs:
                    return self.get_cached(slugs=slugs, portal_id=portal_id)
                return []  # We rely on the slug and id maps being up to date
        return []
Example #20
0
 def cache_get_many(self, keys):
     return cache.get_many(keys)
Example #21
0
 def count(self, request, ip=True, field=None, period=60):
     counters = dict((key, 0) for key in self._keys(request, ip, field))
     counters.update(cache.get_many(counters.keys()))
     for key in counters:
         counters[key] += 1
     cache.set_many(counters, timeout=period)
Example #22
0
def get_instance(model, instance_or_pk, timeout=None, using=None):
    """
    Returns the ``model`` instance with a primary key of ``instance_or_pk``.

    If the data is cached it will be returned from there, otherwise the regular
    Django ORM is queried for this instance and the data stored in the cache.

    If omitted, the timeout value defaults to
    ``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).

    Example::

        >>> get_instance(User, 1) # Cache miss
        <User: lamby>
        >>> get_instance(User, 1) # Cache hit
        <User: lamby>
        >>> User.objects.get(pk=1) == get_instance(User, 1)
        True
    """

    pk = getattr(instance_or_pk, "pk", instance_or_pk)

    primary_model = model
    descriptors = getattr(primary_model, "_cache_fetch_related", ())
    models = [model, *(d.related.field.model for d in descriptors)]
    # Note: we're assuming that the relations are primary key foreign keys, and
    # so all have the same primary key. This matches the assumption which
    # `cache_relation` makes.
    keys_to_models = {instance_key(model, instance_or_pk): model for model in models}

    data_map = cache.get_many(tuple(keys_to_models.keys()))
    instance_map = {}

    if data_map.keys() == keys_to_models.keys():
        try:
            for key, data in data_map.items():
                model = keys_to_models[key]
                instance_map[key] = deserialise(model, data, pk, using)
        except:
            # Error when deserialising - remove from the cache; we will
            # fallback and return the underlying instance
            cache.delete_many(tuple(keys_to_models.keys()))

        else:
            key = instance_key(primary_model, instance_or_pk)
            primary_instance = instance_map[key]

            for descriptor in descriptors:
                related_instance = instance_map[
                    instance_key(
                        descriptor.related.field.model,
                        instance_or_pk,
                    )
                ]
                related_cache_name = get_related_cache_name(
                    get_related_name(descriptor),
                )
                setattr(primary_instance, related_cache_name, related_instance)

            return primary_instance

    related_names = [d.related.field.related_query_name() for d in descriptors]

    # Use the default manager so we are never filtered by a .get_query_set()
    queryset = primary_model._default_manager.using(using)
    if related_names:
        # NB: select_related without args selects all it can find, which we don't want.
        queryset = queryset.select_related(*related_names)
    primary_instance = queryset.get(pk=pk)

    instances = [
        primary_instance,
        *(getattr(primary_instance, x, None) for x in related_names),
    ]

    cache_data = {}
    for instance in instances:
        if instance is None:
            continue

        key = instance_key(instance._meta.model, instance)
        cache_data[key] = serialise(instance)

    if timeout is None:
        timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT

    cache.set_many(cache_data, timeout)

    return primary_instance
Example #23
0
 def limit(self, request, ip=True, field=None, count=5):
     counters = cache.get_many(self._keys(request, ip, field))
     return any((v > count) for v in counters.values())
 def get_counters(self, request):
     return cache.get_many(self.keys_to_check(request))
Example #25
0
def get_users_for_authors(organization_id, authors, user=None):
    """
    Returns a dictionary of author_id => user, if a Sentry
    user object exists for that email. If there is no matching
    Sentry user, a {user, email} dict representation of that
    author is returned.
    e.g.
    {
        1: serialized(<User id=1>),
        2: {email: '*****@*****.**', name: 'dunno'},
        ...
    }
    """
    results = {}

    fetched = cache.get_many([
        _user_to_author_cache_key(organization_id, author)
        for author in authors
    ])
    if fetched:
        missed = []
        for author in authors:
            fetched_user = fetched.get(
                _user_to_author_cache_key(organization_id, author))
            if fetched_user is None:
                missed.append(author)
            else:
                results[str(author.id)] = fetched_user
    else:
        missed = authors

    if missed:
        # Filter users based on the emails provided in the commits
        user_emails = list(
            UserEmail.objects.filter(
                in_iexact("email", [a.email for a in missed])).order_by("id"))

        # Filter users belonging to the organization associated with
        # the release
        users = User.objects.filter(
            id__in={ue.user_id
                    for ue in user_emails},
            is_active=True,
            sentry_orgmember_set__organization_id=organization_id,
        )
        users = serialize(list(users), user)
        users_by_id = {user["id"]: user for user in users}
        # Figure out which email address matches to a user
        users_by_email = {}
        for email in user_emails:
            # force emails to lower case so we can do case insensitive matching
            lower_email = email.email.lower()
            if lower_email not in users_by_email:
                user = users_by_id.get(str(email.user_id), None)
                # user can be None if there's a user associated
                # with user_email in separate organization
                if user:
                    users_by_email[lower_email] = user
        to_cache = {}
        for author in missed:
            results[str(author.id)] = users_by_email.get(
                author.email.lower(), {
                    "name": author.name,
                    "email": author.email
                })
            to_cache[_user_to_author_cache_key(
                organization_id, author)] = results[str(author.id)]
        cache.set_many(to_cache)

    metrics.incr("sentry.release.get_users_for_authors.missed",
                 amount=len(missed))
    metrics.incr("sentry.tasks.process_suspect_commits.total",
                 amount=len(results))
    return results
Example #26
0
def should_limit(key, minutes=3, rate=20, now=None, howmuch=1):
    """ Increments the event count for the current key+minute, and returns True
    if 'key' has had more than 'rate' events in the past 'minutes' minutes. """
    keys = _recent_keys(key, minutes, now)
    _incr_key(keys[0], minutes * 60, howmuch=howmuch)
    return sum(cache.get_many(keys).values()) > rate
Example #27
0
    def diff(self, request, new_imgid, old_imgid):

        new_img = Image.objects.select_related(
            "container_repo").prefetch_related(
                "repo", "container_repo__platform").get(pk=new_imgid)
        old_img = Image.objects.select_related(
            "container_repo").prefetch_related(
                "repo", "container_repo__platform").get(pk=old_imgid)
        live_diff = (False, False)

        new_repo = new_img.container_repo
        old_repo = old_img.container_repo
        issue_ref = []
        names = []
        for repo in new_img.repo.all():
            for i in repo.platform.issuetrackers.all():
                if i.name not in names:
                    issue_ref.append({
                        'name': i.name,
                        're': i.re,
                        'url': i.url
                    })
                    names.append(i.name)
        filter_repos = set(request.GET.getlist("repo"))
        filter_meta = _get_filter_meta(request.GET)

        cachekey = "%s%s%s" % ("repodiff", new_repo.id, old_repo.id)
        cached = cache.get_many([cachekey, cachekey + 'ts'])
        diff = cached.get(cachekey)
        diffts = cached.get(cachekey + 'ts')

        if diff is None:
            diff = _diff_sacks(new_repo, old_repo)
            diffts = datetime.datetime.now()
            cachelife = (60 *
                         3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
                                                                    24)
            cache.set_many({
                cachekey: diff,
                cachekey + 'ts': diffts
            }, cachelife)

        diff = _sort_filter_diff(
            diff,
            pkgs=list(set(new_img.packages) | set(old_img.packages)),
            repos=filter_repos,
            meta=filter_meta,
        )

        title = "Comparing Images"
        is_popup = request.GET.get('is_popup', False)

        full_path = "%s?" % request.path
        for query, values in request.GET.lists():
            full_path += "&".join(['%s=%s' % (query, val) for val in values])

        full_path += "&"

        context = {
            "title":
            title,
            "opts":
            self.model._meta,
            "app_label":
            self.model._meta.app_label,
            'diff':
            diff,
            'diffts':
            diffts,
            'live_diff':
            live_diff,
            'new_obj':
            new_img,
            'old_obj':
            old_img,
            'is_popup':
            is_popup,
            'issue_ref':
            json.dumps(issue_ref),
            'packagemetatypes':
            list(PackageMetaType.objects.all().prefetch_related("choices")),
            "full_path":
            full_path,
        }

        return TemplateResponse(
            request,
            'diff.html',
            context=context,
        )
Example #28
0
 def send(self):
     data = cache.get_many([self.data_key, self.counter_key])
     if not data:
         return
     cache.delete_many([self.data_key, self.counter_key])
     self.mail(data[self.counter_key], data[self.data_key])
Example #29
0
def get_online_count():
    online_ips = cache.get("online_ips", [])
    if online_ips:
        online_ips = cache.get_many(online_ips).keys()
        return len(online_ips)
    return 0
Example #30
0
    def diff(self, request, new_repoid, old_repoid):
        start = datetime.datetime.now()

        is_popup = request.GET.get('is_popup', False)
        do_regen = request.GET.get('do_regen', False)
        progress_id = request.GET.get('progress_id', None)

        if not request.is_ajax() and request.method != 'POST':
            progress_id = uuid.uuid4()
            context = {
                "title": "",
                "opts": self.model._meta,
                "app_label": self.model._meta.app_label,
                "progress_id": progress_id,
                "do_regen": do_regen,
            }
            return TemplateResponse(
                request,
                'diff.html',
                context=context,
            )

        def progress_cb(msg):
            if progress_id is not None:
                cache.set(progress_id, msg, 60 * 5)

        progress_cb("Initializing repositories")
        new_repo = Repo.objects.select_related(
            "server",
            "platform").prefetch_related("projects", "components",
                                         "containers").get(pk=new_repoid)
        old_repo = Repo.objects.select_related(
            "server",
            "platform").prefetch_related("projects", "components",
                                         "containers").get(pk=old_repoid)
        live_diff = (new_repo.is_live, old_repo.is_live)

        end = datetime.datetime.now() - start
        context = {
            "title": "",
            "opts": self.model._meta,
            "app_label": self.model._meta.app_label,
            'is_popup': is_popup,
            'new_obj': new_repo,
            'old_obj': old_repo,
            'live_diff': live_diff,
            'processing_time': end.total_seconds(),
        }

        if request.method == 'POST':
            progress_cb("Creating request")
            if not (live_diff[0] and live_diff[1]):
                raise ValidationError("Can only creq on live repos")
            submit = request.POST.getlist('submit')
            delete = request.POST.getlist('delete')
            comment = "Request by %s from repodiff of %s to %s" % (
                request.user, new_repo, old_repo)
            creq_msg = request.POST.get('creq_msg')
            if creq_msg:
                comment = "%s\n%s" % (comment, creq_msg)
            mesgs, errors = _creq(new_repo, old_repo, submit, delete, comment)
            for msg in mesgs:
                messages.info(request, msg, extra_tags="safe")
            for err in errors:
                messages.error(request, err, extra_tags="safe")
            progress_cb("Done")

            return TemplateResponse(
                request,
                'diff_noprogress.html',
                context=context,
            )

        progress_cb("Generating repository diff")
        cachekey = "%s%s%s" % ("repodiff", new_repoid, old_repoid)
        cached = cache.get_many([cachekey, cachekey + 'ts'])
        diff = cached.get(cachekey)
        diffts = cached.get(cachekey + 'ts')

        if diff is None or do_regen:
            diff = _diff_sacks(new_repo, old_repo, progress_cb)
            diffts = datetime.datetime.now()
            cachelife = (60 *
                         3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
                                                                    24)
            cache.set_many({
                cachekey: diff,
                cachekey + 'ts': diffts
            }, cachelife)

        filter_repos = set(request.GET.getlist("repo", None))
        filter_meta = _get_filter_meta(request.GET)
        diff = _sort_filter_diff(diff, repos=filter_repos, meta=filter_meta)

        issue_ref = []
        names = []
        for i in new_repo.platform.issuetrackers.all():
            if i.name not in names:
                issue_ref.append({'name': i.name, 're': i.re, 'url': i.url})
                names.append(i.name)

        full_path = "%s?" % request.path
        for query, values in request.GET.lists():
            full_path += "&".join(['%s=%s' % (query, val) for val in values])

        full_path += "&"

        end = datetime.datetime.now() - start
        context.update({
            'title': "Comparing repositories",
            'packagemetatypes': list(PackageMetaType.objects.all()),
            'diff': diff,
            'diffts': diffts,
            'issue_ref': json.dumps(issue_ref),
            'full_path': full_path,
            'processing_time': end.total_seconds()
        })

        progress_cb("Done")
        return TemplateResponse(
            request,
            "diff_content.html",
            context=context,
        )
Example #31
0
 def test_delete_with_prefix_with_no_reverse_works(self):
     cache.set_many({'K1': 'value', 'K2': 'value2', 'B2': 'Anothervalue'})
     assert cache.delete_with_prefix('K') == 2
     assert cache.get_many(['K1', 'K2', 'B2']) == {'B2': 'Anothervalue'}
Example #32
0
 def _run_get_many(self, keys):
     result = cache.get_many([self._prefix_key(key) for key in keys])
     for key in keys:
         self._cache_data[key] = result.get(self._prefix_key(key))
Example #33
0
def handler(cache_name,
            function=None,
            page_key="",
            force_update_cache=False,
            cache_duration=DEFAULT_CACHE_DURATION,
            *args,
            **kwargs):
    """Handles caching for results and data.

    Args:
        cache: Name of the cache to use.
        function: A function reference that returns some value to be cached. This function must only use **kwargs.
        page_key: The page name or page number to use as a key value.
        force_update_cache:
        cache_duration:
        **kwargs: Any parameters that need to be passed into "function".
    """

    cached_results = None
    # Looks through cache and will perform a search if needed.
    try:
        log.handler(
            cache_name + " - Accessed.",
            log.DEBUG,
            __logger,
        )

        # If the function was actually a list, then use set_many and/or get_many
        # All items must belong to the same cache
        # { page_key: {
        #               "function": function_value,
        #               "kwargs": {kwargs_value},
        #               "args": [args_values],
        #             },
        # ... }
        if isinstance(function, dict):
            if len(function) == 0:
                # Nothing was passed in
                return None
            else:
                # Obtain all the keys from the passed in dictionary
                requested_keys = []
                for key, value in function.items():
                    cache_key = generate_cache_key(cache_name, value["args"],
                                                   value["kwargs"], key)
                    log.handler(
                        cache_name +
                        " - Multi-execution generated cache key " + cache_key,
                        log.DEBUG,
                        __logger,
                    )
                    requested_keys.append(cache_key)

                # Search cache for all keys
                cached_results = cache.get_many(requested_keys)
                log.handler(
                    cache_name + " - Multi-execution detected " +
                    str(len(cached_results)) + " available keys.",
                    log.INFO,
                    __logger,
                )

                # If nothing was in cache, or cache was expired, run function()
                thread_list = []
                for cache_key in requested_keys:
                    if not cached_results.__contains__(cache_key):
                        key = obtain_key_from_cache_key(cache_key)
                        thread = ReturnThread(
                            target=function[key]["function"],
                            args=function[key]["args"],
                            kwargs=function[key]["kwargs"],
                        )
                        thread.start()
                        thread_list.append((cache_key, thread))

                missing_keys = {}
                for key, thread in thread_list:
                    missing_keys[key] = thread.join()

                # Set values in cache for any newly executed functions
                if bool(missing_keys):
                    log.handler(
                        cache_name + " - Multi-execution detected " +
                        str(len(missing_keys)) + " missing keys.",
                        log.INFO,
                        __logger,
                    )
                    cache.set_many(missing_keys, cache_duration)

                # Return all results
                cached_results.update(missing_keys)

                # If results were none, log it.
                if cached_results is None:
                    log.handler(
                        cache_name +
                        " - Multi-execution generated no results!",
                        log.WARNING,
                        __logger,
                    )

                return cached_results

        # Get the cached value
        cache_key = generate_cache_key(cache_name, args, kwargs, page_key)

        log.handler(
            cache_name + " - Generated cache key " + cache_key,
            log.DEBUG,
            __logger,
        )
        cached_results = cache.get(cache_key)

        # No function was provided, just return bare cache value
        if function is None:
            log.handler(
                cache_name + " - Requested raw cache values.",
                log.DEBUG,
                __logger,
            )
            return cached_results

        # If the user wants to force update the cache, nothing
        # was in cache, or cache was expired, run function()
        if cached_results is None or force_update_cache:
            function_results = function(*args, **kwargs)
            log.handler(
                cache_name + " - Function " + function.__name__ +
                " has been executed!",
                log.INFO,
                __logger,
            )
            cache.set(cache_key, function_results, cache_duration)
            return function_results

        if cached_results is None:
            log.handler(
                cache_name + " - No cached results found!",
                log.INFO,
                __logger,
            )

        # If a value was in cache and not expired, return that value
        return cached_results

    except:
        # If the function threw an exception, return none.
        if isinstance(function, dict):
            log.handler(
                "Function list failed to execute!",
                log.ERROR,
                __logger,
            )
        else:
            log.handler(
                "Function " + function.__name__ + " failed to execute!",
                log.ERROR,
                __logger,
            )

        return None
Example #34
0
def get_pathways(site, pathway_id=None):
    """
    Read pathways from the cache.
    The cache is populated by a management command, cache_programs.

    Arguments:
        site (Site): django.contrib.sites.models object

    Keyword Arguments:
        pathway_id (string): id identifying a specific pathway to read from the cache.

    Returns:
        list of dict, representing pathways.
        dict, if a specific pathway is requested.
    """
    missing_details_msg_tpl = 'Failed to get details for credit pathway {id} from the cache.'

    if pathway_id:
        pathway = cache.get(PATHWAY_CACHE_KEY_TPL.format(id=pathway_id))
        if not pathway:
            logger.warning(missing_details_msg_tpl.format(id=pathway_id))

        return pathway
    pathway_ids = cache.get(
        SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), [])
    if not pathway_ids:
        logger.warning('Failed to get credit pathway ids from the cache.')

    pathways = cache.get_many([
        PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
        for pathway_id in pathway_ids
    ])
    pathways = list(pathways.values())

    # The get_many above sometimes fails to bring back details cached on one or
    # more Memcached nodes. It doesn't look like these keys are being evicted.
    # 99% of the time all keys come back, but 1% of the time all the keys stored
    # on one or more nodes are missing from the result of the get_many. One
    # get_many may fail to bring these keys back, but a get_many occurring
    # immediately afterwards will succeed in bringing back all the keys. This
    # behavior can be mitigated by trying again for the missing keys, which is
    # what we do here. Splitting the get_many into smaller chunks may also help.
    missing_ids = set(pathway_ids) - {pathway['id'] for pathway in pathways}
    if missing_ids:
        logger.info(
            f'Failed to get details for {len(missing_ids)} pathways. Retrying.'
        )

        retried_pathways = cache.get_many([
            PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
            for pathway_id in missing_ids
        ])
        pathways += list(retried_pathways.values())

        still_missing_ids = set(pathway_ids) - {
            pathway['id']
            for pathway in pathways
        }
        for missing_id in still_missing_ids:
            logger.warning(missing_details_msg_tpl.format(id=missing_id))

    return pathways
Example #35
0
def get_programs(site=None, uuid=None, course=None):  # pylint: disable=redefined-outer-name
    """Read programs from the cache.

    The cache is populated by a management command, cache_programs.

    Keyword Arguments:
        site (Site): django.contrib.sites.models object
        uuid (string): UUID identifying a specific program to read from the cache.
        course (string): course id identifying a specific course run to read from the cache.

    Returns:
        list of dict, representing programs.
        dict, if a specific program is requested.
    """
    if len([arg for arg in (site, uuid, course) if arg is not None]) != 1:
        raise TypeError('get_programs takes exactly one argument')

    missing_details_msg_tpl = u'Failed to get details for program {uuid} from the cache.'

    if uuid:
        program = cache.get(PROGRAM_CACHE_KEY_TPL.format(uuid=uuid))
        if not program:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

        return program
    elif course:
        uuids = cache.get(
            COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course))
        if not uuids:
            # Currently, the cache does not differentiate between a cache miss and a course
            # without programs. After this is changed, log any cache misses here.
            return []
    else:
        uuids = cache.get(
            SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
        if not uuids:
            logger.warning(
                u'Failed to get program UUIDs from the cache for site {}.'.
                format(site.domain))

    programs = cache.get_many(
        [PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in uuids])
    programs = list(programs.values())

    # The get_many above sometimes fails to bring back details cached on one or
    # more Memcached nodes. It doesn't look like these keys are being evicted.
    # 99% of the time all keys come back, but 1% of the time all the keys stored
    # on one or more nodes are missing from the result of the get_many. One
    # get_many may fail to bring these keys back, but a get_many occurring
    # immediately afterwards will succeed in bringing back all the keys. This
    # behavior can be mitigated by trying again for the missing keys, which is
    # what we do here. Splitting the get_many into smaller chunks may also help.
    missing_uuids = set(uuids) - set(program['uuid'] for program in programs)
    if missing_uuids:
        logger.info(
            u'Failed to get details for {count} programs. Retrying.'.format(
                count=len(missing_uuids)))

        retried_programs = cache.get_many([
            PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids
        ])
        programs += list(retried_programs.values())

        still_missing_uuids = set(uuids) - set(program['uuid']
                                               for program in programs)
        for uuid in still_missing_uuids:
            logger.warning(missing_details_msg_tpl.format(uuid=uuid))

    return programs