def get_many_for_update(namespaces, keys):
"""Get cached data, full keys and create namespace counter where necessary.
Arguments namespaces and keys should be a lists of the same size.
Returns a pair (cached_data, full_keys),
where cached_data is a dictionary {full_keys: data}
and full_keys is a list of full keys, in the same order as
given namespaces and keys.
Use cache.set_many({full_key: value}) to finish updating.
It is assumed namespace counters won't change in between. (if they do,
maybe it is even better to immediately 'mark' data as invalid...)
"""
counters = cache.get_many(namespaces)
# Create new counters
new_counters = {x: 1 for x in namespaces if x not in counters}
cache.set_many(new_counters)
# Generate list of full_keys and list of keys to read from cache.
full_keys = []
to_get = []
for namespace, key in zip(namespaces, keys):
counter = counters.get(namespace, None)
if counter:
full_key = make_full_key(namespace, counter, key)
full_keys.append(full_key)
to_get.append(full_key)
else:
full_keys.append(make_full_key(namespace, 1, key))
return cache.get_many(to_get), full_keys
def get_programs_by_uuids(uuids):
"""
Gets a list of programs for the provided uuids
"""
# a list of UUID objects would be a perfectly reasonable parameter to provide
uuid_strings = [six.text_type(handle) for handle in uuids]
programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=handle) for handle in uuid_strings])
programs = list(programs.values())
# The get_many above sometimes fails to bring back details cached on one or
# more Memcached nodes. It doesn't look like these keys are being evicted.
# 99% of the time all keys come back, but 1% of the time all the keys stored
# on one or more nodes are missing from the result of the get_many. One
# get_many may fail to bring these keys back, but a get_many occurring
# immediately afterwards will succeed in bringing back all the keys. This
# behavior can be mitigated by trying again for the missing keys, which is
# what we do here. Splitting the get_many into smaller chunks may also help.
missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
if missing_uuids:
logger.info(
u'Failed to get details for {count} programs. Retrying.'.format(count=len(missing_uuids))
)
retried_programs = cache.get_many([PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids])
programs += list(retried_programs.values())
still_missing_uuids = set(uuid_strings) - set(program['uuid'] for program in programs)
for uuid in still_missing_uuids:
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return programs
def test_get_many(self):
# Multiple cache keys can be returned using get_many
cache.set("a", "a")
cache.set("b", "b")
cache.set("c", "c")
cache.set("d", "d")
self.assertEqual(cache.get_many(["a", "c", "d"]), {"a": "a", "c": "c", "d": "d"})
self.assertEqual(cache.get_many(["a", "b", "e"]), {"a": "a", "b": "b"})
def test_get_many(self):
# get_many
cache.set('a', 'a')
cache.set('b', 'b')
cache.set('c', 'c')
cache.set('d', 'd')
self.assertEqual(cache.get_many(['a', 'c', 'd']), {'a' : 'a', 'c' : 'c', 'd' : 'd'})
self.assertEqual(cache.get_many(['a', 'b', 'e']), {'a' : 'a', 'b' : 'b'})
def test_get_many(self):
# Multiple cache keys can be returned using get_many
cache.set('a', 'a')
cache.set('b', 'b')
cache.set('c', 'c')
cache.set('d', 'd')
self.assertEqual(cache.get_many(['a', 'c', 'd']), {'a': 'a', 'c': 'c', 'd': 'd'})
self.assertEqual(cache.get_many(['a', 'b', 'e']), {'a': 'a', 'b': 'b'})
def iterator(self):
cache_key = self.model._generate_cache_key("QUERY:%s" %
self._get_query_hash())
on_cache_query_attr = self.model.value_to_list_on_cache_query()
to_return = None
to_cache = {}
with_aggregates = len(self.query.aggregates) > 0
key_list = self._fetch_from_query_cache(cache_key)
if key_list is None:
if not with_aggregates:
values_list = [on_cache_query_attr]
if len(self.query.extra):
values_list += self.query.extra.keys()
key_list = [v[0] for v in self.values_list(*values_list)]
to_cache[cache_key] = (datetime.datetime.now(), key_list)
else:
to_return = list(super(CachedQuerySet, self).iterator())
to_cache[cache_key] = (datetime.datetime.now(), [
(row.__dict__[on_cache_query_attr],
dict([(k, row.__dict__[k])
for k in self.query.aggregates.keys()]))
for row in to_return
])
elif with_aggregates:
tmp = key_list
key_list = [k[0] for k in tmp]
with_aggregates = [k[1] for k in tmp]
del tmp
if (not to_return) and key_list:
row_keys = [
self.model.infer_cache_key({on_cache_query_attr: attr})
for attr in key_list
]
cached = cache.get_many(row_keys)
to_return = [(ck in cached) and self.obj_from_datadict(cached[ck])
or ToFetch(force_unicode(key_list[i]))
for i, ck in enumerate(row_keys)]
if len(cached) != len(row_keys):
to_fetch = [
unicode(tr) for tr in to_return if isinstance(tr, ToFetch)
]
fetched = dict([
(force_unicode(r.__dict__[on_cache_query_attr]), r)
for r in models.query.QuerySet(self.model).filter(
**{"%s__in" % on_cache_query_attr: to_fetch})
])
to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)]
or tr) for tr in to_return]
to_cache.update(
dict([(self.model.infer_cache_key(
{on_cache_query_attr: attr}), r._as_dict())
for attr, r in fetched.items()]))
if with_aggregates:
for i, r in enumerate(to_return):
r.__dict__.update(with_aggregates[i])
if len(to_cache):
cache.set_many(to_cache, 60 * 60)
if to_return:
for row in to_return:
if hasattr(row, 'leaf'):
row = row.leaf
row.reset_original_state()
yield row
def bulk_get_rule_status(
self, rules: Sequence[Rule]) -> Mapping[int, GroupRuleStatus]:
keys = [self._build_rule_status_cache_key(rule.id) for rule in rules]
cache_results: Mapping[str, GroupRuleStatus] = cache.get_many(keys)
missing_rule_ids: Set[int] = set()
rule_statuses: Mapping[int, GroupRuleStatus] = {}
for key, rule in zip(keys, rules):
rule_status = cache_results.get(key)
if not rule_status:
missing_rule_ids.add(rule.id)
else:
rule_statuses[rule.id] = rule_status
if missing_rule_ids:
# If not cached, attempt to fetch status from the database
statuses = GroupRuleStatus.objects.filter(
group=self.group, rule_id__in=missing_rule_ids)
to_cache: Sequence[GroupRuleStatus] = list()
for status in statuses:
rule_statuses[status.rule_id] = status
missing_rule_ids.remove(status.rule_id)
to_cache.append(status)
# We might need to create some statuses if they don't already exist
if missing_rule_ids:
# We use `ignore_conflicts=True` here to avoid race conditions where the statuses
# might be created between when we queried above and attempt to create the rows now.
GroupRuleStatus.objects.bulk_create(
[
GroupRuleStatus(rule_id=rule_id,
group=self.group,
project=self.project)
for rule_id in missing_rule_ids
],
ignore_conflicts=True,
)
# Using `ignore_conflicts=True` prevents the pk from being set on the model
# instances. Re-query the database to fetch the rows, they should all exist at this
# point.
statuses = GroupRuleStatus.objects.filter(
group=self.group, rule_id__in=missing_rule_ids)
for status in statuses:
rule_statuses[status.rule_id] = status
missing_rule_ids.remove(status.rule_id)
to_cache.append(status)
if missing_rule_ids:
# Shouldn't happen, but log just in case
self.logger.error(
"Failed to fetch some GroupRuleStatuses in RuleProcessor",
extra={
"missing_rule_ids": missing_rule_ids,
"group_id": self.group.id
},
)
if to_cache:
cache.set_many({
self._build_rule_status_cache_key(item.rule_id): item
for item in to_cache
})
return rule_statuses
def list(self, request):
stats_keys = utils_stats.DEFAULTS.keys()
cached_stats = cache.get_many(stats_keys)
stats = utils_stats.DEFAULTS.copy()
stats.update(cached_stats)
return Response(stats)
def post(self, request, request_id):
try:
data = json.loads(request.body)
validate(data, ANALYSIS_ID_SCHEMA)
ids: Optional[Ids] = cache.get(f'{request_id}/analysis_ids')
if ids is None:
return HttpResponseNotFound('Query Analysis Ids not available')
data = {(tuple(idx[0]), idx[1]): opt for idx, opt in data}
if not any(v['show'] for v in data.values()):
return HttpResponseBadRequest("cannot hide all analyses")
for key in ids:
try:
if ids[key]['name'] != data[key]['name']:
ids[key] = {
**data[key], 'version': ids[key]['version'] + 1
}
else:
ids[key] = {
**data[key], 'version': ids[key]['version']
}
except KeyError:
pass
cache.set(f'{request_id}/analysis_ids', ids)
cached_result = cache.get_many([
f'{request_id}/target_genes',
f'{request_id}/tabular_output_unfiltered'
])
result = cached_result[f'{request_id}/tabular_output_unfiltered']
result = filter_df_by_ids(result, ids)
try:
user_lists = cached_result[f'{request_id}/target_genes']
result = result[result.index.str.upper().isin(
user_lists[0].index.str.upper())].dropna(axis=1, how='all')
if result.empty:
raise QueryError(
"Empty result (user list too restrictive).")
result = reorder_data(result)
except KeyError:
pass
cache.set(f'{request_id}/tabular_output',
result) # refresh filtered tabular output
# delete cache keys and refresh cache here.
cache.delete_many([
# formatted output
f'{request_id}/formatted_tabular_output',
# network
f'{request_id}/network',
# AUPR curve
f'{request_id}/figure',
f'{request_id}/figure_data',
# network stats
f'{request_id}/stats',
# Gene list enrichment
f'{request_id}/list_enrichment',
f'{request_id}/list_enrichment_legend',
f'{request_id}/list_enrichment_data',
# motif enrichment
*(f'{request_id}/{r}_enrich' for r in MOTIFS.regions),
# analysis enrichment
f'{request_id}/analysis_enrichment',
# summary
f'{request_id}/summary'
])
return JsonResponse(list(ids.items()),
status=201,
safe=False,
encoder=PandasJSONEncoder)
except (json.JSONDecodeError, ValidationError, QueryError) as e:
return HttpResponseBadRequest(e)
except KeyError:
return HttpResponseNotFound(
'Query does not exist. Please start a new query.')
def get_cached_objects(pks, model=None, timeout=CACHE_TIMEOUT, missing=RAISE):
"""
Return a list of objects with given PKs using cache.
Params:
pks - list of Primary Key values to look up or list of content_type_id, pk tuples
model - ContentType instance representing the model's class or the model class itself
timeout - TTL for the items in cache, defaults to CACHE_TIMEOUT
Throws:
model.DoesNotExist is propagated from content_type.get_object_for_this_type
"""
if model is not None:
if not isinstance(model, ContentType):
model = ContentType.objects.get_for_model(model)
pks = [(model, pk) for pk in pks]
else:
pks = [(ContentType.objects.get_for_id(ct_id), pk) for (ct_id, pk) in pks]
keys = [_get_key(KEY_PREFIX, model, pk=pk) for (model, pk) in pks]
cached = cache.get_many(keys)
# keys not in cache
keys_to_set = set(keys) - set(cached.keys())
if keys_to_set:
# build lookup to get model and pks from the key
lookup = dict(zip(keys, pks))
to_get = {}
# group lookups by CT so we can do in_bulk
for k in keys_to_set:
ct, pk = lookup[k]
to_get.setdefault(ct, {})[int(pk)] = k
# take out all the publishables
publishable_ct = ContentType.objects.get_for_model(get_model('core', 'publishable'))
if publishable_ct in to_get:
publishable_keys = to_get.pop(publishable_ct)
models = publishable_ct.model_class()._default_manager.values('content_type_id', 'id').filter(id__in=publishable_keys.keys())
for m in models:
ct = ContentType.objects.get_for_id(m['content_type_id'])
pk = m['id']
# and put them back as their native content_type
to_get.setdefault(ct, {})[pk] = publishable_keys[pk]
to_set = {}
# retrieve all the models from DB
for ct, vals in to_get.items():
models = ct.model_class()._default_manager.in_bulk(vals.keys())
for pk, m in models.items():
k = vals[pk]
cached[k] = to_set[k] = m
if not isinstance(cache, DummyCache):
# write them into cache
cache.set_many(to_set, timeout=timeout)
out = []
for k in keys:
try:
out.append(cached[k])
except KeyError:
if missing == NONE:
out.append(None)
elif missing == SKIP:
pass
elif missing == RAISE:
ct = ContentType.objects.get_for_id(int(k.split(':')[1]))
raise ct.model_class().DoesNotExist(
'%s matching query does not exist.' % ct.model_class()._meta.object_name)
return out
def query_request(kb_ids, issue, kb_vers_map):
"""
requirement: (1)cache中查找知识库的地址，cache中没有的，则为其在cache中没有被其他kb占用的box中选取BOX，并异步写入其TARGET
(2)遍历/B/vm/下的子节点，找到空闲节点，为其赋一个知识库，单独起进程发送请求，请求结果放入queue
param: kb_ids 列表
return:
"""
# 取cache中已有的kb
logger.debug('start get add')
start_get_add = time()
kb_add_dict = cache.get_many(kb_ids)
no_kbs = set(kb_ids) - set(kb_add_dict.keys())
logger.debug('no_kbs:%s' % no_kbs)
# 为cache中没有的kb赋予box
boxs = [('/B/83c4ee846cf2/B70/', '192.168.30.187:8000/70'),
('/B/83c4ee846cf2/B74/', '192.168.30.187:8000/74'),
('/B/83c4ee846cf2/B73/', '192.168.30.187:8000/73'),
('/B/83c4ee846cf2/B72/', '192.168.30.187:8000/72'),
('/B/83c4ee846cf2/B71/', '192.168.30.187:8000/71'),
('/B/83c4ee846cf2/B30/', '192.168.30.187:8000/30'),
('/B/83c4ee846cf2/B23/', '192.168.30.187:8000/23'),
('/B/83c4ee846cf2/B22/', '192.168.30.187:8000/22'),
('/B/83c4ee846cf2/B21/', '192.168.30.187:8000/21'),
('/B/83c4ee846cf2/B20/', '192.168.30.187:8000/20'),
('/B/83c4ee846cf2/B27/', '192.168.30.187:8000/27'),
('/B/83c4ee846cf2/B26/', '192.168.30.187:8000/26'),
('/B/83c4ee846cf2/B25/', '192.168.30.187:8000/25'),
('/B/83c4ee846cf2/B24/', '192.168.30.187:8000/24'),
('/B/83c4ee846cf2/B66/', '192.168.30.187:8000/66'),
('/B/83c4ee846cf2/B67/', '192.168.30.187:8000/67'),
('/B/83c4ee846cf2/B64/', '192.168.30.187:8000/64'),
('/B/83c4ee846cf2/B29/', '192.168.30.187:8000/29'),
('/B/83c4ee846cf2/B65/', '192.168.30.187:8000/65'),
('/B/83c4ee846cf2/B28/', '192.168.30.187:8000/28'),
('/B/83c4ee846cf2/B68/', '192.168.30.187:8000/68'),
('/B/83c4ee846cf2/B69/', '192.168.30.187:8000/69'),
('/B/83c4ee846cf2/B5/', '192.168.30.187:8000/5'),
('/B/83c4ee846cf2/B4/', '192.168.30.187:8000/4'),
('/B/83c4ee846cf2/B81/', '192.168.30.187:8000/81'),
('/B/83c4ee846cf2/B3/', '192.168.30.187:8000/3'),
('/B/83c4ee846cf2/B80/', '192.168.30.187:8000/80'),
('/B/83c4ee846cf2/B2/', '192.168.30.187:8000/2'),
('/B/83c4ee846cf2/B83/', '192.168.30.187:8000/83'),
('/B/83c4ee846cf2/B9/', '192.168.30.187:8000/9'),
('/B/83c4ee846cf2/B82/', '192.168.30.187:8000/82'),
('/B/83c4ee846cf2/B8/', '192.168.30.187:8000/8'),
('/B/83c4ee846cf2/B85/', '192.168.30.187:8000/85'),
('/B/83c4ee846cf2/B7/', '192.168.30.187:8000/7'),
('/B/83c4ee846cf2/B84/', '192.168.30.187:8000/84'),
('/B/83c4ee846cf2/B6/', '192.168.30.187:8000/6'),
('/B/83c4ee846cf2/B40/', '192.168.30.187:8000/40'),
('/B/83c4ee846cf2/B41/', '192.168.30.187:8000/41'),
('/B/83c4ee846cf2/B32/', '192.168.30.187:8000/32'),
('/B/83c4ee846cf2/B31/', '192.168.30.187:8000/31'),
('/B/83c4ee846cf2/B34/', '192.168.30.187:8000/34'),
('/B/83c4ee846cf2/B33/', '192.168.30.187:8000/33'),
('/B/83c4ee846cf2/B36/', '192.168.30.187:8000/36'),
('/B/83c4ee846cf2/B35/', '192.168.30.187:8000/35'),
('/B/83c4ee846cf2/B38/', '192.168.30.187:8000/38'),
('/B/83c4ee846cf2/B37/', '192.168.30.187:8000/37'),
('/B/83c4ee846cf2/B75/', '192.168.30.187:8000/75'),
('/B/83c4ee846cf2/B76/', '192.168.30.187:8000/76'),
('/B/83c4ee846cf2/B39/', '192.168.30.187:8000/39'),
('/B/83c4ee846cf2/B77/', '192.168.30.187:8000/77'),
('/B/83c4ee846cf2/B78/', '192.168.30.187:8000/78'),
('/B/83c4ee846cf2/B79/', '192.168.30.187:8000/79'),
('/B/83c4ee846cf2/B1/', '192.168.30.187:8000/1'),
('/B/83c4ee846cf2/B19/', '192.168.30.187:8000/19'),
('/B/83c4ee846cf2/B17/', '192.168.30.187:8000/17'),
('/B/83c4ee846cf2/B18/', '192.168.30.187:8000/18'),
('/B/83c4ee846cf2/B90/', '192.168.30.187:8000/90'),
('/B/83c4ee846cf2/B51/', '192.168.30.187:8000/51'),
('/B/83c4ee846cf2/B11/', '192.168.30.187:8000/11'),
('/B/83c4ee846cf2/B52/', '192.168.30.187:8000/52'),
('/B/83c4ee846cf2/B12/', '192.168.30.187:8000/12'),
('/B/83c4ee846cf2/B50/', '192.168.30.187:8000/50'),
('/B/83c4ee846cf2/B10/', '192.168.30.187:8000/10'),
('/B/83c4ee846cf2/B15/', '192.168.30.187:8000/15'),
('/B/83c4ee846cf2/B16/', '192.168.30.187:8000/16'),
('/B/83c4ee846cf2/B13/', '192.168.30.187:8000/13'),
('/B/83c4ee846cf2/B14/', '192.168.30.187:8000/14'),
('/B/83c4ee846cf2/B49/', '192.168.30.187:8000/49'),
('/B/83c4ee846cf2/B48/', '192.168.30.187:8000/48'),
('/B/83c4ee846cf2/B47/', '192.168.30.187:8000/47'),
('/B/83c4ee846cf2/B46/', '192.168.30.187:8000/46'),
('/B/83c4ee846cf2/B45/', '192.168.30.187:8000/45'),
('/B/83c4ee846cf2/B44/', '192.168.30.187:8000/44'),
('/B/83c4ee846cf2/B43/', '192.168.30.187:8000/43'),
('/B/83c4ee846cf2/B42/', '192.168.30.187:8000/42'),
('/B/83c4ee846cf2/B88/', '192.168.30.187:8000/88'),
('/B/83c4ee846cf2/B89/', '192.168.30.187:8000/89'),
('/B/83c4ee846cf2/B86/', '192.168.30.187:8000/86'),
('/B/83c4ee846cf2/B87/', '192.168.30.187:8000/87'),
('/B/83c4ee846cf2/B60/', '192.168.30.187:8000/60'),
('/B/83c4ee846cf2/B61/', '192.168.30.187:8000/61'),
('/B/83c4ee846cf2/B62/', '192.168.30.187:8000/62'),
('/B/83c4ee846cf2/B63/', '192.168.30.187:8000/63'),
('/B/83c4ee846cf2/B58/', '192.168.30.187:8000/58'),
('/B/83c4ee846cf2/B57/', '192.168.30.187:8000/57'),
('/B/83c4ee846cf2/B59/', '192.168.30.187:8000/59'),
('/B/83c4ee846cf2/B54/', '192.168.30.187:8000/54'),
('/B/83c4ee846cf2/B53/', '192.168.30.187:8000/53'),
('/B/83c4ee846cf2/B56/', '192.168.30.187:8000/56'),
('/B/83c4ee846cf2/B55/', '192.168.30.187:8000/55'),
('/B/d204c1d12b8a/B70/', '192.168.30.186:8000/70'),
('/B/d204c1d12b8a/B74/', '192.168.30.186:8000/74'),
('/B/d204c1d12b8a/B73/', '192.168.30.186:8000/73'),
('/B/d204c1d12b8a/B72/', '192.168.30.186:8000/72'),
('/B/d204c1d12b8a/B71/', '192.168.30.186:8000/71'),
('/B/d204c1d12b8a/B30/', '192.168.30.186:8000/30'),
('/B/d204c1d12b8a/B23/', '192.168.30.186:8000/23'),
('/B/d204c1d12b8a/B22/', '192.168.30.186:8000/22'),
('/B/d204c1d12b8a/B21/', '192.168.30.186:8000/21'),
('/B/d204c1d12b8a/B20/', '192.168.30.186:8000/20'),
('/B/d204c1d12b8a/B27/', '192.168.30.186:8000/27'),
('/B/d204c1d12b8a/B26/', '192.168.30.186:8000/26'),
('/B/d204c1d12b8a/B25/', '192.168.30.186:8000/25'),
('/B/d204c1d12b8a/B24/', '192.168.30.186:8000/24'),
('/B/d204c1d12b8a/B66/', '192.168.30.186:8000/66'),
('/B/d204c1d12b8a/B67/', '192.168.30.186:8000/67'),
('/B/d204c1d12b8a/B64/', '192.168.30.186:8000/64'),
('/B/d204c1d12b8a/B29/', '192.168.30.186:8000/29'),
('/B/d204c1d12b8a/B65/', '192.168.30.186:8000/65'),
('/B/d204c1d12b8a/B28/', '192.168.30.186:8000/28'),
('/B/d204c1d12b8a/B68/', '192.168.30.186:8000/68'),
('/B/d204c1d12b8a/B69/', '192.168.30.186:8000/69'),
('/B/d204c1d12b8a/B5/', '192.168.30.186:8000/5'),
('/B/d204c1d12b8a/B4/', '192.168.30.186:8000/4'),
('/B/d204c1d12b8a/B81/', '192.168.30.186:8000/81'),
('/B/d204c1d12b8a/B3/', '192.168.30.186:8000/3'),
('/B/d204c1d12b8a/B80/', '192.168.30.186:8000/80'),
('/B/d204c1d12b8a/B2/', '192.168.30.186:8000/2'),
('/B/d204c1d12b8a/B83/', '192.168.30.186:8000/83'),
('/B/d204c1d12b8a/B9/', '192.168.30.186:8000/9'),
('/B/d204c1d12b8a/B82/', '192.168.30.186:8000/82'),
('/B/d204c1d12b8a/B8/', '192.168.30.186:8000/8'),
('/B/d204c1d12b8a/B85/', '192.168.30.186:8000/85'),
('/B/d204c1d12b8a/B7/', '192.168.30.186:8000/7'),
('/B/d204c1d12b8a/B84/', '192.168.30.186:8000/84'),
('/B/d204c1d12b8a/B6/', '192.168.30.186:8000/6'),
('/B/d204c1d12b8a/B40/', '192.168.30.186:8000/40'),
('/B/d204c1d12b8a/B41/', '192.168.30.186:8000/41'),
('/B/d204c1d12b8a/B32/', '192.168.30.186:8000/32'),
('/B/d204c1d12b8a/B31/', '192.168.30.186:8000/31'),
('/B/d204c1d12b8a/B34/', '192.168.30.186:8000/34'),
('/B/d204c1d12b8a/B33/', '192.168.30.186:8000/33'),
('/B/d204c1d12b8a/B36/', '192.168.30.186:8000/36'),
('/B/d204c1d12b8a/B35/', '192.168.30.186:8000/35'),
('/B/d204c1d12b8a/B38/', '192.168.30.186:8000/38'),
('/B/d204c1d12b8a/B37/', '192.168.30.186:8000/37'),
('/B/d204c1d12b8a/B75/', '192.168.30.186:8000/75'),
('/B/d204c1d12b8a/B76/', '192.168.30.186:8000/76'),
('/B/d204c1d12b8a/B39/', '192.168.30.186:8000/39'),
('/B/d204c1d12b8a/B77/', '192.168.30.186:8000/77'),
('/B/d204c1d12b8a/B78/', '192.168.30.186:8000/78'),
('/B/d204c1d12b8a/B79/', '192.168.30.186:8000/79'),
('/B/d204c1d12b8a/B1/', '192.168.30.186:8000/1'),
('/B/d204c1d12b8a/B19/', '192.168.30.186:8000/19'),
('/B/d204c1d12b8a/B17/', '192.168.30.186:8000/17'),
('/B/d204c1d12b8a/B18/', '192.168.30.186:8000/18'),
('/B/d204c1d12b8a/B90/', '192.168.30.186:8000/90'),
('/B/d204c1d12b8a/B51/', '192.168.30.186:8000/51'),
('/B/d204c1d12b8a/B11/', '192.168.30.186:8000/11'),
('/B/d204c1d12b8a/B52/', '192.168.30.186:8000/52'),
('/B/d204c1d12b8a/B12/', '192.168.30.186:8000/12'),
('/B/d204c1d12b8a/B50/', '192.168.30.186:8000/50'),
('/B/d204c1d12b8a/B10/', '192.168.30.186:8000/10'),
('/B/d204c1d12b8a/B15/', '192.168.30.186:8000/15'),
('/B/d204c1d12b8a/B16/', '192.168.30.186:8000/16'),
('/B/d204c1d12b8a/B13/', '192.168.30.186:8000/13'),
('/B/d204c1d12b8a/B14/', '192.168.30.186:8000/14'),
('/B/d204c1d12b8a/B49/', '192.168.30.186:8000/49'),
('/B/d204c1d12b8a/B48/', '192.168.30.186:8000/48'),
('/B/d204c1d12b8a/B47/', '192.168.30.186:8000/47'),
('/B/d204c1d12b8a/B46/', '192.168.30.186:8000/46'),
('/B/d204c1d12b8a/B45/', '192.168.30.186:8000/45'),
('/B/d204c1d12b8a/B44/', '192.168.30.186:8000/44'),
('/B/d204c1d12b8a/B43/', '192.168.30.186:8000/43'),
('/B/d204c1d12b8a/B42/', '192.168.30.186:8000/42'),
('/B/d204c1d12b8a/B88/', '192.168.30.186:8000/88'),
('/B/d204c1d12b8a/B89/', '192.168.30.186:8000/89'),
('/B/d204c1d12b8a/B86/', '192.168.30.186:8000/86'),
('/B/d204c1d12b8a/B87/', '192.168.30.186:8000/87'),
('/B/d204c1d12b8a/B60/', '192.168.30.186:8000/60'),
('/B/d204c1d12b8a/B61/', '192.168.30.186:8000/61'),
('/B/d204c1d12b8a/B62/', '192.168.30.186:8000/62'),
('/B/d204c1d12b8a/B63/', '192.168.30.186:8000/63'),
('/B/d204c1d12b8a/B58/', '192.168.30.186:8000/58'),
('/B/d204c1d12b8a/B57/', '192.168.30.186:8000/57'),
('/B/d204c1d12b8a/B59/', '192.168.30.186:8000/59'),
('/B/d204c1d12b8a/B54/', '192.168.30.186:8000/54'),
('/B/d204c1d12b8a/B53/', '192.168.30.186:8000/53'),
('/B/d204c1d12b8a/B56/', '192.168.30.186:8000/56'),
('/B/d204c1d12b8a/B55/', '192.168.30.186:8000/55')]
boxs_free = []
if kb_add_dict:
boxs_free = set(dict(boxs).keys()) - set(
dict(kb_add_dict.values()).keys())
else:
boxs_free = set(dict(boxs).keys())
if len(boxs_free) < len(no_kbs):
rest_kbs = no_kbs[len(boxs_free):]
kb_ids = set(kb_ids) - set(rest_kbs)
# 写入cache
boxs_free_info = filter(lambda x: x[0] in boxs_free, boxs)
temp_kb_box_list = list(zip(no_kbs, boxs_free_info))
cache_ret = map(lambda x: cache.set(x[0], x[1], 30 * 60), temp_kb_box_list)
logger.debug('cache_ret:%s' % list(cache_ret))
kb_add_dict = cache.get_many(kb_ids)
logger.debug('kb_add_dict:%s' % kb_add_dict)
logger.debug('------get address time:%.5f' % (time() - start_get_add))
logger.debug('start box-request ')
start_request = time()
num = len(kb_ids)
q = Manager().Queue()
p_list = []
for i in range(0, num):
kb = kb_ids[i]
version = kb_vers_map[kb]
add = kb_add_dict[kb][1]
logger.debug('Target:%s Add:%s' % (kb, add))
temp_p = Process(target=_writer, args=(q, kb, version, add, issue))
p_list.append(temp_p)
temp_p.start()
for pr in p_list:
pr.join()
logger.debug('------box-request time:%.5f' % (time() - start_request))
start_get_msg = time()
i = 0
ret = {'no_box': [], 'ans': [], 'not_match': [], 'fail': []}
while not q.empty():
msg = q.get()
if 'not_match' in msg.keys():
ret['not_match'].append(msg['not_match'])
elif 'fail' in msg.keys():
ret['fail'].append(msg['fail'])
else:
ret['ans'].append(msg)
logger.debug('------%d msg:%s' % (i, msg))
i += 1
logger.debug('------get answers time:%.5f' % (time() - start_get_msg))
# 异步写入zk
# set_box_zk.delay(temp_kb_box_list)
return ret
def get(document, cache_control, base_url, timeout=None):
"""Perform a kumascript GET request for a document locale and slug."""
if not cache_control:
# Default to the configured max-age for cache control.
max_age = config.KUMASCRIPT_MAX_AGE
cache_control = 'max-age=%s' % max_age
if not base_url:
site = Site.objects.get_current()
base_url = 'http://%s' % site.domain
if not timeout:
timeout = config.KUMASCRIPT_TIMEOUT
document_locale = document.locale
document_slug = document.slug
max_age = config.KUMASCRIPT_MAX_AGE
# 1063580 - Kumascript converts template name calls to lower case and bases
# caching keys off of that.
document_slug_for_kumascript = document_slug
body, errors = None, None
try:
url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
url = unicode(url_tmpl).format(path=u'%s/%s' %
(document_locale,
document_slug_for_kumascript))
cache_keys = build_cache_keys(document_slug, document_locale)
etag_key, modified_key, body_key, errors_key = cache_keys
headers = {
'X-FireLogger': '1.2',
'Cache-Control': cache_control,
}
# Create the file interface
files = []
for attachment in document.files.select_related('current_revision'):
files.append(_get_attachment_metadata_dict(attachment))
# Assemble some KumaScript env vars
# TODO: See dekiscript vars for future inspiration
# http://developer.mindtouch.com/en/docs/DekiScript/Reference/
# Wiki_Functions_and_Variables
path = document.get_absolute_url()
# TODO: Someday merge with _get_document_for_json in views.py
# where most of this is duplicated code.
env_vars = dict(
path=path,
url=urljoin(base_url, path),
id=document.pk,
revision_id=document.current_revision.pk,
locale=document.locale,
title=document.title,
files=files,
slug=document.slug,
tags=list(document.tags.names()),
review_tags=list(document.current_revision.review_tags.names()),
modified=time.mktime(document.modified.timetuple()),
cache_control=cache_control,
)
add_env_headers(headers, env_vars)
# Set up for conditional GET, if we have the details cached.
cached_meta = cache.get_many([etag_key, modified_key])
if etag_key in cached_meta:
headers['If-None-Match'] = cached_meta[etag_key]
if modified_key in cached_meta:
headers['If-Modified-Since'] = cached_meta[modified_key]
# Finally, fire off the request.
response = requests.get(url, headers=headers, timeout=timeout)
if response.status_code == 304:
# Conditional GET was a pass, so use the cached content.
result = cache.get_many([body_key, errors_key])
body = result.get(body_key, '').decode('utf-8')
errors = result.get(errors_key, None)
elif response.status_code == 200:
body = process_body(response)
errors = process_errors(response)
# Cache the request for conditional GET, but use the max_age for
# the cache timeout here too.
headers = response.headers
cache.set(etag_key, headers.get('etag'), timeout=max_age)
cache.set(modified_key, headers.get('last-modified'), timeout=max_age)
cache.set(body_key, body.encode('utf-8'), timeout=max_age)
if errors:
cache.set(errors_key, errors, timeout=max_age)
elif response.status_code is None:
errors = KUMASCRIPT_TIMEOUT_ERROR
else:
errors = [
{
"level": "error",
"message": "Unexpected response from Kumascript service: %s" %
response.status_code,
"args": ["UnknownError"],
},
]
except Exception as exc:
# Last resort: Something went really haywire. Kumascript server died
# mid-request, or something. Try to report at least some hint.
errors = [
{
"level": "error",
"message": "Kumascript service failed unexpectedly: %s" % exc,
"args": ["UnknownError"],
},
]
return (body, errors)
def get(self, request, project):
"""
Get list with release setup progress for a project
1. tag an error
2. link a repo
3. associate commits
4. tell sentry about a deploy
"""
tag_key = "onboard_tag:1:%s" % (project.id)
repo_key = "onboard_repo:1:%s" % (project.organization_id)
commit_key = "onboard_commit:1:%s" % hash_values(
[project.organization_id, project.id])
deploy_key = "onboard_deploy:1:%s" % hash_values(
[project.organization_id, project.id])
onboard_cache = cache.get_many(
[tag_key, repo_key, commit_key, deploy_key])
tag = onboard_cache.get(tag_key)
if tag is None:
tag = Group.objects.filter(project=project.id,
first_release__isnull=False).exists()
cache.set(tag_key, tag, 3600 if tag else 60)
repo = onboard_cache.get(repo_key)
if repo is None:
repo = Repository.objects.filter(
organization_id=project.organization_id).exists()
cache.set(repo_key, repo, 3600 if repo else 60)
commit = onboard_cache.get(commit_key)
if commit is None:
commit = ReleaseCommit.objects.filter(
organization_id=project.organization_id,
release__projects=project.id).exists()
cache.set(commit_key, commit, 3600 if commit else 60)
deploy = onboard_cache.get(deploy_key)
if deploy is None:
deploy = Deploy.objects.filter(
organization_id=project.organization_id,
release__projects=project.id).exists()
cache.set(deploy_key, deploy, 3600 if deploy else 60)
return Response([
{
"step": "tag",
"complete": bool(tag)
},
{
"step": "repo",
"complete": bool(repo)
},
{
"step": "commit",
"complete": bool(commit)
},
{
"step": "deploy",
"complete": bool(deploy)
},
])
def query_request_new(kb_ids, issue, kb_vers_map):
"""
tips: cache形式：cache.set(box ,(kb, 0/1), 过期时间)
requirement: (1)查看boxs列表, 对比cache中的boxs, 若有空闲的box，则为其赋一个知识库；
若没有空闲box，则在cache中寻找该KB已有的box。
(2)取出请求中所有知识库对应的BOX的地址，单独起进程发送请求，请求结果放入queue
param: kb_ids 列表
return:
"""
# 取cache中已有的kb
logger.debug('start get add')
start_get_add = time()
kb_add_dict = cache.get_many(kb_ids)
no_kbs = set(kb_ids) - set(kb_add_dict.keys())
boxs = cache.get('boxs')
box_addr_dict = cache.get('box_infos')
box_kb_dict = cache.get_many(boxs)
box_kb_rest = list(
filter(lambda x: (x[1][0] in boxs) and (not x[1][1]),
box_kb_dict.items()))
boxs_idle = list(filter(lambda x: not cache.get(x), boxs))
logger.debug('boxs_idle:%s' % boxs_idle)
# 为cache中没有的kb赋予box
boxs_free = []
if kb_add_dict:
boxs_free = set(dict(boxs).keys()) - set(
dict(kb_add_dict.values()).keys())
else:
boxs_free = set(dict(boxs).keys())
if len(boxs_free) < len(no_kbs):
rest_kbs = no_kbs[len(boxs_free):]
kb_ids = set(kb_ids) - set(rest_kbs)
# 写入cache
boxs_free_info = filter(lambda x: x[0] in boxs_free, boxs)
temp_kb_box_list = list(zip(no_kbs, boxs_free_info))
cache_ret = map(lambda x: cache.set(x[0], x[1], 30 * 60), temp_kb_box_list)
logger.debug('cache_ret:%s' % list(cache_ret))
kb_add_dict = cache.get_many(kb_ids)
logger.debug('kb_add_dict:%s' % kb_add_dict)
logger.debug('------get address time:%.5f' % (time() - start_get_add))
logger.debug('start box-request ')
start_request = time()
num = len(kb_ids)
q = Manager().Queue()
p_list = []
for i in range(0, num):
kb = kb_ids[i]
version = kb_vers_map[kb]
add = kb_add_dict[kb][1]
logger.debug('Target:%s Add:%s' % (kb, add))
temp_p = Process(target=_writer, args=(q, kb, version, add, issue))
p_list.append(temp_p)
temp_p.start()
for pr in p_list:
pr.join()
logger.debug('------box-request time:%.5f' % (time() - start_request))
start_get_msg = time()
i = 0
ret = {'no_box': [], 'ans': [], 'not_match': [], 'fail': []}
while not q.empty():
msg = q.get()
if 'not_match' in msg.keys():
ret['not_match'].append(msg['not_match'])
elif 'fail' in msg.keys():
ret['fail'].append(msg['fail'])
else:
ret['ans'].append(msg)
logger.debug('------%d msg:%s' % (i, msg))
i += 1
logger.debug('------get answers time:%.5f' % (time() - start_get_msg))
# 异步写入zk
# set_box_zk.delay(temp_kb_box_list)
return ret
def _get_cached_long_urls(self, long_urls):
return cache.get_many(long_urls)
def get(document, cache_control, base_url, timeout=None):
"""Perform a kumascript GET request for a document locale and slug."""
if not cache_control:
# Default to the configured max-age for cache control.
max_age = constance.config.KUMASCRIPT_MAX_AGE
cache_control = 'max-age=%s' % max_age
if not base_url:
site = Site.objects.get_current()
base_url = 'http://%s' % site.domain
if not timeout:
timeout = constance.config.KUMASCRIPT_TIMEOUT
document_locale = document.locale
document_slug = document.slug
max_age = constance.config.KUMASCRIPT_MAX_AGE
resp_body, resp_errors = None, None
try:
url_tmpl = settings.KUMASCRIPT_URL_TEMPLATE
url = unicode(url_tmpl).format(path=u'%s/%s' %
(document_locale, document_slug))
ck_etag, ck_modified, ck_body, ck_errors = (build_cache_keys(
document_slug, document_locale))
headers = {'X-FireLogger': '1.2', 'Cache-Control': cache_control}
# Create the file interface
files = []
for attachment in document.attachments.all():
files.append({
'title': attachment.title,
'description': attachment.current_revision.description,
'filename': attachment.current_revision.filename(),
'size': attachment.current_revision.file.size,
'author': attachment.current_revision.creator.username,
'mime': attachment.current_revision.mime_type,
'url': attachment.get_file_url(),
})
# Assemble some KumaScript env vars
# TODO: See dekiscript vars for future inspiration
# http://developer.mindtouch.com/en/docs/DekiScript/Reference/
# Wiki_Functions_and_Variables
path = document.get_absolute_url()
env_vars = dict(
path=path,
url=urljoin(base_url, path),
id=document.pk,
locale=document.locale,
title=document.title,
files=files,
attachments=files, # Just for sake of verbiage?
slug=document.slug,
tags=[x.name for x in document.tags.all()],
modified=time.mktime(document.modified.timetuple()),
cache_control=cache_control,
)
add_env_headers(headers, env_vars)
# Set up for conditional GET, if we have the details cached.
c_meta = cache.get_many([ck_etag, ck_modified])
if ck_etag in c_meta:
headers['If-None-Match'] = c_meta[ck_etag]
if ck_modified in c_meta:
headers['If-Modified-Since'] = c_meta[ck_modified]
# Finally, fire off the request.
statsd.incr('wiki.ks_get')
with statsd.timer('wiki.ks_get'):
resp = requests.get(url, headers=headers, timeout=timeout)
if resp.status_code == 304:
# Conditional GET was a pass, so use the cached content.
c_result = cache.get_many([ck_body, ck_errors])
resp_body = c_result.get(ck_body, '').decode('utf-8')
resp_errors = c_result.get(ck_errors, None)
elif resp.status_code == 200:
resp_body = process_body(resp)
resp_errors = process_errors(resp)
# Cache the request for conditional GET, but use the max_age for
# the cache timeout here too.
cache.set(ck_etag, resp.headers.get('etag'), timeout=max_age)
cache.set(ck_modified,
resp.headers.get('last-modified'),
timeout=max_age)
cache.set(ck_body, resp_body.encode('utf-8'), timeout=max_age)
if resp_errors:
cache.set(ck_errors, resp_errors, timeout=max_age)
elif resp.status_code == None:
resp_errors = KUMASCRIPT_TIMEOUT_ERROR
else:
resp_errors = [{
"level":
"error",
"message":
"Unexpected response from Kumascript service: %s" %
resp.status_code,
"args": ["UnknownError"]
}]
except Exception, e:
# Last resort: Something went really haywire. Kumascript server died
# mid-request, or something. Try to report at least some hint.
resp_errors = [{
"level":
"error",
"message":
"Kumascript service failed unexpectedly: %s" % type(e),
"args": ["UnknownError"]
}]
from django.core.cache import cache # 导入的cache就是配置的默认缓存
from django.core.cache import caches # caches相当于全部缓存集
from django.views.decorators.cache import never_cache, cache_page, cache_control
def_cache = caches['default']
rds_cache = caches['redis']
assert def_cache == cache
assert rds_cache != cache
print('def_cache', def_cache)
print('rds_cache', rds_cache)
# 2）设置，如果将exp过期时间设置0或负值，就是删除缓存
# cache.set('key', 'value', exp=1000)
cache.set('key', 'value')
print(cache.set_many({'a': 1, 'b': 2, 'c': 3}))
print(cache.get_many(['a', 'b', 'c']))
# 3）获取
cache.get('key')
cache.set('num', 1)
cache.incr('num')
cache.incr('num', 10)
cache.decr('num')
cache.decr('num', 5)
cache.clear()
@cache_page(60 * 15, cache="redis") # 可以选用缓存方式
@cache_page(60 * 15, key_prefix="site1")
@cache_control(max_age=3600)
@never_cache
def myview(request):
def test_cache_versioning_get_set_many(self):
# set, using default version = 1
cache.set_many({'ford1': 37, 'arthur1': 42})
assert (
cache.get_many(['ford1', 'arthur1']) ==
{'ford1': 37, 'arthur1': 42}
)
assert (
cache.get_many(['ford1', 'arthur1'], version=1) ==
{'ford1': 37, 'arthur1': 42}
)
assert cache.get_many(['ford1', 'arthur1'], version=2) == {}
assert caches['v2'].get_many(['ford1', 'arthur1']) == {}
assert (
caches['v2'].get_many(['ford1', 'arthur1'], version=1) ==
{'ford1': 37, 'arthur1': 42}
)
assert caches['v2'].get_many(['ford1', 'arthur1'], version=2) == {}
# set, default version = 1, but manually override version = 2
cache.set_many({'ford2': 37, 'arthur2': 42}, version=2)
assert cache.get_many(['ford2', 'arthur2']) == {}
assert cache.get_many(['ford2', 'arthur2'], version=1) == {}
assert (
cache.get_many(['ford2', 'arthur2'], version=2) ==
{'ford2': 37, 'arthur2': 42}
)
assert (
caches['v2'].get_many(['ford2', 'arthur2']) ==
{'ford2': 37, 'arthur2': 42}
)
assert caches['v2'].get_many(['ford2', 'arthur2'], version=1) == {}
assert (
caches['v2'].get_many(['ford2', 'arthur2'], version=2) ==
{'ford2': 37, 'arthur2': 42}
)
# v2 set, using default version = 2
caches['v2'].set_many({'ford3': 37, 'arthur3': 42})
assert cache.get_many(['ford3', 'arthur3']) == {}
assert cache.get_many(['ford3', 'arthur3'], version=1) == {}
assert (
cache.get_many(['ford3', 'arthur3'], version=2) ==
{'ford3': 37, 'arthur3': 42}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3']) ==
{'ford3': 37, 'arthur3': 42}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3'], version=1) ==
{}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3'], version=2) ==
{'ford3': 37, 'arthur3': 42}
)
# v2 set, default version = 2, but manually override version = 1
caches['v2'].set_many({'ford4': 37, 'arthur4': 42}, version=1)
assert (
cache.get_many(['ford4', 'arthur4']) ==
{'ford4': 37, 'arthur4': 42}
)
assert (
cache.get_many(['ford4', 'arthur4'], version=1) ==
{'ford4': 37, 'arthur4': 42}
)
assert cache.get_many(['ford4', 'arthur4'], version=2) == {}
assert caches['v2'].get_many(['ford4', 'arthur4']) == {}
assert (
caches['v2'].get_many(['ford4', 'arthur4'], version=1) ==
{'ford4': 37, 'arthur4': 42}
)
assert caches['v2'].get_many(['ford4', 'arthur4'], version=2) == {}
def get_cached(self,
slugs=None,
pks=None,
select_related_media_tag=True,
portal_id=None):
"""
Gets all ideas defined by either `slugs` or `pks`.
`slugs` and `pks` may be a list or tuple of identifiers to use for
request where the elements are of type string / unicode or int,
respectively. You may provide a single string / unicode or int directly
to query only one object.
:returns: An instance or a list of instances of :class:`CosinnusGroup`.
:raises: If a single object is defined a `CosinnusGroup.DoesNotExist`
will be raised in case the requested object does not exist.
"""
if portal_id is None:
portal_id = CosinnusPortal.get_current().id
# Check that at most one of slugs and pks is set
assert not (slugs and pks)
assert not (slugs or pks)
if slugs is not None:
if isinstance(slugs, six.string_types):
# We request a single idea
slugs = [slugs]
# We request multiple ideas by slugs
keys = [self._IDEAS_SLUG_CACHE_KEY % (portal_id, s) for s in slugs]
ideas = cache.get_many(keys)
missing = [key.split('/')[-1] for key in keys if key not in ideas]
if missing:
# we can only find ideas via this function that are in the same portal we run in
query = self.get_queryset().filter(portal__id=portal_id,
is_active=True,
slug__in=missing)
if select_related_media_tag:
query = query.select_related('media_tag')
for idea in query:
ideas[self._IDEAS_SLUG_CACHE_KEY %
(portal_id, idea.slug)] = idea
cache.set_many(ideas, settings.COSINNUS_IDEA_CACHE_TIMEOUT)
# sort by a good sorting function that acknowldges umlauts, etc, case insensitive
idea_list = list(ideas.values())
idea_list = sorted(idea_list, key=sort_key_strcoll_attr('name'))
return idea_list
elif pks is not None:
if isinstance(pks, int):
pks = [pks]
else:
# We request multiple ideas
cached_pks = self.get_pks(portal_id=portal_id)
slugs = [
_f for _f in (cached_pks.get(pk, []) for pk in pks) if _f
]
if slugs:
return self.get_cached(slugs=slugs, portal_id=portal_id)
return [] # We rely on the slug and id maps being up to date
return []
def cache_get_many(self, keys):
return cache.get_many(keys)
def count(self, request, ip=True, field=None, period=60):
counters = dict((key, 0) for key in self._keys(request, ip, field))
counters.update(cache.get_many(counters.keys()))
for key in counters:
counters[key] += 1
cache.set_many(counters, timeout=period)
def get_instance(model, instance_or_pk, timeout=None, using=None):
"""
Returns the ``model`` instance with a primary key of ``instance_or_pk``.
If the data is cached it will be returned from there, otherwise the regular
Django ORM is queried for this instance and the data stored in the cache.
If omitted, the timeout value defaults to
``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).
Example::
>>> get_instance(User, 1) # Cache miss
<User: lamby>
>>> get_instance(User, 1) # Cache hit
<User: lamby>
>>> User.objects.get(pk=1) == get_instance(User, 1)
True
"""
pk = getattr(instance_or_pk, "pk", instance_or_pk)
primary_model = model
descriptors = getattr(primary_model, "_cache_fetch_related", ())
models = [model, *(d.related.field.model for d in descriptors)]
# Note: we're assuming that the relations are primary key foreign keys, and
# so all have the same primary key. This matches the assumption which
# `cache_relation` makes.
keys_to_models = {instance_key(model, instance_or_pk): model for model in models}
data_map = cache.get_many(tuple(keys_to_models.keys()))
instance_map = {}
if data_map.keys() == keys_to_models.keys():
try:
for key, data in data_map.items():
model = keys_to_models[key]
instance_map[key] = deserialise(model, data, pk, using)
except:
# Error when deserialising - remove from the cache; we will
# fallback and return the underlying instance
cache.delete_many(tuple(keys_to_models.keys()))
else:
key = instance_key(primary_model, instance_or_pk)
primary_instance = instance_map[key]
for descriptor in descriptors:
related_instance = instance_map[
instance_key(
descriptor.related.field.model,
instance_or_pk,
)
]
related_cache_name = get_related_cache_name(
get_related_name(descriptor),
)
setattr(primary_instance, related_cache_name, related_instance)
return primary_instance
related_names = [d.related.field.related_query_name() for d in descriptors]
# Use the default manager so we are never filtered by a .get_query_set()
queryset = primary_model._default_manager.using(using)
if related_names:
# NB: select_related without args selects all it can find, which we don't want.
queryset = queryset.select_related(*related_names)
primary_instance = queryset.get(pk=pk)
instances = [
primary_instance,
*(getattr(primary_instance, x, None) for x in related_names),
]
cache_data = {}
for instance in instances:
if instance is None:
continue
key = instance_key(instance._meta.model, instance)
cache_data[key] = serialise(instance)
if timeout is None:
timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT
cache.set_many(cache_data, timeout)
return primary_instance
def limit(self, request, ip=True, field=None, count=5):
counters = cache.get_many(self._keys(request, ip, field))
return any((v > count) for v in counters.values())
def get_counters(self, request):
return cache.get_many(self.keys_to_check(request))
def get_users_for_authors(organization_id, authors, user=None):
"""
Returns a dictionary of author_id => user, if a Sentry
user object exists for that email. If there is no matching
Sentry user, a {user, email} dict representation of that
author is returned.
e.g.
{
1: serialized(<User id=1>),
2: {email: '*****@*****.**', name: 'dunno'},
...
}
"""
results = {}
fetched = cache.get_many([
_user_to_author_cache_key(organization_id, author)
for author in authors
])
if fetched:
missed = []
for author in authors:
fetched_user = fetched.get(
_user_to_author_cache_key(organization_id, author))
if fetched_user is None:
missed.append(author)
else:
results[str(author.id)] = fetched_user
else:
missed = authors
if missed:
# Filter users based on the emails provided in the commits
user_emails = list(
UserEmail.objects.filter(
in_iexact("email", [a.email for a in missed])).order_by("id"))
# Filter users belonging to the organization associated with
# the release
users = User.objects.filter(
id__in={ue.user_id
for ue in user_emails},
is_active=True,
sentry_orgmember_set__organization_id=organization_id,
)
users = serialize(list(users), user)
users_by_id = {user["id"]: user for user in users}
# Figure out which email address matches to a user
users_by_email = {}
for email in user_emails:
# force emails to lower case so we can do case insensitive matching
lower_email = email.email.lower()
if lower_email not in users_by_email:
user = users_by_id.get(str(email.user_id), None)
# user can be None if there's a user associated
# with user_email in separate organization
if user:
users_by_email[lower_email] = user
to_cache = {}
for author in missed:
results[str(author.id)] = users_by_email.get(
author.email.lower(), {
"name": author.name,
"email": author.email
})
to_cache[_user_to_author_cache_key(
organization_id, author)] = results[str(author.id)]
cache.set_many(to_cache)
metrics.incr("sentry.release.get_users_for_authors.missed",
amount=len(missed))
metrics.incr("sentry.tasks.process_suspect_commits.total",
amount=len(results))
return results
def should_limit(key, minutes=3, rate=20, now=None, howmuch=1):
""" Increments the event count for the current key+minute, and returns True
if 'key' has had more than 'rate' events in the past 'minutes' minutes. """
keys = _recent_keys(key, minutes, now)
_incr_key(keys[0], minutes * 60, howmuch=howmuch)
return sum(cache.get_many(keys).values()) > rate
def diff(self, request, new_imgid, old_imgid):
new_img = Image.objects.select_related(
"container_repo").prefetch_related(
"repo", "container_repo__platform").get(pk=new_imgid)
old_img = Image.objects.select_related(
"container_repo").prefetch_related(
"repo", "container_repo__platform").get(pk=old_imgid)
live_diff = (False, False)
new_repo = new_img.container_repo
old_repo = old_img.container_repo
issue_ref = []
names = []
for repo in new_img.repo.all():
for i in repo.platform.issuetrackers.all():
if i.name not in names:
issue_ref.append({
'name': i.name,
're': i.re,
'url': i.url
})
names.append(i.name)
filter_repos = set(request.GET.getlist("repo"))
filter_meta = _get_filter_meta(request.GET)
cachekey = "%s%s%s" % ("repodiff", new_repo.id, old_repo.id)
cached = cache.get_many([cachekey, cachekey + 'ts'])
diff = cached.get(cachekey)
diffts = cached.get(cachekey + 'ts')
if diff is None:
diff = _diff_sacks(new_repo, old_repo)
diffts = datetime.datetime.now()
cachelife = (60 *
3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
24)
cache.set_many({
cachekey: diff,
cachekey + 'ts': diffts
}, cachelife)
diff = _sort_filter_diff(
diff,
pkgs=list(set(new_img.packages) | set(old_img.packages)),
repos=filter_repos,
meta=filter_meta,
)
title = "Comparing Images"
is_popup = request.GET.get('is_popup', False)
full_path = "%s?" % request.path
for query, values in request.GET.lists():
full_path += "&".join(['%s=%s' % (query, val) for val in values])
full_path += "&"
context = {
"title":
title,
"opts":
self.model._meta,
"app_label":
self.model._meta.app_label,
'diff':
diff,
'diffts':
diffts,
'live_diff':
live_diff,
'new_obj':
new_img,
'old_obj':
old_img,
'is_popup':
is_popup,
'issue_ref':
json.dumps(issue_ref),
'packagemetatypes':
list(PackageMetaType.objects.all().prefetch_related("choices")),
"full_path":
full_path,
}
return TemplateResponse(
request,
'diff.html',
context=context,
)
def send(self):
data = cache.get_many([self.data_key, self.counter_key])
if not data:
return
cache.delete_many([self.data_key, self.counter_key])
self.mail(data[self.counter_key], data[self.data_key])
def get_online_count():
online_ips = cache.get("online_ips", [])
if online_ips:
online_ips = cache.get_many(online_ips).keys()
return len(online_ips)
return 0
def diff(self, request, new_repoid, old_repoid):
start = datetime.datetime.now()
is_popup = request.GET.get('is_popup', False)
do_regen = request.GET.get('do_regen', False)
progress_id = request.GET.get('progress_id', None)
if not request.is_ajax() and request.method != 'POST':
progress_id = uuid.uuid4()
context = {
"title": "",
"opts": self.model._meta,
"app_label": self.model._meta.app_label,
"progress_id": progress_id,
"do_regen": do_regen,
}
return TemplateResponse(
request,
'diff.html',
context=context,
)
def progress_cb(msg):
if progress_id is not None:
cache.set(progress_id, msg, 60 * 5)
progress_cb("Initializing repositories")
new_repo = Repo.objects.select_related(
"server",
"platform").prefetch_related("projects", "components",
"containers").get(pk=new_repoid)
old_repo = Repo.objects.select_related(
"server",
"platform").prefetch_related("projects", "components",
"containers").get(pk=old_repoid)
live_diff = (new_repo.is_live, old_repo.is_live)
end = datetime.datetime.now() - start
context = {
"title": "",
"opts": self.model._meta,
"app_label": self.model._meta.app_label,
'is_popup': is_popup,
'new_obj': new_repo,
'old_obj': old_repo,
'live_diff': live_diff,
'processing_time': end.total_seconds(),
}
if request.method == 'POST':
progress_cb("Creating request")
if not (live_diff[0] and live_diff[1]):
raise ValidationError("Can only creq on live repos")
submit = request.POST.getlist('submit')
delete = request.POST.getlist('delete')
comment = "Request by %s from repodiff of %s to %s" % (
request.user, new_repo, old_repo)
creq_msg = request.POST.get('creq_msg')
if creq_msg:
comment = "%s\n%s" % (comment, creq_msg)
mesgs, errors = _creq(new_repo, old_repo, submit, delete, comment)
for msg in mesgs:
messages.info(request, msg, extra_tags="safe")
for err in errors:
messages.error(request, err, extra_tags="safe")
progress_cb("Done")
return TemplateResponse(
request,
'diff_noprogress.html',
context=context,
)
progress_cb("Generating repository diff")
cachekey = "%s%s%s" % ("repodiff", new_repoid, old_repoid)
cached = cache.get_many([cachekey, cachekey + 'ts'])
diff = cached.get(cachekey)
diffts = cached.get(cachekey + 'ts')
if diff is None or do_regen:
diff = _diff_sacks(new_repo, old_repo, progress_cb)
diffts = datetime.datetime.now()
cachelife = (60 *
3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
24)
cache.set_many({
cachekey: diff,
cachekey + 'ts': diffts
}, cachelife)
filter_repos = set(request.GET.getlist("repo", None))
filter_meta = _get_filter_meta(request.GET)
diff = _sort_filter_diff(diff, repos=filter_repos, meta=filter_meta)
issue_ref = []
names = []
for i in new_repo.platform.issuetrackers.all():
if i.name not in names:
issue_ref.append({'name': i.name, 're': i.re, 'url': i.url})
names.append(i.name)
full_path = "%s?" % request.path
for query, values in request.GET.lists():
full_path += "&".join(['%s=%s' % (query, val) for val in values])
full_path += "&"
end = datetime.datetime.now() - start
context.update({
'title': "Comparing repositories",
'packagemetatypes': list(PackageMetaType.objects.all()),
'diff': diff,
'diffts': diffts,
'issue_ref': json.dumps(issue_ref),
'full_path': full_path,
'processing_time': end.total_seconds()
})
progress_cb("Done")
return TemplateResponse(
request,
"diff_content.html",
context=context,
)
def test_delete_with_prefix_with_no_reverse_works(self):
cache.set_many({'K1': 'value', 'K2': 'value2', 'B2': 'Anothervalue'})
assert cache.delete_with_prefix('K') == 2
assert cache.get_many(['K1', 'K2', 'B2']) == {'B2': 'Anothervalue'}
def _run_get_many(self, keys):
result = cache.get_many([self._prefix_key(key) for key in keys])
for key in keys:
self._cache_data[key] = result.get(self._prefix_key(key))
def handler(cache_name,
function=None,
page_key="",
force_update_cache=False,
cache_duration=DEFAULT_CACHE_DURATION,
*args,
**kwargs):
"""Handles caching for results and data.
Args:
cache: Name of the cache to use.
function: A function reference that returns some value to be cached. This function must only use **kwargs.
page_key: The page name or page number to use as a key value.
force_update_cache:
cache_duration:
**kwargs: Any parameters that need to be passed into "function".
"""
cached_results = None
# Looks through cache and will perform a search if needed.
try:
log.handler(
cache_name + " - Accessed.",
log.DEBUG,
__logger,
)
# If the function was actually a list, then use set_many and/or get_many
# All items must belong to the same cache
# { page_key: {
# "function": function_value,
# "kwargs": {kwargs_value},
# "args": [args_values],
# },
# ... }
if isinstance(function, dict):
if len(function) == 0:
# Nothing was passed in
return None
else:
# Obtain all the keys from the passed in dictionary
requested_keys = []
for key, value in function.items():
cache_key = generate_cache_key(cache_name, value["args"],
value["kwargs"], key)
log.handler(
cache_name +
" - Multi-execution generated cache key " + cache_key,
log.DEBUG,
__logger,
)
requested_keys.append(cache_key)
# Search cache for all keys
cached_results = cache.get_many(requested_keys)
log.handler(
cache_name + " - Multi-execution detected " +
str(len(cached_results)) + " available keys.",
log.INFO,
__logger,
)
# If nothing was in cache, or cache was expired, run function()
thread_list = []
for cache_key in requested_keys:
if not cached_results.__contains__(cache_key):
key = obtain_key_from_cache_key(cache_key)
thread = ReturnThread(
target=function[key]["function"],
args=function[key]["args"],
kwargs=function[key]["kwargs"],
)
thread.start()
thread_list.append((cache_key, thread))
missing_keys = {}
for key, thread in thread_list:
missing_keys[key] = thread.join()
# Set values in cache for any newly executed functions
if bool(missing_keys):
log.handler(
cache_name + " - Multi-execution detected " +
str(len(missing_keys)) + " missing keys.",
log.INFO,
__logger,
)
cache.set_many(missing_keys, cache_duration)
# Return all results
cached_results.update(missing_keys)
# If results were none, log it.
if cached_results is None:
log.handler(
cache_name +
" - Multi-execution generated no results!",
log.WARNING,
__logger,
)
return cached_results
# Get the cached value
cache_key = generate_cache_key(cache_name, args, kwargs, page_key)
log.handler(
cache_name + " - Generated cache key " + cache_key,
log.DEBUG,
__logger,
)
cached_results = cache.get(cache_key)
# No function was provided, just return bare cache value
if function is None:
log.handler(
cache_name + " - Requested raw cache values.",
log.DEBUG,
__logger,
)
return cached_results
# If the user wants to force update the cache, nothing
# was in cache, or cache was expired, run function()
if cached_results is None or force_update_cache:
function_results = function(*args, **kwargs)
log.handler(
cache_name + " - Function " + function.__name__ +
" has been executed!",
log.INFO,
__logger,
)
cache.set(cache_key, function_results, cache_duration)
return function_results
if cached_results is None:
log.handler(
cache_name + " - No cached results found!",
log.INFO,
__logger,
)
# If a value was in cache and not expired, return that value
return cached_results
except:
# If the function threw an exception, return none.
if isinstance(function, dict):
log.handler(
"Function list failed to execute!",
log.ERROR,
__logger,
)
else:
log.handler(
"Function " + function.__name__ + " failed to execute!",
log.ERROR,
__logger,
)
return None
def get_pathways(site, pathway_id=None):
"""
Read pathways from the cache.
The cache is populated by a management command, cache_programs.
Arguments:
site (Site): django.contrib.sites.models object
Keyword Arguments:
pathway_id (string): id identifying a specific pathway to read from the cache.
Returns:
list of dict, representing pathways.
dict, if a specific pathway is requested.
"""
missing_details_msg_tpl = 'Failed to get details for credit pathway {id} from the cache.'
if pathway_id:
pathway = cache.get(PATHWAY_CACHE_KEY_TPL.format(id=pathway_id))
if not pathway:
logger.warning(missing_details_msg_tpl.format(id=pathway_id))
return pathway
pathway_ids = cache.get(
SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not pathway_ids:
logger.warning('Failed to get credit pathway ids from the cache.')
pathways = cache.get_many([
PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
for pathway_id in pathway_ids
])
pathways = list(pathways.values())
# The get_many above sometimes fails to bring back details cached on one or
# more Memcached nodes. It doesn't look like these keys are being evicted.
# 99% of the time all keys come back, but 1% of the time all the keys stored
# on one or more nodes are missing from the result of the get_many. One
# get_many may fail to bring these keys back, but a get_many occurring
# immediately afterwards will succeed in bringing back all the keys. This
# behavior can be mitigated by trying again for the missing keys, which is
# what we do here. Splitting the get_many into smaller chunks may also help.
missing_ids = set(pathway_ids) - {pathway['id'] for pathway in pathways}
if missing_ids:
logger.info(
f'Failed to get details for {len(missing_ids)} pathways. Retrying.'
)
retried_pathways = cache.get_many([
PATHWAY_CACHE_KEY_TPL.format(id=pathway_id)
for pathway_id in missing_ids
])
pathways += list(retried_pathways.values())
still_missing_ids = set(pathway_ids) - {
pathway['id']
for pathway in pathways
}
for missing_id in still_missing_ids:
logger.warning(missing_details_msg_tpl.format(id=missing_id))
return pathways
def get_programs(site=None, uuid=None, course=None): # pylint: disable=redefined-outer-name
"""Read programs from the cache.
The cache is populated by a management command, cache_programs.
Keyword Arguments:
site (Site): django.contrib.sites.models object
uuid (string): UUID identifying a specific program to read from the cache.
course (string): course id identifying a specific course run to read from the cache.
Returns:
list of dict, representing programs.
dict, if a specific program is requested.
"""
if len([arg for arg in (site, uuid, course) if arg is not None]) != 1:
raise TypeError('get_programs takes exactly one argument')
missing_details_msg_tpl = u'Failed to get details for program {uuid} from the cache.'
if uuid:
program = cache.get(PROGRAM_CACHE_KEY_TPL.format(uuid=uuid))
if not program:
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return program
elif course:
uuids = cache.get(
COURSE_PROGRAMS_CACHE_KEY_TPL.format(course_run_id=course))
if not uuids:
# Currently, the cache does not differentiate between a cache miss and a course
# without programs. After this is changed, log any cache misses here.
return []
else:
uuids = cache.get(
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [])
if not uuids:
logger.warning(
u'Failed to get program UUIDs from the cache for site {}.'.
format(site.domain))
programs = cache.get_many(
[PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in uuids])
programs = list(programs.values())
# The get_many above sometimes fails to bring back details cached on one or
# more Memcached nodes. It doesn't look like these keys are being evicted.
# 99% of the time all keys come back, but 1% of the time all the keys stored
# on one or more nodes are missing from the result of the get_many. One
# get_many may fail to bring these keys back, but a get_many occurring
# immediately afterwards will succeed in bringing back all the keys. This
# behavior can be mitigated by trying again for the missing keys, which is
# what we do here. Splitting the get_many into smaller chunks may also help.
missing_uuids = set(uuids) - set(program['uuid'] for program in programs)
if missing_uuids:
logger.info(
u'Failed to get details for {count} programs. Retrying.'.format(
count=len(missing_uuids)))
retried_programs = cache.get_many([
PROGRAM_CACHE_KEY_TPL.format(uuid=uuid) for uuid in missing_uuids
])
programs += list(retried_programs.values())
still_missing_uuids = set(uuids) - set(program['uuid']
for program in programs)
for uuid in still_missing_uuids:
logger.warning(missing_details_msg_tpl.format(uuid=uuid))
return programs