def getProfiles(user_ids): """Fetches profiles of multiple users at once. Args: user_ids: A list of user_id objects we want to fetch profiles for. Returns A mapping of user_id > profile for each matched user. """ keynames = [] for user_id in user_ids: keynames.append("profile:%s" % (user_id)) cache_mapping = memcache.get_multi(keynames) for user_id in user_ids: keyname = "profile:%s" % (user_id) if keyname not in cache_mapping: profile = model.profile.Profile.GetProfile(user_id) cache_mapping[keyname] = profile if profile: memcache.set(keyname, profile) memcache.set_multi(cache_mapping) profiles = {} for user_id in user_ids: profiles[user_id] = cache_mapping["profile:%s" % (user_id)] return profiles
def _Dynamic_Get(self, request, response): """Intercepts get requests and returns them from cache if available.""" logging.info("Tx: %s, Keys: %s", request.has_transaction(), [str(x) for x in request.key_list()]) if request.has_transaction(): self.CallWrappedStub('Get', request, response) return new_request = datastore_pb.GetRequest() new_response = datastore_pb.GetResponse() encoded_keys = [k.Encode() for k in request.key_list()] cached = memcache.get_multi(encoded_keys) for key, encoded_key in itertools.izip(request.key_list(), encoded_keys): if encoded_key not in cached: new_request.add_key().CopyFrom(key) if new_request.key_size() > 0: self.CallWrappedStub('Get', new_request, new_response) entity_iter = iter(new_response.entity_list()) to_put = dict() for encoded_key in encoded_keys: entity = cached.get(encoded_key, None) if entity: response.add_entity().mutable_entity().CopyFrom(entity) else: entity = entity_iter.next() if entity.entity().IsInitialized(): # self.entity_cache[encoded_key] = entity.entity() to_put[encoded_key] = entity.entity() response.add_entity().CopyFrom(entity) if to_put: memcache.set_multi(to_put)
def get_games(): game_slugs = get_game_slugs() if game_slugs: serialized_games = memcache.get_multi(game_slugs, key_prefix='game') # Checking game in slug and not in cache missing_slugs = list(game_slugs) games = [] if serialized_games: for slug, game in serialized_games.iteritems(): games.append(deserialize(game)) missing_slugs.remove(slug) if missing_slugs: query = Game.all() query.filter('active =', True) query.filter('slug in', missing_slugs) missing_games = query.fetch(limit=None) serialized_games = {} for game in missing_games: games.append(game) serialized_games[game.slug] = serialize(game) memcache.set_multi(serialized_games, key_prefix='game') return games
def get(self): # Memcache doesn't support saving values > 1MB. Break up features into chunks # and save those to memcache. if self.MODEL_CLASS == models.FeatureObserver: keys = self.PROPERTY_CLASS.get_property_chunk_memcache_keys( self.PROPERTY_CLASS, self.MEMCACHE_KEY) properties = memcache.get_multi(keys) if len(properties.keys()) != len(properties) or not properties: properties = self.__query_metrics_for_properties() # Memcache doesn't support saving values > 1MB. Break up list into chunks. chunk_keys = self.PROPERTY_CLASS.set_property_chunk_memcache_keys(self.MEMCACHE_KEY, properties) memcache.set_multi(chunk_keys, time=CACHE_AGE) else: temp_list = [] for key in sorted(properties.keys()): temp_list.extend(properties[key]) properties = temp_list else: properties = memcache.get(self.MEMCACHE_KEY) if properties is None: properties = self.__query_metrics_for_properties() memcache.set(self.MEMCACHE_KEY, properties, time=CACHE_AGE) properties = self._clean_data(properties) # Metrics json shouldn't be cached by intermediary caches because users # see different data when logged in. Set Cache-Control: private. super(FeatureHandler, self).get(properties, public=False)
def primeDesireCache(step, startKey = None, stopKey = None): import princeFunc if stopKey is None and startKey is None: startKey = 1 stopKey = int(meSchema.tradeCue.all(keys_only=True).order('-__key__').get().name()) elif stopKey is None or startKey is None: raise(BaseException('Must define both startKey and stopKey, or both must be None!')) memdict = {} clockKeyStep = step queryStr = princeFunc.getDesireQueryStr(max(step-405,0),step) desires = db.GqlQuery(queryStr).fetch(20000) for desire in desires: desirekey = desire.key().name() stckID = meTools.getStckID(desire.Symbol) cueKey = desirekey.split("_")[-2] # Extract cueKey from middle. memkey = 'desire_' + cueKey + '_' + str(stckID) step = int(desirekey.split("_")[0]) # Extract step from front part of desirekey. if not memdict.__contains__(memkey): memdict[memkey] = step elif memdict[memkey] < step: memdict[memkey] = step memcache.set_multi(memdict) cachepy.set_multi(memdict, priority = 1) cachepy.set('stepclock_' + str(startKey) + '_' + str(stopKey), clockKeyStep) # Manually syncing stepclock until get saner method.
def perma_cache(id, update=False): post = memcache.get(id) if post is None or update: logging.error("DB QUERY") post = Blog.get_by_id(int(id)) memcache.set_multi({id: post, 'perma_' + id: time.time()}) return post
def news(request, id): needData = ['news10', 'new_%s' % id] result = memcache.get_multi(['news10', 'new_%s' % id]) needData = [k for k in needData if k not in result] toCache = {} newsFlag = True newFlag = True for key in needData: if key == 'news10': news10 = News.gql('ORDER BY added_on DESC LIMIT 10') news = [] for new in news10: news.append(new) result[m.news3] = news toCache['news10'] = news newsFlag = False elif key == 'new_%s' % id: result['new'] = News.get_by_id(int(id)) toCache['new_%s' % id] = result['new'] newFlag = False if len(toCache) > 0: memcache.set_multi(toCache) if newFlag: result['new'] = result['new_%s' % id] if newsFlag: result[m.news3] = result['news10'] return result
def front_page(update=False, post_id=''): global last_queried, last_all_queried # post = '' if post_id: last_queried, post = multi_memcache_get('LQ' + str(post_id), str(post_id)) # print(last_queried, 'sdfasdfasdfeWEWQERQWEREWQR', post) if not (last_queried and post): # print('DB Q\'ed for single post', last_queried) last_queried, post = datetime.today(), Posts.by_id(post_id) # memcache.set('LQ' + str(post_id), last_queried) # memcache.set(str(post_id), post) memcache.set_multi({'LQ' + str(post_id): last_queried, str(post_id): post}) return post else: return post key = 'front' blog = memcache.get(key) if not all([last_all_queried, not update, blog]): last_all_queried = datetime.today() memcache.set('last all queried', last_all_queried) logging.error('DB GOT the QUERY') blog = db.GqlQuery("SELECT * " "FROM Posts " "ORDER BY created DESC " "LIMIT 10") blog = list(blog) memcache.set(key, blog) return blog
def _Dynamic_Get(self, request, response): """Intercepts get requests and returns them from cache if available.""" if request.has_transaction(): self.CallWrappedStub('Get', request, response) return new_request = datastore_pb.GetRequest() new_response = datastore_pb.GetResponse() encoded_keys = [k.Encode() for k in request.key_list()] cached = memcache.get_multi(encoded_keys) for key, encoded_key in itertools.izip(request.key_list(), encoded_keys): if encoded_key not in cached: new_request.add_key().CopyFrom(key) if new_request.key_size() > 0: logging.info("SHIM: keys are not in memcache: %s" % request.key_list()) self.CallWrappedStub('Get', new_request, new_response) entity_iter = iter(new_response.entity_list()) to_put = dict() for encoded_key in encoded_keys: entity = cached.get(encoded_key, None) if entity: response.add_entity().mutable_entity().CopyFrom(entity) else: entity = entity_iter.next() if entity.entity().IsInitialized(): # self.entity_cache[encoded_key] = entity.entity() to_put[encoded_key] = entity.entity() response.add_entity().CopyFrom(entity) if to_put: memcache.set_multi(to_put)
def get_keys(keys): """ Fetch and save the given keys into memcache. """ # Perform the first load from memcache. urlsafes = [key.urlsafe() for key in keys] cache = memcache.get_multi(urlsafes, key_prefix=RTC) results = {} missing = [] for urlsafe, key in zip(urlsafes, keys): if urlsafe in cache: results[key] = protobuf_to_ndb_entity(cache[urlsafe]) else: missing.append(key) # Record cache stats. memcache.incr(MISSES, delta=len(missing), initial_value=0) memcache.incr(HITS, delta=len(results), initial_value=0) # Fill anything not yet in cache. if missing: writeback = {} for key, entity in zip(missing, ndb.get_multi(missing)): results[key] = entity writeback[key.urlsafe()] = ndb_entity_to_protobuf(entity) memcache.set_multi(writeback, key_prefix=RTC) return [results[key] for key in keys]
def UpdateStats(cls, category, stats): """Update the summary stats in memory and the datastore. This will only update part of a summary score row. Args: category: a category string like 'network' stats: a dict of browser stats (see CategoryStatsManager.GetStats) Returns: The summary stats that have been updated by the given stats. (Used by GetStats.) """ browsers = [b for b in stats.keys() if b != 'total_runs'] update_summary_stats = memcache.get_multi( browsers, namespace=cls.MEMCACHE_NAMESPACE) for browser in browsers: ua_summary_stats = update_summary_stats.setdefault(browser, { 'results': {}}) ua_summary_stats['results'][category] = { 'score': stats[browser]['summary_score'], 'display': stats[browser]['summary_display'], 'total_runs': stats[browser]['total_runs'], } if category == 'acid3': ua_summary_stats['results']['acid3']['display'] = ( stats[browser]['results']['score']['display']) memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE) return update_summary_stats
def UpdateStatsCache(cls, category, browsers): """Update the memcache of stats for all the tests for each browser. This is also where the summary stats get updated. Args: category: a category string like 'network' browsers: a list of browsers like ['Firefox 3.6', 'IE 8.0'] Returns: a list of browsers that were not processed due to a timeout. """ test_set = all_test_sets.GetTestSet(category) test_keys = [t.key for t in test_set.VisibleTests()] ua_stats = {} unhandled_browsers = [] is_timed_out = False for browser in browsers: try: medians, num_scores = test_set.GetMediansAndNumScores(browser) except db.Timeout: is_timed_out = True if is_timed_out: logging.info('Timed out \'%s\' in UpdateStatsCache doing ' 'GetMediansAndNumScores for %s', category, browser) unhandled_browsers.append(browser) else: stats = test_set.GetStats(test_keys, medians, num_scores) ua_stats[browser] = stats memcache.set_multi(ua_stats, **cls.MemcacheParams(category)) if not is_timed_out: SummaryStatsManager.UpdateStats(category, ua_stats) return unhandled_browsers
def get(self): # Memcache doesn't support saving values > 1MB. Break up features into chunks # and save those to memcache. if self.MODEL_CLASS == models.FeatureObserver: keys = self.PROPERTY_CLASS.get_property_chunk_memcache_keys( self.PROPERTY_CLASS, self.MEMCACHE_KEY) properties = memcache.get_multi(keys) if len(properties.keys()) != len(properties) or not properties: properties = self.__query_metrics_for_properties() # Memcache doesn't support saving values > 1MB. Break up list into chunks. chunk_keys = self.PROPERTY_CLASS.set_property_chunk_memcache_keys( self.MEMCACHE_KEY, properties) memcache.set_multi(chunk_keys, time=CACHE_AGE) else: temp_list = [] for key in sorted(properties.keys()): temp_list.extend(properties[key]) properties = temp_list else: properties = memcache.get(self.MEMCACHE_KEY) if properties is None: properties = self.__query_metrics_for_properties() memcache.set(self.MEMCACHE_KEY, properties, time=CACHE_AGE) properties = self._clean_data(properties) # Metrics json shouldn't be cached by intermediary caches because users # see different data when logged in. Set Cache-Control: private. super(FeatureHandler, self).get(properties, public=False)
def get(self): tree = memcache.get('pretty_tree_tree') calc = memcache.get('pretty_tree_calc') if not tree or not calc: data_tree = dtmm_utils.get_modules(self) tree = [] break_on = 3 header_diff = 20 width = 900 calc = { 'width': width, 'cell_height': 80, # in pixels :D 'margin_width': width / 2 } for fragment_num, fragment in enumerate(data_tree): logging.info('Fragment; %s' % fragment) logging.info('Fragment_num; %s' % fragment_num) cur_module = dtmm_utils.get_live_module_data(self, fragment) cur_module.update({ 'filename': dtmm_utils.rpart(fragment['path']), 'row': fragment_num % break_on == 0, 'width': calc['width'] / break_on, 'index': fragment_num }) tree.append(cur_module) rows = len(filter(itemgetter('row'), tree)) calc['height'] = (rows * calc['cell_height']) + header_diff logging.info('This many rows; %s' % (rows)) calc['margin_height'] = calc['height'] / 2 calc['outer_container_height'] = calc['height'] if len(tree) % break_on != 0: remainer = len(tree) % break_on for i in range(1, remainer + 1): tree[-i]['width'] = calc['width'] / remainer tree[0]['row'] = False memcache.set_multi({ 'pretty_tree_tree': tree, 'pretty_tree_calc': calc }) # we want the colours to be different everytime colours = pretty_colours(len(tree)) for idx, fragment in enumerate(tree): fragment.update({'background': colours[idx]}) self.dorender( 'tree_pretty.html', { 'tree': tree, 'calc': calc } )
def generate_feed(uid): subscriptions = Subscription.query(Subscription.uid == uid).fetch(200) subscription_urls = [sub.url for sub in subscriptions if sub.url] if len(subscription_urls) > 0: sources = Source.query(Source.url.IN(subscription_urls)).order(-Source.most_recent_article_added_date).fetch(len(subscription_urls)) source_jsons = {} for source_json in memcache.get_multi([source.feed_cache_key() for source in sources]).itervalues(): source_jsons[source_json['id']] = source_json to_fetch = [source for source in sources if source.key.id() not in source_jsons] print 'HITS {0} TO_FETCH {1}'.format(len(source_jsons), len(to_fetch)) if len(to_fetch): source_promises = [src.json(include_articles=True, article_limit=FEED_ARTICLE_LIMIT, return_promise=True) for src in to_fetch] for promise in source_promises: data = promise() source_jsons[data['id']] = data # put the cache keys: if len(to_fetch): memcache.set_multi({source.feed_cache_key(): source_jsons[source.key.id()] for source in to_fetch if (source.key.id() in source_jsons)}) source_json = [source_jsons[source.key.id()] for source in sources if source.key.id() in source_jsons] else: source_json = [] return { "sources": source_json }
def set_many(self, data, timeout=0): safe_data = {} for key, value in data.items(): if isinstance(value, unicode): value = value.encode('utf-8') safe_data[smart_str(key)] = value memcache.set_multi(mapping=safe_data, time=timeout)
def memcache_usage(): email = get_user_email() fname = '' lname = '' username = '' if email: if memcache.get("user_"): return True else: qry = accountModel.get_by_id(users.get_current_user().user_id()) if qry: fname = qry.firstName lname = qry.lastName username = qry.username else: lm = lobbyModel.query(lobbyModel.ownerID == "118168406204694893029 ").get() if lm: publob = lobbyAccessModel(lobbyID=lm.key, userID=users.get_current_user().user_id()) lobbyAccessModel.put(publob) user = accountModel(id=users.get_current_user().user_id(), firstName=fname, lastName=lname, username=email.split("@", 1)[0]) accountModel.put(user) memcache.set_multi({"fname": fname, "lname": lname, "username": username}, key_prefix="user_", time=3600) return True return False
def get(self): self.response.headers['Content-Type'] = 'text/plain' rates_url = 'http://www.twilio.com/resources/rates/international-rates.csv' rates_data = '' # Get the data from Twilio try: remote_data = urlfetch.fetch(rates_url) if remote_data.status_code != 200: raise Exception('') rates_csv = remote_data.content rates_data = csv.reader(cStringIO.StringIO(rates_csv)) except: self.response.out.write('{"status": "ERROR"}') return # Save the data into memcache (expire in 10 days) for row in rates_data: country = row[0] rate = row[1] numbers = row[2].split(' ') d = {} for number in numbers: d[number] = '{"country": "' + country + '", "rate": "' + rate + '"}' #memKey = number #memVal = '{"country": "' + country + '", "rate": "' + rate + '"}' #memcache.set(key = memKey, value = memVal, time = 864000) memcache.set_multi(d, time = 864000) self.response.out.write('{"status": "OK"}')
def _Dynamic_Next(self, request, response): """Intercepts query results and caches the returned entities.""" self.CallWrappedStub('Next', request, response) if not response.keys_only(): to_put = dict([(e.key().Encode(), e) for e in response.result_list()]) memcache.set_multi(to_put)
def cache_multiset(data_dict, time=settings.DEFAULT_CACHE_TIME): """ Sends multiple objects to a memory cache.""" data = {} for k, v in data_dict: data[k if isinstance(k, basestring) else "|".join(k)] = v memcache.set_multi(data, time, namespace=settings.CACHE_NAMESPACE) return data_dict
def perma_cache(id, update = False): post = memcache.get(id) if post is None or update: logging.error("DB QUERY") post = Blog.get_by_id(int(id)) memcache.set_multi({id: post, 'perma_'+id:time.time()}) return post
def cancel_cards(userid): current_val = memcache.get(key=userid) if current_val is None: logging.warn('Nothing scheduled for user %s', current_val) return memcache_values = { card:1 for card in current_val.split('__') } memcache.set_multi(memcache_values, time=60*60)
def get_api(self): memcache_key_prefix = 'refresh_item_api_' memcache_keys = ['page', 'pages'] page_info = memcache.get_multi(memcache_keys, key_prefix = memcache_key_prefix) if 'page' not in page_info or 'pages' not in page_info: page = pages = 1 else: page, pages = page_info['page'], page_info['pages'] put_rpcs = [] while page <= pages: memcache.set_multi({'page' : page, 'pages' : pages}, key_prefix = memcache_key_prefix) logging.info('Getting page = %d / %d ...' % (page, pages)) method = '/simple/auctions.list' content = fetch_api(method, {'defs' : 1, 'per_page' : 1000, 'page' : page}) auctions_info = parse_json(content, None) page = auctions_info['page'] + 1 pages = auctions_info['pages'] items_info = auctions_info['items'] for item_info in items_info.itervalues(): item_def = item_info['item_def'] image = Image(key_name = item_def['class_tsid'], url = item_def['iconic_url'], width = 40, height = 40) put_rpcs.append(put_async(image)) item = Item(key_name = item_def['class_tsid'], category = item_def['category'], name = item_def['name_single'], image = image) put_rpcs.append(put_async(item)) memcache.delete_multi(memcache_keys, key_prefix = memcache_key_prefix) return put_rpcs
def getTopThreadsPerForum(forums): """For each forum, returns the 'top' thread, which we'll display in the forum list page. The 'top' thread is the most-recently created thread. When you click through to the forum, the top thread will actually be the thread with the most recent reply, so it's slightly different.""" keynames = [] for forum in forums: keynames.append("forum:%s:top-thread" % (forum.slug)) cache_mapping = memcache.get_multi(keynames) # fetch any from the data store that weren't cached for forum in forums: keyname = "forum:%s:top-thread" % (forum.slug) if keyname not in cache_mapping: query = model.forum.ForumThread.all().filter("forum", forum).order("-posted").fetch(1) for forum_thread in query: cache_mapping[keyname] = forum_thread break memcache.set_multi(cache_mapping) # convert from our (internal) memcache key names to a more reasonable key top_threads = {} for forum in forums: keyname = "forum:%s:top-thread" % (forum.slug) if keyname in cache_mapping: top_threads[forum.slug] = cache_mapping[keyname] return top_threads
def memGetPercentReturns(memkeylist, prefix): EntityDict = {} newMemEntities = {} memCacheEntities = {} memEntities = cachepy.get_multi(memkeylist) missingKeys = meTools.getMissingKeys(memkeylist,memEntities) memCacheEntities = memcache.get_multi(missingKeys) cachepy.set_multi(memCacheEntities) missingKeys = meTools.getMissingKeys(missingKeys,memCacheEntities) memEntities.update(memCacheEntities) if missingKeys: missingKeys = [key.replace(prefix,'') for key in missingKeys] if prefix == 'BTR-': Entities = meSchema.backTestResult.get_by_key_name(missingKeys) elif prefix == 'LAR-': Entities = meSchema.liveAlg.get_by_key_name(missingKeys) for entity in Entities: if entity: memkey = prefix + entity.key().name() pReturn = entity.percentReturn newMemEntities[memkey] = pReturn memEntities[memkey] = pReturn memcache.set_multi(newMemEntities) cachepy.set_multi(newMemEntities) return memEntities
def _render(self, year): year = int(year) avatars = [] shards = memcache.get_multi(['{}avatars_{}'.format(year, i) for i in xrange(10)]) if len(shards) == 10: # If missing a shard, must refetch all for _, shard in sorted(shards.items(), key=lambda kv: kv[0]): avatars += shard if not avatars: avatars_future = Media.query(Media.media_type_enum == MediaType.AVATAR, Media.year == year).fetch_async() avatars = sorted(avatars_future.get_result(), key=lambda a: int(a.references[0].id()[3:])) shards = {} size = len(avatars) / 10 + 1 for i in xrange(10): start = i * size end = start + size shards['{}avatars_{}'.format(year, i)] = avatars[start:end] memcache.set_multi(shards, 60*60*24) self.template_values.update({ 'year': year, 'avatars': avatars, }) return jinja2_engine.render('avatars.html', self.template_values)
def _render(self, year): year = int(year) avatars = [] shards = memcache.get_multi( ['{}avatars_{}'.format(year, i) for i in xrange(10)]) if len(shards) == 10: # If missing a shard, must refetch all for _, shard in sorted(shards.items(), key=lambda kv: kv[0]): avatars += shard if not avatars: avatars_future = Media.query( Media.media_type_enum == MediaType.AVATAR, Media.year == year).fetch_async() avatars = sorted(avatars_future.get_result(), key=lambda a: int(a.references[0].id()[3:])) shards = {} size = len(avatars) / 10 + 1 for i in xrange(10): start = i * size end = start + size shards['{}avatars_{}'.format(year, i)] = avatars[start:end] memcache.set_multi(shards, 60 * 60 * 24) self.template_values.update({ 'year': year, 'avatars': avatars, }) return jinja2_engine.render('avatars.html', self.template_values)
def get_status_dict(self): # It will returns a serialized status in mem_cache serialized_status = memcache.get_multi(self.status_ids, key_prefix='status') # Just copying my status_ids list to make some changes localy missing_status_ids = list(self.status_ids) game_status = {} for status_id, status in serialized_status.iteritems(): game_status[status_id] = deserialize(status) missing_status_ids.remove(status_id) # Taking the missing status in database and add them in memcache if missing_status_ids: missing_status = Status.get(missing_status_ids) serialized_status = {} for status in missing_status: game_status[status_id] = deserialize(status) serialized_status[status.id] = serialize(status) memcache.set_multi(serialized_status, key_prefix='status') # I really dunno why, but the game_status list in this function # works like a list of string, and when this function pass to some # function or when it returns, game_status assume its really identity # that is a list of status, not a list of strings... (crazy, I know) self.actualise_status(game_status) return game_status # Returns a random list of Status playing this game
def get_image(self): #TODO: support multiple badge sizes im = get_cached_issue_image(self.number) if not im: im, width = draw_image(self.number, self.vote_count) memcache_key = "issue_%d_image" % self.number memcache.set_multi({memcache_key: im.tostring(), memcache_key + "_width" : width}) return im
def get_multi_test(): memcache.set_multi(DATA, 30, key_prefix='memcache_get_multi_test') now = time() memcache.get_multi(MULTI_KEYS, key_prefix='memcache_get_multi_test') result = time() - now memcache.delete_multi(MULTI_KEYS, key_prefix='memcache_get_multi_test') return result
def storeAssociation(self, server_url, association): data = association.serialize() key1, key2 = self.getAssociationKeys(server_url, association.handle) memcache.set_multi({ key1: data, key2: data }, namespace=MEMCACHE_NAMESPACE)
def get(self): self.test_memcache() self.test_memcache() memcache.set_multi({'key': 'value', 'other': 'value'}) memcache.get_multi(['key', 'other', 'thing']) self.response.out.write('Hello world')
def mem_set(key, value, chunksize=900000): serialized = pickle.dumps(value, 2) values = {} for i in xrange(0, len(serialized), chunksize): values['%s.%s' % (key, i//chunksize)] = serialized[i : i+chunksize] logging.info("cached: " + ('%s.%s' % (key, i//chunksize))+ str(values['%s.%s' % (key, i//chunksize)])); memcache.set_multi(values) memcache.set(key, len(values))
def ExecuteApiQueryTask(api_query): """Executes a refresh of an API Query from the task queue. Attempts to fetch and update an API Query and will also log any errors. Schedules the API Query for next execution. Args: api_query: The API Query to refresh. Returns: A boolean. True if the API refresh was a success and False if the API Query is not valid or an error was logged. """ if api_query: query_id = str(api_query.key()) api_query.in_queue = False api_response_content = FetchApiQueryResponse(api_query) if not api_response_content or api_response_content.get('error'): InsertApiQueryError(api_query, api_response_content) if api_query.is_error_limit_reached: api_query.is_scheduled = False SaveApiQuery(api_query) # Since it failed, execute the query again unless the refresh interval of # query is less than the random countdown, then schedule it normally. if api_query.refresh_interval < co.MAX_RANDOM_COUNTDOWN: schedule_helper.ScheduleApiQuery(api_query) # Run at normal interval. else: schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0) return False else: SaveApiQueryResponse(api_query, api_response_content) # Check that public endpoint wasn't disabled after task added to queue. if api_query.is_active: memcache.set_multi({'api_query': api_query, co.DEFAULT_FORMAT: api_response_content}, key_prefix=query_id, time=api_query.refresh_interval) # Delete the transformed content in memcache since it will be updated # at the next request. delete_keys = set(co.SUPPORTED_FORMATS) - set([co.DEFAULT_FORMAT]) memcache.delete_multi(list(delete_keys), key_prefix=query_id) SaveApiQuery(api_query) schedule_helper.ScheduleApiQuery(api_query) return True # Save the query state if the user has disabled it # while it was in the task queue. if api_query.is_active is False: SaveApiQuery(api_query) return False
def blackhole(self, urls): """Blackholes a set of URLs by domain for the rest of the current period. Args: urls: Iterable of URLs to blackhole. """ values = dict(('failure:' + get_url_domain(u), self.min_requests) for u in urls) memcache.set_multi(values, key_prefix=self.prefix)
def save_objects(self, keys_to_objects): if not keys_to_objects: return memcache_set = {} for k, v in keys_to_objects.iteritems(): if self._is_cacheable(k, v): cache_key = self.key_to_cache_key(k) memcache_set[cache_key] = v memcache.set_multi(memcache_set, 2 * 3600)
def front_cache(update=False): key = 'blog' posts = memcache.get(key) if posts is None or update: logging.error("DB QUERY") posts = db.GqlQuery("SELECT * FROM Blog ORDER BY created DESC") posts = list(posts) memcache.set_multi({key: posts, 'front_queried': time.time()}) return posts
def set_multi_models(models): # This uses dict comprehension to create a dictionary that looks like that: # {'model1.key()': [model1], # 'model2.key()': [model2], ...} # # memcache.set_multi() receives this mapping and puts multiple values at the # same time. mapping = {str(model.key()): model for model in models} memcache.set_multi(mapping)
def ExecuteApiQueryTask(api_query): """Executes a refresh of an API Query from the task queue. Attempts to fetch and update an API Query and will also log any errors. Schedules the API Query for next execution. Args: api_query: The API Query to refresh. Returns: A boolean. True if the API refresh was a success and False if the API Query is not valid or an error was logged. """ if api_query: query_id = str(api_query.key()) api_query.in_queue = False api_response_content = FetchApiQueryResponse(api_query) if not api_response_content or api_response_content.get('error'): InsertApiQueryError(api_query, api_response_content) if api_query.is_error_limit_reached: api_query.is_scheduled = False SaveApiQuery(api_query) # Since it failed, execute the query again unless the refresh interval of # query is less than the random countdown, then schedule it normally. if api_query.refresh_interval < co.MAX_RANDOM_COUNTDOWN: schedule_helper.ScheduleApiQuery(api_query) # Run at normal interval. else: schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0) return False else: SaveApiQueryResponse(api_query, api_response_content) # Check that public endpoint wasn't disabled after task added to queue. if api_query.is_active: memcache.set_multi({'api_query': api_query, co.DEFAULT_FORMAT: api_response_content}, key_prefix=query_id, time=api_query.refresh_interval) # Delete the transformed content in memcache since it will be updated # at the next request. delete_keys = set(co.SUPPORTED_FORMATS) - set([co.DEFAULT_FORMAT]) memcache.delete_multi(list(delete_keys), key_prefix=query_id) SaveApiQuery(api_query) schedule_helper.ScheduleApiQuery(api_query) return True # Save the query state just in case the user disabled it # while it was in the task queue. SaveApiQuery(api_query) return False
def put(self, key, value): logs.log('MemcacheLarge put %s.' + key) # Make JSON representation as compact as possible (don't use spaces). string_value = json.dumps(value, separators=(',', ':')) keys_and_values = {key: len(string_value)} for chunk_start in xrange(0, len(string_value), self.CHUNK_LEN): full_key = '%s-%s-%s' % (self.MAGIC_STR, key, chunk_start) keys_and_values[full_key] = string_value[chunk_start:chunk_start + self.CHUNK_LEN] memcache.set_multi(keys_and_values)
def get_image(self): #TODO: support multiple badge sizes im = get_cached_issue_image(self.number) if not im: im, width = draw_image(self.number, self.vote_count) memcache_key = "issue_%d_image" % self.number memcache.set_multi({ memcache_key: im.tostring(), memcache_key + "_width": width }) return im
def Set(cls, key, value): """Sets a value in memcache.""" serialized_parts = _Serialize(value) if len(serialized_parts) > _MAX_NUM_PARTS: logging.error('Max number of parts reached.') return cached_values = {} cached_values[cls._GetCacheKey(key)] = len(serialized_parts) for i in xrange(len(serialized_parts)): cached_values[cls._GetCacheKey(key, i)] = serialized_parts[i] memcache.set_multi(cached_values)
def UrlsToItemIds(urls): """Returns a map from url to item id.""" if not urls: return {} url_to_item_id = memcache.get_multi(urls, key_prefix='u2i:') urls_not_in_cache = [url for url in urls if url not in url_to_item_id] if urls_not_in_cache: new_items = {} for item in Item.query(Item.url.IN(urls_not_in_cache)).fetch(): url_to_item_id[item.url] = item.ItemId() new_items[item.url] = item.ItemId() memcache.set_multi(new_items, key_prefix='u2i:') return url_to_item_id
def get_multi(self, keys): result = {} not_local = [] for item_key in keys: try: result[item_key] = self._cache[item_key] except KeyError: not_local.append(item_key) if not_local: to_get = [] to_get.extend(not_local) to_get.extend(self._prefetch) self._prefetch = set() try: cached = memcache.get_multi(to_get, self._prefix) except DeadlineExceededError: cached = {} to_compute = [] for item_key in to_get: try: r = cached[item_key] except KeyError: to_compute.append(item_key) continue self._cache[item_key] = r result[item_key] = r if to_compute: to_cache = {} for item_key, r in zip(to_compute, self._compute(to_compute)): if r is not None: to_cache[item_key] = r self._cache[item_key] = r result[item_key] = r if to_cache: try: if self._timeout is None: memcache.set_multi(to_cache, key_prefix = self._prefix) else: memcache.set_multi(to_cache, time = self._timeout, key_prefix = self._prefix) except DeadlineExceededError: pass return result
def testMulti(self): """Stores multiple keys' values at once.""" memcache.set_multi({'map_key_one': 1, 'map_key_two': u'some value'}) values = memcache.get_multi(['map_key_one', 'map_key_two']) assert {'map_key_one': 1, 'map_key_two': u'some value'} == values memcache.add_multi({ 'map_key_one': 'one', 'map_key_two': 2, 'three': u'trois' }) values = memcache.get_multi(['map_key_two', 'three']) assert {'map_key_two': u'some value', 'three': u'trois'} == values
def _memcache_put(models, time=0): '''Put given models to memcache in serialized form with expiration in seconds Returns: List of db.Keys of the models that were put ''' to_put = _to_dict(models) for key, model in to_put.iteritems(): to_put[key] = _serialize(model) memcache.set_multi(to_put, time) return [model.key() for model in models]
def _PutTilesToMemcache(self): mapping = {} for tile in self.tiles.values(): key = self._GetGameTileKeyName(tile.Id()) mapping[key] = db.model_to_protobuf(tile) for email in self.players: key = self._GetPlayerTileLocationKeyName(email) mapping[key] = self.players[email] if len(mapping): logging.debug("Putting %d game tiles to memcache." % len(mapping)) memcache.set_multi(mapping) else: logging.debug("Not putting any game tiles to memcache.")
def get_queues(bot_root_key): """Returns the known task queues as integers. This function is called to get the task queues to poll, as the bot is trying to reap a task, any task. It is also called while the bot is running a task, to refresh the task queues. Arguments: bot_root_key: ndb.Key to bot_management.BotRoot Returns: dimensions_hashes: list of dimension_hash for the bot """ bot_id = bot_root_key.string_id() dimensions_hashes = memcache.get(bot_id, namespace='task_queues') if dimensions_hashes is not None: # Note: This may return stale queues. We may want to change the format to # include the expiration. logging.debug('get_queues(%s): can run from %d queues (memcache)\n%s', bot_id, len(dimensions_hashes), dimensions_hashes) # Refresh all the keys. memcache.set_multi({str(d): True for d in dimensions_hashes}, time=61, namespace='task_queues_tasks') return dimensions_hashes # Retrieve all the dimensions_hash that this bot could run that have # actually been triggered in the past. Since this is under a root entity, this # should be fast. now = utils.utcnow() dimensions_hashes = sorted( obj.key.integer_id() for obj in BotTaskDimensions.query(ancestor=bot_root_key) if obj.valid_until_ts >= now) memcache.set(bot_id, dimensions_hashes, namespace='task_queues', time=_EXPIRATION_TIME_TASK_QUEUES) logging.info('get_queues(%s): Query in %.3fs: can run from %d queues\n%s', bot_id, (utils.utcnow() - now).total_seconds(), len(dimensions_hashes), dimensions_hashes) memcache.set_multi({str(d): True for d in dimensions_hashes}, time=61, namespace='task_queues_tasks') return dimensions_hashes
def test_put_clears_cache(self): pc_id = 'ProjectCohort_foo' survey_id = 'Survey_foo' code = 'foo bar' start1 = datetime.datetime.today() end1 = start1 + datetime.timedelta(days=1) start2 = start1 + datetime.timedelta(days=2) end2 = start1 + datetime.timedelta(days=3) cache_data = { ParticipantData.participation_cache_key(pc_id): { ParticipantData.date_key(start1, end1): ['result1'], ParticipantData.date_key(start2, end2): ['result2'], }, ParticipantData.participation_cache_key(survey_id): { ParticipantData.date_key(start1, end1): ['result1'], ParticipantData.date_key(start2, end2): ['result2'], }, ParticipantData.participation_by_pc_cache_key(pc_id): { ParticipantData.date_key(start1, end1): ['result1'], ParticipantData.date_key(start2, end2): ['result2'], }, ParticipantData.participation_by_pc_cache_key(code): { ParticipantData.date_key(start1, end1): ['result1'], ParticipantData.date_key(start2, end2): ['result2'], }, } memcache.set_multi(cache_data) # Write a pd that relates to the pc and survey, falling in the first # date range. That date range should clear, the other should remain. pd = ParticipantData.create( key='progress', value=1, participant_id='Participant_foo', program_label='demo-program', project_id='Project_foo', cohort_label='2019', project_cohort_id=pc_id, code=code, survey_id=survey_id, survey_ordinal=1, ) ParticipantData.put_for_index(pd, 'participant-survey-key') for cache_key in cache_data.keys(): self.assertEqual(len(memcache.get(cache_key)), 1)
def get_problem_list(key_list): """ Returns list of problems in datastore as per the key_list. Uses and updates memcache as necessary. """ id_list = [key.id() for key in key_list] retrieved_dict = memcache.get_multi(id_list, namespace='p') #these returns a dict of (id, entity) retreived from memcache retrieved_problems = retrieved_dict.values() retrieved_ids = retrieved_dict.keys() not_retrieved_ids = list(set(id_list) - set(retrieved_ids)) if not not_retrieved_ids: #saves 1 memcache set rpc return retrieved_problems not_retrieved_keys = [ndb.Key(Problem, id) for id in not_retrieved_ids] not_retrieved_problems = ndb.get_multi(not_retrieved_keys) not_retrieved_dict = dict(zip(not_retrieved_ids, not_retrieved_problems)) memcache.set_multi(not_retrieved_dict, namespace='p') return retrieved_problems + not_retrieved_problems
def get(self): # [START batch] values = {'comment': 'I did not ... ', 'comment_by': 'Bill Holiday'} if not memcache.set_multi(values): logging.error('Unable to set Memcache values') tvalues = memcache.get_multi(('comment', 'comment_by')) self.response.write(tvalues)
def set_multi_async(self, mapping, time=0, key_prefix='', min_compress_len=0, namespace=None, rpc=None): prefixed_mapping = {} for key, value in mapping.items(): prefixed_mapping[default_key_func(key, KEY_PREFIX, VERSION)] = value if self.sync_mode: # We don't call up, because set_multi calls set_multi_async return memcache.set_multi(prefixed_mapping, time=time, key_prefix=key_prefix, min_compress_len=min_compress_len, namespace=namespace) else: return super(KeyPrefixedClient, self).set_multi_async( prefixed_mapping, time=time, key_prefix=key_prefix, min_compress_len=min_compress_len, namespace=namespace, rpc=rpc)
def _put_tasklet(self, todo): assert todo # TODO: What if the same entity is being put twice? # TODO: What if two entities with the same key are being put? # TODO: Clear entities from memcache before starting the write? # TODO: Attempt to prevent dogpile effect while keeping cache consistent? ents = [ent for (_, ent) in todo] results = yield self._conn.async_put(None, ents) for key, (fut, ent) in zip(results, todo): if key != ent.key: assert ent.key is None or not list(ent.key.flat())[-1] ent.key = key fut.set_result(key) # Now update memcache. # TODO: Could we update memcache *before* calling async_put()? # (Hm, not for new entities but possibly for updated ones.) mapping = {} for _, ent in todo: if self.should_memcache(ent.key): pb = self._conn.adapter.entity_to_pb(ent) mapping[ent.key.urlsafe()] = pb if mapping: # TODO: Optionally set the memcache expiration time; # maybe configurable based on key (or even entity). failures = memcache.set_multi(mapping) if failures: badkeys = [] for failure in failures: badkeys.append(mapping[failure].key) logging.info('memcache failed to set %d out of %d keys: %s', len(failures), len(mapping), badkeys)