Example #1
0
def getProfiles(user_ids):
  """Fetches profiles of multiple users at once.

  Args:
    user_ids: A list of user_id objects we want to fetch profiles for.
  Returns
    A mapping of user_id > profile for each matched user.
  """
  keynames = []
  for user_id in user_ids:
    keynames.append("profile:%s" % (user_id))
  cache_mapping = memcache.get_multi(keynames)

  for user_id in user_ids:
    keyname = "profile:%s" % (user_id)
    if keyname not in cache_mapping:
      profile = model.profile.Profile.GetProfile(user_id)
      cache_mapping[keyname] = profile

      if profile:
        memcache.set(keyname, profile)

  memcache.set_multi(cache_mapping)

  profiles = {}
  for user_id in user_ids:
    profiles[user_id] = cache_mapping["profile:%s" % (user_id)]
  return profiles
    def _Dynamic_Get(self, request, response):
        """Intercepts get requests and returns them from cache if available."""
        logging.info("Tx: %s, Keys: %s", request.has_transaction(),
                     [str(x) for x in request.key_list()])
        if request.has_transaction():
            self.CallWrappedStub('Get', request, response)
            return

        new_request = datastore_pb.GetRequest()
        new_response = datastore_pb.GetResponse()
        encoded_keys = [k.Encode() for k in request.key_list()]
        cached = memcache.get_multi(encoded_keys)

        for key, encoded_key in itertools.izip(request.key_list(),
                                               encoded_keys):
            if encoded_key not in cached:
                new_request.add_key().CopyFrom(key)

        if new_request.key_size() > 0:
            self.CallWrappedStub('Get', new_request, new_response)

        entity_iter = iter(new_response.entity_list())
        to_put = dict()
        for encoded_key in encoded_keys:
            entity = cached.get(encoded_key, None)
            if entity:
                response.add_entity().mutable_entity().CopyFrom(entity)
            else:
                entity = entity_iter.next()
                if entity.entity().IsInitialized():
                    # self.entity_cache[encoded_key] = entity.entity()
                    to_put[encoded_key] = entity.entity()
                response.add_entity().CopyFrom(entity)
        if to_put:
            memcache.set_multi(to_put)
Example #3
0
def get_games():
  game_slugs = get_game_slugs()
  if game_slugs:
    serialized_games = memcache.get_multi(game_slugs, key_prefix='game')
    
    # Checking game in slug and not in cache
    missing_slugs = list(game_slugs)
    games = []
    if serialized_games:
      for slug, game in serialized_games.iteritems():
        games.append(deserialize(game))
        missing_slugs.remove(slug)
    
    if missing_slugs:
      query = Game.all()
      query.filter('active =', True)
      query.filter('slug in', missing_slugs)
      missing_games = query.fetch(limit=None)
      serialized_games = {}
      for game in missing_games:
        games.append(game)
        serialized_games[game.slug] = serialize(game)
      memcache.set_multi(serialized_games, key_prefix='game')
    
    return games
  def get(self):
    # Memcache doesn't support saving values > 1MB. Break up features into chunks
    # and save those to memcache.
    if self.MODEL_CLASS == models.FeatureObserver:
      keys = self.PROPERTY_CLASS.get_property_chunk_memcache_keys(
          self.PROPERTY_CLASS, self.MEMCACHE_KEY)
      properties = memcache.get_multi(keys)

      if len(properties.keys()) != len(properties) or not properties:
        properties = self.__query_metrics_for_properties()

        # Memcache doesn't support saving values > 1MB. Break up list into chunks.
        chunk_keys = self.PROPERTY_CLASS.set_property_chunk_memcache_keys(self.MEMCACHE_KEY, properties)
        memcache.set_multi(chunk_keys, time=CACHE_AGE)
      else:
        temp_list = []
        for key in sorted(properties.keys()):
          temp_list.extend(properties[key])
        properties = temp_list
    else:
      properties = memcache.get(self.MEMCACHE_KEY)
      if properties is None:
        properties = self.__query_metrics_for_properties()
        memcache.set(self.MEMCACHE_KEY, properties, time=CACHE_AGE)

    properties = self._clean_data(properties)
    # Metrics json shouldn't be cached by intermediary caches because users
    # see different data when logged in. Set Cache-Control: private.
    super(FeatureHandler, self).get(properties, public=False)
Example #5
0
def primeDesireCache(step, startKey = None, stopKey = None):
    import princeFunc
    
    if stopKey is None and startKey is None:
        startKey = 1
        stopKey = int(meSchema.tradeCue.all(keys_only=True).order('-__key__').get().name())
    elif stopKey is None or startKey is None:
        raise(BaseException('Must define both startKey and stopKey, or both must be None!'))
    
    memdict = {}
    clockKeyStep = step
    queryStr = princeFunc.getDesireQueryStr(max(step-405,0),step)
    desires = db.GqlQuery(queryStr).fetch(20000)
    for desire in desires:
        desirekey = desire.key().name()
        stckID = meTools.getStckID(desire.Symbol)
        cueKey = desirekey.split("_")[-2]      # Extract cueKey from middle.
        memkey = 'desire_' + cueKey + '_' + str(stckID)
        step = int(desirekey.split("_")[0])    # Extract step from front part of desirekey.
        if not memdict.__contains__(memkey):
            memdict[memkey] = step
        elif memdict[memkey] < step:
            memdict[memkey] = step
    memcache.set_multi(memdict)
    cachepy.set_multi(memdict, priority = 1)
    cachepy.set('stepclock_' + str(startKey) + '_' + str(stopKey), clockKeyStep) # Manually syncing stepclock until get saner method.
Example #6
0
def perma_cache(id, update=False):
    post = memcache.get(id)
    if post is None or update:
        logging.error("DB QUERY")
        post = Blog.get_by_id(int(id))
        memcache.set_multi({id: post, 'perma_' + id: time.time()})
    return post
Example #7
0
def news(request, id):
	needData = ['news10', 'new_%s' % id]
	result = memcache.get_multi(['news10', 'new_%s' % id])
	needData = [k for k in needData if k not in result]

	toCache = {}
	newsFlag = True
	newFlag = True
	for key in needData:
		if key == 'news10':
			news10 = News.gql('ORDER BY added_on DESC LIMIT 10')
			news = []
			for new in news10:
				news.append(new)
			result[m.news3] = news
			toCache['news10'] = news
			newsFlag = False
		elif key == 'new_%s' % id:
			result['new'] = News.get_by_id(int(id))
			toCache['new_%s' % id] = result['new']
			newFlag = False
	if len(toCache) > 0:
		memcache.set_multi(toCache)
	if newFlag:
		result['new'] = result['new_%s' % id]
	if newsFlag:
		result[m.news3] = result['news10']
	return result
Example #8
0
def front_page(update=False, post_id=''):
    global last_queried, last_all_queried
    # post = ''
    if post_id:

        last_queried, post = multi_memcache_get('LQ' + str(post_id), str(post_id))
        # print(last_queried, 'sdfasdfasdfeWEWQERQWEREWQR', post)
        if not (last_queried and post):
            # print('DB Q\'ed for single post', last_queried)
            last_queried, post = datetime.today(), Posts.by_id(post_id)
            # memcache.set('LQ' + str(post_id), last_queried)
            # memcache.set(str(post_id), post)
            memcache.set_multi({'LQ' + str(post_id): last_queried,
                                       str(post_id): post})
            return post
        else:
            return post

    key = 'front'
    blog = memcache.get(key)
    if not all([last_all_queried, not update, blog]):
        last_all_queried = datetime.today()
        memcache.set('last all queried', last_all_queried)
        logging.error('DB GOT the QUERY')
        blog = db.GqlQuery("SELECT * "
                           "FROM Posts "
                           "ORDER BY created DESC "
                           "LIMIT 10")
        blog = list(blog)
        memcache.set(key, blog)
    return blog
Example #9
0
  def _Dynamic_Get(self, request, response):
    """Intercepts get requests and returns them from cache if available."""
    if request.has_transaction():
      self.CallWrappedStub('Get', request, response)
      return

    new_request = datastore_pb.GetRequest()
    new_response = datastore_pb.GetResponse()
    encoded_keys = [k.Encode() for k in request.key_list()]
    cached = memcache.get_multi(encoded_keys)

    for key, encoded_key in itertools.izip(request.key_list(), encoded_keys):
      if encoded_key not in cached:
        new_request.add_key().CopyFrom(key)

    if new_request.key_size() > 0:
      logging.info("SHIM: keys are not in memcache: %s" % request.key_list())
      self.CallWrappedStub('Get', new_request, new_response)

    entity_iter = iter(new_response.entity_list())
    to_put = dict()
    for encoded_key in encoded_keys:
      entity = cached.get(encoded_key, None)
      if entity:
        response.add_entity().mutable_entity().CopyFrom(entity)
      else:
        entity = entity_iter.next()
        if entity.entity().IsInitialized():
          # self.entity_cache[encoded_key] = entity.entity()
          to_put[encoded_key] = entity.entity()
        response.add_entity().CopyFrom(entity)
    if to_put:
      memcache.set_multi(to_put)
Example #10
0
def get_keys(keys):
    """
    Fetch and save the given keys into memcache.
    """

    # Perform the first load from memcache.
    urlsafes = [key.urlsafe() for key in keys]
    cache = memcache.get_multi(urlsafes, key_prefix=RTC)
    results = {}

    missing = []
    for urlsafe, key in zip(urlsafes, keys):
        if urlsafe in cache:
            results[key] = protobuf_to_ndb_entity(cache[urlsafe])
        else:
            missing.append(key)

    # Record cache stats.
    memcache.incr(MISSES, delta=len(missing), initial_value=0)
    memcache.incr(HITS, delta=len(results), initial_value=0)

    # Fill anything not yet in cache.
    if missing:
        writeback = {}
        for key, entity in zip(missing, ndb.get_multi(missing)):
            results[key] = entity
            writeback[key.urlsafe()] = ndb_entity_to_protobuf(entity)
        memcache.set_multi(writeback, key_prefix=RTC)

    return [results[key] for key in keys]
Example #11
0
    def UpdateStats(cls, category, stats):
        """Update the summary stats in memory and the datastore.

        This will only update part of a summary score row.

        Args:
            category: a category string like 'network'
            stats: a dict of browser stats (see CategoryStatsManager.GetStats)
        Returns:
            The summary stats that have been updated by the given stats.
            (Used by GetStats.)
        """
        browsers = [b for b in stats.keys() if b != 'total_runs']
        update_summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        for browser in browsers:
            ua_summary_stats = update_summary_stats.setdefault(browser, {
                    'results': {}})
            ua_summary_stats['results'][category] = {
                    'score': stats[browser]['summary_score'],
                    'display': stats[browser]['summary_display'],
                    'total_runs': stats[browser]['total_runs'],
                    }
            if category == 'acid3':
                ua_summary_stats['results']['acid3']['display'] = (
                        stats[browser]['results']['score']['display'])
        memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE)
        return update_summary_stats
Example #12
0
    def UpdateStatsCache(cls, category, browsers):
        """Update the memcache of stats for all the tests for each browser.

        This is also where the summary stats get updated.

        Args:
            category: a category string like 'network'
            browsers: a list of browsers like ['Firefox 3.6', 'IE 8.0']
        Returns:
            a list of browsers that were not processed due to a timeout.
        """
        test_set = all_test_sets.GetTestSet(category)
        test_keys = [t.key for t in test_set.VisibleTests()]
        ua_stats = {}
        unhandled_browsers = []
        is_timed_out = False
        for browser in browsers:
            try:
                medians, num_scores = test_set.GetMediansAndNumScores(browser)
            except db.Timeout:
                is_timed_out = True
            if is_timed_out:
                logging.info('Timed out \'%s\' in UpdateStatsCache doing '
                                         'GetMediansAndNumScores for %s', category, browser)
                unhandled_browsers.append(browser)
            else:
                stats = test_set.GetStats(test_keys, medians, num_scores)
                ua_stats[browser] = stats
        memcache.set_multi(ua_stats, **cls.MemcacheParams(category))
        if not is_timed_out:
            SummaryStatsManager.UpdateStats(category, ua_stats)
        return unhandled_browsers
Example #13
0
    def get(self):
        # Memcache doesn't support saving values > 1MB. Break up features into chunks
        # and save those to memcache.
        if self.MODEL_CLASS == models.FeatureObserver:
            keys = self.PROPERTY_CLASS.get_property_chunk_memcache_keys(
                self.PROPERTY_CLASS, self.MEMCACHE_KEY)
            properties = memcache.get_multi(keys)

            if len(properties.keys()) != len(properties) or not properties:
                properties = self.__query_metrics_for_properties()

                # Memcache doesn't support saving values > 1MB. Break up list into chunks.
                chunk_keys = self.PROPERTY_CLASS.set_property_chunk_memcache_keys(
                    self.MEMCACHE_KEY, properties)
                memcache.set_multi(chunk_keys, time=CACHE_AGE)
            else:
                temp_list = []
                for key in sorted(properties.keys()):
                    temp_list.extend(properties[key])
                properties = temp_list
        else:
            properties = memcache.get(self.MEMCACHE_KEY)
            if properties is None:
                properties = self.__query_metrics_for_properties()
                memcache.set(self.MEMCACHE_KEY, properties, time=CACHE_AGE)

        properties = self._clean_data(properties)
        # Metrics json shouldn't be cached by intermediary caches because users
        # see different data when logged in. Set Cache-Control: private.
        super(FeatureHandler, self).get(properties, public=False)
Example #14
0
    def get(self):
        tree = memcache.get('pretty_tree_tree')
        calc = memcache.get('pretty_tree_calc')
        if not tree or not calc:
            data_tree = dtmm_utils.get_modules(self)

            tree = []
            break_on = 3
            header_diff = 20
            width = 900
            calc = {
                'width': width,
                'cell_height': 80,  # in pixels :D
                'margin_width': width / 2
            }

            for fragment_num, fragment in enumerate(data_tree):
                logging.info('Fragment; %s' % fragment)
                logging.info('Fragment_num; %s' % fragment_num)

                cur_module = dtmm_utils.get_live_module_data(self, fragment)

                cur_module.update({
                    'filename': dtmm_utils.rpart(fragment['path']),
                    'row': fragment_num % break_on == 0,
                    'width': calc['width'] / break_on,
                    'index': fragment_num
                })

                tree.append(cur_module)

            rows = len(filter(itemgetter('row'), tree))
            calc['height'] = (rows * calc['cell_height']) + header_diff
            logging.info('This many rows; %s' % (rows))
            calc['margin_height'] = calc['height'] / 2
            calc['outer_container_height'] = calc['height']

            if len(tree) % break_on != 0:
                remainer = len(tree) % break_on
                for i in range(1, remainer + 1):
                    tree[-i]['width'] = calc['width'] / remainer

            tree[0]['row'] = False
            memcache.set_multi({
                'pretty_tree_tree': tree,
                'pretty_tree_calc': calc
            })

        # we want the colours to be different everytime
        colours = pretty_colours(len(tree))
        for idx, fragment in enumerate(tree):
            fragment.update({'background': colours[idx]})

        self.dorender(
            'tree_pretty.html',
            {
                'tree': tree,
                'calc': calc
            }
        )
Example #15
0
def generate_feed(uid):
    subscriptions = Subscription.query(Subscription.uid == uid).fetch(200)
    subscription_urls = [sub.url for sub in subscriptions if sub.url]
    if len(subscription_urls) > 0:
        sources = Source.query(Source.url.IN(subscription_urls)).order(-Source.most_recent_article_added_date).fetch(len(subscription_urls))
        
        source_jsons = {}
        for source_json in memcache.get_multi([source.feed_cache_key() for source in sources]).itervalues():
            source_jsons[source_json['id']] = source_json
        
        to_fetch = [source for source in sources if source.key.id() not in source_jsons]
        print 'HITS {0} TO_FETCH {1}'.format(len(source_jsons), len(to_fetch))
        if len(to_fetch):
            source_promises = [src.json(include_articles=True, article_limit=FEED_ARTICLE_LIMIT, return_promise=True) for src in to_fetch]
            for promise in source_promises:
                data = promise()
                source_jsons[data['id']] = data
        
        # put the cache keys:
        if len(to_fetch):
            memcache.set_multi({source.feed_cache_key(): source_jsons[source.key.id()] for source in to_fetch if (source.key.id() in source_jsons)})
        
        source_json = [source_jsons[source.key.id()] for source in sources if source.key.id() in source_jsons]
    else:
        source_json = []
    return {
        "sources": source_json
    }
Example #16
0
    def UpdateStats(cls, category, stats):
        """Update the summary stats in memory and the datastore.

        This will only update part of a summary score row.

        Args:
            category: a category string like 'network'
            stats: a dict of browser stats (see CategoryStatsManager.GetStats)
        Returns:
            The summary stats that have been updated by the given stats.
            (Used by GetStats.)
        """
        browsers = [b for b in stats.keys() if b != 'total_runs']
        update_summary_stats = memcache.get_multi(
                browsers, namespace=cls.MEMCACHE_NAMESPACE)
        for browser in browsers:
            ua_summary_stats = update_summary_stats.setdefault(browser, {
                    'results': {}})
            ua_summary_stats['results'][category] = {
                    'score': stats[browser]['summary_score'],
                    'display': stats[browser]['summary_display'],
                    'total_runs': stats[browser]['total_runs'],
                    }
            if category == 'acid3':
                ua_summary_stats['results']['acid3']['display'] = (
                        stats[browser]['results']['score']['display'])
        memcache.set_multi(update_summary_stats, namespace=cls.MEMCACHE_NAMESPACE)
        return update_summary_stats
Example #17
0
 def set_many(self, data, timeout=0):
     safe_data = {}
     for key, value in data.items():
         if isinstance(value, unicode):
             value = value.encode('utf-8')
         safe_data[smart_str(key)] = value
     memcache.set_multi(mapping=safe_data, time=timeout)
Example #18
0
def memcache_usage():
	email = get_user_email()
	fname = ''
	lname = ''
	username = ''
	if email:
		if memcache.get("user_"):
			return True
		else:
			qry = accountModel.get_by_id(users.get_current_user().user_id())
			if qry:
				fname = qry.firstName
				lname = qry.lastName
				username = qry.username
			else:  
				lm = lobbyModel.query(lobbyModel.ownerID == "118168406204694893029 ").get()
				if lm:
					publob = lobbyAccessModel(lobbyID=lm.key, userID=users.get_current_user().user_id())
					lobbyAccessModel.put(publob)
				user = accountModel(id=users.get_current_user().user_id(), firstName=fname, lastName=lname, username=email.split("@", 1)[0])
				accountModel.put(user)
				
			memcache.set_multi({"fname": fname,
						"lname": lname,
						"username": username},
						key_prefix="user_", time=3600)
			return True
	return False
Example #19
0
    def UpdateStatsCache(cls, category, browsers):
        """Update the memcache of stats for all the tests for each browser.

        This is also where the summary stats get updated.

        Args:
            category: a category string like 'network'
            browsers: a list of browsers like ['Firefox 3.6', 'IE 8.0']
        Returns:
            a list of browsers that were not processed due to a timeout.
        """
        test_set = all_test_sets.GetTestSet(category)
        test_keys = [t.key for t in test_set.VisibleTests()]
        ua_stats = {}
        unhandled_browsers = []
        is_timed_out = False
        for browser in browsers:
            try:
                medians, num_scores = test_set.GetMediansAndNumScores(browser)
            except db.Timeout:
                is_timed_out = True
            if is_timed_out:
                logging.info('Timed out \'%s\' in UpdateStatsCache doing '
                                         'GetMediansAndNumScores for %s', category, browser)
                unhandled_browsers.append(browser)
            else:
                stats = test_set.GetStats(test_keys, medians, num_scores)
                ua_stats[browser] = stats
        memcache.set_multi(ua_stats, **cls.MemcacheParams(category))
        if not is_timed_out:
            SummaryStatsManager.UpdateStats(category, ua_stats)
        return unhandled_browsers
Example #20
0
	def get(self):
		self.response.headers['Content-Type'] = 'text/plain'

		rates_url = 'http://www.twilio.com/resources/rates/international-rates.csv'
		rates_data = ''

		# Get the data from Twilio
		try:
			remote_data = urlfetch.fetch(rates_url)
			if remote_data.status_code != 200:
				raise Exception('')
			rates_csv = remote_data.content
			rates_data = csv.reader(cStringIO.StringIO(rates_csv))
		except:
			self.response.out.write('{"status": "ERROR"}')
			return

		# Save the data into memcache (expire in 10 days)
		for row in rates_data:
			country = row[0]
			rate = row[1]
			numbers = row[2].split(' ')
			d = {}
			for number in numbers:
				d[number] = '{"country": "' + country + '", "rate": "' + rate + '"}'
				#memKey = number
				#memVal = '{"country": "' + country + '", "rate": "' + rate + '"}'
				#memcache.set(key = memKey, value = memVal, time = 864000)
			memcache.set_multi(d, time = 864000)

		self.response.out.write('{"status": "OK"}')
Example #21
0
  def _Dynamic_Next(self, request, response):
    """Intercepts query results and caches the returned entities."""
    self.CallWrappedStub('Next', request, response)

    if not response.keys_only():
      to_put = dict([(e.key().Encode(), e) for e in response.result_list()])
      memcache.set_multi(to_put)
Example #22
0
def cache_multiset(data_dict, time=settings.DEFAULT_CACHE_TIME):
    """ Sends multiple objects to a memory cache."""
    data = {}
    for k, v in data_dict:
        data[k if isinstance(k, basestring) else "|".join(k)] = v
    memcache.set_multi(data, time, namespace=settings.CACHE_NAMESPACE)
    return data_dict
Example #23
0
def perma_cache(id, update = False):
    post = memcache.get(id)
    if post is None or update:
        logging.error("DB QUERY")
        post = Blog.get_by_id(int(id))
        memcache.set_multi({id: post, 'perma_'+id:time.time()})
    return post
Example #24
0
def cancel_cards(userid):
    current_val = memcache.get(key=userid)
    if current_val is None:
        logging.warn('Nothing scheduled for user %s', current_val)
        return
    memcache_values = { card:1 for card in current_val.split('__') }
    memcache.set_multi(memcache_values, time=60*60)
Example #25
0
	def get_api(self):
		memcache_key_prefix = 'refresh_item_api_'
		memcache_keys = ['page', 'pages']

		page_info = memcache.get_multi(memcache_keys, key_prefix = memcache_key_prefix)
		if 'page' not in page_info or 'pages' not in page_info:
			page = pages = 1
		else:
			page, pages = page_info['page'], page_info['pages']

		put_rpcs = []
		while page <= pages:
			memcache.set_multi({'page' : page, 'pages' : pages}, key_prefix = memcache_key_prefix)
			logging.info('Getting page = %d / %d ...' % (page, pages))
			method = '/simple/auctions.list'
			content = fetch_api(method, {'defs' : 1, 'per_page' : 1000, 'page' : page})
			auctions_info = parse_json(content, None)
			page = auctions_info['page'] + 1
			pages = auctions_info['pages']
			items_info = auctions_info['items']
			for item_info in items_info.itervalues():
				item_def = item_info['item_def']
				image = Image(key_name = item_def['class_tsid'], url = item_def['iconic_url'],
						width = 40, height = 40)
				put_rpcs.append(put_async(image))
				item = Item(key_name = item_def['class_tsid'], category = item_def['category'],
						name = item_def['name_single'], image = image)
				put_rpcs.append(put_async(item))
		memcache.delete_multi(memcache_keys, key_prefix = memcache_key_prefix)
		return put_rpcs
Example #26
0
def getTopThreadsPerForum(forums):
  """For each forum, returns the 'top' thread, which we'll display in the forum list page.

  The 'top' thread is the most-recently created thread. When you click through to the forum, the
  top thread will actually be the thread with the most recent reply, so it's slightly different."""
  keynames = []
  for forum in forums:
    keynames.append("forum:%s:top-thread" % (forum.slug))
  cache_mapping = memcache.get_multi(keynames)

  # fetch any from the data store that weren't cached
  for forum in forums:
    keyname = "forum:%s:top-thread" % (forum.slug)
    if keyname not in cache_mapping:
      query = model.forum.ForumThread.all().filter("forum", forum).order("-posted").fetch(1)
      for forum_thread in query:
        cache_mapping[keyname] = forum_thread
        break

  memcache.set_multi(cache_mapping)

  # convert from our (internal) memcache key names to a more reasonable key
  top_threads = {}
  for forum in forums:
    keyname = "forum:%s:top-thread" % (forum.slug)
    if keyname in cache_mapping:
      top_threads[forum.slug] = cache_mapping[keyname]

  return top_threads
def memGetPercentReturns(memkeylist, prefix):
    EntityDict = {}
    newMemEntities = {}
    memCacheEntities = {}
    
    memEntities = cachepy.get_multi(memkeylist)
    missingKeys = meTools.getMissingKeys(memkeylist,memEntities)
    
    memCacheEntities = memcache.get_multi(missingKeys)
    cachepy.set_multi(memCacheEntities)
    missingKeys = meTools.getMissingKeys(missingKeys,memCacheEntities)

    memEntities.update(memCacheEntities)
    if missingKeys:
        missingKeys = [key.replace(prefix,'') for key in missingKeys]
        if prefix == 'BTR-':
            Entities = meSchema.backTestResult.get_by_key_name(missingKeys)
        elif prefix == 'LAR-':
            Entities = meSchema.liveAlg.get_by_key_name(missingKeys)
        for entity in Entities:
            if entity:
                memkey = prefix + entity.key().name()
                pReturn = entity.percentReturn
                newMemEntities[memkey] = pReturn
                memEntities[memkey] = pReturn
        memcache.set_multi(newMemEntities)
        cachepy.set_multi(newMemEntities)
    return memEntities
    def _render(self, year):
        year = int(year)
        avatars = []
        shards = memcache.get_multi(['{}avatars_{}'.format(year, i) for i in xrange(10)])
        if len(shards) == 10:  # If missing a shard, must refetch all
            for _, shard in sorted(shards.items(), key=lambda kv: kv[0]):
                avatars += shard

        if not avatars:
            avatars_future = Media.query(Media.media_type_enum == MediaType.AVATAR, Media.year == year).fetch_async()
            avatars = sorted(avatars_future.get_result(), key=lambda a: int(a.references[0].id()[3:]))

            shards = {}
            size = len(avatars) / 10 + 1
            for i in xrange(10):
                start = i * size
                end = start + size
                shards['{}avatars_{}'.format(year, i)] = avatars[start:end]
            memcache.set_multi(shards, 60*60*24)

        self.template_values.update({
            'year': year,
            'avatars': avatars,
        })
        return jinja2_engine.render('avatars.html', self.template_values)
    def _render(self, year):
        year = int(year)
        avatars = []
        shards = memcache.get_multi(
            ['{}avatars_{}'.format(year, i) for i in xrange(10)])
        if len(shards) == 10:  # If missing a shard, must refetch all
            for _, shard in sorted(shards.items(), key=lambda kv: kv[0]):
                avatars += shard

        if not avatars:
            avatars_future = Media.query(
                Media.media_type_enum == MediaType.AVATAR,
                Media.year == year).fetch_async()
            avatars = sorted(avatars_future.get_result(),
                             key=lambda a: int(a.references[0].id()[3:]))

            shards = {}
            size = len(avatars) / 10 + 1
            for i in xrange(10):
                start = i * size
                end = start + size
                shards['{}avatars_{}'.format(year, i)] = avatars[start:end]
            memcache.set_multi(shards, 60 * 60 * 24)

        self.template_values.update({
            'year': year,
            'avatars': avatars,
        })
        return jinja2_engine.render('avatars.html', self.template_values)
Example #30
0
 def get_status_dict(self):
   # It will returns a serialized status in mem_cache
   serialized_status = memcache.get_multi(self.status_ids,
                                               key_prefix='status')
   # Just copying my status_ids list to make some changes localy
   missing_status_ids = list(self.status_ids)
   game_status = {}
   for status_id, status in serialized_status.iteritems():
     game_status[status_id] = deserialize(status)
     missing_status_ids.remove(status_id)
   
   # Taking the missing status in database and add them in memcache
   if missing_status_ids:
     missing_status = Status.get(missing_status_ids)
     serialized_status = {}
     for status in missing_status:
       game_status[status_id] = deserialize(status)
       serialized_status[status.id] = serialize(status)
       
     memcache.set_multi(serialized_status, key_prefix='status')
   
   # I really dunno why, but the game_status list in this function
   # works like a list of string, and when this function pass to some
   # function or when it returns, game_status assume its really identity
   # that is a list of status, not a list of strings... (crazy, I know)
   self.actualise_status(game_status)
   
   return game_status # Returns a random list of Status playing this game
Example #31
0
	def get_image(self):
		#TODO: support multiple badge sizes
		im = get_cached_issue_image(self.number)
		if not im:
			im, width = draw_image(self.number, self.vote_count)
			memcache_key = "issue_%d_image" % self.number
			memcache.set_multi({memcache_key: im.tostring(), memcache_key + "_width" : width})
		return im
Example #32
0
def get_multi_test():

    memcache.set_multi(DATA, 30, key_prefix='memcache_get_multi_test')
    now = time()
    memcache.get_multi(MULTI_KEYS, key_prefix='memcache_get_multi_test')
    result = time() - now
    memcache.delete_multi(MULTI_KEYS, key_prefix='memcache_get_multi_test')
    return result
Example #33
0
 def storeAssociation(self, server_url, association):
     data = association.serialize()
     key1, key2 = self.getAssociationKeys(server_url, association.handle)
     memcache.set_multi({
         key1: data,
         key2: data
     },
                        namespace=MEMCACHE_NAMESPACE)
Example #34
0
    def get(self):
        self.test_memcache()
        self.test_memcache()

        memcache.set_multi({'key': 'value', 'other': 'value'})
        memcache.get_multi(['key', 'other', 'thing'])

        self.response.out.write('Hello world')
Example #35
0
def mem_set(key, value, chunksize=900000):
  serialized = pickle.dumps(value, 2)
  values = {}
  for i in xrange(0, len(serialized), chunksize):
    values['%s.%s' % (key, i//chunksize)] = serialized[i : i+chunksize]
    logging.info("cached: " + ('%s.%s' % (key, i//chunksize))+ str(values['%s.%s' % (key, i//chunksize)]));
  memcache.set_multi(values)
  memcache.set(key, len(values))
def ExecuteApiQueryTask(api_query):
  """Executes a refresh of an API Query from the task queue.

    Attempts to fetch and update an API Query and will also log any errors.
    Schedules the API Query for next execution.

  Args:
    api_query: The API Query to refresh.

  Returns:
    A boolean. True if the API refresh was a success and False if the API
    Query is not valid or an error was logged.
  """
  if api_query:
    query_id = str(api_query.key())
    api_query.in_queue = False

    api_response_content = FetchApiQueryResponse(api_query)

    if not api_response_content or api_response_content.get('error'):
      InsertApiQueryError(api_query, api_response_content)

      if api_query.is_error_limit_reached:
        api_query.is_scheduled = False

      SaveApiQuery(api_query)

      # Since it failed, execute the query again unless the refresh interval of
      # query is less than the random countdown, then schedule it normally.
      if api_query.refresh_interval < co.MAX_RANDOM_COUNTDOWN:
        schedule_helper.ScheduleApiQuery(api_query)  # Run at normal interval.
      else:
        schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
      return False

    else:
      SaveApiQueryResponse(api_query, api_response_content)

      # Check that public  endpoint wasn't disabled after task added to queue.
      if api_query.is_active:
        memcache.set_multi({'api_query': api_query,
                            co.DEFAULT_FORMAT: api_response_content},
                           key_prefix=query_id,
                           time=api_query.refresh_interval)
        # Delete the transformed content in memcache since it will be updated
        # at the next request.
        delete_keys = set(co.SUPPORTED_FORMATS) - set([co.DEFAULT_FORMAT])
        memcache.delete_multi(list(delete_keys), key_prefix=query_id)

        SaveApiQuery(api_query)
        schedule_helper.ScheduleApiQuery(api_query)
        return True

      # Save the query state if the user has disabled it
      # while it was in the task queue.
      if api_query.is_active is False:
        SaveApiQuery(api_query)
  return False
Example #37
0
  def blackhole(self, urls):
    """Blackholes a set of URLs by domain for the rest of the current period.

    Args:
      urls: Iterable of URLs to blackhole.
    """
    values = dict(('failure:' + get_url_domain(u), self.min_requests)
                  for u in urls)
    memcache.set_multi(values, key_prefix=self.prefix)
Example #38
0
 def save_objects(self, keys_to_objects):
     if not keys_to_objects:
         return
     memcache_set = {}
     for k, v in keys_to_objects.iteritems():
         if self._is_cacheable(k, v):
             cache_key = self.key_to_cache_key(k)
             memcache_set[cache_key] = v
     memcache.set_multi(memcache_set, 2 * 3600)
Example #39
0
  def blackhole(self, urls):
    """Blackholes a set of URLs by domain for the rest of the current period.

    Args:
      urls: Iterable of URLs to blackhole.
    """
    values = dict(('failure:' + get_url_domain(u), self.min_requests)
                  for u in urls)
    memcache.set_multi(values, key_prefix=self.prefix)
Example #40
0
def front_cache(update=False):
    key = 'blog'
    posts = memcache.get(key)
    if posts is None or update:
        logging.error("DB QUERY")
        posts = db.GqlQuery("SELECT * FROM Blog ORDER BY created DESC")
        posts = list(posts)
        memcache.set_multi({key: posts, 'front_queried': time.time()})
    return posts
 def set_multi_models(models):
     # This uses dict comprehension to create a dictionary that looks like that:
     # {'model1.key()': [model1],
     #  'model2.key()': [model2], ...}
     #
     # memcache.set_multi() receives this mapping and puts multiple values at the
     # same time.
     mapping = {str(model.key()): model for model in models}
     memcache.set_multi(mapping)
def ExecuteApiQueryTask(api_query):
  """Executes a refresh of an API Query from the task queue.

    Attempts to fetch and update an API Query and will also log any errors.
    Schedules the API Query for next execution.

  Args:
    api_query: The API Query to refresh.

  Returns:
    A boolean. True if the API refresh was a success and False if the API
    Query is not valid or an error was logged.
  """
  if api_query:
    query_id = str(api_query.key())
    api_query.in_queue = False

    api_response_content = FetchApiQueryResponse(api_query)

    if not api_response_content or api_response_content.get('error'):
      InsertApiQueryError(api_query, api_response_content)

      if api_query.is_error_limit_reached:
        api_query.is_scheduled = False

      SaveApiQuery(api_query)

      # Since it failed, execute the query again unless the refresh interval of
      # query is less than the random countdown, then schedule it normally.
      if api_query.refresh_interval < co.MAX_RANDOM_COUNTDOWN:
        schedule_helper.ScheduleApiQuery(api_query)  # Run at normal interval.
      else:
        schedule_helper.ScheduleApiQuery(api_query, randomize=True, countdown=0)
      return False

    else:
      SaveApiQueryResponse(api_query, api_response_content)

      # Check that public  endpoint wasn't disabled after task added to queue.
      if api_query.is_active:
        memcache.set_multi({'api_query': api_query,
                            co.DEFAULT_FORMAT: api_response_content},
                           key_prefix=query_id,
                           time=api_query.refresh_interval)
        # Delete the transformed content in memcache since it will be updated
        # at the next request.
        delete_keys = set(co.SUPPORTED_FORMATS) - set([co.DEFAULT_FORMAT])
        memcache.delete_multi(list(delete_keys), key_prefix=query_id)

        SaveApiQuery(api_query)
        schedule_helper.ScheduleApiQuery(api_query)
        return True

      # Save the query state just in case the user disabled it
      # while it was in the task queue.
      SaveApiQuery(api_query)
  return False
Example #43
0
    def put(self, key, value):
        logs.log('MemcacheLarge put %s.' + key)
        # Make JSON representation as compact as possible (don't use spaces).
        string_value = json.dumps(value, separators=(',', ':'))
        keys_and_values = {key: len(string_value)}
        for chunk_start in xrange(0, len(string_value), self.CHUNK_LEN):
            full_key = '%s-%s-%s' % (self.MAGIC_STR, key, chunk_start)
            keys_and_values[full_key] = string_value[chunk_start:chunk_start +
                                                     self.CHUNK_LEN]

        memcache.set_multi(keys_and_values)
Example #44
0
 def get_image(self):
     #TODO: support multiple badge sizes
     im = get_cached_issue_image(self.number)
     if not im:
         im, width = draw_image(self.number, self.vote_count)
         memcache_key = "issue_%d_image" % self.number
         memcache.set_multi({
             memcache_key: im.tostring(),
             memcache_key + "_width": width
         })
     return im
Example #45
0
    def Set(cls, key, value):
        """Sets a value in memcache."""
        serialized_parts = _Serialize(value)
        if len(serialized_parts) > _MAX_NUM_PARTS:
            logging.error('Max number of parts reached.')
            return

        cached_values = {}
        cached_values[cls._GetCacheKey(key)] = len(serialized_parts)
        for i in xrange(len(serialized_parts)):
            cached_values[cls._GetCacheKey(key, i)] = serialized_parts[i]
        memcache.set_multi(cached_values)
def UrlsToItemIds(urls):
    """Returns a map from url to item id."""
    if not urls:
        return {}
    url_to_item_id = memcache.get_multi(urls, key_prefix='u2i:')
    urls_not_in_cache = [url for url in urls if url not in url_to_item_id]
    if urls_not_in_cache:
        new_items = {}
        for item in Item.query(Item.url.IN(urls_not_in_cache)).fetch():
            url_to_item_id[item.url] = item.ItemId()
            new_items[item.url] = item.ItemId()
        memcache.set_multi(new_items, key_prefix='u2i:')
    return url_to_item_id
Example #47
0
  def get_multi(self, keys):
    result = {}

    not_local = []
    for item_key in keys:
      try:
        result[item_key] = self._cache[item_key]
      except KeyError:
        not_local.append(item_key)

    if not_local:
      to_get = []
      to_get.extend(not_local)
      to_get.extend(self._prefetch)
      self._prefetch = set()

      try:
        cached = memcache.get_multi(to_get, self._prefix)
      except DeadlineExceededError:
        cached = {}

      to_compute = []
      for item_key in to_get:
        try:
          r = cached[item_key]
        except KeyError:
          to_compute.append(item_key)
          continue
        self._cache[item_key] = r
        result[item_key] = r

      if to_compute:
        to_cache = {}
        for item_key, r in zip(to_compute, self._compute(to_compute)):
          if r is not None:
            to_cache[item_key] = r
          self._cache[item_key] = r
          result[item_key] = r

        if to_cache:
          try:
            if self._timeout is None:
              memcache.set_multi(to_cache,
                                 key_prefix = self._prefix)
            else:
              memcache.set_multi(to_cache,
                                 time = self._timeout,
                                 key_prefix = self._prefix)
          except DeadlineExceededError:
            pass
    return result
Example #48
0
    def testMulti(self):
        """Stores multiple keys' values at once."""

        memcache.set_multi({'map_key_one': 1, 'map_key_two': u'some value'})
        values = memcache.get_multi(['map_key_one', 'map_key_two'])
        assert {'map_key_one': 1, 'map_key_two': u'some value'} == values

        memcache.add_multi({
            'map_key_one': 'one',
            'map_key_two': 2,
            'three': u'trois'
        })
        values = memcache.get_multi(['map_key_two', 'three'])
        assert {'map_key_two': u'some value', 'three': u'trois'} == values
Example #49
0
def _memcache_put(models, time=0):
    '''Put given models to memcache in serialized form
   with expiration in seconds
     
  Returns:
    List of  db.Keys of the models that were put
  '''
    to_put = _to_dict(models)

    for key, model in to_put.iteritems():
        to_put[key] = _serialize(model)

    memcache.set_multi(to_put, time)
    return [model.key() for model in models]
Example #50
0
  def _PutTilesToMemcache(self):
    mapping = {}
    for tile in self.tiles.values():
      key = self._GetGameTileKeyName(tile.Id())
      mapping[key] = db.model_to_protobuf(tile)

    for email in self.players:
      key = self._GetPlayerTileLocationKeyName(email)
      mapping[key] = self.players[email]
      
    if len(mapping):
      logging.debug("Putting %d game tiles to memcache." % len(mapping))
      memcache.set_multi(mapping)
    else:
      logging.debug("Not putting any game tiles to memcache.")
Example #51
0
def get_queues(bot_root_key):
    """Returns the known task queues as integers.

  This function is called to get the task queues to poll, as the bot is trying
  to reap a task, any task.

  It is also called while the bot is running a task, to refresh the task queues.

  Arguments:
    bot_root_key: ndb.Key to bot_management.BotRoot

  Returns:
    dimensions_hashes: list of dimension_hash for the bot
  """
    bot_id = bot_root_key.string_id()
    dimensions_hashes = memcache.get(bot_id, namespace='task_queues')
    if dimensions_hashes is not None:
        # Note: This may return stale queues. We may want to change the format to
        # include the expiration.
        logging.debug('get_queues(%s): can run from %d queues (memcache)\n%s',
                      bot_id, len(dimensions_hashes), dimensions_hashes)
        # Refresh all the keys.
        memcache.set_multi({str(d): True
                            for d in dimensions_hashes},
                           time=61,
                           namespace='task_queues_tasks')
        return dimensions_hashes

    # Retrieve all the dimensions_hash that this bot could run that have
    # actually been triggered in the past. Since this is under a root entity, this
    # should be fast.
    now = utils.utcnow()
    dimensions_hashes = sorted(
        obj.key.integer_id()
        for obj in BotTaskDimensions.query(ancestor=bot_root_key)
        if obj.valid_until_ts >= now)
    memcache.set(bot_id,
                 dimensions_hashes,
                 namespace='task_queues',
                 time=_EXPIRATION_TIME_TASK_QUEUES)
    logging.info('get_queues(%s): Query in %.3fs: can run from %d queues\n%s',
                 bot_id, (utils.utcnow() - now).total_seconds(),
                 len(dimensions_hashes), dimensions_hashes)
    memcache.set_multi({str(d): True
                        for d in dimensions_hashes},
                       time=61,
                       namespace='task_queues_tasks')
    return dimensions_hashes
Example #52
0
    def test_put_clears_cache(self):
        pc_id = 'ProjectCohort_foo'
        survey_id = 'Survey_foo'
        code = 'foo bar'

        start1 = datetime.datetime.today()
        end1 = start1 + datetime.timedelta(days=1)
        start2 = start1 + datetime.timedelta(days=2)
        end2 = start1 + datetime.timedelta(days=3)

        cache_data = {
            ParticipantData.participation_cache_key(pc_id): {
                ParticipantData.date_key(start1, end1): ['result1'],
                ParticipantData.date_key(start2, end2): ['result2'],
            },
            ParticipantData.participation_cache_key(survey_id): {
                ParticipantData.date_key(start1, end1): ['result1'],
                ParticipantData.date_key(start2, end2): ['result2'],
            },
            ParticipantData.participation_by_pc_cache_key(pc_id): {
                ParticipantData.date_key(start1, end1): ['result1'],
                ParticipantData.date_key(start2, end2): ['result2'],
            },
            ParticipantData.participation_by_pc_cache_key(code): {
                ParticipantData.date_key(start1, end1): ['result1'],
                ParticipantData.date_key(start2, end2): ['result2'],
            },
        }
        memcache.set_multi(cache_data)

        # Write a pd that relates to the pc and survey, falling in the first
        # date range. That date range should clear, the other should remain.
        pd = ParticipantData.create(
            key='progress',
            value=1,
            participant_id='Participant_foo',
            program_label='demo-program',
            project_id='Project_foo',
            cohort_label='2019',
            project_cohort_id=pc_id,
            code=code,
            survey_id=survey_id,
            survey_ordinal=1,
        )
        ParticipantData.put_for_index(pd, 'participant-survey-key')

        for cache_key in cache_data.keys():
            self.assertEqual(len(memcache.get(cache_key)), 1)
Example #53
0
def get_problem_list(key_list):
	"""
	Returns list of problems in datastore as per the key_list. Uses and updates memcache as necessary.
	"""
	id_list = [key.id() for key in key_list]
	retrieved_dict = memcache.get_multi(id_list, namespace='p') #these returns a dict of (id, entity) retreived from memcache
	retrieved_problems = retrieved_dict.values()
	retrieved_ids = retrieved_dict.keys()
	not_retrieved_ids = list(set(id_list) - set(retrieved_ids))
	if not not_retrieved_ids: #saves 1 memcache set rpc
		return retrieved_problems
	not_retrieved_keys = [ndb.Key(Problem, id) for id in not_retrieved_ids]
	not_retrieved_problems = ndb.get_multi(not_retrieved_keys)
	not_retrieved_dict = dict(zip(not_retrieved_ids, not_retrieved_problems))
	memcache.set_multi(not_retrieved_dict, namespace='p')
	return retrieved_problems + not_retrieved_problems
Example #54
0
 def get(self):
     # [START batch]
     values = {'comment': 'I did not ... ', 'comment_by': 'Bill Holiday'}
     if not memcache.set_multi(values):
         logging.error('Unable to set Memcache values')
     tvalues = memcache.get_multi(('comment', 'comment_by'))
     self.response.write(tvalues)
Example #55
0
    def set_multi_async(self,
                        mapping,
                        time=0,
                        key_prefix='',
                        min_compress_len=0,
                        namespace=None,
                        rpc=None):
        prefixed_mapping = {}
        for key, value in mapping.items():
            prefixed_mapping[default_key_func(key, KEY_PREFIX,
                                              VERSION)] = value

        if self.sync_mode:
            # We don't call up, because set_multi calls set_multi_async
            return memcache.set_multi(prefixed_mapping,
                                      time=time,
                                      key_prefix=key_prefix,
                                      min_compress_len=min_compress_len,
                                      namespace=namespace)
        else:
            return super(KeyPrefixedClient, self).set_multi_async(
                prefixed_mapping,
                time=time,
                key_prefix=key_prefix,
                min_compress_len=min_compress_len,
                namespace=namespace,
                rpc=rpc)
Example #56
0
 def _put_tasklet(self, todo):
   assert todo
   # TODO: What if the same entity is being put twice?
   # TODO: What if two entities with the same key are being put?
   # TODO: Clear entities from memcache before starting the write?
   # TODO: Attempt to prevent dogpile effect while keeping cache consistent?
   ents = [ent for (_, ent) in todo]
   results = yield self._conn.async_put(None, ents)
   for key, (fut, ent) in zip(results, todo):
     if key != ent.key:
       assert ent.key is None or not list(ent.key.flat())[-1]
       ent.key = key
     fut.set_result(key)
   # Now update memcache.
   # TODO: Could we update memcache *before* calling async_put()?
   # (Hm, not for new entities but possibly for updated ones.)
   mapping = {}
   for _, ent in todo:
     if self.should_memcache(ent.key):
       pb = self._conn.adapter.entity_to_pb(ent)
       mapping[ent.key.urlsafe()] = pb
   if mapping:
     # TODO: Optionally set the memcache expiration time;
     # maybe configurable based on key (or even entity).
     failures = memcache.set_multi(mapping)
     if failures:
       badkeys = []
       for failure in failures:
         badkeys.append(mapping[failure].key)
       logging.info('memcache failed to set %d out of %d keys: %s',
                    len(failures), len(mapping), badkeys)