Beispiel #1
0
 def addIfNew(self, names):
   new_names = []
   fresh = None
   for name in names:
     if name not in self.known_counter_names_:
       if fresh == None:
         fresh = self.all()
         if name not in fresh:
           new_names += [name] 
       else:
         new_names += [name] 
   if new_names:
     namespace = namespace_manager.get_namespace()
     try:
       namespace_manager.set_namespace(self.namespace)
       if self.last_shard_ == None:
         CounterNamesShard.get_or_insert("0")
         self.last_shard_ = 0
       ADD_SUCCESS = 1
       ADD_FULL = 2
       ADD_FAIL = 3
       def addNames(key, names):
         record = db.get(key)
         local_names = set(record.names)
         for name in names:
           if name not in local_names:
             local_names.add(name)
             record.names += [name]
         try:
           record.put()
           return ADD_SUCCESS
         except (RequestTooLargeError, ValueError):
           if len(names) == len(record.names):
             return ADD_FAIL
           else:            
             return ADD_FULL
       result = None
       try:
         result = db.run_in_transaction(addNames,
             db.Key.from_path('CounterNamesShard', str(self.last_shard_)),
             new_names)
       except:
         result = ADD_FAIL          
       if result == ADD_FULL:
         CounterNamesShard.get_or_insert(str(self.last_shard_ + 1))
         self.addIfNew(names)
       if result == ADD_SUCCESS:
         logging.info("Registered new counter names: %s." %
                      ",".join(new_names))
       else:
         logging.warning(
             "Coudn't register counter names: %s. (Will retry next time)" %
             ",".join(new_names))
     finally:
       namespace_manager.set_namespace(namespace)
   return (not not fresh, len(new_names))  
Beispiel #2
0
 def delete(self, regexp):
   # NOTE(andrin): When a counter is deleted and used again afterwards
   #               it will not be harvested in the following case:
   #               - Instance A deletes counter C.
   #               - Instance B has C still in self.known_counter_names_
   #                 and increments C without updating the CounterNamesShards.
   #               - Instance A generates the harvest report and wont find C.
   #               To fix this issue, we reload the DefaultCounterNamesManager
   #               once a day.
   #               To fix this issue manually, you can just deploy a new
   #               version of your app.
   namespace = namespace_manager.get_namespace()
   try:
     namespace_manager.set_namespace(self.namespace)
     to_delete = []
     matcher = re.compile("(" + regexp + ")$")
     for name in self.all():
       if matcher.match(name):
         to_delete += [name]
     for shard in CounterNamesShard.all():
       dirty = False
       for name in to_delete:
         if name in shard.names:
           shard.names.remove(name)
           dirty = True
       if dirty:
         shard.put()
     for name in to_delete:
       if name in self.known_counter_names_:
         self.known_counter_names_.remove(name)
     return to_delete
   finally:
     namespace_manager.set_namespace(namespace)
Beispiel #3
0
 def all(self, force_reload=False):
   namespace = namespace_manager.get_namespace()
   try:
     namespace_manager.set_namespace(self.namespace)
     query = CounterNamesShard.all()
     if force_reload:
       self.known_counter_names_ = set([])
     elif self.last_update_:
       query.filter("timestamp >= ",
                    self.last_update_-
                    datetime.timedelta(0, config.MAX_CLOCK_SKEW))
     for record in query:
       self.known_counter_names_ = self.known_counter_names_.union(
                                       record.names)
       self.last_shard_ = max(self.last_shard_, int(record.key().name()))
       self.last_update_ = datetime.datetime.now()
     return self.known_counter_names_
   finally:
     namespace_manager.set_namespace(namespace)