def addIfNew(self, names):
new_names = []
fresh = None
for name in names:
if name not in self.known_counter_names_:
if fresh == None:
fresh = self.all()
if name not in fresh:
new_names += [name]
else:
new_names += [name]
if new_names:
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
if self.last_shard_ == None:
CounterNamesShard.get_or_insert("0")
self.last_shard_ = 0
ADD_SUCCESS = 1
ADD_FULL = 2
ADD_FAIL = 3
def addNames(key, names):
record = db.get(key)
local_names = set(record.names)
for name in names:
if name not in local_names:
local_names.add(name)
record.names += [name]
try:
record.put()
return ADD_SUCCESS
except (RequestTooLargeError, ValueError):
if len(names) == len(record.names):
return ADD_FAIL
else:
return ADD_FULL
result = None
try:
result = db.run_in_transaction(
addNames,
db.Key.from_path('CounterNamesShard',
str(self.last_shard_)), new_names)
except:
result = ADD_FAIL
if result == ADD_FULL:
CounterNamesShard.get_or_insert(str(self.last_shard_ + 1))
self.addIfNew(names)
if result == ADD_SUCCESS:
logging.info("Registered new counter names: %s." %
",".join(new_names))
else:
logging.warning(
"Coudn't register counter names: %s. (Will retry next time)"
% ",".join(new_names))
finally:
namespace_manager.set_namespace(namespace)
return (not not fresh, len(new_names))
def addIfNew(self, names):
new_names = []
fresh = None
for name in names:
if name not in self.known_counter_names_:
if fresh == None:
fresh = self.all()
if name not in fresh:
new_names += [name]
else:
new_names += [name]
if new_names:
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
if self.last_shard_ == None:
CounterNamesShard.get_or_insert("0")
self.last_shard_ = 0
ADD_SUCCESS = 1
ADD_FULL = 2
ADD_FAIL = 3
def addNames(key, names):
record = db.get(key)
local_names = set(record.names)
for name in names:
if name not in local_names:
local_names.add(name)
record.names += [name]
try:
record.put()
return ADD_SUCCESS
except (RequestTooLargeError, ValueError):
if len(names) == len(record.names):
return ADD_FAIL
else:
return ADD_FULL
result = None
try:
result = db.run_in_transaction(addNames,
db.Key.from_path('CounterNamesShard', str(self.last_shard_)),
new_names)
except:
result = ADD_FAIL
if result == ADD_FULL:
CounterNamesShard.get_or_insert(str(self.last_shard_ + 1))
self.addIfNew(names)
if result == ADD_SUCCESS:
logging.info("Registered new counter names: %s." %
",".join(new_names))
else:
logging.warning(
"Coudn't register counter names: %s. (Will retry next time)" %
",".join(new_names))
finally:
namespace_manager.set_namespace(namespace)
return (not not fresh, len(new_names))
def delete(self, regexp):
# NOTE(andrin): When a counter is deleted and used again afterwards
# it will not be harvested in the following case:
# - Instance A deletes counter C.
# - Instance B has C still in self.known_counter_names_
# and increments C without updating the CounterNamesShards.
# - Instance A generates the harvest report and wont find C.
# To fix this issue, we reload the DefaultCounterNamesManager
# once a day.
# To fix this issue manually, you can just deploy a new
# version of your app.
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
to_delete = []
matcher = re.compile("(" + regexp + ")$")
for name in self.all():
if matcher.match(name):
to_delete += [name]
for shard in CounterNamesShard.all():
dirty = False
for name in to_delete:
if name in shard.names:
shard.names.remove(name)
dirty = True
if dirty:
shard.put()
for name in to_delete:
if name in self.known_counter_names_:
self.known_counter_names_.remove(name)
return to_delete
finally:
namespace_manager.set_namespace(namespace)
def delete(self, regexp):
# NOTE(andrin): When a counter is deleted and used again afterwards
# it will not be harvested in the following case:
# - Instance A deletes counter C.
# - Instance B has C still in self.known_counter_names_
# and increments C without updating the CounterNamesShards.
# - Instance A generates the harvest report and wont find C.
# To fix this issue, we reload the DefaultCounterNamesManager
# once a day.
# To fix this issue manually, you can just deploy a new
# version of your app.
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
to_delete = []
matcher = re.compile("(" + regexp + ")$")
for name in self.all():
if matcher.match(name):
to_delete += [name]
for shard in CounterNamesShard.all():
dirty = False
for name in to_delete:
if name in shard.names:
shard.names.remove(name)
dirty = True
if dirty:
shard.put()
for name in to_delete:
if name in self.known_counter_names_:
self.known_counter_names_.remove(name)
return to_delete
finally:
namespace_manager.set_namespace(namespace)
def all(self, force_reload=False):
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
query = CounterNamesShard.all()
if force_reload:
self.known_counter_names_ = set([])
elif self.last_update_:
query.filter("timestamp >= ",
self.last_update_-
datetime.timedelta(0, config.MAX_CLOCK_SKEW))
for record in query:
self.known_counter_names_ = self.known_counter_names_.union(
record.names)
self.last_shard_ = max(self.last_shard_, int(record.key().name()))
self.last_update_ = datetime.datetime.now()
return self.known_counter_names_
finally:
namespace_manager.set_namespace(namespace)
def all(self, force_reload=False):
namespace = namespace_manager.get_namespace()
try:
namespace_manager.set_namespace(self.namespace)
query = CounterNamesShard.all()
if force_reload:
self.known_counter_names_ = set([])
elif self.last_update_:
query.filter(
"timestamp >= ", self.last_update_ -
datetime.timedelta(0, config.MAX_CLOCK_SKEW))
for record in query:
self.known_counter_names_ = self.known_counter_names_.union(
record.names)
self.last_shard_ = max(self.last_shard_,
int(record.key().name()))
self.last_update_ = datetime.datetime.now()
return self.known_counter_names_
finally:
namespace_manager.set_namespace(namespace)
