def process_pending(self):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
try:
for host_id in self.cluster.hosts.iterkeys():
conn = self.cluster.get_local_client(host_id)
keys = conn.zrange(self.pending_key, 0, -1)
if not keys:
continue
keycount = 0
for key in keys:
keycount += 1
process_incr.apply_async(kwargs={
'key': key,
})
pipe = conn.pipeline()
pipe.zrem(self.pending_key, *keys)
pipe.execute()
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def process_pending(self):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
try:
keycount = 0
with self.cluster.all() as conn:
results = conn.zrange(self.pending_key, 0, -1)
with self.cluster.all() as conn:
for host_id, keys in six.iteritems(results.value):
if not keys:
continue
keycount += len(keys)
for key in keys:
process_incr.apply_async(kwargs={
'key': key,
})
conn.target([host_id]).zrem(self.pending_key, *keys)
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def process_pending(self):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
try:
for host_id in self.cluster.hosts.iterkeys():
conn = self.cluster.get_local_client(host_id)
keys = conn.zrange(self.pending_key, 0, -1)
if not keys:
continue
keycount = 0
for key in keys:
keycount += 1
process_incr.apply_async(kwargs={
'key': key,
})
pipe = conn.pipeline()
pipe.zrem(self.pending_key, *keys)
pipe.execute()
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def incr(self, model, columns, filters, extra=None):
"""
>>> incr(Group, columns={'times_seen': 1}, filters={'pk': group.pk})
"""
process_incr.apply_async(
kwargs={"model": model, "columns": columns, "filters": filters, "extra": extra}
)
def process_pending(self):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
try:
keycount = 0
with self.cluster.all() as conn:
results = conn.zrange(self.pending_key, 0, -1)
with self.cluster.all() as conn:
for host_id, keys in six.iteritems(results.value):
if not keys:
continue
keycount += len(keys)
for key in keys:
process_incr.apply_async(kwargs={
'key': key,
})
conn.target([host_id]).zrem(self.pending_key, *keys)
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def incr(self, model, columns, filters, extra=None):
"""
>>> incr(Group, columns={'times_seen': 1}, filters={'pk': group.pk})
"""
process_incr.apply_async(kwargs={
'model': model,
'columns': columns,
'filters': filters,
'extra': extra,
})
def incr(self, model, columns, filters, extra=None):
"""
>>> incr(Group, columns={'times_seen': 1}, filters={'pk': group.pk})
"""
process_incr.apply_async(kwargs={
'model': model,
'columns': columns,
'filters': filters,
'extra': extra,
})
def process_pending(self):
for conn in self.conn.hosts.itervalues():
keys = conn.zrange(self.pending_key, 0, -1)
if not keys:
continue
for key in keys:
process_incr.apply_async(kwargs={
'key': key,
})
conn.zrem(self.pending_key, *keys)
def process_pending(self, partition=None):
if partition is None and self.pending_partitions > 1:
# If we're using partitions, this one task fans out into
# N subtasks instead.
for i in range(self.pending_partitions):
process_pending.apply_async(kwargs={'partition': i})
# Explicitly also run over the unpartitioned buffer as well
# to ease in transition. In practice, this should just be
# super fast and is fine to do redundantly.
pending_key = self._make_pending_key(partition)
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
pending_buffer = PendingBuffer(self.incr_batch_size)
try:
keycount = 0
with self.cluster.all() as conn:
results = conn.zrange(pending_key, 0, -1)
with self.cluster.all() as conn:
for host_id, keys in six.iteritems(results.value):
if not keys:
continue
keycount += len(keys)
for key in keys:
pending_buffer.append(key)
if pending_buffer.full():
process_incr.apply_async(
kwargs={
'batch_keys': pending_buffer.flush(),
}
)
conn.target([host_id]).zrem(pending_key, *keys)
# queue up remainder of pending keys
if not pending_buffer.empty():
process_incr.apply_async(kwargs={
'batch_keys': pending_buffer.flush(),
})
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def incr(self, model, columns, filters, extra=None, signal_only=None):
"""
>>> incr(Group, columns={'times_seen': 1}, filters={'pk': group.pk})
signal_only - added to indicate that `process` should only call the complete
signal handler with the updated model and skip creates/updates in the database. this
is useful in cases where we need to do additional processing before writing to the
database and opt to do it in a `buffer_incr_complete` receiver.
"""
process_incr.apply_async(
kwargs={
"model": model,
"columns": columns,
"filters": filters,
"extra": extra,
"signal_only": signal_only,
})
def process_pending(self, partition=None):
if partition is None and self.pending_partitions > 1:
# If we're using partitions, this one task fans out into
# N subtasks instead.
for i in range(self.pending_partitions):
process_pending.apply_async(kwargs={'partition': i})
# Explicitly also run over the unpartitioned buffer as well
# to ease in transition. In practice, this should just be
# super fast and is fine to do redundantly.
pending_key = self._make_pending_key(partition)
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(pending_key)
# prevent a stampede due to celerybeat + periodic task
if not client.set(lock_key, '1', nx=True, ex=60):
return
pending_buffer = PendingBuffer(self.incr_batch_size)
try:
keycount = 0
with self.cluster.all() as conn:
results = conn.zrange(pending_key, 0, -1)
with self.cluster.all() as conn:
for host_id, keys in six.iteritems(results.value):
if not keys:
continue
keycount += len(keys)
for key in keys:
pending_buffer.append(key)
if pending_buffer.full():
process_incr.apply_async(
kwargs={
'batch_keys': pending_buffer.flush(),
})
conn.target([host_id]).zrem(pending_key, *keys)
# queue up remainder of pending keys
if not pending_buffer.empty():
process_incr.apply_async(kwargs={
'batch_keys': pending_buffer.flush(),
})
metrics.timing('buffer.pending-size', keycount)
finally:
client.delete(lock_key)
def process_pending(self):
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not self.conn.set(lock_key, '1', nx=True, ex=60):
return
try:
for conn in self.conn.hosts.itervalues():
keys = conn.zrange(self.pending_key, 0, -1)
if not keys:
continue
for key in keys:
process_incr.apply_async(kwargs={
'key': key,
})
pipe = conn.pipeline()
pipe.zrem(self.pending_key, *keys)
pipe.execute()
finally:
self.conn.delete(lock_key)
def process_pending(self):
lock_key = self._make_lock_key(self.pending_key)
# prevent a stampede due to celerybeat + periodic task
if not self.conn.set(lock_key, '1', nx=True, ex=60):
return
try:
for conn in self.conn.hosts.itervalues():
keys = conn.zrange(self.pending_key, 0, -1)
if not keys:
continue
for key in keys:
process_incr.apply_async(kwargs={
'key': key,
})
pipe = conn.pipeline()
pipe.zrem(self.pending_key, *keys)
pipe.execute()
finally:
self.conn.delete(lock_key)
