def list(self):
count = 0
while True:
queues = self._redis.xreadgroup(groupname=STREAM_GROUP, consumername=CONSUMER_NAME,
streams={self._name: '>'}, count=1, block=round(self._timeout_ms))
if not queues:
queues = self._claim_olds()
stats.set_gauge(['redis', self._name_str, 'nb_messages'], 0)
if queues is None and self._stop_if_empty:
break
if queues:
for redis_message in queues:
queue_name, queue_messages = redis_message
assert queue_name == self._name
for message in queue_messages:
id_, body = message
try:
tile = decode_message(body[b'message'], from_redis=True, sqs_message=id_)
yield tile
except Exception:
logger.warning('Failed decoding the Redis message', exc_info=True)
stats.increment_counter(['redis', self._name_str, 'decode_error'])
count += 1
if count % 100 == 0:
stats.set_gauge(['redis', self._name_str, 'nb_messages'], self._redis.xlen(name=self._name))
def list(self):
count = 0
while True:
queues = self._redis.xreadgroup(groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
streams={self._name: '>'},
count=1,
block=round(self._timeout_ms))
if not queues:
queues = self._claim_olds()
stats.set_gauge(['redis', self._name_str, 'nb_messages'], 0)
if queues is None and self._stop_if_empty:
break
if queues:
for redis_message in queues:
queue_name, queue_messages = redis_message
assert queue_name == self._name
for message in queue_messages:
id_, body = message
try:
tile = decode_message(body[b'message'],
from_redis=True,
sqs_message=id_)
yield tile
except Exception:
logger.warning('Failed decoding the Redis message',
exc_info=True)
stats.increment_counter(
['redis', self._name_str, 'decode_error'])
count += 1
if count % 100 == 0:
stats.set_gauge(['redis', self._name_str, 'nb_messages'],
self._redis.xlen(name=self._name))
def _claim_olds(self):
pendings = self._master.xpending_range(
name=self._name, groupname=STREAM_GROUP, min="-", max="+", count=10
)
if not pendings:
# None means there is nothing pending at all
return None
to_steal = []
to_drop = []
for pending in pendings:
if int(pending["time_since_delivered"]) >= self._pending_timeout_ms:
id_ = pending["message_id"]
nb_retries = int(pending["times_delivered"])
if nb_retries < self._max_retries:
logger.info(
"A message has been pending for too long. Stealing it (retry #%d): %s",
nb_retries,
id_,
)
to_steal.append(id_)
else:
logger.warning(
"A message has been pending for too long and retried too many times. Dropping it: %s",
id_,
)
to_drop.append(id_)
if to_drop:
drop_messages = self._master.xclaim(
name=self._name,
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms,
message_ids=to_drop,
)
drop_ids = [drop_message[0] for drop_message in drop_messages]
self._master.xack(self._name, STREAM_GROUP, *drop_ids)
self._master.xdel(self._name, *drop_ids)
for _, drop_message in drop_messages:
tile = decode_message(drop_message[b"message"])
self._master.xadd(
name=self._errors_name,
fields=dict(tilecoord=str(tile.tilecoord)),
maxlen=self._max_errors_nb,
)
stats.increment_counter(["redis", self._name_str, "dropped"], len(to_drop))
if to_steal:
messages = self._master.xclaim(
name=self._name,
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms,
message_ids=to_steal,
)
stats.increment_counter(["redis", self._name_str, "stolen"], len(to_steal))
return [[self._name, messages]]
else:
# Empty means there are pending jobs, but they are not old enough to be stolen
return []
def list(self):
while True:
redis_message = self._redis.blpop(self._name,
timeout=self._timeout)
if not redis_message:
if self._stop_if_empty:
break
else:
try:
tile = decode_message(redis_message[1], from_redis=True)
yield tile
except Exception:
logger.warning('Failed decoding the Redis message',
exc_info=True)
def list(self) -> Iterator[Tile]:
count = 0
while True:
queues = self._master.xreadgroup(
groupname=STREAM_GROUP,
consumername=CONSUMER_NAME,
streams={self._name: ">"},
count=1,
block=round(self._timeout_ms),
)
logger.debug("Get %d new elements", len(queues))
if not queues:
queues = self._claim_olds()
if queues is None:
stats.set_gauge(["redis", self._name_str, "nb_messages"],
0)
stats.set_gauge(["redis", self._name_str, "pending"], 0)
if queues is None and self._stop_if_empty:
break
if queues:
for redis_message in queues:
queue_name, queue_messages = redis_message
assert queue_name == self._name
for message in queue_messages:
id_, body = message
try:
tile = decode_message(body[b"message"],
from_redis=True,
sqs_message=id_)
yield tile
except Exception:
logger.warning("Failed decoding the Redis message",
exc_info=True)
stats.increment_counter(
["redis", self._name_str, "decode_error"])
count += 1
if count % 10 == 0:
stats.set_gauge(
["redis", self._name_str, "nb_messages"],
self._slave.xlen(name=self._name),
)
pending = self._slave.xpending(
self._name, STREAM_GROUP) # type: ignore
stats.set_gauge(["redis", self._name_str, "pending"],
pending["pending"])
def list(self):
while True:
try:
sqs_messages = self.queue.receive_messages(MaxNumberOfMessages=BATCH_SIZE)
except botocore.exceptions.EndpointConnectionError:
logger.warning("Error fetching SQS messages", exc_info=True)
sqs_messages = []
if not sqs_messages:
if self.on_empty(self.queue):
break
else:
for sqs_message in sqs_messages:
try:
tile = decode_message(sqs_message.body.encode('utf-8'), sqs_message=sqs_message)
yield tile
except Exception:
logger.warning('Failed decoding the SQS message', exc_info=True)
sqs_message.delete()
def _claim_olds(self):
pendings = self._redis.xpending_range(name=self._name, groupname=STREAM_GROUP, min='-', max='+', count=10)
if not pendings:
# None means there is nothing pending at all
return None
to_steal = []
to_drop = []
for pending in pendings:
if int(pending['time_since_delivered']) >= self._pending_timeout_ms:
id_ = pending['message_id']
nb_retries = int(pending['times_delivered'])
if nb_retries < self._max_retries:
logger.info('A message has been pending for too long. Stealing it (retry #%d): %s', nb_retries, id_)
to_steal.append(id_)
else:
logger.warning(
'A message has been pending for too long and retried too many times. Dropping it: %s', id_)
to_drop.append(id_)
if to_drop:
drop_messages = self._redis.xclaim(name=self._name, groupname=STREAM_GROUP, consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms, message_ids=to_drop)
drop_ids = [drop_message[0] for drop_message in drop_messages]
self._redis.xack(self._name, STREAM_GROUP, *drop_ids)
self._redis.xdel(self._name, *drop_ids)
for drop_id, drop_message in drop_messages:
tile = decode_message(drop_message[b'message'])
self._redis.xadd(name=self._errors_name, fields=dict(tilecoord=str(tile.tilecoord)),
maxlen=self._max_errors_nb)
stats.increment_counter(['redis', self._name_str, 'dropped'], len(to_drop))
if to_steal:
messages = self._redis.xclaim(name=self._name, groupname=STREAM_GROUP, consumername=CONSUMER_NAME,
min_idle_time=self._pending_timeout_ms, message_ids=to_steal)
stats.increment_counter(['redis', self._name_str, 'stolen'], len(to_steal))
return [[self._name, messages]]
else:
# Empty means there are pending jobs, but they are not old enough to be stolen
return []
