def consume(topic, follow, fetch_size):
topic = Topic(StrictRedis(), topic)
start = time.time()
cursor = 0
n = 0
try:
while True:
lower = cursor
cursor, batch = topic.consume(cursor, fetch_size)
logger.debug('Retrieved %s items from %s to %s.', len(batch), lower, cursor)
if not batch:
if not follow:
logger.debug('Retrieved empty batch (end of stream.)')
break
else:
logger.debug('Retrieved empty batch.')
time.sleep(0.1)
for n, (offset, item) in enumerate(batch, n + 1):
print offset, item
except KeyboardInterrupt:
pass
stop = time.time()
logger.info(
'Consumed %s records in %s seconds (%s records/second.)',
n,
stop - start,
n / (stop - start),
)
def benchmark(count, topic, batch_size, page_size, payload_length):
client = StrictRedis()
batch = ["x" * payload_length] * batch_size
topic = Topic(client, topic)
try:
topic.create(page_size)
except Exception:
pass
gc.disable()
start = time.time()
generator = xrange(1, count + 1) if count else itertools.count(1)
i = 0
try:
for i in generator:
topic.produce(batch)
if i % 100 == 0:
print "Produced", i, "batches."
except KeyboardInterrupt:
pass
end = time.time()
print "Produced", i * batch_size, payload_length / 1024.0, "KB records in", end - start, "seconds"
print (i * batch_size) / (end - start), "messages/sec"
def test_ttl(client):
name = 'example'
size = 10
ttl = 60
topic = Topic(client, name)
topic.create(size, ttl=ttl)
for i in xrange(0, size + 1):
topic.produce((i,))
assert ttl - 1 <= client.ttl('{}/pages/{}'.format(name, 0)) <= ttl
assert client.ttl('{}/pages/{}'.format(name, 1)) == -1
def test_produce(client):
name = 'example'
size = 10
items = []
generator = itertools.imap(
lambda i: (i, str(i)),
itertools.count(),
)
topic = Topic(client, name)
topic.create(size)
payload, offset = generator.next()
topic.produce((payload,))
items.append((payload, offset))
assert client.zrangebyscore('{}/pages'.format(name), '-inf', 'inf', withscores=True) == [('0', 0.0)]
assert list(enumerate(client.lrange('{}/pages/{}'.format(name, 0), 0, size))) == items
for payload, offset in itertools.islice(generator, size):
topic.produce((payload,))
items.append((payload, offset))
assert client.zrangebyscore('{}/pages'.format(name), '-inf', 'inf', withscores=True) == [('0', 0.0), ('1', float(size))]
assert list(enumerate(client.lrange('{}/pages/{}'.format(name, 0), 0, size))) == items[:size]
assert list(enumerate(client.lrange('{}/pages/{}'.format(name, 1), 0, size), size)) == items[size:]
def test_consume_page_sizes(client):
name = 'example'
size = 10
items = []
generator = itertools.imap(
lambda i: [i, str(i)],
itertools.count(),
)
topic = Topic(client, name)
topic.create(size)
for offset, payload in itertools.islice(generator, size + 1):
topic.produce((payload,))
items.append([offset, payload])
offset, batch = list(topic.consume(0, limit=size))
assert items[:size] == batch
offset, batch = list(topic.consume(offset, limit=size))
assert items[size:] == batch
def test_consume_across_pages(client):
name = 'example'
size = 10
items = []
generator = itertools.imap(
lambda i: [i, str(i)],
itertools.count(),
)
topic = Topic(client, name)
topic.create(size)
for offset, payload in itertools.islice(generator, size + 1):
topic.produce((payload,))
items.append([offset, payload])
# Check with batches crossing pages.
offset, batch = topic.consume(5)
assert batch == items[5:]
def create(topic, page_size, ttl, max):
topic = Topic(StrictRedis(), topic)
topic.create(page_size, ttl=ttl, max=max)
def test_create(client):
topic = Topic(client, 'topic')
topic.create()
with pytest.raises(Exception):
topic.create()
