def test_invalid_key():
with RedisRunnerContext():
client = ProtonPack.redis_connect()
test_stream_name = uuid4().hex
client.set(test_stream_name, "ima-lima-bean")
with pytest.raises(Exception, match="invalid stream key"):
ProtonPack.create_stream(test_stream_name)
def test_create_consumer_group():
with RedisRunnerContext():
test_stream_name = uuid4().hex
streams = ProtonPack.list_streams()
assert test_stream_name not in streams
grp = ProtonPack.create_consumer_group(test_stream_name, "group")
assert grp
streams = ProtonPack.list_streams()
assert test_stream_name in streams
def test_list_consumer_groups_create_if():
with RedisRunnerContext():
test_stream_name = uuid4().hex
streams = ProtonPack.list_streams()
assert test_stream_name not in streams
groups = ProtonPack.list_consumer_groups(test_stream_name, True)
assert len(groups) == 0
streams = ProtonPack.list_streams()
assert test_stream_name in streams
def test_list_streams():
with RedisRunnerContext():
streams = ProtonPack.list_streams()
assert streams is not None
assert isinstance(streams, set)
test_stream_name = uuid4().hex
stream_count = len(streams)
ProtonPack.create_stream(test_stream_name)
streams = ProtonPack.list_streams()
assert len(streams) == stream_count + 1
def do_poll(self):
batch = ProtonPack.poll_events(self.stream_name, self.consumer_group,
self.consumer_id, self.batch_size,
self.block_time)
self.log(action="worker_batch", batch_len=len(batch))
for item in batch:
if self.handle_event(item):
ProtonPack.acknowledge_events(self.stream_name,
self.consumer_group,
item.event_id)
def test_send_event():
with RedisRunnerContext():
client = ProtonPack.redis_connect()
test_stream_name = uuid4().hex
ProtonPack.create_stream(test_stream_name)
length = client.xlen(test_stream_name)
assert length == 1
evt = Event(topic="test", evt_type="TEST", activity="test_send_event")
ProtonPack.send_event(test_stream_name, evt)
length = client.xlen(test_stream_name)
assert length == 2
def __init__(self,
stream_name: str,
consumer_group: str,
consumer_id: str = None,
janitor: bool = False,
batch_size: int = 10,
block_time: int = 1000):
self.stream_name = stream_name
self.consumer_group = consumer_group
self.consumer_id = consumer_id or uuid4().hex
self.batch_size = batch_size
self.block_time = block_time
self.janitor = janitor
self.log(action="worker_init")
ProtonPack.create_consumer_group(stream_name, consumer_group)
def get_streams():
streams = ProtonPack.list_streams()
return make_response(
json_dumps({
"status": "ok",
"streams": list(streams)
}), 200)
def get_stream_consumer_groups(stream):
groups = ProtonPack.list_consumer_groups(stream)
return make_response(
json_dumps({
"status": "ok",
"stream": stream,
"groups": [group.to_dict() for group in groups]
}), 200)
def test_list_consumer_groups_and_create_consumer_group():
with RedisRunnerContext():
test_stream_name = uuid4().hex
ProtonPack.create_stream(test_stream_name)
groups = ProtonPack.list_consumer_groups(test_stream_name)
assert len(groups) == 0
ProtonPack.create_consumer_group(test_stream_name, "consumers_1")
groups = ProtonPack.list_consumer_groups(test_stream_name)
assert len(groups) == 1
assert "consumers_1" in [group.name for group in groups]
ProtonPack.create_consumer_group(test_stream_name, "consumers_2")
groups = ProtonPack.list_consumer_groups(test_stream_name)
assert len(groups) == 2
assert "consumers_2" in [group.name for group in groups]
def put_stream_consumer_group(stream):
data = request.get_json(force=True)
group_name = data.get("group_name")
group = ProtonPack.create_consumer_group(stream, group_name)
return make_response(
json_dumps({
"status": "ok",
"stream": stream,
"group_name": group_name,
"created": group
}), 200)
def get_stream_info(stream):
if request.method == "GET":
info = ProtonPack.stream_info(stream)
return make_response(
json_dumps({
"status": "ok",
"stream": stream,
"info": info
}), 200)
elif request.method == "POST":
raw = request.get_json(force=True)
if isinstance(raw, collections.Mapping):
event = Event.from_dict(raw)
evt_id = ProtonPack.send_event(stream, event)
raw["event_id"] = evt_id
app.logger.debug({"action": "event_posted", "event": raw})
return make_response(
json_dumps({
"status": "ok",
"stream": stream,
"events": [evt_id]
}), 200)
else:
evt_ids = []
for evt in raw:
event = Event.from_dict(evt)
evt_id = ProtonPack.send_event(stream, event)
evt["event_id"] = evt_id
evt_ids.append(evt_id)
app.logger.debug({"action": "event_posted", "event": evt})
return make_response(
json_dumps({
"status": "ok",
"stream": stream,
"events": evt_ids
}), 200)
def test_workers_unite():
httpretty.register_uri(
httpretty.POST,
"http://example.com",
body='{"origin": "127.0.0.1"}',
status=200,
)
with RedisRunnerContext():
# -- create a subscriber that is mocked with httpretty
subscriber = Subscriber(stream=uuid4().hex,
subscriber_name=uuid4().hex,
topics=["system"],
evt_types=["SYS"],
activities=["init", "verb"],
protocol=Protocol.HTTP,
endpoint="http://example.com")
ok = SubscriberManager.put_subscriber(subscriber)
assert ok
# -- prepare an event to send
event = Event(
topic="system",
evt_type="SYS",
activity="init",
ref_id="00",
)
ok = ProtonPack.send_event(subscriber.stream, event)
assert ok
# -- start up a worker and have it consume the stream
gb = GhostBuster(subscriber.stream, "consumer_group", "consumer_id")
gb.start(False)
# -- assert the handler uri is called
called = httpretty.last_request()
print(dir(called))
assert called.body is not None
assert called.body != ""
body = json.loads(called.body)
assert body["stream"] == subscriber.stream
assert body["event"]["topic"] == event.topic
assert body["event"]["evt_type"] == event.evt_type
assert body["event"]["activity"] == event.activity
assert body["event"]["ref_id"] == event.ref_id
assert body["consumer_group"] == "consumer_group"
assert body["consumer_id"] == "consumer_id"
assert body["subscriber_name"] == subscriber.subscriber_name
def test_acknowledge_events():
with RedisRunnerContext():
groupname = "group"
consumername = "consumer"
test_stream_name = uuid4().hex
evt = Event(topic="test", evt_type="TEST", activity="test_send_event")
evt_id = ProtonPack.send_event(test_stream_name, evt)
assert evt_id is not None
ProtonPack.create_consumer_group(test_stream_name, groupname)
ProtonPack.poll_events(test_stream_name, groupname, consumername, 100, 10000)
ack = ProtonPack.acknowledge_events(test_stream_name, groupname, evt_id)
assert ack == 1
def test_stream_exists():
with RedisRunnerContext():
test_stream_name = uuid4().hex
assert not ProtonPack.stream_exists(test_stream_name)
ProtonPack.create_stream(test_stream_name)
assert ProtonPack.stream_exists(test_stream_name)