def get_event_class_and_attrs(self, topic, state):
# Resolve topic to event class.
domain_event_class = resolve_topic(topic)
# Decrypt state.
if self.cipher:
state = self.cipher.decrypt(state)
# Deserialize data.
event_attrs = json_loads(state, cls=self.json_decoder_class)
return domain_event_class, event_attrs
def from_topic_and_data(self, topic, data):
# Resolve topic to event class.
domain_event_class = resolve_topic(topic)
# Decrypt data.
if self.cipher:
data = self.cipher.decrypt(data)
# Deserialize data.
event_attrs = json_loads(data, cls=self.json_decoder_class)
# Reconstruct domain event object.
return reconstruct_object(domain_event_class, event_attrs)
def run(self, prompt=None, advance_by=None):
if prompt:
assert isinstance(prompt, Prompt)
upstream_names = [prompt.process_name]
else:
upstream_names = self.readers.keys()
notification_count = 0
for upstream_name in upstream_names:
if not self.is_reader_position_ok[upstream_name]:
self.del_notification_generator(upstream_name)
self.set_reader_position_from_tracking_records(upstream_name)
self.is_reader_position_ok[upstream_name] = True
while True:
with self._policy_lock:
# Get notification generator.
generator = self.get_notification_generator(upstream_name, advance_by)
try:
notification = next(generator)
except StopIteration:
self.del_notification_generator(upstream_name)
break
notification_count += 1
# Get domain event from notification.
event = self.get_event_from_notification(notification)
# Decode causal dependencies of the domain event.
causal_dependencies = notification.get('causal_dependencies') or '[]'
causal_dependencies = json_loads(causal_dependencies) or []
# Check causal dependencies are satisfied.
for causal_dependency in causal_dependencies:
pipeline_id = causal_dependency['pipeline_id']
notification_id = causal_dependency['notification_id']
_manager = self.event_store.record_manager
has_tracking_record = _manager.has_tracking_record(
upstream_application_name=upstream_name,
pipeline_id=pipeline_id,
notification_id=notification_id
)
if not has_tracking_record:
# Invalidate reader position.
self.is_reader_position_ok[upstream_name] = False
# Raise exception.
raise CausalDependencyFailed({
'application_name': self.name,
'upstream_name': upstream_name,
'pipeline_id': pipeline_id,
'notification_id': notification_id
})
# Wait on the clock event, if there is one.
if self.clock_event is not None:
self.clock_event.wait()
# print("Processing upstream event: ", event)
new_events = self.process_upstream_event(event, notification['id'], upstream_name)
self.take_snapshots(new_events)
# Publish a prompt if there are new notifications.
# Todo: Optionally send events as prompts, saves pulling event if it arrives in order.
if any([event.__notifiable__ for event in new_events]):
self.publish_prompt()
return notification_count
def run(self, prompt=None, advance_by=None):
if prompt is None:
readers_items = self.readers.items()
else:
assert isinstance(prompt, Prompt)
reader = self.readers[prompt.process_name]
readers_items = [(prompt.process_name, reader)]
notification_count = 0
for upstream_application_name, reader in readers_items:
if not self.is_reader_position_ok[upstream_application_name]:
self.set_reader_position_from_tracking_records(reader, upstream_application_name)
self.is_reader_position_ok[upstream_application_name] = True
for notification in reader.read(advance_by=advance_by):
# Todo: Put this on a queue and then get the next one.
notification_count += 1
# Todo: Get this from a queue and do it in a different thread?
# Domain event from notification.
event = self.event_store.sequenced_item_mapper.from_topic_and_data(
topic=notification['event_type'],
data=notification['state']
)
# Wait for causal dependencies to be satisfied.
upstream_causal_dependencies = notification.get('causal_dependencies')
if upstream_causal_dependencies is not None:
upstream_causal_dependencies = json_loads(upstream_causal_dependencies)
if upstream_causal_dependencies is None:
upstream_causal_dependencies = []
for causal_dependency in upstream_causal_dependencies:
pipeline_id = causal_dependency['pipeline_id']
notification_id = causal_dependency['notification_id']
if not self.tracking_record_manager.has_tracking_record(
application_name=self.name,
upstream_application_name=upstream_application_name,
pipeline_id=pipeline_id,
notification_id=notification_id
):
self.is_reader_position_ok[upstream_application_name] = False
raise CausalDependencyFailed({
'application_name': self.name,
'upstream_application_name': upstream_application_name,
'pipeline_id': pipeline_id,
'notification_id': notification_id
})
# Call policy with the event.
all_aggregates, causal_dependencies = self.call_policy(event)
# Record new events.
try:
new_events = self.record_new_events(
all_aggregates, notification, upstream_application_name, causal_dependencies
)
except Exception as e:
self.is_reader_position_ok[upstream_application_name] = False
# self._cached_entities = {}
raise e
else:
# Publish a prompt if there are new notifications.
# Todo: Optionally send events as prompts, saves pulling event if it arrives in order.
if len(new_events):
self.publish_prompt()
return notification_count
def test_json_loads(self):
# Check raises ValueError when JSON string is invalid.
with self.assertRaises(ValueError):
json_loads('{')
def test_causal_dependencies(self):
# Try to process an event that has unresolved causal dependencies.
pipeline_id1 = 0
pipeline_id2 = 1
# Create two events, one has causal dependency on the other.
core1 = Process(
'core',
persist_event_type=ExampleAggregate.Created,
setup_tables=True,
pipeline_id=pipeline_id1,
)
core2 = Process(
'core',
pipeline_id=pipeline_id2,
policy=example_policy,
session=core1.session
)
# First event in partition 1.
aggregate = ExampleAggregate.__create__()
aggregate.__save__()
# Second event in partition 2.
# - it's important this is done in a policy so the causal dependency is identified
core2.follow('core', core1.notification_log)
core2.run()
# Check the aggregate exists.
self.assertTrue(aggregate.id in core1.repository)
# Check the aggregate has been "moved on".
self.assertTrue(core1.repository[aggregate.id].is_moved_on)
# Check the events have different partition IDs.
records = core1.event_store.record_manager.get_records(aggregate.id)
self.assertEqual(2, len(records))
self.assertEqual(pipeline_id1, records[0].pipeline_id)
# Check the causal dependencies have been constructed.
self.assertEqual(None, records[0].causal_dependencies)
self.assertTrue({
'notification_id': 1,
'pipeline_id': pipeline_id1
}, json_loads(records[1].causal_dependencies))
# Setup downstream process.
downstream1 = Process(
'downstream',
pipeline_id=pipeline_id1,
session=core1.session,
policy=event_logging_policy,
)
downstream1.follow('core', core1.notification_log)
downstream2 = Process(
'downstream',
pipeline_id=pipeline_id2,
session=core1.session,
policy=event_logging_policy,
)
downstream2.follow('core', core2.notification_log)
# Try to process pipeline 2, should fail due to causal dependency.
with self.assertRaises(CausalDependencyFailed):
downstream2.run()
self.assertEqual(0, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(0, len(downstream2.event_store.record_manager.get_notifications()))
# Try to process pipeline 1, should work.
downstream1.run()
self.assertEqual(1, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(0, len(downstream2.event_store.record_manager.get_notifications()))
# Try again to process pipeline 2, should work this time.
downstream2.run()
self.assertEqual(1, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(1, len(downstream2.event_store.record_manager.get_notifications()))
core1.close()
core2.close()
downstream1.close()
downstream2.close()
def test_causal_dependencies(self):
# Try to process an event that has unresolved causal dependencies.
pipeline_id1 = 0
pipeline_id2 = 1
# Create two events, one has causal dependency on the other.
process_class = ProcessApplication.mixin(self.process_class)
core1 = process_class(
name='core',
# persist_event_type=ExampleAggregate.Created,
persist_event_type=BaseAggregateRoot.Event,
setup_table=True,
pipeline_id=pipeline_id1,
)
core1.use_causal_dependencies = True
# Needed for SQLAlchemy only.
kwargs = {
'session': core1.session
} if hasattr(core1, 'session') else {}
core2 = process_class(name='core',
pipeline_id=pipeline_id2,
policy=example_policy,
**kwargs)
core2.use_causal_dependencies = True
# First event in pipeline 1.
aggregate = ExampleAggregate.__create__()
aggregate.__save__()
# Second event in pipeline 2.
# - it's important this is done in a policy so the causal dependencies are identified
core2.follow('core', core1.notification_log)
core2.run()
# Check the aggregate exists.
self.assertTrue(aggregate.id in core1.repository)
# Check the aggregate has been "moved on".
aggregate = core1.repository[aggregate.id]
self.assertTrue(aggregate.is_moved_on)
self.assertTrue(aggregate.second_id)
self.assertIn(aggregate.second_id, core1.repository)
# Check the events have different pipeline IDs.
aggregate_records = core1.event_store.record_manager.get_records(
aggregate.id)
second_entity_records = core1.event_store.record_manager.get_records(
aggregate.second_id)
self.assertEqual(2, len(aggregate_records))
self.assertEqual(1, len(second_entity_records))
self.assertEqual(pipeline_id1, aggregate_records[0].pipeline_id)
self.assertEqual(pipeline_id2, aggregate_records[1].pipeline_id)
self.assertEqual(pipeline_id2, second_entity_records[0].pipeline_id)
# Check the causal dependencies have been constructed.
# - the first 'Created' event doesn't have an causal dependencies
self.assertFalse(aggregate_records[0].causal_dependencies)
# - the second 'Created' event depends on the Created event in another pipeline.
expect = [{'notification_id': 1, 'pipeline_id': pipeline_id1}]
actual = json_loads(second_entity_records[0].causal_dependencies)
self.assertEqual(expect, actual)
# - the 'AttributeChanged' event depends on the second Created,
# which is in the same pipeline, so expect no causal dependencies.
self.assertFalse(aggregate_records[1].causal_dependencies)
# Setup downstream process.
downstream1 = process_class(name='downstream',
pipeline_id=pipeline_id1,
policy=event_logging_policy,
**kwargs)
downstream1.follow('core', core1.notification_log)
downstream2 = process_class(name='downstream',
pipeline_id=pipeline_id2,
policy=event_logging_policy,
**kwargs)
downstream2.follow('core', core2.notification_log)
# Try to process pipeline 2, should fail due to causal dependency.
with self.assertRaises(CausalDependencyFailed):
downstream2.run()
self.assertEqual(
0, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(
0, len(downstream2.event_store.record_manager.get_notifications()))
# Try to process pipeline 1, should work.
downstream1.run()
self.assertEqual(
1, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(
0, len(downstream2.event_store.record_manager.get_notifications()))
# Try again to process pipeline 2, should work this time.
downstream2.run()
self.assertEqual(
1, len(downstream1.event_store.record_manager.get_notifications()))
self.assertEqual(
2, len(downstream2.event_store.record_manager.get_notifications()))
core1.close()
core2.close()
downstream1.close()
downstream2.close()
