def test_handle_prompt_failed(self):
process = ProcessApplication.mixin(self.infrastructure_class)(
name="test",
policy=example_policy,
persist_event_type=ExampleAggregate.Event,
setup_table=True,
)
def raise_exception(_):
raise Exception()
def raise_prompt_failed(_):
raise PromptFailed()
subscribe(raise_exception)
try:
with self.assertRaises(PromptFailed):
process.publish_prompt()
finally:
unsubscribe(raise_exception)
subscribe(raise_prompt_failed)
try:
with self.assertRaises(PromptFailed):
process.publish_prompt()
finally:
unsubscribe(raise_prompt_failed)
try:
process.publish_prompt()
finally:
process.close()
def close(self):
super(InProcessRunner, self).close()
unsubscribe(
predicate=self.system.is_prompt,
handler=self.handle_prompt,
)
def test_handle_prompt_failed(self):
process = Process(
'test',
policy=example_policy,
persist_event_type=ExampleAggregate.Event,
setup_tables=True,
)
def raise_exception(_):
raise Exception()
def raise_prompt_failed(_):
raise PromptFailed()
subscribe(raise_exception)
try:
with self.assertRaises(PromptFailed):
process.publish_prompt()
finally:
unsubscribe(raise_exception)
subscribe(raise_prompt_failed)
try:
with self.assertRaises(PromptFailed):
process.publish_prompt()
finally:
unsubscribe(raise_prompt_failed)
try:
process.publish_prompt()
finally:
process.close()
def close(self):
if self.persistence_policy:
unsubscribe(
predicate=self.persistence_policy.is_event,
handler=self.publish_prompt,
)
super(ProcessApplication, self).close()
def run(self):
# Construct process application object.
self.process = self.application_process_class(
pipeline_id=self.pipeline_id,
notification_log_section_size=self.notification_log_section_size,
pool_size=self.pool_size,
setup_table=self.setup_tables,
)
# Follow upstream notification logs.
for upstream_name in self.upstream_names:
# Obtain a notification log object (local or remote) for the upstream process.
if upstream_name == self.process.name:
# Upstream is this process's application,
# so use own notification log.
notification_log = self.process.notification_log
else:
# For a different application, we need to construct a notification
# log with a record manager that has the upstream application ID.
# Currently assumes all applications are using the same database
# and record manager class. If it wasn't the same database,we would
# to use a remote notification log, and upstream would need to provide
# an API from which we can pull. It's not unreasonable to have a fixed
# number of application processes connecting to the same database.
record_manager = self.process.event_store.record_manager
notification_log = RecordManagerNotificationLog(
record_manager=record_manager.clone(
application_name=upstream_name,
pipeline_id=self.pipeline_id),
section_size=self.process.notification_log_section_size)
# Todo: Support upstream partition IDs different from self.pipeline_id.
# Todo: Support combining partitions. Read from different partitions but write to the same partition,
# could be one os process that reads from many logs of the same upstream app, or many processes each
# reading one partition with contention writing to the same partition).
# Todo: Support dividing partitions Read from one but write to many. Maybe one process per
# upstream partition, round-robin to pick partition for write. Or have many processes reading
# with each taking it in turn to skip processing somehow.
# Todo: Dividing partitions would allow a stream to flow at the same rate through slower
# process applications.
# Todo: Support merging results from "replicated state machines" - could have a command
# logging process that takes client commands and presents them in a notification log.
# Then the system could be deployed in different places, running independently, receiving
# the same commands, and running the same processes. The command logging process could
# be accompanied with a result logging process that reads results from replicas as they
# are available. Not sure what to do if replicas return different things. If one replica
# goes down, then it could resume by pulling events from another? Not sure what to do.
# External systems could be modelled as commands.
# Make the process follow the upstream notification log.
self.process.follow(upstream_name, notification_log)
# Subscribe to broadcast prompts published by the process application.
subscribe(handler=self.broadcast_prompt, predicate=self.is_prompt)
try:
self.loop_on_prompts()
finally:
unsubscribe(handler=self.broadcast_prompt,
predicate=self.is_prompt)
def test_publish_subscribe_unsubscribe(self):
# Check subscribing event handlers with predicates.
# - when predicate is True, handler should be called
event = mock.Mock()
predicate = mock.Mock()
handler = mock.Mock()
# When predicate is True, handler should be called ONCE.
subscribe(event_predicate=predicate, subscriber=handler)
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# When predicate is True, after unsubscribing, handler should NOT be called again.
unsubscribe(event_predicate=predicate, subscriber=handler)
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# When predicate is False, handler should NOT be called.
predicate = lambda x: False
handler = mock.Mock()
subscribe(event_predicate=predicate, subscriber=handler)
publish(event)
self.assertEqual(0, handler.call_count)
def stop(self):
"""
Stops the process.
"""
self.has_been_stopped.set()
self.process_application.close()
unsubscribe(handler=self._enqueue_prompt_to_pull,
predicate=is_prompt_to_pull)
def close(self):
unsubscribe(handler=self.broadcast_prompt, predicate=self.is_prompt)
for os_process in self.os_processes:
os_process.inbox.put('QUIT')
for os_process in self.os_processes:
os_process.join(timeout=10)
for os_process in self.os_processes:
os_process.is_alive() and os_process.terminate()
self.os_processes = None
self.manager = None
def init_process(self, msg):
self.pipeline_actor = msg.pipeline_actor
self.downstream_actors = msg.downstream_actors
self.pipeline_id = msg.pipeline_id
self.upstream_application_names = msg.upstream_application_names
# Construct the process application class.
process_class = msg.process_application_class
if msg.infrastructure_class:
process_class = process_class.mixin(msg.infrastructure_class)
# Reset the database connection (for Django).
process_class.reset_connection_after_forking()
# Construct the process application.
self.process = process_class(pipeline_id=self.pipeline_id)
assert isinstance(self.process, ProcessApplication)
# Subscribe the slave actor's send_prompt() method.
# - the process application will call publish_prompt()
# and the actor will receive the prompt and send it
# as a message.
subscribe(predicate=self.is_my_prompt, handler=self.send_prompt)
# Close the process application persistence policy.
# - slave actor process application doesn't publish
# events, so we don't need this
self.process.persistence_policy.close()
# Unsubscribe process application's publish_prompt().
# - slave actor process application doesn't publish
# events, so we don't need this
unsubscribe(
predicate=self.process.persistence_policy.is_event,
handler=self.process.publish_prompt_for_events,
)
# Construct and follow upstream notification logs.
for upstream_application_name in self.upstream_application_names:
record_manager = self.process.event_store.record_manager
# assert isinstance(record_manager, ACIDRecordManager), type(record_manager)
notification_log = RecordManagerNotificationLog(
record_manager=record_manager.clone(
application_name=upstream_application_name,
pipeline_id=self.pipeline_id,
),
section_size=self.process.notification_log_section_size,
)
self.process.follow(upstream_application_name, notification_log)
def close(self) -> None:
super(MultiprocessRunner, self).close()
unsubscribe(handler=self.broadcast_prompt, predicate=is_prompt_to_pull)
for os_process in self.os_processes:
os_process.inbox.put(PromptToQuit())
for os_process in self.os_processes:
os_process.join(timeout=10)
for os_process in self.os_processes:
if os_process.is_alive():
os_process.terminate()
self.os_processes.clear()
def test_publish_subscribe_unsubscribe(self):
# Check subscribing event handlers with predicates.
# - when predicate is True, handler should be called
event = mock.Mock()
predicate = mock.Mock()
handler = mock.Mock()
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
# When predicate is True, handler should be called ONCE.
subscribe(handler=handler, predicate=predicate)
# Check we can assert there are event handlers subscribed.
self.assertRaises(EventHandlersNotEmptyError,
assert_event_handlers_empty)
# Check what happens when an event is published.
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# When predicate is True, after unsubscribing, handler should NOT be called again.
unsubscribe(handler=handler, predicate=predicate)
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
# When predicate is False, handler should NOT be called.
predicate = lambda x: False
handler = mock.Mock()
subscribe(handler=handler, predicate=predicate)
publish(event)
self.assertEqual(0, handler.call_count)
# Unsubscribe.
unsubscribe(handler=handler, predicate=predicate)
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
def test_publish_subscribe_unsubscribe(self):
# Check subscribing event handlers with predicates.
# - when predicate is True, handler should be called
event = mock.Mock()
predicate = mock.Mock()
handler = mock.Mock()
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
# When predicate is True, handler should be called ONCE.
subscribe(handler=handler, predicate=predicate)
# Check we can assert there are event handlers subscribed.
self.assertRaises(EventHandlersNotEmptyError, assert_event_handlers_empty)
# Check what happens when an event is published.
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# When predicate is True, after unsubscribing, handler should NOT be called again.
unsubscribe(handler=handler, predicate=predicate)
publish(event)
predicate.assert_called_once_with(event)
handler.assert_called_once_with(event)
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
# When predicate is False, handler should NOT be called.
predicate = lambda x: False
handler = mock.Mock()
subscribe(handler=handler, predicate=predicate)
publish(event)
self.assertEqual(0, handler.call_count)
# Unsubscribe.
unsubscribe(handler=handler, predicate=predicate)
# Check we can assert there are no event handlers subscribed.
assert_event_handlers_empty()
def stop(self):
"""
Stops the process.
"""
# print("%s actor stopping %s" % (os.getpid(), datetime.datetime.now()))
self.has_been_stopped.set()
# print("%s actor joining db_jobs_thread %s" % (os.getpid(),
# datetime.datetime.now()))
self.db_jobs_queue.put(None)
self.upstream_event_queue.put(None)
self.downstream_prompt_queue.put(None)
self._has_been_prompted.set()
self.positions_initialised.set()
self.db_jobs_thread.join(timeout=1)
assert not self.db_jobs_thread.is_alive(), (
"DB jobs thread still alive")
# print("%s actor joining process_events_thread %s" % (os.getpid(),
# datetime.datetime.now()))
self.process_events_thread.join(timeout=1)
assert not self.process_events_thread.is_alive(), (
"Process events thread still alive")
# print("%s actor joining process_prompts_thread %s" % (os.getpid(),
# datetime.datetime.now()))
self.process_prompts_thread.join(timeout=1)
assert not self.process_prompts_thread.is_alive(), (
"Process prompts thread still alive")
# print("%s actor joining push_prompts_thread %s" % (os.getpid(),
# datetime.datetime.now()))
self.push_prompts_thread.join(timeout=1)
assert not self.push_prompts_thread.is_alive(), (
"Push prompts thread still alive")
self.process_application.close()
unsubscribe(handler=self._enqueue_prompt_to_pull,
predicate=is_prompt_to_pull)
# print("%s actor stopped %s" % (os.getpid(), datetime.datetime.now()))
ray.actor.exit_actor()
def close(self):
unsubscribe(self.contract_specification_created,
self.generate_dependency_graph)
def close(self):
unsubscribe(self.add_list_to_collection, self.is_list_started)
unsubscribe(self.remove_list_from_collection, self.is_list_discarded)
def tearDown(self):
unsubscribe(*self.subscription)
assert_event_handlers_empty()
def close(self):
"""Stops all the actors running a system of process applications."""
super(ActorModelRunner, self).close()
unsubscribe(handler=self.forward_prompt, predicate=self.is_prompt)
if self.shutdown_on_close:
self.shutdown()
def close(self):
unsubscribe(self.market_simulation_created,
self.generate_simulated_prices_for_market_simulation)
def close(self):
unsubscribe(predicate=self.is_upstream_prompt, handler=self.run)
unsubscribe(predicate=self.persistence_policy.is_event, handler=self.publish_prompt_from_event)
super(ProcessApplication, self).close()
def close(self):
unsubscribe(self.is_call_dependencies_created, self.cache_dependencies)
unsubscribe(self.is_call_dependents_created, self.cache_dependents)
unsubscribe(self.is_call_result_created, self.cache_result)
unsubscribe(self.is_call_result_discarded, self.purge_result)
def close(self):
unsubscribe(self.store_event, self.is_event)
def unsubscribe(self):
unsubscribe(self.is_call_requirement_created, self.print_compilation_progress)
unsubscribe(self.is_calculating, self.check_is_timed_out)
unsubscribe(self.is_calculating, self.check_is_interrupted)
unsubscribe(self.is_evaluation_complete, self.set_is_finished)
unsubscribe(self.is_result_value_computed, self.inc_result_value_computed_count)
unsubscribe(self.is_result_value_computed, self.print_evaluation_progress)
unsubscribe(self.is_call_result_created, self.inc_call_result_count)
unsubscribe(self.is_call_result_created, self.print_evaluation_progress)
def close(self):
unsubscribe(self.is_domain_event, self.store_domain_event)
def close(self):
unsubscribe(self.is_call_dependencies_created, self.cache_dependencies)
unsubscribe(self.is_call_dependents_created, self.cache_dependents)
unsubscribe(self.is_call_result_created, self.cache_result)
unsubscribe(self.is_call_result_discarded, self.purge_result)
def close(self):
unsubscribe(self.add_user_to_collection, self.is_user_created)
unsubscribe(self.remove_user_from_collection, self.is_user_discarded)
def close(self):
unsubscribe(predicate=self.trigger, handler=self.take_snapshot)
def close(self):
unsubscribe(self.store_event, self.is_event)
def close(self) -> None:
super(InProcessRunner, self).close()
unsubscribe(predicate=is_prompt_to_pull, handler=self.handle_prompt)
def close(self) -> None:
unsubscribe(self.add_ticket, self.is_ticket_created)
unsubscribe(self.update_ticket, self.is_ticket_updated)
unsubscribe(self.delete_ticket, self.is_ticket_deleted)
def close(self):
unsubscribe(self.is_contract_valuation_created, self.generate_contract_valuation)
def close(self):
unsubscribe(self.contract_specification_created, self.generate_dependency_graph)
unsubscribe(self.call_requirement_created, self.limit_calls)
def test_attribute(self):
# Check we get an error when called with something other than a function.
self.assertRaises(ProgrammingError, attribute, 'not a getter')
self.assertRaises(ProgrammingError, attribute, 123)
self.assertRaises(ProgrammingError, attribute, None)
# Call the decorator with a function.
getter = lambda: None
p = attribute(getter)
# Check we got a property object.
self.assertIsInstance(p, property)
# Check the property object has both setter and getter functions.
self.assertTrue(p.fset)
self.assertTrue(p.fget)
# Pretend we decorated an object.
entity_id = uuid4()
o = VersionedEntity(id=entity_id, __version__=0)
o.__dict__['_<lambda>'] = 'value1'
# Call the property's getter function.
value = p.fget(o)
self.assertEqual(value, 'value1')
# Call the property's setter function.
p.fset(o, 'value2')
# Check the attribute has changed.
value = p.fget(o)
self.assertEqual(value, 'value2')
# Check the property's getter function isn't the getter function we passed in.
self.assertNotEqual(p.fget, getter)
# Define a class that uses the decorator.
class Aaa(VersionedEntity):
"An event sourced entity."
def __init__(self, a, *args, **kwargs):
super(Aaa, self).__init__(*args, **kwargs)
self._a = a
@attribute
def a(self):
"A mutable event sourced property."
# Instantiate the class and check assigning to the property publishes an event and updates the object state.
published_events = []
subscription = (lambda x: True, lambda x: published_events.append(x))
subscribe(*subscription)
entity_id = uuid4()
try:
aaa = Aaa(id=entity_id, __version__=1, a=1)
self.assertEqual(aaa.a, 1)
aaa.a = 'value1'
self.assertEqual(aaa.a, 'value1')
finally:
unsubscribe(*subscription)
# Check an event was published.
self.assertEqual(len(published_events), 1)
# Check the published event was an AttributeChanged event, with the expected attribute values.
published_event = published_events[0]
self.assertIsInstance(published_event, AttributeChanged)
self.assertEqual(published_event.name, '_a')
self.assertEqual(published_event.value, 'value1')
self.assertTrue(published_event.originator_version, 1)
self.assertEqual(published_event.originator_id, entity_id)
def tearDown(self):
unsubscribe(*self.subscription)
assert_event_handlers_empty()
def close(self):
unsubscribe(predicate=self.is_my_prompt, handler=self.send_prompt)
self.process.close()
def close(self):
unsubscribe(self.contract_specification_created, self.generate_dependency_graph)
def test_attribute(self):
# Check we get an error when called with something other than a function.
self.assertRaises(ProgrammingError, attribute, 'not a getter')
self.assertRaises(ProgrammingError, attribute, 123)
self.assertRaises(ProgrammingError, attribute, None)
# Call the decorator with a function.
getter = lambda: None
p = attribute(getter)
# Check we got a property object.
self.assertIsInstance(p, property)
# Check the property object has both setter and getter functions.
self.assertTrue(p.fset)
self.assertTrue(p.fget)
# Pretend we decorated an object.
entity_id = uuid4()
o = VersionedEntity(originator_id=entity_id, originator_version=0)
o.__dict__['_<lambda>'] = 'value1'
# Call the property's getter function.
value = p.fget(o)
self.assertEqual(value, 'value1')
# Call the property's setter function.
p.fset(o, 'value2')
# Check the attribute has changed.
value = p.fget(o)
self.assertEqual(value, 'value2')
# Check the property's getter function isn't the getter function we passed in.
self.assertNotEqual(p.fget, getter)
# Define a class that uses the decorator.
class Aaa(VersionedEntity):
"An event sourced entity."
def __init__(self, a, *args, **kwargs):
super(Aaa, self).__init__(*args, **kwargs)
self._a = a
@attribute
def a(self):
"A mutable event sourced property."
# Instantiate the class and check assigning to the property publishes an event and updates the object state.
published_events = []
subscription = (lambda x: True, lambda x: published_events.append(x))
subscribe(*subscription)
entity_id = uuid4()
try:
aaa = Aaa(originator_id=entity_id, originator_version=1, a=1)
self.assertEqual(aaa.a, 1)
aaa.a = 'value1'
self.assertEqual(aaa.a, 'value1')
finally:
unsubscribe(*subscription)
# Check an event was published.
self.assertEqual(len(published_events), 1)
# Check the published event was an AttributeChanged event, with the expected attribute values.
published_event = published_events[0]
self.assertIsInstance(published_event, AttributeChanged)
self.assertEqual(published_event.name, '_a')
self.assertEqual(published_event.value, 'value1')
self.assertTrue(published_event.originator_version, 1)
self.assertEqual(published_event.originator_id, entity_id)
def close(self):
unsubscribe(self.is_contract_valuation_created,
self.generate_contract_valuation)
def close(self):
unsubscribe(predicate=self.condition, handler=self.take_snapshot)
def close(self):
unsubscribe(self.market_simulation_created, self.generate_simulated_prices_for_market_simulation)