def __push_prompts(self):
try:
item = self.downstream_prompt_queue.get() # timeout=1)
self.downstream_prompt_queue.task_done()
# Todo: Instead, drain the queue and consolidate prompts.
except Empty:
self._print_timecheck(
"timed out getting item from downstream prompt "
"queue")
if self.has_been_stopped.is_set():
return
else:
# self.print_timecheck("task done on downstream prompt queue")
if item is None or self.has_been_stopped.is_set():
return
elif isinstance(item, PromptToPull):
if item.head_notification_id:
head_notification_id = item.head_notification_id
else:
head_notification_id = self._get_max_notification_id()
prompt = RayPrompt(
self.process_application.name,
self.process_application.pipeline_id,
head_notification_id,
)
else:
prompt = item
# self._print_timecheck('pushing prompt with', prompt.notification_ids)
prompt_response_ids = []
# self.print_timecheck("pushing prompts", prompt)
for downstream_name, ray_process in self.downstream_processes.items(
):
prompt_response_ids.append(ray_process.prompt.remote(prompt))
if self.has_been_stopped.is_set():
return
# self._print_timecheck("pushed prompt to", downstream_name)
ray.get(prompt_response_ids)
def __process_events(self):
try:
queue_item = self.upstream_event_queue.get() # timeout=5)
self.upstream_event_queue.task_done()
except Empty:
if self.has_been_stopped.is_set():
return
else:
if queue_item is None or self.has_been_stopped.is_set():
return
for (domain_event, notification_id, upstream_name) in queue_item:
# print("Processing upstream event:", (domain_event,
# notification_id, upstream_name))
new_events, new_records = (), ()
while not self.has_been_stopped.is_set():
try:
new_events, new_records = self.do_db_job(
method=self.process_application.
process_upstream_event,
args=(domain_event, notification_id,
upstream_name),
kwargs={},
)
break
except Exception as e:
print(traceback.format_exc())
self._print_timecheck(
"Retrying to reprocess event after error:", e)
sleep(1)
# Todo: Forever? What if this is the wrong event?
if self.has_been_stopped.is_set():
return
# if new_events:
# self._print_timecheck("new events", len(new_events), new_events)
notifications = ()
notification_ids = ()
notifiable_events = [e for e in new_events if e.__notifiable__]
if len(notifiable_events):
if PROMPT_WITH_NOTIFICATION_IDS or PROMPT_WITH_NOTIFICATION_OBJS:
manager = self.process_application.event_store.record_manager
assert isinstance(manager,
RecordManagerWithNotifications)
notification_id_name = manager.notification_id_name
notifications = []
for record in new_records:
if isinstance(
getattr(record, notification_id_name,
None), int):
notifications.append(
manager.create_notification_from_record(
record))
if len(notifications):
head_notification_id = notifications[-1]["id"]
if PROMPT_WITH_NOTIFICATION_IDS:
notification_ids = self._put_notifications_in_ray_object_store(
notifications)
# Clear the notifications, avoid sending with IDs.
notifications = ()
else:
head_notification_id = self._get_max_notification_id(
)
else:
head_notification_id = self._get_max_notification_id()
prompt = RayPrompt(
self.process_application.name,
self.process_application.pipeline_id,
head_notification_id,
notification_ids,
notifications,
)
# self.print_timecheck(
# "putting prompt on downstream " "prompt queue",
# self.downstream_prompt_queue.qsize(),
# )
self.downstream_prompt_queue.put(prompt)
sleep(MICROSLEEP)
def test_ray_process(self):
# Create process with Orders application.
ray_orders_process = RayProcess.remote(
application_process_class=Orders,
infrastructure_class=self.infrastructure_class,
setup_tables=True,
)
# Initialise the ray process.
ray.get(ray_orders_process.init.remote({}, {}))
# Create a new order, within the ray process.
order_id = ray.get(ray_orders_process.call.remote("create_new_order"))
# Check a UUID is returned.
self.assertIsInstance(order_id, UUID)
# Get range of notifications.
notifications = ray.get(ray_orders_process.get_notifications.remote(1, 1000))
self.assertIsInstance(notifications, list)
self.assertEqual(len(notifications), 1)
self.assertEqual(notifications[0]["id"], 1)
# Check the range is working.
notifications = ray.get(ray_orders_process.get_notifications.remote(2, 2))
self.assertIsInstance(notifications, list)
self.assertEqual(len(notifications), 0, notifications)
# Create process with Reservations application.
ray_reservations_process = RayProcess.remote(
application_process_class=Reservations,
infrastructure_class=self.infrastructure_class,
setup_tables=True,
)
# Make the reservations follow the orders.
ray.get(
ray_reservations_process.init.remote({"orders": ray_orders_process}, {})
)
# Get range of notifications.
notifications = ray.get(
ray_reservations_process.get_notifications.remote(1, 1000)
)
self.assertIsInstance(notifications, list)
self.assertEqual(len(notifications), 0)
# Prompt the process.
prompt = RayPrompt("orders", 0)
rayid = ray_reservations_process.prompt.remote(prompt)
ray.get(rayid)
# Check a reservation was created.
retries = 10
while retries:
sleep(0.1)
# Get range of notifications.
notifications = ray.get(
ray_reservations_process.get_notifications.remote(1, 1000)
)
self.assertIsInstance(notifications, list)
try:
self.assertEqual(len(notifications), 1)
except AssertionError:
if retries:
retries -= 1
print("Retrying...", retries)
else:
raise
else:
break
# Add reservations as downstream process of orders.
ray_orders_process.add_downstream_process.remote(
"reservations", ray_reservations_process
)
# Create a new order, within the ray process.
order_id = ray.get(ray_orders_process.call.remote("create_new_order"))
# Get range of notifications.
notifications = ray.get(ray_orders_process.get_notifications.remote(1, 1000))
self.assertIsInstance(notifications, list)
self.assertEqual(len(notifications), 2)
# Check another reservation was created.
retries = 10
while True:
sleep(0.1)
# Get a section of the notification log.
# Get range of notifications.
notifications = ray.get(
ray_reservations_process.get_notifications.remote(1, 1000)
)
self.assertIsInstance(notifications, list)
try:
self.assertEqual(len(notifications), 2)
except AssertionError:
if retries:
retries -= 1
print("Retrying...", retries)
else:
raise
else:
break