def get_index_data(config):
not_before = tz.utc_to_timestamp(config.time)
return IndexData(
notBefore=not_before,
name=config.index_name,
type=config.doc_type,
keys=config.keys
)
def load(self):
"""Load chat model from database.
Raises:
Exception (sqlalchemy)
"""
if not self.loaded_event.is_set():
try:
session = self.service_handler.get_database_session()
model = session.query(ChatModel)\
.filter_by(token=self.token)\
.one()
self.id = model.id
session.commit()
self.state.maxDuration = model.max_duration
self.state.maxParticipants = model.max_participants
if model.end:
self.state.status = ChatStatus.ENDED
self.state.startTimestamp = tz.utc_to_timestamp(
model.start)
self.state.endTimesamp = tz.utc_to_timestamp(model.end)
elif model.start:
self.state.status = ChatStatus.STARTED
self.state.startTimestamp = tz.utc_to_timestamp(
model.start)
except NoResultFound:
self.loaded_event.set()
self.loaded_event.clear()
except Exception as error:
logging.exception(error)
self.loaded_event.set()
self.loaded_event.clear()
raise
finally:
session.close()
self.loaded_event.set()
def load(self):
"""Load chat model from database.
Raises:
Exception (sqlalchemy)
"""
if not self.loaded_event.is_set():
try:
session = self.service_handler.get_database_session()
model = session.query(ChatModel)\
.filter_by(token=self.token)\
.one()
self.id = model.id
session.commit()
self.state.maxDuration = model.max_duration
self.state.maxParticipants = model.max_participants
if model.end:
self.state.status = ChatStatus.ENDED
self.state.startTimestamp = tz.utc_to_timestamp(model.start)
self.state.endTimesamp = tz.utc_to_timestamp(model.end)
elif model.start:
self.state.status = ChatStatus.STARTED
self.state.startTimestamp = tz.utc_to_timestamp(model.start)
except NoResultFound:
self.loaded_event.set()
self.loaded_event.clear()
except Exception as error:
logging.exception(error)
self.loaded_event.set()
self.loaded_event.clear()
raise
finally:
session.close()
self.loaded_event.set()
def _validate_indexjob_model(self, model, index_action, index_data):
"""Encapsulate code to validate an IndexJob model.
Args:
model: the IndexJob model to validate
index_action: IndexAction object
index_data: the Thrift IndexData the model was created from
"""
self.assertEqual(model.context, self.context)
self.assertEqual(model.data, IndexOp(index_action, index_data).to_json())
self.assertAlmostEqual(index_data.notBefore, tz.utc_to_timestamp(model.not_before), places=7)
self.assertIsNotNone(model.created)
self.assertIsNone(model.start)
self.assertIsNone(model.end)
self.assertIsNone(model.owner)
self.assertIsNone(model.successful)
self.assertEqual(self.max_retry_attempts, model.retries_remaining)
def test_createModels(self):
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMarkerHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id,
chat_data.topic_collection)
marker_handler = message_handler.chat_marker_handler
# Process messages
for deserialized_msg in chat_data.message_list:
message_handler.process(deserialized_msg)
# Retrieve all speaking marker models
speaking_models = marker_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_marker_models),
len(speaking_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in speaking_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreater(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_marker_models
for index, model in enumerate(speaking_models):
self.assertIsNotNone(model.start)
self.assertIsNotNone(model.end)
self.assertEqual(expected_models[index].user_id, model.user_id)
self.assertEqual(
expected_models[index].chat_minute.chat_session_id,
model.chat_minute.chat_session_id)
self.assertEqual(expected_models[index].chat_minute.topic_id,
model.chat_minute.topic_id)
self.assertEqual(expected_models[index].chat_minute.start,
model.chat_minute.start)
self.assertEqual(expected_models[index].chat_minute.end,
model.chat_minute.end)
def _validate_indexjob_model(self, model, index_action, index_data):
"""Encapsulate code to validate an IndexJob model.
Args:
model: the IndexJob model to validate
index_action: IndexAction object
index_data: the Thrift IndexData the model was created from
"""
self.assertEqual(model.context, self.context)
self.assertEqual(model.data,
IndexOp(index_action, index_data).to_json())
self.assertAlmostEqual(index_data.notBefore,
tz.utc_to_timestamp(model.not_before),
places=7)
self.assertIsNotNone(model.created)
self.assertIsNone(model.start)
self.assertIsNone(model.end)
self.assertIsNone(model.owner)
self.assertIsNone(model.successful)
self.assertEqual(self.max_retry_attempts, model.retries_remaining)
def _validate_notificationjob_model(self, model, notification, expected_recipient_id, expected_retries_remaining):
"""Encapsulate code to validate a NotificationJob model.
Args:
model: the NotificationJob model to validate
notification: the Thrift Notification the model was created from
expected_recipient_id: the ID of the expected recipient
expected_retries_remaining: the expected number of retries remaining
"""
self.assertEqual(expected_recipient_id, model.recipient_id)
self.assertEqual(
NOTIFICATION_PRIORITY_VALUES[NotificationPriority._VALUES_TO_NAMES[notification.priority]],
model.priority)
self.assertAlmostEqual(notification.notBefore, tz.utc_to_timestamp(model.not_before), places=7)
self.assertIsNotNone(model.created)
self.assertIsNone(model.start)
self.assertIsNone(model.end)
self.assertIsNone(model.owner)
self.assertIsNone(model.successful)
self.assertEqual(expected_retries_remaining, model.retries_remaining)
def test_updateModels(self):
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMinuteHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id,
chat_data.topic_collection)
minute_handler = message_handler.chat_minute_handler
# Set the start time on the chat minutes (Root, Topic1)
message = chat_data.message_list[27]
message_handler.process(message)
# Since the end-time's have not been written, an exception should be raised
with self.assertRaises(InvalidChatMinuteException):
minute_handler.finalize()
# Set the end time on the chat minutes (Root, Topic1)
message = chat_data.message_list[64]
message_handler.process(message)
minute_models = minute_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_minute_models),
len(minute_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in minute_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreaterEqual(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_minute_models
for index, model in enumerate(minute_models):
self.assertEqual(expected_models[index].chat_session_id,
model.chat_session_id)
self.assertEqual(expected_models[index].topic_id, model.topic_id)
self.assertEqual(expected_models[index].start, model.start)
self.assertEqual(expected_models[index].end, model.end)
def test_chatMinuteHandler(self):
""" Sends input deserialized Thrift messages into
the ChatMessageHandler and verifies the output
ChatMinute objects.
"""
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMinuteHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id,
chat_data.topic_collection)
minute_handler = message_handler.chat_minute_handler
# Process messages
for deserialized_msg in chat_data.message_list:
message_handler.process(deserialized_msg)
# Retrieve all chat minute models ordered by topic rank
minute_models = minute_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_minute_models),
len(minute_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in minute_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreaterEqual(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_minute_models
for index, model in enumerate(minute_models):
self.assertEqual(expected_models[index].chat_session_id,
model.chat_session_id)
self.assertEqual(expected_models[index].topic_id, model.topic_id)
self.assertEqual(expected_models[index].start, model.start)
self.assertEqual(expected_models[index].end, model.end)
def finalize(self):
"""
Hook to allow any operations to be completed
after all chat messages have been consumed.
This method is also responsible for returning
all models to persist. It should be called
after processing all messages in a chat.
Returns:
List of models to persist.
"""
models_to_persist = []
for message_model_data_obj in self.all_markers.values():
model = message_model_data_obj.get_model()
if model is not None:
models_to_persist.append(model)
# Sort list by timestamp and return
models_to_persist.sort(key=lambda model: tz.utc_to_timestamp(model.start))
return models_to_persist
def test_updateModels(self):
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMinuteHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id, chat_data.topic_collection)
minute_handler = message_handler.chat_minute_handler
# Set the start time on the chat minutes (Root, Topic1)
message = chat_data.message_list[27]
message_handler.process(message)
# Since the end-time's have not been written, an exception should be raised
with self.assertRaises(InvalidChatMinuteException):
minute_handler.finalize()
# Set the end time on the chat minutes (Root, Topic1)
message = chat_data.message_list[64]
message_handler.process(message)
minute_models = minute_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_minute_models), len(minute_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in minute_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreaterEqual(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_minute_models
for index, model in enumerate(minute_models):
self.assertEqual(expected_models[index].chat_session_id, model.chat_session_id)
self.assertEqual(expected_models[index].topic_id, model.topic_id)
self.assertEqual(expected_models[index].start, model.start)
self.assertEqual(expected_models[index].end, model.end)
def _validate_notificationjob_model(self, model, notification,
expected_recipient_id,
expected_retries_remaining):
"""Encapsulate code to validate a NotificationJob model.
Args:
model: the NotificationJob model to validate
notification: the Thrift Notification the model was created from
expected_recipient_id: the ID of the expected recipient
expected_retries_remaining: the expected number of retries remaining
"""
self.assertEqual(expected_recipient_id, model.recipient_id)
self.assertEqual(
NOTIFICATION_PRIORITY_VALUES[NotificationPriority._VALUES_TO_NAMES[
notification.priority]], model.priority)
self.assertAlmostEqual(notification.notBefore,
tz.utc_to_timestamp(model.not_before),
places=7)
self.assertIsNotNone(model.created)
self.assertIsNone(model.start)
self.assertIsNone(model.end)
self.assertIsNone(model.owner)
self.assertIsNone(model.successful)
self.assertEqual(expected_retries_remaining, model.retries_remaining)
def finalize(self):
"""
Hook to allow any operations to be completed
after all chat messages have been consumed.
This method is also responsible for returning
all models to persist. It should be called
after processing all messages in a chat.
Returns:
List of models to persist.
"""
models_to_persist = []
for message_model_data_obj in self.all_markers.values():
model = message_model_data_obj.get_model()
if model is not None:
models_to_persist.append(model)
# Sort list by timestamp and return
models_to_persist.sort(
key=lambda model: tz.utc_to_timestamp(model.start))
return models_to_persist
def test_chatMinuteHandler(self):
""" Sends input deserialized Thrift messages into
the ChatMessageHandler and verifies the output
ChatMinute objects.
"""
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMinuteHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id, chat_data.topic_collection)
minute_handler = message_handler.chat_minute_handler
# Process messages
for deserialized_msg in chat_data.message_list:
message_handler.process(deserialized_msg)
# Retrieve all chat minute models ordered by topic rank
minute_models = minute_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_minute_models), len(minute_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in minute_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreaterEqual(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_minute_models
for index, model in enumerate(minute_models):
self.assertEqual(expected_models[index].chat_session_id, model.chat_session_id)
self.assertEqual(expected_models[index].topic_id, model.topic_id)
self.assertEqual(expected_models[index].start, model.start)
self.assertEqual(expected_models[index].end, model.end)
def test_createModels(self):
# Get chat data
chat_data = self.test_chat_datasets[0]
# Create ChatMarkerHandler
message_handler = ChatMessageHandler(chat_data.chat_session_id, chat_data.topic_collection)
marker_handler = message_handler.chat_marker_handler
# Process messages
for deserialized_msg in chat_data.message_list:
message_handler.process(deserialized_msg)
# Retrieve all speaking marker models
speaking_models = marker_handler.finalize()
# Ensure number of created models
self.assertEqual(len(chat_data.expected_marker_models), len(speaking_models))
# Ensure models are returned in chronological order
prev_model_start = 0
for model in speaking_models:
model_start = tz.utc_to_timestamp(model.start)
self.assertGreater(model_start, prev_model_start)
prev_model_start = model_start
# Check model data
expected_models = chat_data.expected_marker_models
for index, model in enumerate(speaking_models):
self.assertIsNotNone(model.start)
self.assertIsNotNone(model.end)
self.assertEqual(expected_models[index].user_id, model.user_id)
self.assertEqual(expected_models[index].chat_minute.chat_session_id, model.chat_minute.chat_session_id)
self.assertEqual(expected_models[index].chat_minute.topic_id, model.chat_minute.topic_id)
self.assertEqual(expected_models[index].chat_minute.start, model.chat_minute.start)
self.assertEqual(expected_models[index].chat_minute.end, model.chat_minute.end)
def write_datetime(self, value):
context = self.context_stack[-1]
if value:
value = tz.utc_to_timestamp(value)
context.write(value)
def get_index_data(config):
not_before = tz.utc_to_timestamp(config.time)
return IndexData(notBefore=not_before,
name=config.index_name,
type=config.doc_type,
keys=config.keys)