def setUpClass(cls):
#IntegrationTestCase.setUpClass()
#cls.db_session = cls.service.handler.get_database_session()
# Get chat data
chat_data = ChatTestDataSets()
cls.test_chat_datasets = chat_data.get_list()
# Dummy Chat Minute
cls.dummy_chat_minute = ChatMinute(
chat_session_id='dummy_session_id',
topic_id=777, # Random ID
start=tz.timestamp_to_utc(1345643963),
end=tz.timestamp_to_utc(1345643999))
def setUpClass(cls):
#IntegrationTestCase.setUpClass()
#cls.db_session = cls.service.handler.get_database_session()
# Get chat data
chat_data = ChatTestDataSets()
cls.test_chat_datasets = chat_data.get_list()
# Dummy Chat Minute
cls.dummy_chat_minute = ChatMinute(
chat_session_id='dummy_session_id',
topic_id=777, # Random ID
start=tz.timestamp_to_utc(1345643963),
end = tz.timestamp_to_utc(1345643999))
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
TopicIdDoesNotExistException
"""
# Create models from message
if self._is_valid_create_message(message):
topic_id = message.minuteCreateMessage.topicId
start_time = tz.timestamp_to_utc(message.minuteCreateMessage.startTimestamp)
# Update this topic's start time
minute = self.topic_minute_map[topic_id]
minute.start = start_time
self._set_active_minute(minute)
# Update parent topic's start time
self._start_parent_topic_minutes(topic_id, start_time)
# Update previous topic's end-times
self._end_previous_topic_minutes(topic_id, start_time)
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
DuplicateTagIdException
NoActiveChatMinuteException
"""
created_model = None
if self._is_valid_create_tag_message(message):
chat_minute = self.chat_message_handler.chat_minute_handler.get_active_minute()
created_model = ChatTag(
user_id=message.header.userId,
time = tz.timestamp_to_utc(message.header.timestamp),
chat_minute=chat_minute,
tag_id=message.tagCreateMessage.tagReferenceId,
name=message.tagCreateMessage.name,
deleted=False)
self._update_tags_to_persist(chat_minute, message)
# Store message and its associated model for tagID look-ups on tag delete messages.
tag_data = MessageModelData(message.tagCreateMessage.tagId, message, created_model)
self.all_tags[tag_data.id] = tag_data
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
DuplicateTagIdException
NoActiveChatMinuteException
"""
created_model = None
if self._is_valid_create_tag_message(message):
chat_minute = self.chat_message_handler.chat_minute_handler.get_active_minute(
)
created_model = ChatTag(
user_id=message.header.userId,
time=tz.timestamp_to_utc(message.header.timestamp),
chat_minute=chat_minute,
tag_id=message.tagCreateMessage.tagReferenceId,
name=message.tagCreateMessage.name,
deleted=False)
self._update_tags_to_persist(chat_minute, message)
# Store message and its associated model for tagID look-ups on tag delete messages.
tag_data = MessageModelData(message.tagCreateMessage.tagId, message,
created_model)
self.all_tags[tag_data.id] = tag_data
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
TopicIdDoesNotExistException
"""
# Create models from message
if self._is_valid_create_message(message):
topic_id = message.minuteCreateMessage.topicId
start_time = tz.timestamp_to_utc(
message.minuteCreateMessage.startTimestamp)
# Update this topic's start time
minute = self.topic_minute_map[topic_id]
minute.start = start_time
self._set_active_minute(minute)
# Update parent topic's start time
self._start_parent_topic_minutes(topic_id, start_time)
# Update previous topic's end-times
self._end_previous_topic_minutes(topic_id, start_time)
def read_datetime(self):
value = None
context = self.context_stack[-1]
timestamp = context.read()
if timestamp:
value = tz.timestamp_to_utc(timestamp)
return value
def _index(self, context, index_action, index_data, index_all=False):
"""Helper function. Pulled out common code from index() & indexAll().
This method creates a job to index the specified input data.
Args:
context: String to identify calling context
index_action: IndexAction object
index_data: Thrift IndexData object
index_all: Boolean indicating if all keys should be acted upon
Returns:
None
Raises:
InvalidDataException if input data to index is invalid.
UnavailableException for any other unexpected error.
"""
try:
# Get a db session
db_session = self.get_database_session()
# Validate inputs
self._validate_index_params(
context,
index_action,
index_data,
index_all
)
# If input specified a start-processing-time
# convert it to UTC DateTime object.
if index_data.notBefore is not None:
processing_start_time = tz.timestamp_to_utc(index_data.notBefore)
else:
processing_start_time = func.current_timestamp()
# Massage input data into format for IndexJob
data = IndexOp(action=index_action, data=index_data)
# Create IndexJob
job = IndexJobModel(
created=func.current_timestamp(),
context=context,
not_before=processing_start_time,
retries_remaining=settings.INDEXER_JOB_MAX_RETRY_ATTEMPTS,
data=json.dumps(data.to_json())
)
db_session.add(job)
db_session.commit()
return
finally:
db_session.close()
def to_python(self, value):
result = None
if value is None:
result = None
elif isinstance(value, datetime.datetime):
result = value.replace(tzinfo=pytz.utc)
elif isinstance(value, datetime.date):
dt = datetime.datetime(value.year, value.month, value.day)
result = dt.replace(tzinfo=pytz.utc)
elif isinstance(value, (int, float)):
result = tz.timestamp_to_utc(value)
elif isinstance(value, basestring):
result = tz.iso_to_utc(value) or tz.now_to_utc(value)
if result is None:
try:
result = tz.timestamp_to_utc(float(value))
except:
raise ValidationError("invalid datetime '%s'" % str(value))
else:
raise ValidationError("invalid datetime '%s'" % str(value))
return result
def to_python(self, value):
result = None
if value is None:
result = None
elif isinstance(value, datetime.datetime):
result = value.replace(tzinfo=pytz.utc)
elif isinstance(value, datetime.date):
dt = datetime.datetime(value.year, value.month, value.day)
result = dt.replace(tzinfo=pytz.utc)
elif isinstance(value, (int, float)):
result = tz.timestamp_to_utc(value)
elif isinstance(value, basestring):
result = tz.iso_to_utc(value) or tz.now_to_utc(value)
if result is None:
try:
result = tz.timestamp_to_utc(float(value))
except:
raise ValidationError("invalid datetime '%s'" % str(value))
else :
raise ValidationError("invalid datetime '%s'" % str(value))
return result
def update_models(self, message):
"""
Update model instance(s) from Thrift Message
This method is responsible for listening for a
MinuteUpdateMessage on the last topic in a chat.
Once received, this method will generate
end-timestamps for the ChatMinute's that require it
(the root topic and any parents of the final leaf topic).
Note that for all other topics, the end-timestamps
are written when a new minuteCreateMessage comes in
via create_models().The reason that messages are processed this
way is that the message order of minuteStart and minuteEnd
messages was not guaranteed to be in chronological order
since the time between these successive messages
was so close (any network latency in one message
but not the other was enough to change the order).
Args:
message: Deserialized Thrift Message
Throws:
TopicIdDoesNotExistException
"""
if self._is_valid_update_message(message):
topic_id = message.minuteUpdateMessage.topicId
end_time = tz.timestamp_to_utc(
message.minuteUpdateMessage.endTimestamp)
# Update this final leaf's end time
minute = self.topic_minute_map[topic_id]
minute.end = end_time
# Update parent's end times
# Expecting at least the root topic to be updated here
parent_topics_to_end = self.minute_end_topic_chain[topic_id]
for topic in parent_topics_to_end:
minute = self.topic_minute_map[topic.id]
minute.end = end_time
def update_models(self, message):
"""
Update model instance(s) from Thrift Message
This method is responsible for listening for a
MinuteUpdateMessage on the last topic in a chat.
Once received, this method will generate
end-timestamps for the ChatMinute's that require it
(the root topic and any parents of the final leaf topic).
Note that for all other topics, the end-timestamps
are written when a new minuteCreateMessage comes in
via create_models().The reason that messages are processed this
way is that the message order of minuteStart and minuteEnd
messages was not guaranteed to be in chronological order
since the time between these successive messages
was so close (any network latency in one message
but not the other was enough to change the order).
Args:
message: Deserialized Thrift Message
Throws:
TopicIdDoesNotExistException
"""
if self._is_valid_update_message(message):
topic_id = message.minuteUpdateMessage.topicId
end_time = tz.timestamp_to_utc(message.minuteUpdateMessage.endTimestamp)
# Update this final leaf's end time
minute = self.topic_minute_map[topic_id]
minute.end = end_time
# Update parent's end times
# Expecting at least the root topic to be updated here
parent_topics_to_end = self.minute_end_topic_chain[topic_id]
for topic in parent_topics_to_end:
minute = self.topic_minute_map[topic.id]
minute.end = end_time
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
NoActiveChatMinuteException
"""
created_model = None
# Listen for a chat-start message so that we can know
# if the active chat minute becomes invalid while
# processing the other marker messages.
#
# Note that it's possible that the first create-chat-minute msg
# will be received prior to the chat-started msg. If this is the
# case, we still properly process marker messages.
#
if not self.is_chat_started and self._is_valid_start_chat_marker_message(message):
self.is_chat_started = True
elif self._is_valid_create_speaking_marker_message(message):
# Only expecting & handling speaking markers
# The general approach for handling speaking markers is to
# listen for a speaking-start marker and then wait for
# the corresponding speaking-end. All other duplicate speaking-start
# markers for the specified user will be ignored until the
# corresponding speaking-end message is received. The reason
# this is done because in a chat with 3 users, the two users
# who are not speaking will generate duplicate speaking marker messages.
# Models are created when the end-speaking marker is received.
# Get user's speaking state
user_id = message.markerCreateMessage.marker.speakingMarker.userId
user_speaking_data = None
if user_id not in self.speaking_state:
user_speaking_data = SpeakingData(user_id)
else:
user_speaking_data = self.speaking_state[user_id]
# Determine if the speaking minute has ended and we need to persist the marker
if message.markerCreateMessage.marker.speakingMarker.isSpeaking:
# msg indicates user started speaking
if not user_speaking_data.is_speaking():
# If user wasn't already speaking, process msg.
# Ignore duplicate speaking_start markers. Once the first speaking start
# message is processed, we will ignore the rest until we receive a speaking
# end message.
user_speaking_data.set_start_timestamp(message.header.timestamp)
user_speaking_data.set_speaking(True)
else:
# msg indicates user stopped speaking
if user_speaking_data.is_speaking():
# If user was already speaking, process msg.
# Ignore duplicate speaking_end markers. Once the first speaking end
# message is processed, we will ignore the rest until we receive a speaking
# start message.
user_speaking_data.set_end_timestamp(message.header.timestamp)
duration = user_speaking_data.calculate_speaking_duration()
if duration > self.SPEAKING_DURATION_THRESHOLD:
chat_minute = self.chat_message_handler.chat_minute_handler.get_active_minute()
start_time = tz.timestamp_to_utc(user_speaking_data.get_start_timestamp())
end_time = tz.timestamp_to_utc(user_speaking_data.get_end_timestamp())
created_model = ChatSpeakingMarker(
user_id=user_id,
chat_minute=chat_minute,
start=start_time,
end=end_time)
# Reset user's speaking state
user_speaking_data.set_start_timestamp(None)
user_speaking_data.set_end_timestamp(None)
user_speaking_data.set_speaking(False)
# Update state
self.speaking_state[user_id] = user_speaking_data
# Store message and its associated model
marker_data = MessageModelData(message.markerCreateMessage.markerId, message, created_model)
self.all_markers[marker_data.id] = marker_data
def end(self):
if self.state.endTimestamp == 0:
result = None
else:
result = tz.timestamp_to_utc(self.state.endTimestamp)
return result
def notify(self, context, notification):
"""Send notification
This method writes the input notification to the db
and creates a job for each recipient for the
notification to process.
This is done to ensure that we don't lose any
information should this service go down.
Args:
context: String to identify calling context
notification: Thrift Notification object.
The notification text fields (subject,
plainText, htmlText) support python
template strings (via string.Template).
Note that all '$' chars will have to be
properly escaped.
The following template strings are supported:
'first_name' : the recipient's first name
'last_name' : the recipient's last name
For example:
'Dear ${first_name}',
Returns:
Thrift Notification object.
If no 'token' attribute was provided in the input
notification object, the returned object will
specify one.
Raises:
InvalidNotificationException if input Notification
object is invalid.
UnavailableException for any other unexpected error.
"""
try:
# Get a db session
db_session = self.get_database_session()
# Validate inputs
# During validation of recipients, populate a list of Users.
users = []
self._validate_notify_params(db_session, users, context, notification)
# If input notification doesn't specify a
# unique token, then generate one.
if not notification.token:
notification.token = uuid.uuid4().hex
# Create Notification Model
notification_model = NotificationModel(
created=func.current_timestamp(),
token=notification.token,
context=context,
priority=NOTIFICATION_PRIORITY_VALUES[
NotificationPriority._VALUES_TO_NAMES[notification.priority]],
recipients=users,
subject=notification.subject,
html_text=notification.htmlText,
plain_text=notification.plainText
)
db_session.add(notification_model)
# If notification specified a start-processing-time
# convert it to UTC DateTime object.
if notification.notBefore is not None:
processing_start_time = tz.timestamp_to_utc(notification.notBefore)
else:
processing_start_time = func.current_timestamp()
# Create NotificationJobs
for user_id in notification.recipientUserIds:
job = NotificationJobModel(
created=func.current_timestamp(),
not_before=processing_start_time,
notification=notification_model,
recipient_id=user_id,
priority=NOTIFICATION_PRIORITY_VALUES[
NotificationPriority._VALUES_TO_NAMES[notification.priority]],
retries_remaining=settings.NOTIFIER_JOB_MAX_RETRY_ATTEMPTS
)
db_session.add(job)
db_session.commit()
return notification
except InvalidNotificationException as error:
self.log.exception(error)
raise InvalidNotificationException()
except Exception as error:
self.log.exception(error)
raise UnavailableException(str(error))
finally:
db_session.close()
def start(self):
if self.state.startTimestamp == 0:
result = None
else:
result = tz.timestamp_to_utc(self.state.startTimestamp)
return result
def create_models(self, message):
"""
Create model instance(s) from input message.
Need to call finalize() after processing
all the message in a chat to get the created
instances.
Args:
message: Deserialized Thrift Message
Throws:
NoActiveChatMinuteException
"""
created_model = None
# Listen for a chat-start message so that we can know
# if the active chat minute becomes invalid while
# processing the other marker messages.
#
# Note that it's possible that the first create-chat-minute msg
# will be received prior to the chat-started msg. If this is the
# case, we still properly process marker messages.
#
if not self.is_chat_started and self._is_valid_start_chat_marker_message(
message):
self.is_chat_started = True
elif self._is_valid_create_speaking_marker_message(message):
# Only expecting & handling speaking markers
# The general approach for handling speaking markers is to
# listen for a speaking-start marker and then wait for
# the corresponding speaking-end. All other duplicate speaking-start
# markers for the specified user will be ignored until the
# corresponding speaking-end message is received. The reason
# this is done because in a chat with 3 users, the two users
# who are not speaking will generate duplicate speaking marker messages.
# Models are created when the end-speaking marker is received.
# Get user's speaking state
user_id = message.markerCreateMessage.marker.speakingMarker.userId
user_speaking_data = None
if user_id not in self.speaking_state:
user_speaking_data = SpeakingData(user_id)
else:
user_speaking_data = self.speaking_state[user_id]
# Determine if the speaking minute has ended and we need to persist the marker
if message.markerCreateMessage.marker.speakingMarker.isSpeaking:
# msg indicates user started speaking
if not user_speaking_data.is_speaking():
# If user wasn't already speaking, process msg.
# Ignore duplicate speaking_start markers. Once the first speaking start
# message is processed, we will ignore the rest until we receive a speaking
# end message.
user_speaking_data.set_start_timestamp(
message.header.timestamp)
user_speaking_data.set_speaking(True)
else:
# msg indicates user stopped speaking
if user_speaking_data.is_speaking():
# If user was already speaking, process msg.
# Ignore duplicate speaking_end markers. Once the first speaking end
# message is processed, we will ignore the rest until we receive a speaking
# start message.
user_speaking_data.set_end_timestamp(
message.header.timestamp)
duration = user_speaking_data.calculate_speaking_duration()
if duration > self.SPEAKING_DURATION_THRESHOLD:
chat_minute = self.chat_message_handler.chat_minute_handler.get_active_minute(
)
start_time = tz.timestamp_to_utc(
user_speaking_data.get_start_timestamp())
end_time = tz.timestamp_to_utc(
user_speaking_data.get_end_timestamp())
created_model = ChatSpeakingMarker(
user_id=user_id,
chat_minute=chat_minute,
start=start_time,
end=end_time)
# Reset user's speaking state
user_speaking_data.set_start_timestamp(None)
user_speaking_data.set_end_timestamp(None)
user_speaking_data.set_speaking(False)
# Update state
self.speaking_state[user_id] = user_speaking_data
# Store message and its associated model
marker_data = MessageModelData(message.markerCreateMessage.markerId,
message, created_model)
self.all_markers[marker_data.id] = marker_data
def start(self):
if self.state.startTimestamp == 0:
result = None
else:
result = tz.timestamp_to_utc(self.state.startTimestamp)
return result
def end(self):
if self.state.endTimestamp == 0:
result = None
else:
result = tz.timestamp_to_utc(self.state.endTimestamp)
return result