def test_get(self):
test_start_time = tz.utcnow()
with self.assertRaises(QueueEmpty):
self.db_queue.get(False)
with self._archive_job(1) as (session, job):
with self.db_queue.get(True, 10) as db_job:
self.assertEqual(job.id, db_job.id)
self.assertEqual(job.chat_session_id, db_job.chat_session_id)
self.assertEqual(job.created, db_job.created)
self.assertEqual(db_job.owner, "unittest")
self.assertIsNone(db_job.end)
self.assertIsNone(db_job.successful)
self.assertTrue(tz.utcnow() > db_job.start)
self.assertTrue(test_start_time < db_job.start)
session.refresh(job)
self.assertTrue(job.successful)
self.assertTrue(job.start < job.end)
def test_get(self):
test_start_time = tz.utcnow()
with self.assertRaises(QueueEmpty):
self.db_queue.get(False)
with self._archive_job(1) as (session, job):
with self.db_queue.get(True, 10) as db_job:
self.assertEqual(job.id, db_job.id)
self.assertEqual(job.chat_session_id, db_job.chat_session_id)
self.assertEqual(job.created, db_job.created)
self.assertEqual(db_job.owner, "unittest")
self.assertIsNone(db_job.end)
self.assertIsNone(db_job.successful)
self.assertTrue(tz.utcnow() > db_job.start)
self.assertTrue(test_start_time < db_job.start)
session.refresh(job)
self.assertTrue(job.successful)
self.assertTrue(job.start < job.end)
def _archive_job(self, chat_session_id, not_before=None):
not_before = not_before or tz.utcnow()
try:
session = self.db_session_factory()
job = ChatArchiveJob(chat_session_id=chat_session_id,
created=tz.utcnow(),
not_before=not_before,
retries_remaining=0)
session.add(job)
session.commit()
yield session, job
except Exception as error:
logging.exception(error)
raise
finally:
if session:
if job:
session.delete(job)
session.commit()
session.close()
def _archive_job(self, chat_session_id, not_before=None):
not_before = not_before or tz.utcnow()
try:
session = self.db_session_factory()
job = ChatArchiveJob(
chat_session_id=chat_session_id,
created=tz.utcnow(),
not_before=not_before,
retries_remaining=0)
session.add(job)
session.commit()
yield session, job
except Exception as error:
logging.exception(error)
raise
finally:
if session:
if job:
session.delete(job)
session.commit()
session.close()
def _start_update_values(self):
"""Model attributes/values to be updated when job is started.
Returns:
dict of model {attribute: value} to be updated when
the job is started.
"""
update_values = {}
if hasattr(self.model_class, "owner"):
update_values["owner"] = self.owner
if hasattr(self.model_class, "start"):
update_values["start"] = tz.utcnow()
return update_values
def _start_update_values(self):
"""Model attributes/values to be updated when job is started.
Returns:
dict of model {attribute: value} to be updated when
the job is started.
"""
update_values = {}
if hasattr(self.model_class, "owner"):
update_values["owner"] = self.owner
if hasattr(self.model_class, "start"):
update_values["start"] = tz.utcnow()
return update_values
def __init__(self, argv):
self.MAX_DAYS = 90
self.preview = False
self.indexjob_context = "index_job_scheduler"
self.days = 1
self.time = tz.utcnow().replace(hour=0,
minute=0,
second=0,
microsecond=0)
self.index_name = None
self.doc_type = None
self.keys = []
try:
options, arguments = getopt.getopt(argv, "hpc:i:t:k:d:T:", [
"help", "preview", "context=", "index=", "type=", "keys=",
"days=", "time="
])
for option, argument in options:
if option in ("-h", "--help"):
raise Usage()
elif option in ("-p", "--preview"):
self.preview = True
elif option in ("-c", "--context"):
self.indexjob_context = argument
elif option in ("-i", "--index"):
self.index_name = argument
elif option in ("-t", "--type"):
self.doc_type = argument
elif option in ("-k", "--keys"):
k = argument.replace(" ", "")
self.keys = k.split(',')
elif option in ("-d", "--days"):
self.days = int(argument)
elif option in ("-T", "--time"):
t = argument.split(':')
hour = int(t[0])
minute = int(t[1])
self.time = self.time.replace(hour=hour, minute=minute)
else:
raise Usage()
if (not self.index_name or not self.doc_type
or self.days > self.MAX_DAYS):
raise Usage()
except Exception as e:
raise Usage()
def test_not_before(self):
not_before = tz.utcnow() + datetime.timedelta(seconds=5)
with self._archive_job(1, not_before) as (session, job):
with self.assertRaises(QueueEmpty):
self.db_queue.get(False)
with self.db_queue.get(True, 10) as db_job:
self.assertEqual(job.id, db_job.id)
self.assertEqual(job.chat_session_id, db_job.chat_session_id)
self.assertEqual(job.created, db_job.created)
self.assertEqual(db_job.owner, "unittest")
self.assertIsNone(db_job.end)
session.refresh(job)
self.assertTrue(job.successful)
self.assertTrue(job.start < job.end)
def test_not_before(self):
not_before = tz.utcnow() + datetime.timedelta(seconds=5)
with self._archive_job(1, not_before) as (session, job):
with self.assertRaises(QueueEmpty):
self.db_queue.get(False)
with self.db_queue.get(True, 10) as db_job:
self.assertEqual(job.id, db_job.id)
self.assertEqual(job.chat_session_id, db_job.chat_session_id)
self.assertEqual(job.created, db_job.created)
self.assertEqual(db_job.owner, "unittest")
self.assertIsNone(db_job.end)
session.refresh(job)
self.assertTrue(job.successful)
self.assertTrue(job.start < job.end)
def __init__(self, argv):
self.MAX_DAYS = 90
self.preview = False
self.indexjob_context = "index_job_scheduler"
self.days = 1
self.time = tz.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
self.index_name = None
self.doc_type = None
self.keys = []
try:
options, arguments = getopt.getopt(argv, "hpc:i:t:k:d:T:",["help", "preview", "context=", "index=", "type=", "keys=", "days=", "time="])
for option, argument in options:
if option in ("-h", "--help"):
raise Usage()
elif option in ("-p", "--preview"):
self.preview = True
elif option in ("-c", "--context"):
self.indexjob_context = argument
elif option in ("-i", "--index"):
self.index_name = argument
elif option in ("-t", "--type"):
self.doc_type = argument
elif option in ("-k", "--keys"):
k = argument.replace(" ", "")
self.keys = k.split(',')
elif option in ("-d", "--days"):
self.days = int(argument)
elif option in ("-T", "--time"):
t = argument.split(':')
hour = int(t[0])
minute = int(t[1])
self.time = self.time.replace(hour=hour, minute=minute)
else:
raise Usage()
if (not self.index_name or
not self.doc_type or
self.days > self.MAX_DAYS):
raise Usage()
except Exception as e:
raise Usage()
def _retry_job(self, failed_job):
"""Create a new IndexJob from a failed job.
This method creates a new IndexJob from a
job that failed processing.
Args:
failed_job: DatabaseJob object, or objected derived from DatabaseJob
Returns:
None
"""
try:
db_session = None
#create new job in db to retry.
if failed_job.retries_remaining > 0:
not_before = tz.utcnow() + datetime.timedelta(
seconds=self.job_retry_seconds)
new_job = IndexJob(
data=failed_job.data,
context=failed_job.context,
created=func.current_timestamp(),
not_before=not_before,
retries_remaining=failed_job.retries_remaining - 1)
# Add job to db
db_session = self.db_session_factory()
db_session.add(new_job)
db_session.commit()
else:
self.log.info("No retries remaining for job for index_job_id=%s"\
% (failed_job.id))
self.log.error("Job for index_job_id=%s failed!"\
% (failed_job.id))
except Exception as e:
self.log.exception(e)
if db_session:
db_session.rollback()
finally:
if db_session:
db_session.close()
def _retry_job(self, failed_job):
"""Create a new IndexJob from a failed job.
This method creates a new IndexJob from a
job that failed processing.
Args:
failed_job: DatabaseJob object, or objected derived from DatabaseJob
Returns:
None
"""
try:
db_session = None
#create new job in db to retry.
if failed_job.retries_remaining > 0:
not_before = tz.utcnow() + datetime.timedelta(seconds=self.job_retry_seconds)
new_job = IndexJob(
data=failed_job.data,
context=failed_job.context,
created=func.current_timestamp(),
not_before=not_before,
retries_remaining=failed_job.retries_remaining-1
)
# Add job to db
db_session = self.db_session_factory()
db_session.add(new_job)
db_session.commit()
else:
self.log.info("No retries remaining for job for index_job_id=%s"\
% (failed_job.id))
self.log.error("Job for index_job_id=%s failed!"\
% (failed_job.id))
except Exception as e:
self.log.exception(e)
if db_session:
db_session.rollback()
finally:
if db_session:
db_session.close()
def _start_chat_persist_job(self):
"""Start processing the chat persist job.
Mark the ChatPersistJob record in the db as started by
updating the 'owner' field and the 'start' field.
"""
self.log.info("Starting chat persist job with job_id=%d ..." %
self.job_id)
try:
db_session = self.create_db_session()
# This query.update generates the following sql:
# UPDATE chat_persist_job SET owner='persistsvc' WHERE
# chat_persist_job.id = 1 AND chat_persist_job.owner IS NULL
num_rows_updated = db_session.query(ChatPersistJob).\
filter(ChatPersistJob.id==self.job_id).\
filter(ChatPersistJob.owner==None).\
update({
ChatPersistJob.owner: 'persistsvc',
ChatPersistJob.start: tz.utcnow()
})
if not num_rows_updated:
raise DuplicatePersistJobException()
db_session.commit()
except DuplicatePersistJobException as e:
# No data was modified in the db so no need to rollback.
raise e
except Exception as e:
self.log.exception(e)
db_session.rollback()
raise e
finally:
if db_session:
db_session.close()
def _start_chat_persist_job(self):
"""Start processing the chat persist job.
Mark the ChatPersistJob record in the db as started by
updating the 'owner' field and the 'start' field.
"""
self.log.info("Starting chat persist job with job_id=%d ..." % self.job_id)
try:
db_session = self.create_db_session()
# This query.update generates the following sql:
# UPDATE chat_persist_job SET owner='persistsvc' WHERE
# chat_persist_job.id = 1 AND chat_persist_job.owner IS NULL
num_rows_updated = db_session.query(ChatPersistJob).\
filter(ChatPersistJob.id==self.job_id).\
filter(ChatPersistJob.owner==None).\
update({
ChatPersistJob.owner: 'persistsvc',
ChatPersistJob.start: tz.utcnow()
})
if not num_rows_updated:
raise DuplicatePersistJobException()
db_session.commit()
except DuplicatePersistJobException as e:
# No data was modified in the db so no need to rollback.
raise e
except Exception as e:
self.log.exception(e)
db_session.rollback()
raise e
finally:
if db_session:
db_session.close()
def _retry_job(self, job):
"""Create a ChatArchiveJob to retry a failed job.
This method will create a new ChatArchiveJob, which
will be delayed by job_retry_seconds, as long as
the number of retries_remaining on the failed
job is greather than 1.
"""
try:
db_session = self.db_session_factory()
if job.retries_remaining:
not_before =tz.utcnow() + \
datetime.timedelta(seconds=self.job_retry_seconds)
self.log.info("Creating retry job for chat_id=%s at %s" \
% (job.chat_id, not_before))
retry = ChatArchiveJob(
chat_id=job.chat_id,
created=func.current_timestamp(),
not_before=not_before,
data=job.data,
retries_remaining=job.retries_remaining-1)
db_session.add(retry)
db_session.commit()
else:
self.log.info("No retries remaining for job for chat_id=%s" \
% (job.chat_id))
self.log.error("Job for chat_id=%s failed!" \
% (job.chat_id))
except Exception as error:
self.log.exception(error)
db_session.rollback()
finally:
if db_session:
db_session.close()
def _retry_job(self, failed_job):
"""Create a new NotificationJob from a failed job.
This method creates a new Notification Job from a
job that failed to process successfully.
"""
try:
db_session = None
#create new job in db to retry.
if failed_job.retries_remaining > 0:
not_before = tz.utcnow() + datetime.timedelta(seconds=self.job_retry_seconds)
new_job = NotificationJob(
created=func.current_timestamp(),
not_before=not_before,
notification_id=failed_job.notification_id,
recipient_id=failed_job.recipient_id,
priority=failed_job.priority,
retries_remaining=failed_job.retries_remaining-1
)
# Add job to db
db_session = self.db_session_factory()
db_session.add(new_job)
db_session.commit()
else:
self.log.info("No retries remaining for job for notification_job_id=%s"\
% (failed_job.id))
self.log.error("Job for notification_job_id=%s failed!"\
% (failed_job.id))
except Exception as e:
self.log.exception(e)
if db_session:
db_session.rollback()
finally:
if db_session:
db_session.close()
def _retry_job(self, job):
"""Create a ChatArchiveJob to retry a failed job.
This method will create a new ChatArchiveJob, which
will be delayed by job_retry_seconds, as long as
the number of retries_remaining on the failed
job is greather than 1.
"""
try:
db_session = self.db_session_factory()
if job.retries_remaining:
not_before =tz.utcnow() + \
datetime.timedelta(seconds=self.job_retry_seconds)
self.log.info("Creating retry job for chat_id=%s at %s" \
% (job.chat_id, not_before))
retry = ChatArchiveJob(
chat_id=job.chat_id,
created=func.current_timestamp(),
not_before=not_before,
data=job.data,
retries_remaining=job.retries_remaining - 1)
db_session.add(retry)
db_session.commit()
else:
self.log.info("No retries remaining for job for chat_id=%s" \
% (job.chat_id))
self.log.error("Job for chat_id=%s failed!" \
% (job.chat_id))
except Exception as error:
self.log.exception(error)
db_session.rollback()
finally:
if db_session:
db_session.close()
def build_all_query(self, **kwargs):
kwargs["end__lt"] = tz.utcnow()
if "order_by" not in kwargs:
kwargs["order_by"] = "id__desc"
return super(ChatManager, self).build_all_query(**kwargs)
def build_all_query(self, **kwargs):
kwargs["end__lt"] = tz.utcnow()
if "order_by" not in kwargs:
kwargs["order_by"] = "id__desc"
return super(ChatManager, self).build_all_query(**kwargs)
def setUp(self):
print '################################################'
print 'Invoking setUp'
print '################################################'
# Need to create the data objs that persist job requires
# 1) Need to write ChatPersistJobs
# 2) Need to write ChatMessages
# Generate chat persist job data
self.test_user_id = 1
# We need to match the topic structure of the test chat data
# that will be generated:
# Root Topic
# --- Subtopic1
self.root_topic = Topic(
parent_id=None,
rank=0,
title="PersisterTestChat",
description="Chat topic used to test the persist service",
duration=60, # secs
public=True,
active=True,
recommended_participants=1,
user_id=self.test_user_id,
type_id=1)
self.topic1 = Topic(
rank=1,
title="subtopic1",
description="subtopic1 description",
duration=60, #secs
public=True,
active=True,
recommended_participants=1,
user_id=self.test_user_id,
type_id=1)
self.root_topic.children.append(self.topic1)
self.chat = Chat(type_id=1,
topic=self.root_topic,
start=tz.utcnow(),
end=tz.utcnow() + datetime.timedelta(minutes=5))
# We can have as many chat_sessions and jobs as needed
# For now, doing 1 chat session and 1 persist job.
self.chat_session_token = 'persister_test_dummy_token'
self.chat_session = ChatSession(
chat=self.chat,
token=self.chat_session_token,
participants=1,
)
self.chat_user = ChatUser(chat_session=self.chat_session,
user_id=self.test_user_id,
participant=1)
self.chat_persist_job = ChatPersistJob(chat_session=self.chat_session,
created=tz.utcnow())
try:
# Write data to db so we can test the persist svc
# Write ChatPersistJob data to the db
db_session = self.service.handler.get_database_session()
db_session.add(self.chat_user)
# Before writing chat session to db, we need
# to add a reference to the test user. This field
# is required for the svc to create ChatHighlightSessions.
user = db_session.query(User).\
filter_by(id=self.test_user_id).\
one()
self.chat_session.users.append(user)
db_session.add(self.chat_session)
db_session.add(self.chat)
db_session.add(self.root_topic)
db_session.add(self.chat_persist_job)
db_session.commit()
# Generate chat message data
# Note the commit() above is required since we need
# real IDs to build and persist ChatMessages.
builder = ChatTestDataBuilder()
chat_data = builder.build(
root_topic_id=self.root_topic.id,
topic1_id=self.topic1.id,
chat_session_id=self.chat_session.id,
chat_session_token=self.chat_session_token,
user_id=self.test_user_id)
# Write ChatMessages to the db for consumption by the persist svc
self.chat_messages = chat_data.serialized_message_list
for chat_message in self.chat_messages:
db_session.add(chat_message)
db_session.commit()
except Exception as e:
logging.exception(e)
if db_session:
db_session.rollback()
finally:
if db_session:
db_session.close()
def setUpClass(cls):
print '################################################'
print 'Invoking setUpClass'
print '################################################'
DistributedTestCase.setUpClass()
cls.test_user_id=1
# Need to create the data objs that persist job requires
# 1) Need to write ChatPersistJobs
# 2) Need to write ChatMessages
# Generate chat persist job data
#
# Match the topic structure of the test chat data
# that was used above to generate chat messages.
cls.root_topic = Topic(
parent_id=None,
rank=0,
title="DistributedTestChat",
description="Chat topic used to test the persist service",
duration=30,
public=True,
active=True,
recommended_participants=1,
user_id=cls.test_user_id,
type_id=1
)
cls.topic1 = Topic(
rank=1,
title="t1",
description="t1 description",
duration=10,
public=True,
active=True,
recommended_participants=1,
user_id=cls.test_user_id,
type_id=1
)
cls.root_topic.children.append(cls.topic1)
cls.chat = Chat(
type_id=1,
topic=cls.root_topic,
start=tz.utcnow(),
end=tz.utcnow()+datetime.timedelta(minutes=5))
# Can have as many chat_sessions and jobs as needed
# Just doing 1 session and 1 persist job for now.
cls.chat_session_token = 'test1_dummy_token'
cls.chat_session = ChatSession(
chat=cls.chat,
token=cls.chat_session_token,
participants=1)
cls.chat_user = ChatUser(
chat_session=cls.chat_session,
user_id=cls.test_user_id,
participant=1
)
cls.chat_persist_job = ChatPersistJob(
chat_session=cls.chat_session,
created=tz.utcnow()
)
try:
# Write ChatPersistJob data to the db
db_session = cls.service.handler.get_database_session()
db_session.add(cls.chat_user)
# Before writing chat session to db, we need
# to add a reference to the test user. This field
# is required for the svc to create ChatHighlightSessions.
user = db_session.query(User).\
filter_by(id=cls.test_user_id).\
one()
cls.chat_session.users.append(user)
db_session.add(cls.chat_session)
db_session.add(cls.chat)
db_session.add(cls.root_topic)
db_session.add(cls.chat_persist_job)
db_session.commit()
# Generate chat message data
# Note the commit() above is required since we need
# real IDs to build and persist ChatMessages.
builder = ChatTestDataBuilder()
cls.chat_data = builder.build(
root_topic_id=cls.root_topic.id,
topic1_id=cls.topic1.id,
chat_session_id=cls.chat_session.id,
chat_session_token=cls.chat_session_token,
user_id=cls.test_user_id
)
# Write ChatMessages to the db for consumption by the persist svc
cls.chat_messages = cls.chat_data.serialized_message_list
for chat_message in cls.chat_messages:
db_session.add(chat_message)
db_session.commit()
except Exception as e:
logging.exception(e)
if db_session:
db_session.rollback()
finally:
db_session.close()
def setUp(self):
print '################################################'
print 'Invoking setUp'
print '################################################'
# Need to create the data objs that persist job requires
# 1) Need to write ChatPersistJobs
# 2) Need to write ChatMessages
# Generate chat persist job data
self.test_user_id = 1
# We need to match the topic structure of the test chat data
# that will be generated:
# Root Topic
# --- Subtopic1
self.root_topic = Topic(
parent_id=None,
rank=0,
title="PersisterTestChat",
description="Chat topic used to test the persist service",
duration=60, # secs
public=True,
active=True,
recommended_participants=1,
user_id=self.test_user_id,
type_id=1
)
self.topic1 = Topic(
rank=1,
title="subtopic1",
description="subtopic1 description",
duration=60, #secs
public=True,
active=True,
recommended_participants=1,
user_id=self.test_user_id,
type_id=1
)
self.root_topic.children.append(self.topic1)
self.chat = Chat(
type_id=1,
topic=self.root_topic,
start=tz.utcnow(),
end=tz.utcnow()+datetime.timedelta(minutes=5))
# We can have as many chat_sessions and jobs as needed
# For now, doing 1 chat session and 1 persist job.
self.chat_session_token = 'persister_test_dummy_token'
self.chat_session = ChatSession(
chat=self.chat,
token=self.chat_session_token,
participants=1,
)
self.chat_user = ChatUser(
chat_session=self.chat_session,
user_id=self.test_user_id,
participant=1
)
self.chat_persist_job = ChatPersistJob(
chat_session=self.chat_session,
created=tz.utcnow()
)
try:
# Write data to db so we can test the persist svc
# Write ChatPersistJob data to the db
db_session = self.service.handler.get_database_session()
db_session.add(self.chat_user)
# Before writing chat session to db, we need
# to add a reference to the test user. This field
# is required for the svc to create ChatHighlightSessions.
user = db_session.query(User).\
filter_by(id=self.test_user_id).\
one()
self.chat_session.users.append(user)
db_session.add(self.chat_session)
db_session.add(self.chat)
db_session.add(self.root_topic)
db_session.add(self.chat_persist_job)
db_session.commit()
# Generate chat message data
# Note the commit() above is required since we need
# real IDs to build and persist ChatMessages.
builder = ChatTestDataBuilder()
chat_data = builder.build(
root_topic_id=self.root_topic.id,
topic1_id=self.topic1.id,
chat_session_id=self.chat_session.id,
chat_session_token=self.chat_session_token,
user_id=self.test_user_id
)
# Write ChatMessages to the db for consumption by the persist svc
self.chat_messages = chat_data.serialized_message_list
for chat_message in self.chat_messages:
db_session.add(chat_message)
db_session.commit()
except Exception as e:
logging.exception(e)
if db_session:
db_session.rollback()
finally:
if db_session:
db_session.close()