def test_batch_insert_if_not_exists(self):
""" tests that batch insertion with if_not_exists work as expected """
id = uuid4()
with BatchQuery() as b:
TestIfNotExistsModel.batch(b).if_not_exists().create(
id=id, count=8, text='123456789')
b = BatchQuery()
TestIfNotExistsModel.batch(b).if_not_exists().create(
id=id, count=9, text='111111111111')
with self.assertRaises(LWTException) as assertion:
b.execute()
self.assertEqual(assertion.exception.existing, {
'count': 8,
'id': id,
'text': '123456789',
'[applied]': False,
})
q = TestIfNotExistsModel.objects(id=id)
self.assertEqual(len(q), 1)
tm = q.first()
self.assertEqual(tm.count, 8)
self.assertEqual(tm.text, '123456789')
def comment_on_video(self, video_id, user_id, comment_id, comment):
#Checking values have been provided
if not video_id:
raise ValueError('video_id should be provided to submit a comment')
elif not user_id:
raise ValueError('user_id should be provided to submit a comment')
elif not comment_id:
raise ValueError(
'comment_id should be provided to submit a comment')
elif not comment:
raise ValueError('comment should be provided to submit a comment')
#Needs to insert into both tables to be successful
batch_query = BatchQuery()
CommentsByVideoModel.batch(batch_query).create(video_id=video_id,
comment_id=comment_id,
user_id=user_id,
comment=comment)
CommentsByUserModel.batch(batch_query).create(user_id=user_id,
comment_id=comment_id,
video_id=video_id,
comment=comment)
batch_query.execute()
#Publish UserCommentedOnVideo event
self.user_comments_publisher.publish_user_comment_added_event(
video_id=video_id, user_id=user_id, comment_id=comment_id)
return
def test_callbacks_tied_to_execute(self):
"""Batch callbacks should NOT fire if batch is not executed in context manager mode"""
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
with BatchQuery() as batch:
batch.add_callback(my_callback)
self.assertEqual(len(call_history), 1)
class SomeError(Exception):
pass
with self.assertRaises(SomeError):
with BatchQuery() as batch:
batch.add_callback(my_callback)
# this error bubbling up through context manager
# should prevent callback runs (along with b.execute())
raise SomeError
# still same call history. Nothing added
self.assertEqual(len(call_history), 1)
# but if execute ran, even with an error bubbling through
# the callbacks also would have fired
with self.assertRaises(SomeError):
with BatchQuery(execute_on_exception=True) as batch:
batch.add_callback(my_callback)
raise SomeError
# updated call history
self.assertEqual(len(call_history), 2)
def test_batch_update_if_exists_success(self):
"""
Tests that batch update with if_exists work as expected
@since 3.1
@jira_ticket PYTHON-432
@expected_result
@test_category object_mapper
"""
id = uuid4()
m = TestIfExistsModel.create(id=id, count=8, text='123456789')
with BatchQuery() as b:
m.text = '111111111'
m.batch(b).if_exists().update()
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist
m.batch(b).if_exists().update()
self.assertEqual(assertion.exception.existing, {
'[applied]': False,
})
q = TestIfExistsModel.objects(id=id)
self.assertEqual(len(q), 1)
tm = q.first()
self.assertEqual(tm.count, 8)
self.assertEqual(tm.text, '111111111')
def test_basic_batch_query(self):
"""
Test Batch queries with connections explicitly set
@since 3.7
@jira_ticket PYTHON-613
@expected_result queries should execute appropriately
@test_category object_mapper
"""
# No connection with a QuerySet (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Explicit connection with a QuerySet
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Get an object from the BD
with ContextQuery(TestModel, connection='cluster') as tm:
obj = tm.objects.get(partition=1, cluster=1)
obj.__connection__ = None
# No connection with a model (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
obj.count = 2
obj.batch(b).save()
# Explicit connection with a model
with BatchQuery(connection='cluster') as b:
obj.count = 2
obj.batch(b).save()
def test_batch_delete_if_exists_success(self):
"""
Tests that batch deletes with if_exists work, and throw proper LWTException when they are are not applied
@since 3.1
@jira_ticket PYTHON-432
@expected_result Deletes will be preformed if they exist, otherwise throw LWTException
@test_category object_mapper
"""
id = uuid4()
m = TestIfExistsModel.create(id=id, count=8, text='123456789')
with BatchQuery() as b:
m.batch(b).if_exists().delete()
q = TestIfExistsModel.objects(id=id)
self.assertEqual(len(q), 0)
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist
m.batch(b).if_exists().delete()
self.assertEqual(assertion.exception.existing, {
'[applied]': False,
})
def test_batch_query_connection_override(self):
"""
Test that we cannot override a BatchQuery connection per model
@since 3.7
@jira_ticket PYTHON-613
@expected_result Proper exceptions should be raised
@test_category object_mapper
"""
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
TestModel.batch(b).using(connection='test').save()
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
TestModel.using(connection='test').batch(b).save()
with ContextQuery(TestModel, AnotherTestModel,
connection='cluster') as (tm, atm):
obj1 = tm.objects.get(partition=1, cluster=1)
obj1.__connection__ = None
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
obj1.using(connection='test').batch(b).save()
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
obj1.batch(b).using(connection='test').save()
def test_insert_success_case(self):
b = BatchQuery()
TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4')
with self.assertRaises(TestMultiKeyModel.DoesNotExist):
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
b.execute()
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def test_batch_consistency(self):
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery(consistency=CL.ALL) as b:
TestConsistencyModel.batch(b).create(text="monkey")
args = m.call_args
self.assertEqual(CL.ALL, args[0][0].consistency_level)
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery() as b:
TestConsistencyModel.batch(b).create(text="monkey")
args = m.call_args
self.assertNotEqual(CL.ALL, args[0][0].consistency_level)
def test_batch_if_not_exists(self):
""" ensure 'IF NOT EXISTS' exists in statement when in batch """
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery() as b:
TestIfNotExistsModel.batch(b).if_not_exists().create(count=8)
self.assertIn("IF NOT EXISTS", m.call_args[0][0].query_string)
def test_update_success_case(self):
inst = TestMultiKeyModel.create(partition=self.pkey, cluster=2, count=3, text='4')
b = BatchQuery()
inst.count = 4
inst.batch(b).save()
inst2 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
self.assertEqual(inst2.count, 3)
b.execute()
inst3 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
self.assertEqual(inst3.count, 4)
def test_instance_update_in_batch(self):
with mock.patch.object(self.session, "execute") as m:
with BatchQuery() as b:
self.instance.batch(b).timestamp(
timedelta(seconds=30)).update(count=2)
query = m.call_args[0][0].query_string
"USING TIMESTAMP".should.be.within(query)
def test_dml_none_success_case(self):
""" Tests that passing None into the batch call clears any batch object """
b = BatchQuery()
q = DMLQuery(TestMultiKeyModel, batch=b)
self.assertEqual(q._batch, b)
q.batch(None)
self.assertIsNone(q._batch)
def test_batch_query_different_connection(self):
"""
Test BatchQuery with Models that have a different connection
@since 3.7
@jira_ticket PYTHON-613
@expected_result queries should execute appropriately
@test_category object_mapper
"""
# Testing on a model class
TestModel.__connection__ = 'cluster'
AnotherTestModel.__connection__ = 'cluster2'
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1,
cluster=1)
TestModel.__connection__ = None
AnotherTestModel.__connection__ = None
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1, cluster=1)
# Testing on a model instance
with ContextQuery(TestModel, AnotherTestModel,
connection='cluster') as (tm, atm):
obj1 = tm.objects.get(partition=1, cluster=1)
obj2 = atm.objects.get(partition=1, cluster=1)
obj1.__connection__ = 'cluster'
obj2.__connection__ = 'cluster2'
obj1.count = 4
obj2.count = 4
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
obj1.batch(b).save()
obj2.batch(b).save()
def test_batch(self):
with mock.patch.object(self.session, "execute") as m:
with BatchQuery() as b:
TestTimestampModel.timestamp(
timedelta(seconds=10)).batch(b).create(count=1)
query = m.call_args[0][0].query_string
query.should.match(r"INSERT.*USING TIMESTAMP")
query.should_not.match(r"TIMESTAMP.*INSERT")
def test_context_manager(self):
with BatchQuery() as b:
for i in range(5):
TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=i, count=3, text='4')
for i in range(5):
with self.assertRaises(TestMultiKeyModel.DoesNotExist):
TestMultiKeyModel.get(partition=self.pkey, cluster=i)
for i in range(5):
TestMultiKeyModel.get(partition=self.pkey, cluster=i)
def test_batch_update_conditional(self):
t = TestConditionalModel.create(text='something', count=5)
id = t.id
with BatchQuery() as b:
t.batch(b).iff(count=5).update(text='something else')
updated = TestConditionalModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
b = BatchQuery()
updated.batch(b).iff(count=6).update(text='and another thing')
with self.assertRaises(LWTException) as assertion:
b.execute()
self.assertEqual(assertion.exception.existing, {
'id': id,
'count': 5,
'[applied]': False,
})
updated = TestConditionalModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
def test_bulk_delete_success_case(self):
for i in range(1):
for j in range(5):
TestMultiKeyModel.create(partition=i, cluster=j, count=i*j, text='{0}:{1}'.format(i,j))
with BatchQuery() as b:
TestMultiKeyModel.objects.batch(b).filter(partition=0).delete()
self.assertEqual(TestMultiKeyModel.filter(partition=0).count(), 5)
self.assertEqual(TestMultiKeyModel.filter(partition=0).count(), 0)
#cleanup
for m in TestMultiKeyModel.all():
m.delete()
def test_callbacks_properly_execute_callables_and_tuples(self):
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
# adding on init:
batch = BatchQuery()
batch.add_callback(my_callback)
batch.add_callback(my_callback, 'more', 'args')
batch.execute()
self.assertEqual(len(call_history), 2)
self.assertEqual([(), ('more', 'args')], call_history)
def test_API_managing_callbacks(self):
# Callbacks can be added at init and after
def my_callback(*args, **kwargs):
pass
# adding on init:
batch = BatchQuery()
batch.add_callback(my_callback)
batch.add_callback(my_callback, 2, named_arg='value')
batch.add_callback(my_callback, 1, 3)
self.assertEqual(batch._callbacks, [(my_callback, (), {}),
(my_callback, (2, ), {
'named_arg': 'value'
}), (my_callback, (1, 3), {})])
def test_batch_execute_on_exception_succeeds(self):
# makes sure if execute_on_exception == True we still apply the batch
drop_table(BatchQueryLogModel)
sync_table(BatchQueryLogModel)
obj = BatchQueryLogModel.objects(k=1)
self.assertEqual(0, len(obj))
try:
with BatchQuery(execute_on_exception=True) as b:
BatchQueryLogModel.batch(b).create(k=1, v=1)
raise Exception("Blah")
except:
pass
obj = BatchQueryLogModel.objects(k=1)
# should be 1 because the batch should execute
self.assertEqual(1, len(obj))
def test_batch_execute_on_exception_skips_if_not_specified(self):
# makes sure if execute_on_exception == True we still apply the batch
drop_table(BatchQueryLogModel)
sync_table(BatchQueryLogModel)
obj = BatchQueryLogModel.objects(k=2)
self.assertEqual(0, len(obj))
try:
with BatchQuery() as b:
BatchQueryLogModel.batch(b).create(k=2, v=2)
raise Exception("Blah")
except:
pass
obj = BatchQueryLogModel.objects(k=2)
# should be 0 because the batch should not execute
self.assertEqual(0, len(obj))
def test_batch_mixed_update_if_exists_success(self):
"""
Tests that batch update with with one bad query will still fail with LWTException
@since 3.1
@jira_ticket PYTHON-432
@expected_result
@test_category object_mapper
"""
m = TestIfExistsModel2.create(id=1, count=8, text='123456789')
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m.text = '111111112'
m.batch(b).if_exists().update() # Does exist
n = TestIfExistsModel2(id=1, count=10,
text="Failure") # Doesn't exist
n.batch(b).if_exists().update()
self.assertEqual(assertion.exception.existing.get('[applied]'), False)
def test_callbacks_work_multiple_times(self):
"""
Tests that multiple executions of execute on a batch statement
logs a warning, and that we don't encounter an attribute error.
@since 3.1
@jira_ticket PYTHON-445
@expected_result warning message is logged
@test_category object_mapper
"""
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
with warnings.catch_warnings(record=True) as w:
with BatchQuery() as batch:
batch.add_callback(my_callback)
batch.execute()
batch.execute()
self.assertEqual(len(w), 2) # package filter setup to warn always
self.assertRegexpMatches(str(w[0].message), r"^Batch.*multiple.*")
def test_batch_delete_mixed(self):
"""
Tests that batch deletes with multiple queries and throw proper LWTException when they are are not all applicable
@since 3.1
@jira_ticket PYTHON-432
@expected_result If one delete clause doesn't exist all should fail.
@test_category object_mapper
"""
m = TestIfExistsModel2.create(id=3, count=8, text='123456789')
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m.batch(b).if_exists().delete() # Does exist
n = TestIfExistsModel2(id=3, count=42,
text='1111111') # Doesn't exist
n.batch(b).if_exists().delete()
self.assertEqual(assertion.exception.existing.get('[applied]'), False)
q = TestIfExistsModel2.objects(id=3, count=8)
self.assertEqual(len(q), 1)
def test_disable_multiple_callback_warning(self):
"""
Tests that multiple executions of a batch statement
don't log a warning when warn_multiple_exec flag is set, and
that we don't encounter an attribute error.
@since 3.1
@jira_ticket PYTHON-445
@expected_result warning message is logged
@test_category object_mapper
"""
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
with patch('dse.cqlengine.query.BatchQuery.warn_multiple_exec', False):
with warnings.catch_warnings(record=True) as w:
with BatchQuery() as batch:
batch.add_callback(my_callback)
batch.execute()
batch.execute()
self.assertFalse(w)
def test_batch_execute_no_timeout(self):
with mock.patch.object(Session, 'execute') as mock_execute:
with BatchQuery() as b:
BatchQueryLogModel.batch(b).create(k=2, v=2)
self.assertEqual(mock_execute.call_args[-1]['timeout'], NOT_SET)
def submit_youtube_video(self, video_id, user_id, name, description, tags,
you_tube_video_id):
# validate inputs
if not video_id:
raise ValueError(
'video_id should be provided for submit youtube video request')
elif not user_id:
raise ValueError(
'user_id should be provided for submit youtube video request')
elif not name:
raise ValueError(
'video name should be provided for submit youtube video request'
)
elif not description:
raise ValueError(
'video description should be provided for submit youtube video request'
)
elif not you_tube_video_id:
raise ValueError(
'video YouTube id should be provided for submit youtube video request'
)
# Formulate the preview location
preview_image_location = '//img.youtube.com/vi/' + you_tube_video_id + '/hqdefault.jpg'
# formulate the time-based values
now = datetime.utcnow()
yyyymmdd = now.strftime('%Y%m%d')
# create and execute batch statement to insert into multiple tables
batch_query = BatchQuery(timestamp=now)
VideosModel.batch(batch_query).create(
video_id=video_id,
user_id=user_id,
name=name,
description=description,
location=you_tube_video_id,
location_type=YOUTUBE,
preview_image_location=preview_image_location,
tags=tags,
added_date=now)
LatestVideosModel.batch(batch_query).create(
yyyymmdd=yyyymmdd,
added_date=now,
video_id=video_id,
user_id=user_id,
name=name,
preview_image_location=preview_image_location)
UserVideosModel.batch(batch_query).create(
user_id=user_id,
added_date=now,
video_id=video_id,
name=name,
preview_image_location=preview_image_location)
batch_query.execute()
# publish YouTubeVideoAdded event
self.video_catalog_publisher.publish_youtube_video_added_event(
video_id=video_id,
user_id=user_id,
name=name,
description=description,
tags=tags,
location=you_tube_video_id,
preview_image_location=preview_image_location,
added_date=now,
timestamp=now)
def test_batch_is_included(self):
with mock.patch.object(self.session, "execute") as m:
with BatchQuery(timestamp=timedelta(seconds=30)) as b:
TestTimestampModel.batch(b).create(count=1)
"USING TIMESTAMP".should.be.within(m.call_args[0][0].query_string)
def test_empty_batch(self):
b = BatchQuery()
b.execute()
with BatchQuery() as b:
pass
