def store_cancer_data(self, cancer_data):
data_group_info = cancer_data.data_group_info
country = data_group_info.country
age = data_group_info.age_gender.age
gender = data_group_info.age_gender.gender
bq = BatchQuery(consistency=ConsistencyLevel.ONE)
for cancer_data_record in cancer_data.cancer_data:
cancer = cancer_data_record.cancer
deaths = cancer_data_record.deaths
crude_rate = cancer_data_record.crude_rate
asr = cancer_data_record.crude_rate
cumulative_risk = cancer_data_record.cumulative_risk
CancerDataEntity.batch(bq).create(
age=age,
gender=gender,
country=country,
cancer=cancer,
asr=asr,
crude_rate=crude_rate,
cumulative_risk=cumulative_risk,
deaths=deaths,
)
bq.execute()
def test_batch_query_different_connection(self):
"""Test BatchQuery with Models that have a different connection"""
# Testing on a model class
TestModel.__connection__ = 'cluster'
AnotherTestModel.__connection__ = 'cluster2'
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1, cluster=1)
TestModel.__connection__ = None
AnotherTestModel.__connection__ = None
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1, cluster=1)
# Testing on a model instance
with ContextQuery(TestModel, AnotherTestModel, connection='cluster') as (tm, atm):
obj1 = tm.objects.get(partition=1, cluster=1)
obj2 = atm.objects.get(partition=1, cluster=1)
obj1.__connection__ = 'cluster'
obj2.__connection__ = 'cluster2'
obj1.count = 4
obj2.count = 4
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
obj1.batch(b).save()
obj2.batch(b).save()
def import_csv(csv_path, batch=False, chunksize=500, skiprows=None):
start_time = dt.datetime.today().timestamp()
for df in pd.read_csv(csv_path,
delimiter='\t',
encoding='utf-8',
dtype=dtype,
converters=converters,
chunksize=chunksize,
skiprows=skiprows,
error_bad_lines=False,
warn_bad_lines=True):
print("transform")
df = df.where((pd.notnull(df)), None)
print("importing rows {} to {}".format(df.index.min(), df.index.max()))
if batch:
b = BatchQuery()
for i, row in df.iterrows():
try:
print(u"DOING {} ; {}".format(i, row['code'].encode('utf-8')))
if row['code'] is None or len(row['code'].strip()) == 0:
print("error with line {0} : code = '{1}'".format(
i, row['code'].encode('utf-8')))
continue
row_converted = {
Product._get_column_by_db_name(cql_name).column_name: value
for cql_name, value in row.items()
if not cql_name.endswith('_datetime') and value is not None
and (type(value) != str or len(value) > 0)
}
product = Product.create(**row_converted)
if not batch:
product.save()
print(u"DONE {} ; {}".format(i, row['code'].encode('utf-8')))
except Exception:
print(u"EXCEPTION {} ; {}".format(i,
row['code'].encode('utf-8')))
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=10,
file=sys.stderr)
if batch:
print("executing batch for rows {} to {}".format(
df.index.min(), df.index.max()))
b.execute()
time_diff = dt.datetime.today().timestamp() - start_time
print("TIMING {} rows/s".format(chunksize / time_diff))
start_time = dt.datetime.today().timestamp()
def test_batch_update_if_exists_success(self):
"""
Tests that batch update with if_exists work as expected
@since 3.1
@jira_ticket PYTHON-432
@expected_result
@test_category object_mapper
"""
id = uuid4()
m = TestIfExistsModel.create(id=id, count=8, text='123456789')
with BatchQuery() as b:
m.text = '111111111'
m.batch(b).if_exists().update()
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist
m.batch(b).if_exists().update()
self.assertEqual(assertion.exception.existing, {
'[applied]': False,
})
q = TestIfExistsModel.objects(id=id)
self.assertEqual(len(q), 1)
tm = q.first()
self.assertEqual(tm.count, 8)
self.assertEqual(tm.text, '111111111')
def test_batch_delete_if_exists_success(self):
"""
Tests that batch deletes with if_exists work, and throw proper LWTException when they are are not applied
@since 3.1
@jira_ticket PYTHON-432
@expected_result Deletes will be preformed if they exist, otherwise throw LWTException
@test_category object_mapper
"""
id = uuid4()
m = TestIfExistsModel.create(id=id, count=8, text='123456789')
with BatchQuery() as b:
m.batch(b).if_exists().delete()
q = TestIfExistsModel.objects(id=id)
self.assertEqual(len(q), 0)
with self.assertRaises(LWTException) as assertion:
with BatchQuery() as b:
m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist
m.batch(b).if_exists().delete()
self.assertEqual(assertion.exception.existing, {
'[applied]': False,
})
def test_batch_insert_if_not_exists_success(self):
""" tests that batch insertion with if_not_exists work as expected """
id = uuid4()
with BatchQuery() as b:
TestIfNotExistsModel.batch(b).if_not_exists().create(id=id, count=8, text='123456789')
b = BatchQuery()
TestIfNotExistsModel.batch(b).if_not_exists().create(id=id, count=9, text='111111111111')
with self.assertRaises(LWTException) as assertion:
b.execute()
self.assertEqual(assertion.exception.existing, {
'count': 8,
'id': id,
'text': '123456789',
'[applied]': False,
})
q = TestIfNotExistsModel.objects(id=id)
self.assertEqual(len(q), 1)
tm = q.first()
self.assertEqual(tm.count, 8)
self.assertEqual(tm.text, '123456789')
def test_cqlengine_batch_type(self):
"""
Tests the different types of `class: cassandra.cqlengine.query.BatchType`
@since 3.13
@jira_ticket PYTHON-88
@expected_result batch query succeeds and the results
are correctly readen
@test_category query
"""
with BatchQuery(batch_type=cqlengine_BatchType.Unlogged) as b:
TestMultiKeyModel.batch(b).create(partition=1, cluster=1)
TestMultiKeyModel.batch(b).create(partition=1, cluster=2)
obj = TestMultiKeyModel.objects(partition=1)
self.assertEqual(2, len(obj))
with BatchQuery(batch_type=cqlengine_BatchType.Counter) as b:
CounterBatchQueryModel.batch(b).create(k=1, v=1)
CounterBatchQueryModel.batch(b).create(k=1, v=2)
CounterBatchQueryModel.batch(b).create(k=1, v=10)
obj = CounterBatchQueryModel.objects(k=1)
self.assertEqual(1, len(obj))
self.assertEqual(obj[0].v, 13)
def test_basic_batch_query(self):
"""
Test Batch queries with connections explicitly set
@since 3.7
@jira_ticket PYTHON-613
@expected_result queries should execute appropriately
@test_category object_mapper
"""
# No connection with a QuerySet (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Explicit connection with a QuerySet
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Get an object from the BD
with ContextQuery(TestModel, connection='cluster') as tm:
obj = tm.objects.get(partition=1, cluster=1)
obj.__connection__ = None
# No connection with a model (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
obj.count = 2
obj.batch(b).save()
# Explicit connection with a model
with BatchQuery(connection='cluster') as b:
obj.count = 2
obj.batch(b).save()
def test_batch_query_connection_override(self):
"""
Test that we cannot override a BatchQuery connection per model
@since 3.7
@jira_ticket PYTHON-613
@expected_result Proper exceptions should be raised
@test_category object_mapper
"""
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
TestModel.batch(b).using(connection='test').save()
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
TestModel.using(connection='test').batch(b).save()
with ContextQuery(TestModel, AnotherTestModel,
connection='cluster') as (tm, atm):
obj1 = tm.objects.get(partition=1, cluster=1)
obj1.__connection__ = None
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
obj1.using(connection='test').batch(b).save()
with self.assertRaises(CQLEngineException):
with BatchQuery(connection='cluster') as b:
obj1.batch(b).using(connection='test').save()
def test_batch_insert_if_not_exists(self):
""" tests that batch insertion with if_not_exists work as expected """
id = uuid4()
with BatchQuery() as b:
TestIfNotExistsModel.batch(b).if_not_exists().create(
id=id, count=8, text='123456789')
b = BatchQuery()
TestIfNotExistsModel.batch(b).if_not_exists().create(
id=id, count=9, text='111111111111')
with self.assertRaises(LWTException) as assertion:
b.execute()
self.assertEqual(assertion.exception.existing, {
'count': 8,
'id': id,
'text': '123456789',
'[applied]': False,
})
q = TestIfNotExistsModel.objects(id=id)
self.assertEqual(len(q), 1)
tm = q.first()
self.assertEqual(tm.count, 8)
self.assertEqual(tm.text, '123456789')
def test_basic_batch_query(self):
"""Test BatchQuery requests"""
# No connection with a QuerySet (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Explicit connection with a QuerySet
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
# Get an object from the BD
with ContextQuery(TestModel, connection='cluster') as tm:
obj = tm.objects.get(partition=1, cluster=1)
obj.__connection__ = None
# No connection with a model (default is a fake one)
with self.assertRaises(NoHostAvailable):
with BatchQuery() as b:
obj.count = 2
obj.batch(b).save()
# Explicit connection with a model
with BatchQuery(connection='cluster') as b:
obj.count = 2
obj.batch(b).save()
def test_callbacks_tied_to_execute(self):
"""Batch callbacks should NOT fire if batch is not executed in context manager mode"""
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
with BatchQuery() as batch:
batch.add_callback(my_callback)
self.assertEqual(len(call_history), 1)
class SomeError(Exception):
pass
with self.assertRaises(SomeError):
with BatchQuery() as batch:
batch.add_callback(my_callback)
# this error bubbling up through context manager
# should prevent callback runs (along with b.execute())
raise SomeError
# still same call history. Nothing added
self.assertEqual(len(call_history), 1)
# but if execute ran, even with an error bubbling through
# the callbacks also would have fired
with self.assertRaises(SomeError):
with BatchQuery(execute_on_exception=True) as batch:
batch.add_callback(my_callback)
raise SomeError
# updated call history
self.assertEqual(len(call_history), 2)
def test_insert_success_case(self):
b = BatchQuery()
inst = TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4')
with self.assertRaises(TestMultiKeyModel.DoesNotExist):
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
b.execute()
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def save_ts(self, ts_list: List[TSData]):
b = BatchQuery()
for ts_data in ts_list:
func = TSFunctionRegistry.find_function(ts_data.ts_type_name)
value_serialized: str = func.serialize(ts_data.values)
TimeSeriesDataModel.batch(b).create(
type=ts_data.ts_type_name,
code=ts_data.code,
visible_time=ts_data.visible_time,
data=value_serialized)
b.execute()
def test_insert_success_case(self):
b = BatchQuery()
inst = TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4')
with self.assertRaises(TestMultiKeyModel.DoesNotExist):
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
b.execute()
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def truncate_models(self):
log.info('Purging data from all tables...')
for model in self.get_cassandra_models():
b = BatchQuery()
for instance in model.objects.all():
instance.batch(b).delete()
b.execute()
log.info('Done!')
def save(self, ts_list: List[TSData]):
b = BatchQuery()
for ts_data in ts_list:
func = TSTypeRegistry.find_function(ts_data.ts_type_name)
if not isinstance(func, HistoryTimeSeriesType):
raise RuntimeError("非法的tsType")
value_serialized: str = func.serialize(ts_data.values)
TimeSeriesDataModel.batch(b).create(
type=ts_data.ts_type_name,
code=ts_data.code,
visible_time=ts_data.visible_time,
data=value_serialized)
b.execute()
def test_async_batch(self):
b = BatchQuery(execute_async=True)
TestMultiKeyModel.batch(b).create(partition=self.pkey,
cluster=2,
count=3,
text='4')
with self.assertRaises(TestMultiKeyModel.DoesNotExist):
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
b.execute()
TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def import_data():
connection.setup(['127.0.0.1'], "geonames", protocol_version=3)
fieldnames = [col for col in Geoname().__dict__['_values']]
with open("geonames/allCountries.txt", encoding='utf8',
newline='') as csvfile:
# Création du DictReader
csv.register_dialect('geoname', delimiter='\t', quoting=csv.QUOTE_NONE)
reader = csv.DictReader(csvfile,
dialect='geoname',
fieldnames=fieldnames)
# Ingestion par lots
count = 0
batch = BatchQuery()
for row in reader:
new_row = clean_row(row)
Geoname.batch(batch).create(**new_row)
count += 1
if not count % 1000:
batch.execute()
batch = BatchQuery()
logger.info('Importés: {}'.format(count))
batch.execute()
def process_create_entry_work(self, rdict, context, do_load):
b = BatchQuery()
# MOSTLY the resource will not exist. So start by calculating the URL and trying to insert the entire record.
if not do_load:
url = u"file://{}{}/{}".format(decode_str(context['local_ip']),
decode_str(context['path']),
decode_str(context['entry']))
else:
with open(context['fullpath'], 'r') as f:
blob = Blob.create_from_file(f, rdict['size'])
if blob:
url = "cassandra://{}".format(blob.id)
else:
return None
try:
# OK -- try to insert ( create ) the record...
t1 = time.time()
resource = Resource.batch(b).create(url=url, **rdict)
msg = u'Resource {} created --> {}'.format(resource.name,
time.time() - t1)
logger.info(msg)
except ResourceConflictError:
# If the create fails, the record already exists... so retrieve it...
t1 = time.time()
resource = Resource.objects().get(container=context['collection'],
name=rdict['name'])
msg = u"{} ::: Fetch Object -> {}".format(resource.name,
time.time() - t1)
logger.info(msg)
# if the url is not correct then update
# TODO: If the URL is a block set that's stored internally, reduce its count so that it can be garbage collected
# t3 = None
if resource.url != url:
t2 = time.time()
# if url.startswith('cassandra://') : tidy up the stored block count...
resource.batch(b).update(url=url)
t3 = time.time()
msg = u"{} ::: update -> {}".format(resource.name, t3 - t2)
logger.info(msg)
# t1 = time.time()
SearchIndex.reset(resource.id)
SearchIndex.index(resource, ['name', 'metadata'])
# msg = "Index Management -> {}".format(time.time() - t1)
# logger.info(msg)
b.execute()
def test_batch_update_transaction(self):
t = TestTransactionModel.create(text='something', count=5)
id = t.id
with BatchQuery() as b:
t.batch(b).iff(count=5).update(text='something else')
updated = TestTransactionModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
b = BatchQuery()
updated.batch(b).iff(count=6).update(text='and another thing')
self.assertRaises(LWTException, b.execute)
updated = TestTransactionModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
def test_update_success_case(self):
inst = TestMultiKeyModel.create(partition=self.pkey, cluster=2, count=3, text='4')
b = BatchQuery()
inst.count = 4
inst.batch(b).save()
inst2 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
self.assertEqual(inst2.count, 3)
b.execute()
inst3 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
self.assertEqual(inst3.count, 4)
def test_batch_consistency(self):
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery(consistency=CL.ALL) as b:
TestConsistencyModel.batch(b).create(text="monkey")
args = m.call_args
self.assertEqual(CL.ALL, args[0][0].consistency_level)
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery() as b:
TestConsistencyModel.batch(b).create(text="monkey")
args = m.call_args
self.assertNotEqual(CL.ALL, args[0][0].consistency_level)
def main():
connection.default()
# Management functions would normally be used in development, and possibly for deployments.
# They are typically not part of a core application.
log.info("### creating keyspace...")
management.create_keyspace_simple(KEYSPACE, 1)
log.info("### syncing model...")
management.sync_table(FamilyMembers)
# default uuid is assigned
simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m')
# add members to his family later
FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m')
sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f')
nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick')
try:
nick.iff(birth_year=1988).update(birth_year=1989)
except LWTException:
print "precondition not met"
# showing validation
try:
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f')
except ValidationError:
log.exception('INTENTIONAL VALIDATION EXCEPTION; Failed creating instance:')
FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', sex='f')
log.info("### add multiple as part of a batch")
# If creating many at one time, can use a batch to minimize round-trips
hogan_id = uuid4()
with BatchQuery() as b:
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Hulk', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Linda', sex='f')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Nick', sex='m')
FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Brooke', sex='f')
log.info("### All members")
for m in FamilyMembers.all():
print m, m.birth_year, m.sex
log.info("### Select by partition key")
for m in FamilyMembers.objects(id=simmons.id):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname):
print m, m.birth_year, m.sex
log.info("### Constrain on clustering key")
kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie'])
log.info("### Delete a record")
FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete()
for m in FamilyMembers.objects(id=hogan_id):
print m, m.birth_year, m.sex
management.drop_keyspace(KEYSPACE)
def test_update_success_case(self):
inst = TestMultiKeyModel.create(partition=self.pkey, cluster=2, count=3, text='4')
b = BatchQuery()
inst.count = 4
inst.batch(b).save()
inst2 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
assert inst2.count == 3
b.execute()
inst3 = TestMultiKeyModel.get(partition=self.pkey, cluster=2)
assert inst3.count == 4
def registration():
reg_form = RegistrationForm()
if reg_form.validate_on_submit():
# handle regsitration here
try:
user_from_db = UserCredentials.objects.get(username=reg_form.username.data)
flash('Username already taken')
return redirect( url_for('index' ))
except:
pass
# create the new user
with BatchQuery() as b:
new_user = Users.batch(b).create(username=reg_form.username.data,
first_name=reg_form.first_name.data,
last_name=reg_form.last_name.data,
created_date=datetime.datetime.now(datetime.timezone.utc))
new_user_creds = UserCredentials.batch(b).create(username=reg_form.username.data,
password=sha256_crypt.hash(reg_form.password.data),
uuid=new_user.uuid)
b.execute()
flash('User created')
return redirect( url_for('index' ))
else:
return render_template('registration.html', form=reg_form)
def test_batch_if_not_exists(self):
""" ensure 'IF NOT EXISTS' exists in statement when in batch """
with mock.patch.object(self.session, 'execute') as m:
with BatchQuery() as b:
TestIfNotExistsModel.batch(b).if_not_exists().create(count=8)
self.assertIn("IF NOT EXISTS", m.call_args[0][0].query_string)
def seen(request):
# Record stats for items marked as seen on a mobile device
# For workload purposes we ignore the posted data, and instead generate
# some random data of our own, cached in memcached
global SAMPLE_COUNT
should_profile = False
if settings.PROFILING:
SAMPLE_COUNT += 1
if SAMPLE_COUNT >= settings.SAMPLE_RATE:
SAMPLE_COUNT = 0
should_profile = True
bundleids = cache.get('bundleids')
if bundleids is None:
bundleids = [uuid.uuid4() for _ in range(1000)]
cache.set('bundleids', bundleids, 24 * 60 * 60)
entryids = cache.get('entryids')
if entryids is None:
entryids = [uuid.uuid4() for _ in range(10000)]
cache.set('entryids', entryids, 24 * 60 * 60)
with statsd.pipeline() as pipe, BatchQuery() as b:
for bundleid in random.sample(bundleids, random.randrange(3)):
if should_profile:
pipe.incr('workloadoutput.bundle.{}.seen'.format(bundleid.hex))
for entryid in random.sample(entryids, random.randrange(5)):
if should_profile:
pipe.incr('workloadoutput.bundle.{}.{}.seen'.format(
bundleid.hex, entryid.hex))
BundleSeenModel(userid=request.user.id,
bundleid=bundleid,
entryid=entryid).save()
return HttpResponse(json.dumps({}), content_type='text/json')
def test_instance_update_in_batch(self):
with mock.patch.object(self.session, "execute") as m:
with BatchQuery() as b:
self.instance.batch(b).timestamp(timedelta(seconds=30)).update(count=2)
query = m.call_args[0][0].query_string
"USING TIMESTAMP".should.be.within(query)
def append(self, sequenced_item_or_items):
if isinstance(sequenced_item_or_items, list):
if len(sequenced_item_or_items):
b = BatchQuery()
for item in sequenced_item_or_items:
assert isinstance(item, self.sequenced_item_class), (type(item), self.sequenced_item_class)
kwargs = self.get_field_kwargs(item)
self.active_record_class.batch(b).if_not_exists().create(**kwargs)
try:
b.execute()
except LWTException as e:
self.raise_sequenced_item_error(sequenced_item_or_items, e)
else:
active_record = self.to_active_record(sequenced_item_or_items)
try:
active_record.save()
except LWTException as e:
self.raise_sequenced_item_error(sequenced_item_or_items, e)
def test_dml_none_success_case(self):
""" Tests that passing None into the batch call clears any batch object """
b = BatchQuery()
q = DMLQuery(TestMultiKeyModel, batch=b)
self.assertEqual(q._batch, b)
q.batch(None)
self.assertIsNone(q._batch)
def test_batch(self):
with mock.patch.object(self.session, "execute") as m:
with BatchQuery() as b:
TestTimestampModel.timestamp(timedelta(seconds=10)).batch(b).create(count=1)
query = m.call_args[0][0].query_string
query.should.match(r"INSERT.*USING TIMESTAMP")
query.should_not.match(r"TIMESTAMP.*INSERT")
def test_dml_none_success_case(self):
""" Tests that passing None into the batch call clears any batch object """
b = BatchQuery()
q = DMLQuery(TestMultiKeyModel, batch=b)
assert q._batch == b
q.batch(None)
assert q._batch is None
def test_batch_execute_no_timeout(self):
with mock.patch.object(Session, 'execute',
autospec=True) as mock_execute:
with BatchQuery() as b:
BatchQueryLogModel.batch(b).create(k=2, v=2)
mock_execute.assert_called_once_with(mock.ANY,
mock.ANY,
mock.ANY,
timeout=NOT_SET)
def test_batch_query_different_connection(self):
"""
Test BatchQuery with Models that have a different connection
@since 3.7
@jira_ticket PYTHON-613
@expected_result queries should execute appropriately
@test_category object_mapper
"""
# Testing on a model class
TestModel.__connection__ = 'cluster'
AnotherTestModel.__connection__ = 'cluster2'
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1,
cluster=1)
TestModel.__connection__ = None
AnotherTestModel.__connection__ = None
with BatchQuery(connection='cluster') as b:
TestModel.objects.batch(b).create(partition=1, cluster=1)
AnotherTestModel.objects.batch(b).create(partition=1, cluster=1)
# Testing on a model instance
with ContextQuery(TestModel, AnotherTestModel,
connection='cluster') as (tm, atm):
obj1 = tm.objects.get(partition=1, cluster=1)
obj2 = atm.objects.get(partition=1, cluster=1)
obj1.__connection__ = 'cluster'
obj2.__connection__ = 'cluster2'
obj1.count = 4
obj2.count = 4
with self.assertRaises(CQLEngineException):
with BatchQuery() as b:
obj1.batch(b).save()
obj2.batch(b).save()
def test_batch_update_conditional_several_rows(self):
sync_table(TestUpdateModel)
self.addCleanup(drop_table, TestUpdateModel)
first_row = TestUpdateModel.create(partition=1, cluster=1, value=5, text="something")
second_row = TestUpdateModel.create(partition=1, cluster=2, value=5, text="something")
b = BatchQuery()
TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=1, value=5, text='something else')
TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=2, value=5, text='something else')
TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=3, value=5, text='something else')
# The response will be more than two rows because two of the inserts will fail
with self.assertRaises(LWTException):
b.execute()
first_row.delete()
second_row.delete()
b.execute()
def parse_csv(filepath, username, trip):
"""
Parse a csv and import to database
"""
with open(filepath, 'r') as csvfile:
reader = csv.DictReader(csvfile.read().split('\n'))
i = 0
b = BatchQuery()
last_dt = None
for i, line in enumerate(reader):
if i % 1000 == 0:
b.execute()
b = BatchQuery()
try:
dt = parser.parse(line['time'])
if dt == last_dt:
continue
pt = {'lon': line['lon'],
'lat': line['lat'],
'accurracy': line['accuracy'],
'username': username,
'created_at': dt,
'trip_id': trip}
last_dt = dt
Point.batch(b).create(**pt)
except ValueError:
continue
b.execute()
return username, trip
def test_batch_update_conditional(self):
t = TestConditionalModel.create(text='something', count=5)
id = t.id
with BatchQuery() as b:
t.batch(b).iff(count=5).update(text='something else')
updated = TestConditionalModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
b = BatchQuery()
updated.batch(b).iff(count=6).update(text='and another thing')
with self.assertRaises(LWTException) as assertion:
b.execute()
self.assertEqual(assertion.exception.existing, {
'id': id,
'count': 5,
'[applied]': False,
})
updated = TestConditionalModel.objects(id=id).first()
self.assertEqual(updated.text, 'something else')
class Batch(BatchQuery):
'''
Performs a batch of insert queries using async connections
'''
def __init__(self, **kwargs):
self.instances = []
self._batch = BatchQuery()
def batch_insert(self, model_instance):
self.instances.append(model_instance)
def __enter__(self):
return self
def add_query(self, query):
self._batch.add_query(query)
def add_callback(self, fn, *args, **kwargs):
raise TypeError('not supported')
def execute(self):
promises = []
session = get_session()
for instance in self.instances:
query = instance.__dmlquery__(instance.__class__, instance)
query.batch(self._batch)
query.save()
for query in self._batch.queries:
statement = SimpleStatement(str(query))
params = query.get_context()
promises.append(session.execute_async(statement, params))
return [r.result() for r in promises]
def __exit__(self, exc_type, exc_val, exc_tb):
self.execute()
def test_callbacks_properly_execute_callables_and_tuples(self):
call_history = []
def my_callback(*args, **kwargs):
call_history.append(args)
# adding on init:
batch = BatchQuery()
batch.add_callback(my_callback)
batch.add_callback(my_callback, 'more', 'args')
batch.execute()
self.assertEqual(len(call_history), 2)
self.assertEqual([(), ('more', 'args')], call_history)
def test_API_managing_callbacks(self):
# Callbacks can be added at init and after
def my_callback(*args, **kwargs):
pass
# adding on init:
batch = BatchQuery()
batch.add_callback(my_callback)
batch.add_callback(my_callback, 2, named_arg='value')
batch.add_callback(my_callback, 1, 3)
self.assertEqual(batch._callbacks, [
(my_callback, (), {}),
(my_callback, (2,), {'named_arg':'value'}),
(my_callback, (1, 3), {})
])
def __init__(self, **kwargs):
self.instances = []
self._batch = BatchQuery()
def test_empty_batch(self):
b = BatchQuery()
b.execute()
with BatchQuery() as b:
pass
