def store_cancer_data(self, cancer_data): data_group_info = cancer_data.data_group_info country = data_group_info.country age = data_group_info.age_gender.age gender = data_group_info.age_gender.gender bq = BatchQuery(consistency=ConsistencyLevel.ONE) for cancer_data_record in cancer_data.cancer_data: cancer = cancer_data_record.cancer deaths = cancer_data_record.deaths crude_rate = cancer_data_record.crude_rate asr = cancer_data_record.crude_rate cumulative_risk = cancer_data_record.cumulative_risk CancerDataEntity.batch(bq).create( age=age, gender=gender, country=country, cancer=cancer, asr=asr, crude_rate=crude_rate, cumulative_risk=cumulative_risk, deaths=deaths, ) bq.execute()
def test_batch_query_different_connection(self): """Test BatchQuery with Models that have a different connection""" # Testing on a model class TestModel.__connection__ = 'cluster' AnotherTestModel.__connection__ = 'cluster2' with self.assertRaises(CQLEngineException): with BatchQuery() as b: TestModel.objects.batch(b).create(partition=1, cluster=1) AnotherTestModel.objects.batch(b).create(partition=1, cluster=1) TestModel.__connection__ = None AnotherTestModel.__connection__ = None with BatchQuery(connection='cluster') as b: TestModel.objects.batch(b).create(partition=1, cluster=1) AnotherTestModel.objects.batch(b).create(partition=1, cluster=1) # Testing on a model instance with ContextQuery(TestModel, AnotherTestModel, connection='cluster') as (tm, atm): obj1 = tm.objects.get(partition=1, cluster=1) obj2 = atm.objects.get(partition=1, cluster=1) obj1.__connection__ = 'cluster' obj2.__connection__ = 'cluster2' obj1.count = 4 obj2.count = 4 with self.assertRaises(CQLEngineException): with BatchQuery() as b: obj1.batch(b).save() obj2.batch(b).save()
def import_csv(csv_path, batch=False, chunksize=500, skiprows=None): start_time = dt.datetime.today().timestamp() for df in pd.read_csv(csv_path, delimiter='\t', encoding='utf-8', dtype=dtype, converters=converters, chunksize=chunksize, skiprows=skiprows, error_bad_lines=False, warn_bad_lines=True): print("transform") df = df.where((pd.notnull(df)), None) print("importing rows {} to {}".format(df.index.min(), df.index.max())) if batch: b = BatchQuery() for i, row in df.iterrows(): try: print(u"DOING {} ; {}".format(i, row['code'].encode('utf-8'))) if row['code'] is None or len(row['code'].strip()) == 0: print("error with line {0} : code = '{1}'".format( i, row['code'].encode('utf-8'))) continue row_converted = { Product._get_column_by_db_name(cql_name).column_name: value for cql_name, value in row.items() if not cql_name.endswith('_datetime') and value is not None and (type(value) != str or len(value) > 0) } product = Product.create(**row_converted) if not batch: product.save() print(u"DONE {} ; {}".format(i, row['code'].encode('utf-8'))) except Exception: print(u"EXCEPTION {} ; {}".format(i, row['code'].encode('utf-8'))) exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=10, file=sys.stderr) if batch: print("executing batch for rows {} to {}".format( df.index.min(), df.index.max())) b.execute() time_diff = dt.datetime.today().timestamp() - start_time print("TIMING {} rows/s".format(chunksize / time_diff)) start_time = dt.datetime.today().timestamp()
def test_batch_update_if_exists_success(self): """ Tests that batch update with if_exists work as expected @since 3.1 @jira_ticket PYTHON-432 @expected_result @test_category object_mapper """ id = uuid4() m = TestIfExistsModel.create(id=id, count=8, text='123456789') with BatchQuery() as b: m.text = '111111111' m.batch(b).if_exists().update() with self.assertRaises(LWTException) as assertion: with BatchQuery() as b: m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist m.batch(b).if_exists().update() self.assertEqual(assertion.exception.existing, { '[applied]': False, }) q = TestIfExistsModel.objects(id=id) self.assertEqual(len(q), 1) tm = q.first() self.assertEqual(tm.count, 8) self.assertEqual(tm.text, '111111111')
def test_batch_delete_if_exists_success(self): """ Tests that batch deletes with if_exists work, and throw proper LWTException when they are are not applied @since 3.1 @jira_ticket PYTHON-432 @expected_result Deletes will be preformed if they exist, otherwise throw LWTException @test_category object_mapper """ id = uuid4() m = TestIfExistsModel.create(id=id, count=8, text='123456789') with BatchQuery() as b: m.batch(b).if_exists().delete() q = TestIfExistsModel.objects(id=id) self.assertEqual(len(q), 0) with self.assertRaises(LWTException) as assertion: with BatchQuery() as b: m = TestIfExistsModel(id=uuid4(), count=42) # Doesn't exist m.batch(b).if_exists().delete() self.assertEqual(assertion.exception.existing, { '[applied]': False, })
def test_batch_insert_if_not_exists_success(self): """ tests that batch insertion with if_not_exists work as expected """ id = uuid4() with BatchQuery() as b: TestIfNotExistsModel.batch(b).if_not_exists().create(id=id, count=8, text='123456789') b = BatchQuery() TestIfNotExistsModel.batch(b).if_not_exists().create(id=id, count=9, text='111111111111') with self.assertRaises(LWTException) as assertion: b.execute() self.assertEqual(assertion.exception.existing, { 'count': 8, 'id': id, 'text': '123456789', '[applied]': False, }) q = TestIfNotExistsModel.objects(id=id) self.assertEqual(len(q), 1) tm = q.first() self.assertEqual(tm.count, 8) self.assertEqual(tm.text, '123456789')
def test_cqlengine_batch_type(self): """ Tests the different types of `class: cassandra.cqlengine.query.BatchType` @since 3.13 @jira_ticket PYTHON-88 @expected_result batch query succeeds and the results are correctly readen @test_category query """ with BatchQuery(batch_type=cqlengine_BatchType.Unlogged) as b: TestMultiKeyModel.batch(b).create(partition=1, cluster=1) TestMultiKeyModel.batch(b).create(partition=1, cluster=2) obj = TestMultiKeyModel.objects(partition=1) self.assertEqual(2, len(obj)) with BatchQuery(batch_type=cqlengine_BatchType.Counter) as b: CounterBatchQueryModel.batch(b).create(k=1, v=1) CounterBatchQueryModel.batch(b).create(k=1, v=2) CounterBatchQueryModel.batch(b).create(k=1, v=10) obj = CounterBatchQueryModel.objects(k=1) self.assertEqual(1, len(obj)) self.assertEqual(obj[0].v, 13)
def test_basic_batch_query(self): """ Test Batch queries with connections explicitly set @since 3.7 @jira_ticket PYTHON-613 @expected_result queries should execute appropriately @test_category object_mapper """ # No connection with a QuerySet (default is a fake one) with self.assertRaises(NoHostAvailable): with BatchQuery() as b: TestModel.objects.batch(b).create(partition=1, cluster=1) # Explicit connection with a QuerySet with BatchQuery(connection='cluster') as b: TestModel.objects.batch(b).create(partition=1, cluster=1) # Get an object from the BD with ContextQuery(TestModel, connection='cluster') as tm: obj = tm.objects.get(partition=1, cluster=1) obj.__connection__ = None # No connection with a model (default is a fake one) with self.assertRaises(NoHostAvailable): with BatchQuery() as b: obj.count = 2 obj.batch(b).save() # Explicit connection with a model with BatchQuery(connection='cluster') as b: obj.count = 2 obj.batch(b).save()
def test_batch_query_connection_override(self): """ Test that we cannot override a BatchQuery connection per model @since 3.7 @jira_ticket PYTHON-613 @expected_result Proper exceptions should be raised @test_category object_mapper """ with self.assertRaises(CQLEngineException): with BatchQuery(connection='cluster') as b: TestModel.batch(b).using(connection='test').save() with self.assertRaises(CQLEngineException): with BatchQuery(connection='cluster') as b: TestModel.using(connection='test').batch(b).save() with ContextQuery(TestModel, AnotherTestModel, connection='cluster') as (tm, atm): obj1 = tm.objects.get(partition=1, cluster=1) obj1.__connection__ = None with self.assertRaises(CQLEngineException): with BatchQuery(connection='cluster') as b: obj1.using(connection='test').batch(b).save() with self.assertRaises(CQLEngineException): with BatchQuery(connection='cluster') as b: obj1.batch(b).using(connection='test').save()
def test_batch_insert_if_not_exists(self): """ tests that batch insertion with if_not_exists work as expected """ id = uuid4() with BatchQuery() as b: TestIfNotExistsModel.batch(b).if_not_exists().create( id=id, count=8, text='123456789') b = BatchQuery() TestIfNotExistsModel.batch(b).if_not_exists().create( id=id, count=9, text='111111111111') with self.assertRaises(LWTException) as assertion: b.execute() self.assertEqual(assertion.exception.existing, { 'count': 8, 'id': id, 'text': '123456789', '[applied]': False, }) q = TestIfNotExistsModel.objects(id=id) self.assertEqual(len(q), 1) tm = q.first() self.assertEqual(tm.count, 8) self.assertEqual(tm.text, '123456789')
def test_basic_batch_query(self): """Test BatchQuery requests""" # No connection with a QuerySet (default is a fake one) with self.assertRaises(NoHostAvailable): with BatchQuery() as b: TestModel.objects.batch(b).create(partition=1, cluster=1) # Explicit connection with a QuerySet with BatchQuery(connection='cluster') as b: TestModel.objects.batch(b).create(partition=1, cluster=1) # Get an object from the BD with ContextQuery(TestModel, connection='cluster') as tm: obj = tm.objects.get(partition=1, cluster=1) obj.__connection__ = None # No connection with a model (default is a fake one) with self.assertRaises(NoHostAvailable): with BatchQuery() as b: obj.count = 2 obj.batch(b).save() # Explicit connection with a model with BatchQuery(connection='cluster') as b: obj.count = 2 obj.batch(b).save()
def test_callbacks_tied_to_execute(self): """Batch callbacks should NOT fire if batch is not executed in context manager mode""" call_history = [] def my_callback(*args, **kwargs): call_history.append(args) with BatchQuery() as batch: batch.add_callback(my_callback) self.assertEqual(len(call_history), 1) class SomeError(Exception): pass with self.assertRaises(SomeError): with BatchQuery() as batch: batch.add_callback(my_callback) # this error bubbling up through context manager # should prevent callback runs (along with b.execute()) raise SomeError # still same call history. Nothing added self.assertEqual(len(call_history), 1) # but if execute ran, even with an error bubbling through # the callbacks also would have fired with self.assertRaises(SomeError): with BatchQuery(execute_on_exception=True) as batch: batch.add_callback(my_callback) raise SomeError # updated call history self.assertEqual(len(call_history), 2)
def test_insert_success_case(self): b = BatchQuery() inst = TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4') with self.assertRaises(TestMultiKeyModel.DoesNotExist): TestMultiKeyModel.get(partition=self.pkey, cluster=2) b.execute() TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def save_ts(self, ts_list: List[TSData]): b = BatchQuery() for ts_data in ts_list: func = TSFunctionRegistry.find_function(ts_data.ts_type_name) value_serialized: str = func.serialize(ts_data.values) TimeSeriesDataModel.batch(b).create( type=ts_data.ts_type_name, code=ts_data.code, visible_time=ts_data.visible_time, data=value_serialized) b.execute()
def test_insert_success_case(self): b = BatchQuery() inst = TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4') with self.assertRaises(TestMultiKeyModel.DoesNotExist): TestMultiKeyModel.get(partition=self.pkey, cluster=2) b.execute() TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def truncate_models(self): log.info('Purging data from all tables...') for model in self.get_cassandra_models(): b = BatchQuery() for instance in model.objects.all(): instance.batch(b).delete() b.execute() log.info('Done!')
def save(self, ts_list: List[TSData]): b = BatchQuery() for ts_data in ts_list: func = TSTypeRegistry.find_function(ts_data.ts_type_name) if not isinstance(func, HistoryTimeSeriesType): raise RuntimeError("非法的tsType") value_serialized: str = func.serialize(ts_data.values) TimeSeriesDataModel.batch(b).create( type=ts_data.ts_type_name, code=ts_data.code, visible_time=ts_data.visible_time, data=value_serialized) b.execute()
def test_async_batch(self): b = BatchQuery(execute_async=True) TestMultiKeyModel.batch(b).create(partition=self.pkey, cluster=2, count=3, text='4') with self.assertRaises(TestMultiKeyModel.DoesNotExist): TestMultiKeyModel.get(partition=self.pkey, cluster=2) b.execute() TestMultiKeyModel.get(partition=self.pkey, cluster=2)
def import_data(): connection.setup(['127.0.0.1'], "geonames", protocol_version=3) fieldnames = [col for col in Geoname().__dict__['_values']] with open("geonames/allCountries.txt", encoding='utf8', newline='') as csvfile: # Création du DictReader csv.register_dialect('geoname', delimiter='\t', quoting=csv.QUOTE_NONE) reader = csv.DictReader(csvfile, dialect='geoname', fieldnames=fieldnames) # Ingestion par lots count = 0 batch = BatchQuery() for row in reader: new_row = clean_row(row) Geoname.batch(batch).create(**new_row) count += 1 if not count % 1000: batch.execute() batch = BatchQuery() logger.info('Importés: {}'.format(count)) batch.execute()
def process_create_entry_work(self, rdict, context, do_load): b = BatchQuery() # MOSTLY the resource will not exist. So start by calculating the URL and trying to insert the entire record. if not do_load: url = u"file://{}{}/{}".format(decode_str(context['local_ip']), decode_str(context['path']), decode_str(context['entry'])) else: with open(context['fullpath'], 'r') as f: blob = Blob.create_from_file(f, rdict['size']) if blob: url = "cassandra://{}".format(blob.id) else: return None try: # OK -- try to insert ( create ) the record... t1 = time.time() resource = Resource.batch(b).create(url=url, **rdict) msg = u'Resource {} created --> {}'.format(resource.name, time.time() - t1) logger.info(msg) except ResourceConflictError: # If the create fails, the record already exists... so retrieve it... t1 = time.time() resource = Resource.objects().get(container=context['collection'], name=rdict['name']) msg = u"{} ::: Fetch Object -> {}".format(resource.name, time.time() - t1) logger.info(msg) # if the url is not correct then update # TODO: If the URL is a block set that's stored internally, reduce its count so that it can be garbage collected # t3 = None if resource.url != url: t2 = time.time() # if url.startswith('cassandra://') : tidy up the stored block count... resource.batch(b).update(url=url) t3 = time.time() msg = u"{} ::: update -> {}".format(resource.name, t3 - t2) logger.info(msg) # t1 = time.time() SearchIndex.reset(resource.id) SearchIndex.index(resource, ['name', 'metadata']) # msg = "Index Management -> {}".format(time.time() - t1) # logger.info(msg) b.execute()
def test_batch_update_transaction(self): t = TestTransactionModel.create(text='something', count=5) id = t.id with BatchQuery() as b: t.batch(b).iff(count=5).update(text='something else') updated = TestTransactionModel.objects(id=id).first() self.assertEqual(updated.text, 'something else') b = BatchQuery() updated.batch(b).iff(count=6).update(text='and another thing') self.assertRaises(LWTException, b.execute) updated = TestTransactionModel.objects(id=id).first() self.assertEqual(updated.text, 'something else')
def test_update_success_case(self): inst = TestMultiKeyModel.create(partition=self.pkey, cluster=2, count=3, text='4') b = BatchQuery() inst.count = 4 inst.batch(b).save() inst2 = TestMultiKeyModel.get(partition=self.pkey, cluster=2) self.assertEqual(inst2.count, 3) b.execute() inst3 = TestMultiKeyModel.get(partition=self.pkey, cluster=2) self.assertEqual(inst3.count, 4)
def test_batch_consistency(self): with mock.patch.object(self.session, 'execute') as m: with BatchQuery(consistency=CL.ALL) as b: TestConsistencyModel.batch(b).create(text="monkey") args = m.call_args self.assertEqual(CL.ALL, args[0][0].consistency_level) with mock.patch.object(self.session, 'execute') as m: with BatchQuery() as b: TestConsistencyModel.batch(b).create(text="monkey") args = m.call_args self.assertNotEqual(CL.ALL, args[0][0].consistency_level)
def main(): connection.default() # Management functions would normally be used in development, and possibly for deployments. # They are typically not part of a core application. log.info("### creating keyspace...") management.create_keyspace_simple(KEYSPACE, 1) log.info("### syncing model...") management.sync_table(FamilyMembers) # default uuid is assigned simmons = FamilyMembers.create(surname='Simmons', name='Gene', birth_year=1949, sex='m') # add members to his family later FamilyMembers.create(id=simmons.id, surname='Simmons', name='Nick', birth_year=1989, sex='m') sophie = FamilyMembers.create(id=simmons.id, surname='Simmons', name='Sophie', sex='f') nick = FamilyMembers.objects(id=simmons.id, surname='Simmons', name='Nick') try: nick.iff(birth_year=1988).update(birth_year=1989) except LWTException: print "precondition not met" # showing validation try: FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', birth_year=1957, sex='f') except ValidationError: log.exception('INTENTIONAL VALIDATION EXCEPTION; Failed creating instance:') FamilyMembers.create(id=simmons.id, surname='Tweed', name='Shannon', sex='f') log.info("### add multiple as part of a batch") # If creating many at one time, can use a batch to minimize round-trips hogan_id = uuid4() with BatchQuery() as b: FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Hulk', sex='m') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Linda', sex='f') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Nick', sex='m') FamilyMembers.batch(b).create(id=hogan_id, surname='Hogan', name='Brooke', sex='f') log.info("### All members") for m in FamilyMembers.all(): print m, m.birth_year, m.sex log.info("### Select by partition key") for m in FamilyMembers.objects(id=simmons.id): print m, m.birth_year, m.sex log.info("### Constrain on clustering key") for m in FamilyMembers.objects(id=simmons.id, surname=simmons.surname): print m, m.birth_year, m.sex log.info("### Constrain on clustering key") kids = FamilyMembers.objects(id=simmons.id, surname=simmons.surname, name__in=['Nick', 'Sophie']) log.info("### Delete a record") FamilyMembers(id=hogan_id, surname='Hogan', name='Linda').delete() for m in FamilyMembers.objects(id=hogan_id): print m, m.birth_year, m.sex management.drop_keyspace(KEYSPACE)
def test_update_success_case(self): inst = TestMultiKeyModel.create(partition=self.pkey, cluster=2, count=3, text='4') b = BatchQuery() inst.count = 4 inst.batch(b).save() inst2 = TestMultiKeyModel.get(partition=self.pkey, cluster=2) assert inst2.count == 3 b.execute() inst3 = TestMultiKeyModel.get(partition=self.pkey, cluster=2) assert inst3.count == 4
def registration(): reg_form = RegistrationForm() if reg_form.validate_on_submit(): # handle regsitration here try: user_from_db = UserCredentials.objects.get(username=reg_form.username.data) flash('Username already taken') return redirect( url_for('index' )) except: pass # create the new user with BatchQuery() as b: new_user = Users.batch(b).create(username=reg_form.username.data, first_name=reg_form.first_name.data, last_name=reg_form.last_name.data, created_date=datetime.datetime.now(datetime.timezone.utc)) new_user_creds = UserCredentials.batch(b).create(username=reg_form.username.data, password=sha256_crypt.hash(reg_form.password.data), uuid=new_user.uuid) b.execute() flash('User created') return redirect( url_for('index' )) else: return render_template('registration.html', form=reg_form)
def test_batch_if_not_exists(self): """ ensure 'IF NOT EXISTS' exists in statement when in batch """ with mock.patch.object(self.session, 'execute') as m: with BatchQuery() as b: TestIfNotExistsModel.batch(b).if_not_exists().create(count=8) self.assertIn("IF NOT EXISTS", m.call_args[0][0].query_string)
def seen(request): # Record stats for items marked as seen on a mobile device # For workload purposes we ignore the posted data, and instead generate # some random data of our own, cached in memcached global SAMPLE_COUNT should_profile = False if settings.PROFILING: SAMPLE_COUNT += 1 if SAMPLE_COUNT >= settings.SAMPLE_RATE: SAMPLE_COUNT = 0 should_profile = True bundleids = cache.get('bundleids') if bundleids is None: bundleids = [uuid.uuid4() for _ in range(1000)] cache.set('bundleids', bundleids, 24 * 60 * 60) entryids = cache.get('entryids') if entryids is None: entryids = [uuid.uuid4() for _ in range(10000)] cache.set('entryids', entryids, 24 * 60 * 60) with statsd.pipeline() as pipe, BatchQuery() as b: for bundleid in random.sample(bundleids, random.randrange(3)): if should_profile: pipe.incr('workloadoutput.bundle.{}.seen'.format(bundleid.hex)) for entryid in random.sample(entryids, random.randrange(5)): if should_profile: pipe.incr('workloadoutput.bundle.{}.{}.seen'.format( bundleid.hex, entryid.hex)) BundleSeenModel(userid=request.user.id, bundleid=bundleid, entryid=entryid).save() return HttpResponse(json.dumps({}), content_type='text/json')
def test_instance_update_in_batch(self): with mock.patch.object(self.session, "execute") as m: with BatchQuery() as b: self.instance.batch(b).timestamp(timedelta(seconds=30)).update(count=2) query = m.call_args[0][0].query_string "USING TIMESTAMP".should.be.within(query)
def append(self, sequenced_item_or_items): if isinstance(sequenced_item_or_items, list): if len(sequenced_item_or_items): b = BatchQuery() for item in sequenced_item_or_items: assert isinstance(item, self.sequenced_item_class), (type(item), self.sequenced_item_class) kwargs = self.get_field_kwargs(item) self.active_record_class.batch(b).if_not_exists().create(**kwargs) try: b.execute() except LWTException as e: self.raise_sequenced_item_error(sequenced_item_or_items, e) else: active_record = self.to_active_record(sequenced_item_or_items) try: active_record.save() except LWTException as e: self.raise_sequenced_item_error(sequenced_item_or_items, e)
def test_dml_none_success_case(self): """ Tests that passing None into the batch call clears any batch object """ b = BatchQuery() q = DMLQuery(TestMultiKeyModel, batch=b) self.assertEqual(q._batch, b) q.batch(None) self.assertIsNone(q._batch)
def test_batch(self): with mock.patch.object(self.session, "execute") as m: with BatchQuery() as b: TestTimestampModel.timestamp(timedelta(seconds=10)).batch(b).create(count=1) query = m.call_args[0][0].query_string query.should.match(r"INSERT.*USING TIMESTAMP") query.should_not.match(r"TIMESTAMP.*INSERT")
def test_dml_none_success_case(self): """ Tests that passing None into the batch call clears any batch object """ b = BatchQuery() q = DMLQuery(TestMultiKeyModel, batch=b) assert q._batch == b q.batch(None) assert q._batch is None
def test_batch_execute_no_timeout(self): with mock.patch.object(Session, 'execute', autospec=True) as mock_execute: with BatchQuery() as b: BatchQueryLogModel.batch(b).create(k=2, v=2) mock_execute.assert_called_once_with(mock.ANY, mock.ANY, mock.ANY, timeout=NOT_SET)
def test_batch_query_different_connection(self): """ Test BatchQuery with Models that have a different connection @since 3.7 @jira_ticket PYTHON-613 @expected_result queries should execute appropriately @test_category object_mapper """ # Testing on a model class TestModel.__connection__ = 'cluster' AnotherTestModel.__connection__ = 'cluster2' with self.assertRaises(CQLEngineException): with BatchQuery() as b: TestModel.objects.batch(b).create(partition=1, cluster=1) AnotherTestModel.objects.batch(b).create(partition=1, cluster=1) TestModel.__connection__ = None AnotherTestModel.__connection__ = None with BatchQuery(connection='cluster') as b: TestModel.objects.batch(b).create(partition=1, cluster=1) AnotherTestModel.objects.batch(b).create(partition=1, cluster=1) # Testing on a model instance with ContextQuery(TestModel, AnotherTestModel, connection='cluster') as (tm, atm): obj1 = tm.objects.get(partition=1, cluster=1) obj2 = atm.objects.get(partition=1, cluster=1) obj1.__connection__ = 'cluster' obj2.__connection__ = 'cluster2' obj1.count = 4 obj2.count = 4 with self.assertRaises(CQLEngineException): with BatchQuery() as b: obj1.batch(b).save() obj2.batch(b).save()
def test_batch_update_conditional_several_rows(self): sync_table(TestUpdateModel) self.addCleanup(drop_table, TestUpdateModel) first_row = TestUpdateModel.create(partition=1, cluster=1, value=5, text="something") second_row = TestUpdateModel.create(partition=1, cluster=2, value=5, text="something") b = BatchQuery() TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=1, value=5, text='something else') TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=2, value=5, text='something else') TestUpdateModel.batch(b).if_not_exists().create(partition=1, cluster=3, value=5, text='something else') # The response will be more than two rows because two of the inserts will fail with self.assertRaises(LWTException): b.execute() first_row.delete() second_row.delete() b.execute()
def parse_csv(filepath, username, trip): """ Parse a csv and import to database """ with open(filepath, 'r') as csvfile: reader = csv.DictReader(csvfile.read().split('\n')) i = 0 b = BatchQuery() last_dt = None for i, line in enumerate(reader): if i % 1000 == 0: b.execute() b = BatchQuery() try: dt = parser.parse(line['time']) if dt == last_dt: continue pt = {'lon': line['lon'], 'lat': line['lat'], 'accurracy': line['accuracy'], 'username': username, 'created_at': dt, 'trip_id': trip} last_dt = dt Point.batch(b).create(**pt) except ValueError: continue b.execute() return username, trip
def test_batch_update_conditional(self): t = TestConditionalModel.create(text='something', count=5) id = t.id with BatchQuery() as b: t.batch(b).iff(count=5).update(text='something else') updated = TestConditionalModel.objects(id=id).first() self.assertEqual(updated.text, 'something else') b = BatchQuery() updated.batch(b).iff(count=6).update(text='and another thing') with self.assertRaises(LWTException) as assertion: b.execute() self.assertEqual(assertion.exception.existing, { 'id': id, 'count': 5, '[applied]': False, }) updated = TestConditionalModel.objects(id=id).first() self.assertEqual(updated.text, 'something else')
class Batch(BatchQuery): ''' Performs a batch of insert queries using async connections ''' def __init__(self, **kwargs): self.instances = [] self._batch = BatchQuery() def batch_insert(self, model_instance): self.instances.append(model_instance) def __enter__(self): return self def add_query(self, query): self._batch.add_query(query) def add_callback(self, fn, *args, **kwargs): raise TypeError('not supported') def execute(self): promises = [] session = get_session() for instance in self.instances: query = instance.__dmlquery__(instance.__class__, instance) query.batch(self._batch) query.save() for query in self._batch.queries: statement = SimpleStatement(str(query)) params = query.get_context() promises.append(session.execute_async(statement, params)) return [r.result() for r in promises] def __exit__(self, exc_type, exc_val, exc_tb): self.execute()
def test_callbacks_properly_execute_callables_and_tuples(self): call_history = [] def my_callback(*args, **kwargs): call_history.append(args) # adding on init: batch = BatchQuery() batch.add_callback(my_callback) batch.add_callback(my_callback, 'more', 'args') batch.execute() self.assertEqual(len(call_history), 2) self.assertEqual([(), ('more', 'args')], call_history)
def test_API_managing_callbacks(self): # Callbacks can be added at init and after def my_callback(*args, **kwargs): pass # adding on init: batch = BatchQuery() batch.add_callback(my_callback) batch.add_callback(my_callback, 2, named_arg='value') batch.add_callback(my_callback, 1, 3) self.assertEqual(batch._callbacks, [ (my_callback, (), {}), (my_callback, (2,), {'named_arg':'value'}), (my_callback, (1, 3), {}) ])
def __init__(self, **kwargs): self.instances = [] self._batch = BatchQuery()
def test_empty_batch(self): b = BatchQuery() b.execute() with BatchQuery() as b: pass