def test_bucket_create(self): self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100)) bucket = self.try_n_times(10, 1, self.bm.get_bucket, "fred") self.assertEqual(bucket['minimum_durability_level'], Durability.NONE)
def test_lookup_kv(sdc_builder, sdc_executor, couchbase, test_name, input, expected_out, expected_error, missing_value_behavior): bucket_name = get_random_string(string.ascii_letters, 10).lower() doc = {'id': 'id1', 'data': 'hello'} raw_dict = dict(id=input) raw_data = json.dumps(raw_dict) cluster = couchbase.cluster try: # populate the database logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) bucket = cluster.bucket(bucket_name) bucket.upsert(doc['id'], doc) # build the pipeline builder = sdc_builder.get_pipeline_builder() origin = builder.add_stage('Dev Raw Data Source') origin.set_attributes(data_format='JSON', stop_after_first_batch=True, raw_data=raw_data) lookup = builder.add_stage('Couchbase Lookup') lookup.set_attributes(authentication_mode='USER', bucket=bucket_name, lookup_type='KV', document_key='${record:value("/id")}', sdc_field='/output', missing_value_behavior=missing_value_behavior) wiretap = builder.add_wiretap() origin >> lookup >> wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) sdc_executor.start_pipeline(pipeline).wait_for_finished() output_records = wiretap.output_records error_records = wiretap.error_records assert len(output_records) == len(expected_out) assert len(error_records) == len(expected_error) if expected_out: assert output_records[0].field == expected_out[0] if expected_error: assert error_records[0].field == expected_error[0] finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_bucket_create_fail(self): settings = CreateBucketSettings(name='fred', bucket_type='couchbase', ram_quota_mb=100) self.bm.create_bucket(settings) self.assertRaises(BucketAlreadyExistsException, self.bm.create_bucket, settings)
def test_object_names_bucket_query(sdc_builder, sdc_executor, couchbase, test_name, bucket_name): document_key_field = 'mydocname' key = 'mydocid' doc = {"data": "hello", document_key_field: key} raw_dict = dict(id=key) raw_data = json.dumps(raw_dict) query = f'SELECT * FROM `{bucket_name}` WHERE {document_key_field}=' + '"${record:value("/id")}"' cluster = couchbase.cluster try: # populate the database logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) bucket = cluster.bucket(bucket_name) bucket.upsert(key, doc) cluster.query(f'CREATE PRIMARY INDEX ON `{bucket_name}`').execute() # build the pipeline builder = sdc_builder.get_pipeline_builder() origin = builder.add_stage('Dev Raw Data Source') origin.set_attributes(data_format='JSON', stop_after_first_batch=True, raw_data=raw_data) lookup = builder.add_stage('Couchbase Lookup') lookup.set_attributes( authentication_mode='USER', bucket=bucket_name, lookup_type='N1QL', n1ql_query=query, n1ql_mappings=[dict(property=bucket_name, sdcField='/output')], missing_value_behavior='ERROR') wiretap = builder.add_wiretap() origin >> lookup >> wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) sdc_executor.start_pipeline(pipeline).wait_for_finished() output_records = wiretap.output_records assert len( output_records ) == 1, 'Number of returned records should equal the number of matching records stored' assert output_records[0].field['output'] == doc finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_data_format_binary(sdc_builder, sdc_executor, couchbase): bucket_name = get_random_string(string.ascii_letters, 10) document_key = 'id' batch_size = 1 cluster = couchbase.cluster # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Data Generator').set_attributes( batch_size=batch_size, fields_to_generate=[{ "type": "BYTE_ARRAY", "field": "data" }]) destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key=document_key, data_format='BINARY', binary_field_path="/data") wiretap = builder.add_wiretap() source >> destination source >= wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline( pipeline).wait_for_pipeline_output_records_count(batch_size) sdc_executor.stop_pipeline(pipeline) history = sdc_executor.get_pipeline_history(pipeline) num_records = history.latest.metrics.counter( 'pipeline.batchInputRecords.counter').count logger.info(f"Wrote {num_records} records") assert num_records == len(wiretap.output_records) bucket = cluster.bucket(bucket_name) assert bucket.get( document_key).value == wiretap.output_records[0].field['data'] finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_bucket_create(self): try: self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100)) finally: self.bm.drop_bucket('fred')
def test_bucket_drop_fail(self): settings = CreateBucketSettings(name='fred', bucket_type='couchbase', ram_quota_mb=100) self.bm.create_bucket(settings) self.try_n_times(10, 1, self.bm.drop_bucket, 'fred') self.assertRaises(BucketDoesNotExistException, self.bm.drop_bucket, 'fred')
def test_bucket_list(self): for bucket, kwargs in BucketManagementTests.buckets_to_add.items(): self.bm.create_bucket( CreateBucketSettings(name=bucket, ram_quota_mb=100, **kwargs)) self.try_n_times(10, 1, self.bm.get_bucket, bucket) self.assertEqual(set(), {"fred", "jane", "sally"}.difference( set(map(lambda x: x.name, self.bm.get_all_buckets()))))
def test_bucket_create_durability(self): min_durability = Durability.MAJORITY_AND_PERSIST_TO_ACTIVE self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100, minimum_durability_level=min_durability)) bucket = self.try_n_times(10, 1, self.bm.get_bucket, "fred") self.assertEqual(bucket['minimum_durability_level'], min_durability)
def test_bucket_create_replica_index(self): self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100, replica_index=True)) bucket = self.try_n_times(10, 1, self.bm.get_bucket, "fred") self.assertTrue(bucket.replica_index) self.try_n_times(10, 1, self.bm.drop_bucket, 'fred') self.assertRaises(BucketDoesNotExistException, self.bm.drop_bucket, 'fred') self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100, replica_index=False)) bucket = self.try_n_times(10, 1, self.bm.get_bucket, "fred") self.assertFalse(bucket.replica_index)
def test_data_types_query(sdc_builder, sdc_executor, couchbase, input, test_name, expected_type, expected_value): if input is None: pytest.skip('Lookup Processor queries deal with null values as errors rather than treating them as values') bucket_name = get_random_string(string.ascii_letters, 10).lower() key = 'mydocid' doc = {'id': key, 'data': input} raw_dict = dict(id=key) raw_data = json.dumps(raw_dict) query = f'SELECT data FROM {bucket_name} WHERE ' + 'id="${record:value("/id")}"' cluster = couchbase.cluster try: # populate the database logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket(CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) bucket = cluster.bucket(bucket_name) bucket.upsert(key, doc) cluster.query(f'CREATE PRIMARY INDEX ON `{bucket_name}`').execute() # build the pipeline builder = sdc_builder.get_pipeline_builder() origin = builder.add_stage('Dev Raw Data Source') origin.set_attributes(data_format='JSON', stop_after_first_batch=True, raw_data=raw_data) lookup = builder.add_stage('Couchbase Lookup') lookup.set_attributes(authentication_mode='USER', bucket=bucket_name, lookup_type='N1QL', n1ql_query=query, n1ql_mappings=[dict(property='data', sdcField='/output')], missing_value_behavior='ERROR') wiretap = builder.add_wiretap() origin >> lookup >> wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) sdc_executor.start_pipeline(pipeline).wait_for_finished() output_records = wiretap.output_records assert len(output_records) == 1 assert output_records[0].field['output'].type == expected_type assert output_records[0].field['output'] == expected_value finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_data_format_protobuf(sdc_builder, sdc_executor, couchbase): bucket_name = get_random_string(string.ascii_letters, 10) document_key = 'id' raw_data = '{"first_name": "Martin","last_name": "Balzamo"}' expected = '\x11\x06Martin\x12\x07Balzamo' cluster = couchbase.cluster # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Raw Data Source').set_attributes( data_format='JSON', raw_data=raw_data, stop_after_first_batch=True) destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key=document_key, data_format='PROTOBUF', message_type='Contact', protobuf_descriptor_file=PROTOBUF_FILE_PATH) source >> destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline(pipeline).wait_for_finished() history = sdc_executor.get_pipeline_history(pipeline) num_records = history.latest.metrics.counter( 'pipeline.batchOutputRecords.counter').count logger.info(f"Wrote {num_records} records") assert num_records == 1, 'Number of records stored should equal the number of records that entered the pipeline' bucket = cluster.bucket(bucket_name) doc_value = bucket.get(document_key).value # Decode the bytes object returned by Couchbase and remove any record separators (newline characters) contents = doc_value.decode('ascii').replace('\n', '') assert contents == expected finally: if pipeline and sdc_executor.get_pipeline_status( pipeline).response.json().get('status') == 'RUNNING': sdc_executor.stop_pipeline(pipeline) try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_multiple_batches(sdc_builder, sdc_executor, couchbase, batch_size): bucket_name = get_random_string(string.ascii_letters, 10) batches = 3 # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Data Generator') source.batch_size = batch_size source.fields_to_generate = [{"type": "LONG_SEQUENCE", "field": "seq"}] cluster = couchbase.cluster destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key='${record:value("/seq")}') wiretap = builder.add_wiretap() source >> destination source >= wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline( pipeline).wait_for_pipeline_output_records_count(batches * batch_size) sdc_executor.stop_pipeline(pipeline) history = sdc_executor.get_pipeline_history(pipeline) num_records = history.latest.metrics.counter( 'pipeline.batchInputRecords.counter').count logger.info(f"Wrote {num_records} records") assert num_records == len(wiretap.output_records) bucket = cluster.bucket(bucket_name) for i in range(num_records): assert bucket.get(str(i)).value == wiretap.output_records[i].field finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_bucket_create_durability(self): if float(self.cluster_version[0:3]) < 6.6: raise SkipTest( "Bucket minimum durability not available on server version < 6.6" ) min_durability = Durability.MAJORITY_AND_PERSIST_TO_ACTIVE self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100, minimum_durability_level=min_durability)) bucket = self.try_n_times(10, 1, self.bm.get_bucket, "fred") self.assertEqual(bucket['minimum_durability_level'], min_durability)
def test_object_names_bucket(sdc_builder, sdc_executor, couchbase, test_name, bucket_generator): """Test using different kinds of bucket names, asserting whether Couchbase receives the data""" bucket_name = bucket_generator() document_key_field = 'mydocname' raw_dict = dict(f1='abc', f2='xyz', f3='lmn') raw_dict[document_key_field] = 'mydocid' raw_data = json.dumps(raw_dict) cluster = couchbase.cluster # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Raw Data Source').set_attributes( data_format='JSON', raw_data=raw_data, stop_after_first_batch=True) destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key="${record:value('/" + document_key_field + "')}") source >> destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name.replace('%', '%25')) sdc_executor.start_pipeline(pipeline).wait_for_finished() # if the bucket name contains a percent sign, it gets interpreted as an escape character in the HTTP call made # by the open_bucket() method, so the ascii for percent (i.e. %25) should be used instead bucket = cluster.bucket(bucket_name.replace('%', '%25')) doc_value = bucket.get(raw_dict[document_key_field]).value assert doc_value == raw_dict finally: if pipeline and sdc_executor.get_pipeline_status( pipeline).response.json().get('status') == 'RUNNING': sdc_executor.stop_pipeline(pipeline) try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_change_ttl(self): # Create the bucket self.bm.create_bucket( CreateBucketSettings(name='fred', ram_quota_mb=100)) self.try_n_times(10, 3, self.bm.get_bucket, 'fred') # change bucket TTL self.try_n_times(10, 3, self.bm.update_bucket, BucketSettings(name='fred', max_ttl=500)) def get_bucket_ttl_equal(name, ttl): if not ttl == self.bm.get_bucket(name).max_ttl: raise Exception("not equal") self.try_n_times(10, 3, get_bucket_ttl_equal, 'fred', 500)
def test_data_format_json(sdc_builder, sdc_executor, couchbase): bucket_name = get_random_string(string.ascii_letters, 10) document_key_field = 'mydocname' raw_dict = dict(f1='abc', f2='xyz', f3='lmn') raw_dict[document_key_field] = 'mydocid' raw_data = json.dumps(raw_dict) cluster = couchbase.cluster # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Raw Data Source').set_attributes( data_format='JSON', raw_data=raw_data, stop_after_first_batch=True) destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key="${record:value('/" + document_key_field + "')}", data_format='JSON') source >> destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline(pipeline).wait_for_finished() bucket = cluster.bucket(bucket_name) doc_value = bucket.get(raw_dict[document_key_field]).value assert doc_value == raw_dict finally: if pipeline and sdc_executor.get_pipeline_status( pipeline).response.json().get('status') == 'RUNNING': sdc_executor.stop_pipeline(pipeline) try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_couchbase_destination(sdc_builder, sdc_executor, couchbase): """ Send simple JSON text into Couchbase destination from Dev Raw Data Source and assert Couchbase has received it. The pipeline looks like: dev_raw_data_source >> couchbase_destination """ bucket_name = get_random_string(string.ascii_letters, 10) document_key_field = 'mydocname' raw_dict = dict(f1='abc', f2='xyz', f3='lmn') raw_dict[document_key_field] = 'mydocid' raw_data = json.dumps(raw_dict) cluster = couchbase.cluster builder = sdc_builder.get_pipeline_builder() dev_raw_data_source = builder.add_stage('Dev Raw Data Source') dev_raw_data_source.set_attributes(data_format='JSON', raw_data=raw_data, stop_after_first_batch=True) couchbase_destination = builder.add_stage('Couchbase', type='destination') couchbase_destination.set_attributes(authentication_mode='USER', document_key="${record:value('/" + document_key_field + "')}", bucket=bucket_name) dev_raw_data_source >> couchbase_destination pipeline = builder.build(title='Couchbase Destination pipeline' ).configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline(pipeline).wait_for_finished() bucket = cluster.bucket(bucket_name) doc_value = bucket.get(raw_dict[document_key_field]).value assert doc_value == raw_dict finally: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name)
def test_bucket_list(self): buckets_to_add = {'fred': {}, 'jane': {}, 'sally': {}} try: for bucket, kwargs in buckets_to_add.items(): self.bm.create_bucket( CreateBucketSettings(name=bucket, ram_quota_mb=100, **kwargs)) self.assertEqual(set(), {"fred", "jane", "sally"}.difference( set(map(lambda x: x.name, self.bm.get_all_buckets())))) finally: for bucket, kwargs in buckets_to_add.items(): try: self.bm.drop_bucket(bucket) except: pass
def setUp(self, *args, **kwargs): if self.config.mock_enabled: raise SkipTest('Real server must be used for admin tests') super(UserManagementTests, self).setUp(*args, **kwargs) self.bm = self.cluster.buckets() try: self.test_bucket = self.bm.get_bucket("default") except BucketDoesNotExistException: self.bm.create_bucket( CreateBucketSettings(name="default", bucket_type="couchbase", ram_quota_mb=100)) self.um = self.cluster.users() if self.supports_groups(): self.um.upsert_group(Group('qweqwe', roles={Role(name='admin')}))
def setUp(self, *args, **kwargs): super(CollectionManagerTestCase, self).setUp() # SkipTest if collections not supported try: self.bucket.collections().get_all_scopes() except NotSupportedException: raise SkipTest('cluster does not support collections') # Need this so we use RBAC. # TODO: lets perhaps move this into the base classes? Then we can maybe not need the default user, etc... self.cluster._cluster.authenticate( username=self.cluster_info.admin_username, password=self.cluster_info.admin_password) self.bm = self.cluster.buckets() # insure other-bucket is gone first try: self.bm.drop_bucket('other-bucket') except: # it maybe isn't there, that's fine pass self.try_n_times_till_exception( 10, 1, self.bm.get_bucket, 'other-bucket', expected_exceptions=(BucketDoesNotExistException, )) # now re-create it fresh (maybe we could just flush, but we may test settings which would not be flushed) self.try_n_times( 10, 1, self.bm.create_bucket, CreateBucketSettings(name='other-bucket', bucket_type='couchbase', ram_quota_mb=100)) self.try_n_times(10, 1, self.bm.get_bucket, 'other-bucket') # we need to get the bucket, but sometimes this fails for a few seconds depending on what # the cluster is doing. So, try_n_times... def get_bucket(name): return self.cluster.bucket(name) self.other_bucket = self.try_n_times(10, 3, get_bucket, 'other-bucket') self.cm = self.other_bucket.collections()
def test_bucket_flush(self): # Create the bucket self.bm.create_bucket( CreateBucketSettings(name='fred', ram_quota_mb=100, flush_enabled=True)) bucket = self.try_n_times(10, 3, self.bm.get_bucket, 'fred') self.assertTrue(bucket.flush_enabled) # flush the bucket self.try_n_times(10, 3, self.bm.flush_bucket, bucket.name) # disable bucket flush self.bm.update_bucket(BucketSettings(name='fred', flush_enabled=False)) bucket = self.try_n_times(10, 3, self.bm.get_bucket, 'fred') self.assertFalse(bucket.flush_enabled) # verify appropriate Exception when flush attempted self.assertRaises(BucketNotFlushableException, self.bm.flush_bucket, 'fred')
def test_bucket_list(self): buckets_to_add = {'fred': {}, 'jane': {}, 'sally': {}} try: for bucket, kwargs in buckets_to_add.items(): self.bm.create_bucket( CreateBucketSettings(name=bucket, ram_quota_mb=100, **kwargs)) self.try_n_times(10, 1, self.bm.get_bucket, bucket) self.assertEqual(set(), {"fred", "jane", "sally"}.difference( set(map(lambda x: x.name, self.bm.get_all_buckets())))) finally: for bucket, kwargs in buckets_to_add.items(): try: self.bm.drop_bucket(bucket) except BucketDoesNotExistException: pass # now be sure it is really gone self.try_n_times_till_exception(10, 3, self.bm.get_bucket, bucket)
def test_actions(self): try: # Remove the bucket, if it exists self.bm.drop_bucket('dummy') except CouchbaseError: pass # Need to explicitly enable admin tests.. # Create the bucket self.bm.create_bucket( CreateBucketSettings(name='dummy', ram_quota_mb=100, bucket_password='******')) self.bm._admin_bucket.wait_ready('dummy', timeout=15.0) # All should be OK, ensure we can connect: connstr = ConnectionString.parse( self.make_connargs()['connection_string']) dummy_bucket = 'dummy' connstr = connstr.encode() args = self.make_connargs() args.pop('connection_string', None) args['bucket'] = dummy_bucket self.factory(connstr, **args) # OK, it exists self.assertRaises(CouchbaseError, self.factory, connstr) # Change the password self.bm.update_bucket(BucketSettings(name=dummy_bucket, max_ttl=500)) self.bm._admin_bucket.wait_ready(dummy_bucket, 10) dummy = self.bm.get_bucket('dummy') self.assertEqual(500, dummy.max_ttl) # Remove the bucket self.bm.drop_bucket('dummy') self.assertRaises(CouchbaseError, self.factory, connstr)
def test_cluster_sees_bucket(self): # Create the bucket self.bm.create_bucket( CreateBucketSettings(name='fred', ram_quota_mb=100)) # cluster should be able to return it (though, not right away) self.try_n_times(10, 3, self.cluster.bucket, 'fred')
def test_data_types(sdc_builder, sdc_executor, couchbase, input, converter_type, expected): bucket_name = get_random_string(string.ascii_letters, 10) document_key_field = 'mydocname' raw_dict = {"value": input, document_key_field: 'mydocid'} raw_data = json.dumps(raw_dict) cluster = couchbase.cluster # Build the pipeline builder = sdc_builder.get_pipeline_builder() source = builder.add_stage('Dev Raw Data Source').set_attributes( data_format='JSON', raw_data=raw_data, stop_after_first_batch=True) converter = builder.add_stage('Field Type Converter') converter.conversion_method = 'BY_FIELD' converter.field_type_converter_configs = [{ 'fields': ['/value'], 'targetType': converter_type, 'dataLocale': 'en,US', 'dateFormat': 'YYYY_MM_DD_HH_MM_SS', 'zonedDateTimeFormat': 'ISO_OFFSET_DATE_TIME', 'scale': 2 }] destination = builder.add_stage('Couchbase', type='destination') destination.set_attributes(authentication_mode='USER', bucket=bucket_name, document_key="${record:value('/" + document_key_field + "')}") source >> converter >> destination pipeline = builder.build().configure_for_environment(couchbase) pipeline.configuration["shouldRetry"] = False sdc_executor.add_pipeline(pipeline) try: logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket( CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) sdc_executor.start_pipeline(pipeline).wait_for_finished() bucket = cluster.bucket(bucket_name) doc_value = bucket.get(raw_dict[document_key_field]).value assert len(doc_value) == len(raw_dict) assert doc_value['value'] == expected finally: if pipeline and sdc_executor.get_pipeline_status( pipeline).response.json().get('status') == 'RUNNING': sdc_executor.stop_pipeline(pipeline) try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_bucket_create(self): self.bm.create_bucket( CreateBucketSettings(name="fred", bucket_type="couchbase", ram_quota_mb=100)) self.try_n_times(10, 1, self.bm.get_bucket, "fred")
authenticator=PasswordAuthenticator("Administrator", "password")) # For Server versions 6.5 or later you do not need to open a bucket here bucket = cluster.bucket("default") collection = bucket.default_collection() bucket_manager = cluster.buckets() # end::create_bucket_mgr[] # tag::create_bucket[] bucket_manager.create_bucket( CreateBucketSettings( name="hello", flush_enabled=False, ram_quota_mb=100, num_replicas=0, bucket_type=BucketType.COUCHBASE, conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER)) # end::create_bucket[] # Another approach to catch if bucket already exists try: bucket_manager.create_bucket( CreateBucketSettings( name="hello", flush_enabled=False, ram_quota_mb=100, num_replicas=0, bucket_type=BucketType.COUCHBASE, conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER))
def test_lookup_query(sdc_builder, sdc_executor, couchbase, test_name, input, expected, multiple_value_behavior, missing_value_behavior): bucket_name = get_random_string(string.ascii_letters, 10).lower() docs = [{'id': 'id1', 'data': 'hello'}, {'id': 'id2', 'data': 'hello'}, {'id': 'id3', 'data': 'hello'}] raw_dict = dict(criteria=input) raw_data = json.dumps(raw_dict) query = f"SELECT id FROM {bucket_name} WHERE " + '${record:value("/criteria")}' cluster = couchbase.cluster try: # populate the database logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket(CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) bucket = cluster.bucket(bucket_name) for doc in docs: bucket.upsert(doc['id'], doc) cluster.query(f'CREATE PRIMARY INDEX ON `{bucket_name}`').execute() # build the pipeline builder = sdc_builder.get_pipeline_builder() origin = builder.add_stage('Dev Raw Data Source') origin.set_attributes(data_format='JSON', stop_after_first_batch=True, raw_data=raw_data) lookup = builder.add_stage('Couchbase Lookup') lookup.set_attributes(authentication_mode='USER', bucket=bucket_name, lookup_type='N1QL', n1ql_query=query, n1ql_mappings=[dict(property='id', sdcField='/output')], multiple_value_behavior=multiple_value_behavior, missing_value_behavior=missing_value_behavior) wiretap = builder.add_wiretap() origin >> lookup >> wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) sdc_executor.start_pipeline(pipeline).wait_for_finished() output_records = wiretap.output_records error_records = wiretap.error_records print('output:', output_records) if missing_value_behavior == 'ERROR': # The input record should pass through to error records without an output field assert len(error_records) == 1 assert 'output' not in error_records[0].field elif not expected: # The input record should pass through to output records without an output field assert len(output_records) == 1 assert 'output' not in output_records[0].field else: assert len(output_records) == len(expected) # Check that the output records are as expected, allowing for reordering output_list = [record.field['output'] for record in output_records] assert Counter(output_list) == Counter(expected) finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")
def test_multiple_batches_query(sdc_builder, sdc_executor, couchbase, batch_size): bucket_name = get_random_string(string.ascii_letters, 10).lower() docs = [{"id": "1", "data": 10}, {"id": "2", "data": 20}, {"id": "3", "data": 30}] batches = 3 query = f'SELECT data FROM {bucket_name} WHERE ' + 'id="${record:value("/lookup")}"' cluster = couchbase.cluster # populate the database logger.info('Creating %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.create_bucket(CreateBucketSettings(name=bucket_name, bucket_type='couchbase', ram_quota_mb=256)) couchbase.wait_for_healthy_bucket(bucket_name) bucket = cluster.bucket(bucket_name) for doc in docs: bucket.upsert(doc["id"], doc) cluster.query(f'CREATE PRIMARY INDEX ON `{bucket_name}`').execute() # build the pipeline builder = sdc_builder.get_pipeline_builder() origin = builder.add_stage('Dev Data Generator') origin.fields_to_generate = [{ "type": "LONG_SEQUENCE", "field": "seq" }] expression = builder.add_stage('Expression Evaluator') expression.field_expressions = [{ 'fieldToSet': '/lookup', 'expression': '${record:value("/seq") % 3 + 1}' }] lookup = builder.add_stage('Couchbase Lookup') lookup.set_attributes(authentication_mode='USER', bucket=bucket_name, lookup_type='N1QL', n1ql_query=query, n1ql_mappings=[dict(property='data', sdcField='/output')], missing_value_behavior='PASS') wiretap = builder.add_wiretap() origin >> expression >> lookup >> wiretap.destination pipeline = builder.build().configure_for_environment(couchbase) sdc_executor.add_pipeline(pipeline) try: # run the pipeline sdc_executor.start_pipeline(pipeline).wait_for_pipeline_output_records_count(batches * batch_size) sdc_executor.stop_pipeline(pipeline) history = sdc_executor.get_pipeline_history(pipeline) record_count = history.latest.metrics.counter('pipeline.batchInputRecords.counter').count logger.info(f"Wrote {record_count} records") records = wiretap.output_records assert len(records) == record_count # Verify each record def sort_func(entry): return entry.field['seq'].value records.sort(key=sort_func) expected_number = 0 for record in records: assert record.field['seq'] == expected_number assert record.field['lookup'] == expected_number % 3 + 1 assert record.field['output'] == (expected_number % 3 + 1) * 10 expected_number = expected_number + 1 finally: try: logger.info('Deleting %s Couchbase bucket ...', bucket_name) couchbase.bucket_manager.drop_bucket(bucket_name) except Exception as e: logger.error(f"Can't delete bucket: {e}")