def test_adb_cb4_connection_no_cert(self): # noinspection PyBroadException try: _credentials_file = os.environ['HOME'] + "/adb-cb4-credentials" # specify the cluster and specify an authenticator containing a # username and password to be passed to the cluster. if not Path(_credentials_file).is_file(): sys.exit("*** credentials_file file " + _credentials_file + " can not be found!") self.get_credentials(_credentials_file) cluster = Cluster( 'couchbase://' + self.cb_host, ClusterOptions( PasswordAuthenticator(self.cb_user, self.cb_password))) # following a successful authentication, a bucket can be opened. # access a bucket in that cluster bucket = cluster.bucket('mdata') collection = bucket.default_collection() ingest_document_result = collection.get( "MD:V01:METAR:stations_ingest") print("test_adb_cb4_connection_no_cert: successfully read ", ingest_document_result.content) except: self.fail( "test_adb_cb4_connection_no_cert: TestConnection.test_connection Exception failure: " + str(sys.exc_info()))
def __init__(self, ssl_mode: str = 'none', n1ql_timeout: int = None, **kwargs): connection_string = 'couchbase://{host}?password={password}&{params}' connstr_params = parse.urlencode(kwargs["connstr_params"]) if ssl_mode == 'data': connection_string = connection_string.replace( 'couchbase', 'couchbases') connection_string += '&certpath=root.pem' connection_string = connection_string.format( host=kwargs['host'], password=kwargs['password'], params=connstr_params) pass_auth = PasswordAuthenticator(kwargs['username'], kwargs['password']) timeout = ClusterTimeoutOptions( kv_timeout=timedelta(seconds=self.TIMEOUT), query_timeout=timedelta( seconds=n1ql_timeout if n1ql_timeout else self.N1QL_TIMEOUT)) options = ClusterOptions(authenticator=pass_auth, timeout_options=timeout) self.cluster = Cluster(connection_string=connection_string, options=options) self.bucket_name = kwargs['bucket'] self.bucket = None self.collections = dict() self.collection = None
def check_mismatched_fcstValidEpoch_to_id(self): """This is a simple ultility test that can be used to see if there are any missmatched fcstValidEpoch values among the observations i.e. the fcstValidEpoch in the id does not match the fcstValidEpoch in the top level fcstValidEpoch field""" try: credentials_file = os.environ["HOME"] + "/adb-cb1-credentials" self.assertTrue( Path(credentials_file).is_file(), "credentials_file Does not exist") f = open(credentials_file) yaml_data = yaml.load(f, yaml.SafeLoader) host = yaml_data["cb_host"] user = yaml_data["cb_user"] password = yaml_data["cb_password"] options = ClusterOptions(PasswordAuthenticator(user, password)) cluster = Cluster("couchbase://" + host, options) result = cluster.query(""" select mdata.fcstValidEpoch, mdata.id FROM mdata WHERE mdata.docType = "obs" AND mdata.subset = "METAR" AND mdata.type = "DD" AND mdata.version = "V01" AND NOT CONTAINS(mdata.id,to_string(mdata.fcstValidEpoch)) """) for row in result: self.fail( "These do not have the same fcstValidEpoch: {0}".format( str(row['fcstValidEpoch']) + row['id'])) except Exception as e: self.fail("TestGsdIngestManager Exception failure: " + str(e))
def uploadDoc(self): # connect to cb cluster try: connection = "couchbase://" + self.server if "ip6" in self.server or self.server.startswith("["): connection = connection + "?ipv6=allow" cluster = Cluster( connection, ClusterOptions( PasswordAuthenticator(self.username, self.password))) # authenticator = PasswordAuthenticator(self.username, self.password) # cluster.authenticate(authenticator) cb = cluster.bucket(self.bucket) cb_coll = cb.default_collection() cb.timeout = 100 except Exception as e: logging.error("Connection error\n" + traceback.format_exc()) json_docs = {} for i in range(self.startseqnum, self.startseqnum + self.num_docs): self.createContent() dockey = self.keyprefix + str(i) json_docs[dockey] = self.json_objs_dict BYTES_PER_BATCH = 1024 * 256 # 256K batches = [] cur_batch = {} cur_size = 0 batches.append(cur_batch) for key, value in list(json_docs.items()): cur_batch[key] = value cur_size += len(key) + len(value) + 24 if cur_size > BYTES_PER_BATCH: cur_batch = {} batches.append(cur_batch) cur_size = 0 num_completed = 0 while batches: batch = batches[-1] try: cb_coll.upsert_multi(batch) num_completed += len(batch) batches.pop() except CouchbaseTransientError as e: logging.error(e) ok, fail = e.split_results() new_batch = {} for key in fail: new_batch[key] = list(json_docs.items())[key] batches.pop() batches.append(new_batch) num_completed += len(ok) logging.info("Retrying {}/{} items".format( len(new_batch), len(ok))) logging.info("Completed {}/{} items".format( num_completed, len(json_docs))) if self.xattrs: logging.info("Upserting xattrs") self.add_xattrs(cb)
def connect(self): db_config = get_db_config() self.host = db_config['host'] self.bucket_name = db_config['bucket'] self.username = db_config['username'] self.password = db_config['password'] connection_str = 'couchbase://{0}'.format(self.host) if db_config['secure'] == 'true': connection_str = 'couchbases://{0}?ssl=no_verify'.format(self.host) print(connection_str) secure = db_config['secure'] print(secure) try: options = ClusterOptions( PasswordAuthenticator(self.username, self.password)) self.__cluster = Cluster(connection_str, options) self.__bucket = self.__cluster.bucket(self.bucket_name) self.__collection = self.__bucket.default_collection() output_message('repository') if len(self.__bucket.name) > 0: output_message( self.__bucket.name, 'repository.py:connect() - connected to bucket:') else: output_message( 'repository.py:connect() - error connecting to bucket.') except Exception as ex: output_message( ex, 'repository.py:connect() - error connecting to bucket.') raise
def get_couchbase_db(): pa = PasswordAuthenticator('cosso_user', 'passw0rd') try: cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa)) return cluster except: print("exception:", sys.exc_info()[0]) return "Could not connect to CouchDB"
def test_async(self): print("blockingtoasync") #tag::blockingtoasync[] cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password"))) bucket = cluster.bucket("travel-sample") # Same API as Bucket, but completely async with asyncio Futures from acouchbase.bucket import Bucket async_bucket=Bucket("couchbase://localhost/default") cluster.disconnect() #end::blockingtoasync[] print("reactivecluster") #tag::reactivecluster[] from acouchbase.bucket import Bucket cluster = Cluster("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")),bucket_class=Bucket) bucket = cluster.bucket("travel-sample") # A reactive cluster's disconnect methods returns a Mono<Void>. # Nothing actually happens until you subscribe to the Mono. # The simplest way to subscribe is to await completion by calling call `block()`. cluster.disconnect() #end::reactivecluster[] print("asynccluster") #tag::asynccluster[] cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password"))) bucket = cluster.bucket("travel-sample") # An async cluster's disconnect methods returns a CompletableFuture<Void>. # The disconnection starts as soon as you call disconnect(). # The simplest way to wait for the disconnect to complete is to call `join()`. cluster.disconnect().join() #end::asynccluster[] print("tls") #tag::tls[] cluster = Cluster("couchbases://localhost",ClusterOptions(PasswordAuthenticator("username","password",cert_path="/path/to/cluster.crt"))) #end::tls[] print("dnssrv") #tag::dnssrv[] env = ClusterEnvironment.builder() \ .ioConfig(IoConfig.enableDnsSrv(true)) \ .build()
def connect_couchbase(bucket_name): # get a reference to our cluster cluster = Cluster(host_url, ClusterOptions(PasswordAuthenticator(user, password))) # get a reference to our bucket cb = cluster.bucket('default') # get a reference to the default collection collection = cb.default_collection() return collection
def get_cb_bucket(cdbname): pa = PasswordAuthenticator(cb_su, cb_pass) try: cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa)) db = cluster.bucket(cdbname) return db except couchbase.exceptions.CouchbaseError as err: print(err) return "an error occurred"
def _createConn(self): try: cluster = Cluster(self.connection_string, ClusterOptions(PasswordAuthenticator(self.bucket, 'password'))) #cluster.authenticate(PasswordAuthenticator(self.bucket, 'password')) self.cb = cluster.bucket(self.bucket) self.default_collection = self.cb.default_collection() except BucketNotFoundError: raise except AuthError: # Try using default user created by the tests, if any, in case there is no user with bucket name in the # cluster. try: cluster = Cluster(self.connection_string, ClusterOptions(PasswordAuthenticator("cbadminbucket", 'password')), bucket_class=CouchbaseBucket) self.cb = cluster.bucket(self.bucket) self.default_collection = self.cb.default_collection() except AuthError: raise
def init_connection(self, args): global couchbase_connection connection_string = 'couchbase://%s' % (args['server']) couchbase_cluster = Cluster( connection_string, ClusterOptions( PasswordAuthenticator(args['username'], args['password']))) couchbase_bucket = couchbase_cluster.bucket(args['bucket']) self.bucket_name = args['bucket'] self.couchbase_connection = couchbase_bucket.default_collection()
def connect_cb(self): # get a reference to our cluster # noinspection PyBroadException try: options = ClusterOptions( PasswordAuthenticator(self.cb_credentials['user'], self.cb_credentials['password'])) self.cluster = Cluster('couchbase://' + self.cb_credentials['host'], options) self.collection = self.cluster.bucket("mdata").default_collection() except: print("*** %s in connect_cb ***" + str(sys.exc_info())) sys.exit("*** Error when connecting to mysql database: ")
def __init__(self): pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET, self.COUCHBASE_PASSWORD) options = ClusterOptions(authenticator=pass_auth) self.cluster = Cluster(connection_string=self.connection_string, options=options) self.bucket = self.cluster.bucket( self.COUCHBASE_BUCKET).default_collection() self.jenkins = JenkinsScanner() self.ps = PerfStore(host=CBMONITOR_HOST) self.weekly = Weekly()
def __init__(self): # CONNECT DB self.cluster = Cluster( 'couchbase://couchbase', ClusterOptions( PasswordAuthenticator(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD')))) # BUCKET self.bucket_items = self.cluster.bucket('items').default_collection() self.bucket_var = self.cluster.bucket('var').default_collection()
def __init__(self, host, bucket, username, password, quiet=True, port=8091): connection_string = 'couchbase://{}?password={}'.format(host, password) pass_auth = PasswordAuthenticator(username, password) timeout = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=self.TIMEOUT)) options = ClusterOptions(authenticator=pass_auth, timeout_options=timeout) self.cluster = Cluster(connection_string=connection_string, options=options) self.bucket = self.cluster.bucket(bucket) self.client = self.bucket.default_collection() self.use_count = 0 self.use_time = 0 self.last_use_time = 0
def __init__(self): pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET, self.COUCHBASE_PASSWORD) options = ClusterOptions(authenticator=pass_auth) self.cluster = Cluster(connection_string=self.connection_string, options=options) self.bucket = self.cluster.bucket( self.COUCHBASE_BUCKET).default_collection() self.jenkins = jenkins.Jenkins(self.JENKINS_URL) self.weekly = Weekly() self.jobs = set()
class CouchbaseHelper: password_authenticator = PasswordAuthenticator(CouchbaseConfig.USERNAME, CouchbaseConfig.PASSWORD) cluster_options = ClusterOptions(password_authenticator) cluster = Cluster(CouchbaseConfig.URL, cluster_options) bucket = cluster.bucket(CouchbaseConfig.BUCKET_NANE) collection = bucket.default_collection() def __init__(self, *args, **kwargs): self.id = str(uuid.uuid4())
def file_loc_processing(): global file_counter # get a reference to our cluster cluster = Cluster( 'couchbase://' + options.host, ClusterOptions(PasswordAuthenticator(options.user, options.password))) file_loc = options.agent_env['HVR_FILE_LOC'] tbl_map = table_file_name_map() client = cluster.bucket(options.database) db = cb_coll = client.default_collection() for t in tbl_map: base_name = table_name_normalize(t[0]) hvr_schema = base_name[0] hvr_base_table = base_name[1] collection = options.collection.format(hvr_schema=hvr_schema, hvr_base_name=hvr_base_table, hvr_tbl_name=t[1]) keys = t[3].split(",") # if collection is absent - no any error # if options.recreate : # trace("Dropping collection '{0}'".format(collection)) # db.drop_collection(collection) for name in tbl_map[t]: full_name = file_loc + '/' + name trace("Reading and parsing file '" + full_name + "' ... ") try: with open(full_name) as json_file: for line in json_file: json_obj = json.loads(line) try: #if options.mode == SetupMode.MODE_SOFTDELETE : #added new pseudo column column_id = '' for column in keys: column_id = column_id + "::" + str( json_obj[column]) #trace("key will equal:" + str(column_id.replace('::', "", 1)) + " /n") mykey = str(hvr_base_table) + str(column_id) trace("key will equal:" + mykey + " /n") result = db.upsert(mykey, str(json_obj)) #trace(str(result.cas) + " mykey:" + str(mykey) + " data:" + str(json_obj)) except Exception as e: trace(e) # remove successfully transmitted file os.remove(full_name) file_counter = file_counter + 1 except IOError as err: raise Exception("Couldn't open file " + full_name)
def connect_cb(self): logging.info(self.threadName + ': data_type_manager - Connecting to couchbase') # get a reference to our cluster # noinspection PyBroadException try: options = ClusterOptions( PasswordAuthenticator(self.cb_credentials['user'], self.cb_credentials['password'])) self.cluster = Cluster('couchbase://' + self.cb_credentials['host'], options) self.collection = self.cluster.bucket("mdata").default_collection() logging.info(self.threadName + ': Couchbase connection success') except Exception as e: logging.error("*** %s in connect_cb ***" + str(e)) sys.exit("*** Error when connecting to mysql database: ")
def connect_cb(self): # noinspection PyBroadException try: _f = open(os.environ['HOME'] + '/adb-credentials-local') _yaml_data = yaml.load(_f, yaml.SafeLoader) _cb_host = _yaml_data['cb_host'] _cb_user = _yaml_data['cb_user'] _cb_password = _yaml_data['cb_password'] options = ClusterOptions( PasswordAuthenticator(_cb_user, _cb_password)) self.cluster = Cluster('couchbase://' + _cb_host, options) self.collection = self.cluster.bucket("mdata").default_collection() except: self.fail("*** %s in connect_cb ***" + str(sys.exc_info()[0])) sys.exit("*** Error when connecting to mysql database: ")
def __init__(self, **kwargs): connection_string = 'couchbase://{host}?password={password}' connection_string = connection_string.format( host=kwargs['host'], password=kwargs['password']) pass_auth = PasswordAuthenticator(kwargs['username'], kwargs['password']) timeout = ClusterTimeoutOptions(kv_timeout=timedelta( seconds=self.TIMEOUT)) options = ClusterOptions(authenticator=pass_auth, timeout_options=timeout) self.cluster = TxCluster(connection_string=connection_string, options=options) self.bucket_name = kwargs['bucket'] self.collections = dict() self.collection = None
class CouchbaseHelper: password_authenticator = PasswordAuthenticator(CouchbaseConfig.USERNAME, CouchbaseConfig.PASSWORD) cluster_options = ClusterOptions(password_authenticator) cluster = Cluster(CouchbaseConfig.URL, cluster_options) bucket = cluster.bucket(CouchbaseConfig.BUCKET_NANE) collection = bucket.default_collection() def __init__(self, *args, **kwargs): self.id = str(uuid.uuid4()) @classmethod def create(cls, obj, schema_cls): schema = schema_cls() result = schema.dump(obj) cls.collection.upsert(str(uuid.uuid4()), result)
def connect_cb(self): """ create a couchbase connection and maintain the collection and cluster objects. """ logging.info("%s: data_type_manager - Connecting to couchbase") # get a reference to our cluster # noinspection PyBroadException try: options = ClusterOptions( PasswordAuthenticator(self.cb_credentials["user"], self.cb_credentials["password"])) self.cluster = Cluster( "couchbase://" + self.cb_credentials["host"], options) self.collection = self.cluster.bucket("mdata").default_collection() logging.info("%s: Couchbase connection success") except Exception as _e: # pylint:disable=broad-except logging.error("*** %s in connect_cb ***", str(_e)) sys.exit("*** Error when connecting to mysql database: ")
def test_simpleconnect(self): #tag::simpleconnect[] cluster = Cluster.connect("127.0.0.1", ClusterOptions(PasswordAuthenticator("username", "password"))) bucket = cluster.bucket("travel-sample") collection = bucket.default_collection() # You can access multiple buckets using the same Cluster object. another_bucket = cluster.bucket("beer-sample") # You can access collections other than the default # if your version of Couchbase Server supports this feature. customer_a = bucket.scope("customer-a") widgets = customer_a.collection("widgets") #end::simpleconnect[] # For a graceful shutdown, disconnect from the cluster when the program ends. cluster.disconnect()
def connection(self, client_ip, bucket_name, user, password): log.info( "Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name, user, \ password)) result = False connection_string = 'couchbase://' + client_ip + '/' + bucket_name + '?username='******'&select_bucket=true' log.info( " Value of connection string is - {0}".format(connection_string)) time.sleep(2) try: cluster = Cluster( 'couchbase://' + client_ip, ClusterOptions(PasswordAuthenticator(user, password))) cb = cluster.bucket(bucket_name) default_collection = cb.default_collection() if cb is not None: result = True return default_collection, result except Exception as ex: log.info("Exception in creating an SDK connection {0}".format(ex)) return result
def connect_to_cluster(host: str, user: str, password: str, bucket: str, services: List[ServiceType] = [ServiceType.Query]): """Creates a connection to a cluster and checks its connected to the given services before returning.""" cluster = Cluster(host, ClusterOptions(PasswordAuthenticator(user, password))) cb = cluster.bucket(bucket) # pylint: disable=unused-variable for _ in range(100): result = cb.ping(PingOptions(service_types=services)) ready = True for service in services: try: if result.endpoints[service][0].state != PingState.OK: ready = False except (KeyError, IndexError) as e: raise AssertionError( f"Service {service.value} not available") from e if ready: return cluster, cb time.sleep(1) raise AssertionError("Failed to connect to cluster")
def __init__(self, num_items, host, bucket, password, collections=None, small=True): if collections: self.use_collection = True else: self.use_collection = False self.num_items = num_items if small: self.kv_cls = KeyValueIterator self.field_cls = NewFieldIterator else: self.kv_cls = KeyLargeValueIterator self.field_cls = NewLargeFieldIterator self.kv_iterator = self.kv_cls(self.num_items) self.field_iterator = self.field_cls(self.num_items) cb_version = pkg_resources.get_distribution("couchbase").version if cb_version[0] == '2': self.cb = Connection(bucket=bucket, host=host, password=password) elif cb_version[0] == '3': connection_string = 'couchbase://{host}?password={password}' connection_string = connection_string.format(host=host, password=password) pass_auth = PasswordAuthenticator(bucket, password) self.cluster = TxCluster(connection_string=connection_string, options=ClusterOptions(pass_auth)) self.bucket = self.cluster.bucket(bucket) self.collection = self.bucket.scope("scope-1").collection( "collection-1") self.fraction = 1 self.iteration = 0
import os import pprint import sys import couchbase import requests from datetime import datetime from couchbase.cluster import Cluster, ClusterOptions from couchbase_core.cluster import PasswordAuthenticator # CONNECT DB cluster = Cluster( 'couchbase://couchbase', ClusterOptions( PasswordAuthenticator(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD')))) # BUCKET ITEMS cb = cluster.bucket('items').default_collection() # CREATION DES INDEX cb.query('CREATE PRIMARY INDEX `items-primary-index` ON `items` USING GSI;' ).execute() cb.query('CREATE INDEX `items-corrupted-index` ON items(corrupted) USING GSI;' ).execute() cb.query( 'CREATE INDEX `items-learning-index` ON items(learningData) USING GSI;' ).execute()
""" [source,python] ---- """ # tag::connect[] # needed for any cluster connection from couchbase.cluster import Cluster, ClusterOptions from couchbase_core.cluster import PasswordAuthenticator # needed to support SQL++ (N1QL) query from couchbase.cluster import QueryOptions # get a reference to our cluster cluster = Cluster( 'couchbase://localhost', ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) # end::connect[] # tag::bucket[] # get a reference to our bucket cb = cluster.bucket('travel-sample') # end::bucket[] # tag::default-collection[] # get a reference to the default collection cb_coll = cb.default_collection() # end::default-collection[] # tag::upsert-func[]
print(result.cas) except Exception as e: print(e) # needed for any cluster connection from couchbase.cluster import Cluster, ClusterOptions from couchbase_core.cluster import PasswordAuthenticator # needed to support SQL++ (N1QL) query from couchbase.cluster import QueryOptions # get a reference to our cluster cluster = Cluster('couchbase://localhost', ClusterOptions( PasswordAuthenticator('Administrator', db_password)), lockmode=2) # added lockmode to avoid locking error # get a reference to our bucket cb = cluster.bucket('events') # get a reference to the default collection cb_coll = cb.default_collection() app = Flask(__name__) @app.route("/webhooks/answer") def answer(): params = request.args pprint(params)