def test_adb_cb4_connection_no_cert(self):
# noinspection PyBroadException
try:
_credentials_file = os.environ['HOME'] + "/adb-cb4-credentials"
# specify the cluster and specify an authenticator containing a
# username and password to be passed to the cluster.
if not Path(_credentials_file).is_file():
sys.exit("*** credentials_file file " + _credentials_file +
" can not be found!")
self.get_credentials(_credentials_file)
cluster = Cluster(
'couchbase://' + self.cb_host,
ClusterOptions(
PasswordAuthenticator(self.cb_user, self.cb_password)))
# following a successful authentication, a bucket can be opened.
# access a bucket in that cluster
bucket = cluster.bucket('mdata')
collection = bucket.default_collection()
ingest_document_result = collection.get(
"MD:V01:METAR:stations_ingest")
print("test_adb_cb4_connection_no_cert: successfully read ",
ingest_document_result.content)
except:
self.fail(
"test_adb_cb4_connection_no_cert: TestConnection.test_connection Exception failure: "
+ str(sys.exc_info()))
def __init__(self,
ssl_mode: str = 'none',
n1ql_timeout: int = None,
**kwargs):
connection_string = 'couchbase://{host}?password={password}&{params}'
connstr_params = parse.urlencode(kwargs["connstr_params"])
if ssl_mode == 'data':
connection_string = connection_string.replace(
'couchbase', 'couchbases')
connection_string += '&certpath=root.pem'
connection_string = connection_string.format(
host=kwargs['host'],
password=kwargs['password'],
params=connstr_params)
pass_auth = PasswordAuthenticator(kwargs['username'],
kwargs['password'])
timeout = ClusterTimeoutOptions(
kv_timeout=timedelta(seconds=self.TIMEOUT),
query_timeout=timedelta(
seconds=n1ql_timeout if n1ql_timeout else self.N1QL_TIMEOUT))
options = ClusterOptions(authenticator=pass_auth,
timeout_options=timeout)
self.cluster = Cluster(connection_string=connection_string,
options=options)
self.bucket_name = kwargs['bucket']
self.bucket = None
self.collections = dict()
self.collection = None
def check_mismatched_fcstValidEpoch_to_id(self):
"""This is a simple ultility test that can be used to see if there are
any missmatched fcstValidEpoch values among the observations i.e. the fcstValidEpoch in the id
does not match the fcstValidEpoch in the top level fcstValidEpoch field"""
try:
credentials_file = os.environ["HOME"] + "/adb-cb1-credentials"
self.assertTrue(
Path(credentials_file).is_file(),
"credentials_file Does not exist")
f = open(credentials_file)
yaml_data = yaml.load(f, yaml.SafeLoader)
host = yaml_data["cb_host"]
user = yaml_data["cb_user"]
password = yaml_data["cb_password"]
options = ClusterOptions(PasswordAuthenticator(user, password))
cluster = Cluster("couchbase://" + host, options)
result = cluster.query("""
select mdata.fcstValidEpoch, mdata.id
FROM mdata
WHERE
mdata.docType = "obs"
AND mdata.subset = "METAR"
AND mdata.type = "DD"
AND mdata.version = "V01"
AND NOT CONTAINS(mdata.id,to_string(mdata.fcstValidEpoch)) """)
for row in result:
self.fail(
"These do not have the same fcstValidEpoch: {0}".format(
str(row['fcstValidEpoch']) + row['id']))
except Exception as e:
self.fail("TestGsdIngestManager Exception failure: " + str(e))
def uploadDoc(self):
# connect to cb cluster
try:
connection = "couchbase://" + self.server
if "ip6" in self.server or self.server.startswith("["):
connection = connection + "?ipv6=allow"
cluster = Cluster(
connection,
ClusterOptions(
PasswordAuthenticator(self.username, self.password)))
# authenticator = PasswordAuthenticator(self.username, self.password)
# cluster.authenticate(authenticator)
cb = cluster.bucket(self.bucket)
cb_coll = cb.default_collection()
cb.timeout = 100
except Exception as e:
logging.error("Connection error\n" + traceback.format_exc())
json_docs = {}
for i in range(self.startseqnum, self.startseqnum + self.num_docs):
self.createContent()
dockey = self.keyprefix + str(i)
json_docs[dockey] = self.json_objs_dict
BYTES_PER_BATCH = 1024 * 256 # 256K
batches = []
cur_batch = {}
cur_size = 0
batches.append(cur_batch)
for key, value in list(json_docs.items()):
cur_batch[key] = value
cur_size += len(key) + len(value) + 24
if cur_size > BYTES_PER_BATCH:
cur_batch = {}
batches.append(cur_batch)
cur_size = 0
num_completed = 0
while batches:
batch = batches[-1]
try:
cb_coll.upsert_multi(batch)
num_completed += len(batch)
batches.pop()
except CouchbaseTransientError as e:
logging.error(e)
ok, fail = e.split_results()
new_batch = {}
for key in fail:
new_batch[key] = list(json_docs.items())[key]
batches.pop()
batches.append(new_batch)
num_completed += len(ok)
logging.info("Retrying {}/{} items".format(
len(new_batch), len(ok)))
logging.info("Completed {}/{} items".format(
num_completed, len(json_docs)))
if self.xattrs:
logging.info("Upserting xattrs")
self.add_xattrs(cb)
def connect(self):
db_config = get_db_config()
self.host = db_config['host']
self.bucket_name = db_config['bucket']
self.username = db_config['username']
self.password = db_config['password']
connection_str = 'couchbase://{0}'.format(self.host)
if db_config['secure'] == 'true':
connection_str = 'couchbases://{0}?ssl=no_verify'.format(self.host)
print(connection_str)
secure = db_config['secure']
print(secure)
try:
options = ClusterOptions(
PasswordAuthenticator(self.username, self.password))
self.__cluster = Cluster(connection_str, options)
self.__bucket = self.__cluster.bucket(self.bucket_name)
self.__collection = self.__bucket.default_collection()
output_message('repository')
if len(self.__bucket.name) > 0:
output_message(
self.__bucket.name,
'repository.py:connect() - connected to bucket:')
else:
output_message(
'repository.py:connect() - error connecting to bucket.')
except Exception as ex:
output_message(
ex, 'repository.py:connect() - error connecting to bucket.')
raise
def get_couchbase_db():
pa = PasswordAuthenticator('cosso_user', 'passw0rd')
try:
cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa))
return cluster
except:
print("exception:", sys.exc_info()[0])
return "Could not connect to CouchDB"
def test_async(self):
print("blockingtoasync")
#tag::blockingtoasync[]
cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")))
bucket = cluster.bucket("travel-sample")
# Same API as Bucket, but completely async with asyncio Futures
from acouchbase.bucket import Bucket
async_bucket=Bucket("couchbase://localhost/default")
cluster.disconnect()
#end::blockingtoasync[]
print("reactivecluster")
#tag::reactivecluster[]
from acouchbase.bucket import Bucket
cluster = Cluster("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")),bucket_class=Bucket)
bucket = cluster.bucket("travel-sample")
# A reactive cluster's disconnect methods returns a Mono<Void>.
# Nothing actually happens until you subscribe to the Mono.
# The simplest way to subscribe is to await completion by calling call `block()`.
cluster.disconnect()
#end::reactivecluster[]
print("asynccluster")
#tag::asynccluster[]
cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")))
bucket = cluster.bucket("travel-sample")
# An async cluster's disconnect methods returns a CompletableFuture<Void>.
# The disconnection starts as soon as you call disconnect().
# The simplest way to wait for the disconnect to complete is to call `join()`.
cluster.disconnect().join()
#end::asynccluster[]
print("tls")
#tag::tls[]
cluster = Cluster("couchbases://localhost",ClusterOptions(PasswordAuthenticator("username","password",cert_path="/path/to/cluster.crt")))
#end::tls[]
print("dnssrv")
#tag::dnssrv[]
env = ClusterEnvironment.builder() \
.ioConfig(IoConfig.enableDnsSrv(true)) \
.build()
def connect_couchbase(bucket_name):
# get a reference to our cluster
cluster = Cluster(host_url, ClusterOptions(PasswordAuthenticator(user, password)))
# get a reference to our bucket
cb = cluster.bucket('default')
# get a reference to the default collection
collection = cb.default_collection()
return collection
def get_cb_bucket(cdbname):
pa = PasswordAuthenticator(cb_su, cb_pass)
try:
cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa))
db = cluster.bucket(cdbname)
return db
except couchbase.exceptions.CouchbaseError as err:
print(err)
return "an error occurred"
def _createConn(self):
try:
cluster = Cluster(self.connection_string, ClusterOptions(PasswordAuthenticator(self.bucket, 'password')))
#cluster.authenticate(PasswordAuthenticator(self.bucket, 'password'))
self.cb = cluster.bucket(self.bucket)
self.default_collection = self.cb.default_collection()
except BucketNotFoundError:
raise
except AuthError:
# Try using default user created by the tests, if any, in case there is no user with bucket name in the
# cluster.
try:
cluster = Cluster(self.connection_string,
ClusterOptions(PasswordAuthenticator("cbadminbucket", 'password')),
bucket_class=CouchbaseBucket)
self.cb = cluster.bucket(self.bucket)
self.default_collection = self.cb.default_collection()
except AuthError:
raise
def init_connection(self, args):
global couchbase_connection
connection_string = 'couchbase://%s' % (args['server'])
couchbase_cluster = Cluster(
connection_string,
ClusterOptions(
PasswordAuthenticator(args['username'], args['password'])))
couchbase_bucket = couchbase_cluster.bucket(args['bucket'])
self.bucket_name = args['bucket']
self.couchbase_connection = couchbase_bucket.default_collection()
def connect_cb(self):
# get a reference to our cluster
# noinspection PyBroadException
try:
options = ClusterOptions(
PasswordAuthenticator(self.cb_credentials['user'], self.cb_credentials['password']))
self.cluster = Cluster('couchbase://' + self.cb_credentials['host'], options)
self.collection = self.cluster.bucket("mdata").default_collection()
except:
print("*** %s in connect_cb ***" + str(sys.exc_info()))
sys.exit("*** Error when connecting to mysql database: ")
def __init__(self):
pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET,
self.COUCHBASE_PASSWORD)
options = ClusterOptions(authenticator=pass_auth)
self.cluster = Cluster(connection_string=self.connection_string,
options=options)
self.bucket = self.cluster.bucket(
self.COUCHBASE_BUCKET).default_collection()
self.jenkins = JenkinsScanner()
self.ps = PerfStore(host=CBMONITOR_HOST)
self.weekly = Weekly()
def __init__(self):
# CONNECT DB
self.cluster = Cluster(
'couchbase://couchbase',
ClusterOptions(
PasswordAuthenticator(os.environ.get('DB_USER'),
os.environ.get('DB_PASSWORD'))))
# BUCKET
self.bucket_items = self.cluster.bucket('items').default_collection()
self.bucket_var = self.cluster.bucket('var').default_collection()
def __init__(self, host, bucket, username, password, quiet=True, port=8091):
connection_string = 'couchbase://{}?password={}'.format(host, password)
pass_auth = PasswordAuthenticator(username, password)
timeout = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=self.TIMEOUT))
options = ClusterOptions(authenticator=pass_auth, timeout_options=timeout)
self.cluster = Cluster(connection_string=connection_string, options=options)
self.bucket = self.cluster.bucket(bucket)
self.client = self.bucket.default_collection()
self.use_count = 0
self.use_time = 0
self.last_use_time = 0
def __init__(self):
pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET,
self.COUCHBASE_PASSWORD)
options = ClusterOptions(authenticator=pass_auth)
self.cluster = Cluster(connection_string=self.connection_string,
options=options)
self.bucket = self.cluster.bucket(
self.COUCHBASE_BUCKET).default_collection()
self.jenkins = jenkins.Jenkins(self.JENKINS_URL)
self.weekly = Weekly()
self.jobs = set()
class CouchbaseHelper:
password_authenticator = PasswordAuthenticator(CouchbaseConfig.USERNAME,
CouchbaseConfig.PASSWORD)
cluster_options = ClusterOptions(password_authenticator)
cluster = Cluster(CouchbaseConfig.URL, cluster_options)
bucket = cluster.bucket(CouchbaseConfig.BUCKET_NANE)
collection = bucket.default_collection()
def __init__(self, *args, **kwargs):
self.id = str(uuid.uuid4())
def file_loc_processing():
global file_counter
# get a reference to our cluster
cluster = Cluster(
'couchbase://' + options.host,
ClusterOptions(PasswordAuthenticator(options.user, options.password)))
file_loc = options.agent_env['HVR_FILE_LOC']
tbl_map = table_file_name_map()
client = cluster.bucket(options.database)
db = cb_coll = client.default_collection()
for t in tbl_map:
base_name = table_name_normalize(t[0])
hvr_schema = base_name[0]
hvr_base_table = base_name[1]
collection = options.collection.format(hvr_schema=hvr_schema,
hvr_base_name=hvr_base_table,
hvr_tbl_name=t[1])
keys = t[3].split(",")
# if collection is absent - no any error
# if options.recreate :
# trace("Dropping collection '{0}'".format(collection))
# db.drop_collection(collection)
for name in tbl_map[t]:
full_name = file_loc + '/' + name
trace("Reading and parsing file '" + full_name + "' ... ")
try:
with open(full_name) as json_file:
for line in json_file:
json_obj = json.loads(line)
try:
#if options.mode == SetupMode.MODE_SOFTDELETE :
#added new pseudo column
column_id = ''
for column in keys:
column_id = column_id + "::" + str(
json_obj[column])
#trace("key will equal:" + str(column_id.replace('::', "", 1)) + " /n")
mykey = str(hvr_base_table) + str(column_id)
trace("key will equal:" + mykey + " /n")
result = db.upsert(mykey, str(json_obj))
#trace(str(result.cas) + " mykey:" + str(mykey) + " data:" + str(json_obj))
except Exception as e:
trace(e)
# remove successfully transmitted file
os.remove(full_name)
file_counter = file_counter + 1
except IOError as err:
raise Exception("Couldn't open file " + full_name)
def connect_cb(self):
logging.info(self.threadName + ': data_type_manager - Connecting to couchbase')
# get a reference to our cluster
# noinspection PyBroadException
try:
options = ClusterOptions(
PasswordAuthenticator(self.cb_credentials['user'], self.cb_credentials['password']))
self.cluster = Cluster('couchbase://' + self.cb_credentials['host'], options)
self.collection = self.cluster.bucket("mdata").default_collection()
logging.info(self.threadName + ': Couchbase connection success')
except Exception as e:
logging.error("*** %s in connect_cb ***" + str(e))
sys.exit("*** Error when connecting to mysql database: ")
def connect_cb(self):
# noinspection PyBroadException
try:
_f = open(os.environ['HOME'] + '/adb-credentials-local')
_yaml_data = yaml.load(_f, yaml.SafeLoader)
_cb_host = _yaml_data['cb_host']
_cb_user = _yaml_data['cb_user']
_cb_password = _yaml_data['cb_password']
options = ClusterOptions(
PasswordAuthenticator(_cb_user, _cb_password))
self.cluster = Cluster('couchbase://' + _cb_host, options)
self.collection = self.cluster.bucket("mdata").default_collection()
except:
self.fail("*** %s in connect_cb ***" + str(sys.exc_info()[0]))
sys.exit("*** Error when connecting to mysql database: ")
def __init__(self, **kwargs):
connection_string = 'couchbase://{host}?password={password}'
connection_string = connection_string.format(
host=kwargs['host'], password=kwargs['password'])
pass_auth = PasswordAuthenticator(kwargs['username'],
kwargs['password'])
timeout = ClusterTimeoutOptions(kv_timeout=timedelta(
seconds=self.TIMEOUT))
options = ClusterOptions(authenticator=pass_auth,
timeout_options=timeout)
self.cluster = TxCluster(connection_string=connection_string,
options=options)
self.bucket_name = kwargs['bucket']
self.collections = dict()
self.collection = None
class CouchbaseHelper:
password_authenticator = PasswordAuthenticator(CouchbaseConfig.USERNAME, CouchbaseConfig.PASSWORD)
cluster_options = ClusterOptions(password_authenticator)
cluster = Cluster(CouchbaseConfig.URL, cluster_options)
bucket = cluster.bucket(CouchbaseConfig.BUCKET_NANE)
collection = bucket.default_collection()
def __init__(self, *args, **kwargs):
self.id = str(uuid.uuid4())
@classmethod
def create(cls, obj, schema_cls):
schema = schema_cls()
result = schema.dump(obj)
cls.collection.upsert(str(uuid.uuid4()), result)
def connect_cb(self):
"""
create a couchbase connection and maintain the collection and cluster objects.
"""
logging.info("%s: data_type_manager - Connecting to couchbase")
# get a reference to our cluster
# noinspection PyBroadException
try:
options = ClusterOptions(
PasswordAuthenticator(self.cb_credentials["user"],
self.cb_credentials["password"]))
self.cluster = Cluster(
"couchbase://" + self.cb_credentials["host"], options)
self.collection = self.cluster.bucket("mdata").default_collection()
logging.info("%s: Couchbase connection success")
except Exception as _e: # pylint:disable=broad-except
logging.error("*** %s in connect_cb ***", str(_e))
sys.exit("*** Error when connecting to mysql database: ")
def test_simpleconnect(self):
#tag::simpleconnect[]
cluster = Cluster.connect("127.0.0.1", ClusterOptions(PasswordAuthenticator("username", "password")))
bucket = cluster.bucket("travel-sample")
collection = bucket.default_collection()
# You can access multiple buckets using the same Cluster object.
another_bucket = cluster.bucket("beer-sample")
# You can access collections other than the default
# if your version of Couchbase Server supports this feature.
customer_a = bucket.scope("customer-a")
widgets = customer_a.collection("widgets")
#end::simpleconnect[]
# For a graceful shutdown, disconnect from the cluster when the program ends.
cluster.disconnect()
def connection(self, client_ip, bucket_name, user, password):
log.info(
"Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name, user, \
password))
result = False
connection_string = 'couchbase://' + client_ip + '/' + bucket_name + '?username='******'&select_bucket=true'
log.info(
" Value of connection string is - {0}".format(connection_string))
time.sleep(2)
try:
cluster = Cluster(
'couchbase://' + client_ip,
ClusterOptions(PasswordAuthenticator(user, password)))
cb = cluster.bucket(bucket_name)
default_collection = cb.default_collection()
if cb is not None:
result = True
return default_collection, result
except Exception as ex:
log.info("Exception in creating an SDK connection {0}".format(ex))
return result
def connect_to_cluster(host: str,
user: str,
password: str,
bucket: str,
services: List[ServiceType] = [ServiceType.Query]):
"""Creates a connection to a cluster and checks its connected to the given services before returning."""
cluster = Cluster(host,
ClusterOptions(PasswordAuthenticator(user, password)))
cb = cluster.bucket(bucket) # pylint: disable=unused-variable
for _ in range(100):
result = cb.ping(PingOptions(service_types=services))
ready = True
for service in services:
try:
if result.endpoints[service][0].state != PingState.OK:
ready = False
except (KeyError, IndexError) as e:
raise AssertionError(
f"Service {service.value} not available") from e
if ready:
return cluster, cb
time.sleep(1)
raise AssertionError("Failed to connect to cluster")
def __init__(self,
num_items,
host,
bucket,
password,
collections=None,
small=True):
if collections:
self.use_collection = True
else:
self.use_collection = False
self.num_items = num_items
if small:
self.kv_cls = KeyValueIterator
self.field_cls = NewFieldIterator
else:
self.kv_cls = KeyLargeValueIterator
self.field_cls = NewLargeFieldIterator
self.kv_iterator = self.kv_cls(self.num_items)
self.field_iterator = self.field_cls(self.num_items)
cb_version = pkg_resources.get_distribution("couchbase").version
if cb_version[0] == '2':
self.cb = Connection(bucket=bucket, host=host, password=password)
elif cb_version[0] == '3':
connection_string = 'couchbase://{host}?password={password}'
connection_string = connection_string.format(host=host,
password=password)
pass_auth = PasswordAuthenticator(bucket, password)
self.cluster = TxCluster(connection_string=connection_string,
options=ClusterOptions(pass_auth))
self.bucket = self.cluster.bucket(bucket)
self.collection = self.bucket.scope("scope-1").collection(
"collection-1")
self.fraction = 1
self.iteration = 0
import os
import pprint
import sys
import couchbase
import requests
from datetime import datetime
from couchbase.cluster import Cluster, ClusterOptions
from couchbase_core.cluster import PasswordAuthenticator
# CONNECT DB
cluster = Cluster(
'couchbase://couchbase',
ClusterOptions(
PasswordAuthenticator(os.environ.get('DB_USER'),
os.environ.get('DB_PASSWORD'))))
# BUCKET ITEMS
cb = cluster.bucket('items').default_collection()
# CREATION DES INDEX
cb.query('CREATE PRIMARY INDEX `items-primary-index` ON `items` USING GSI;'
).execute()
cb.query('CREATE INDEX `items-corrupted-index` ON items(corrupted) USING GSI;'
).execute()
cb.query(
'CREATE INDEX `items-learning-index` ON items(learningData) USING GSI;'
).execute()
"""
[source,python]
----
"""
# tag::connect[]
# needed for any cluster connection
from couchbase.cluster import Cluster, ClusterOptions
from couchbase_core.cluster import PasswordAuthenticator
# needed to support SQL++ (N1QL) query
from couchbase.cluster import QueryOptions
# get a reference to our cluster
cluster = Cluster(
'couchbase://localhost',
ClusterOptions(PasswordAuthenticator('Administrator', 'password')))
# end::connect[]
# tag::bucket[]
# get a reference to our bucket
cb = cluster.bucket('travel-sample')
# end::bucket[]
# tag::default-collection[]
# get a reference to the default collection
cb_coll = cb.default_collection()
# end::default-collection[]
# tag::upsert-func[]
print(result.cas)
except Exception as e:
print(e)
# needed for any cluster connection
from couchbase.cluster import Cluster, ClusterOptions
from couchbase_core.cluster import PasswordAuthenticator
# needed to support SQL++ (N1QL) query
from couchbase.cluster import QueryOptions
# get a reference to our cluster
cluster = Cluster('couchbase://localhost',
ClusterOptions(
PasswordAuthenticator('Administrator', db_password)),
lockmode=2) # added lockmode to avoid locking error
# get a reference to our bucket
cb = cluster.bucket('events')
# get a reference to the default collection
cb_coll = cb.default_collection()
app = Flask(__name__)
@app.route("/webhooks/answer")
def answer():
params = request.args
pprint(params)