def _instantiate_cluster(
self,
connstr_nobucket, # type: str
cluster_class=None, # type: Type[Cluster]
opts=None # type: Any
):
# type: (...) -> ClusterTestCase.T
cluster_class = cluster_class or self.cluster_factory
mock_hack = self.cluster_info.mock_hack_options(self.is_mock)
auth = mock_hack.auth(self.cluster_info.admin_username,
self.cluster_info.admin_password)
if not opts:
opts = ClusterOptions(auth)
else:
opts['authenticator'] = auth
if SLOWCONNECT_PATTERN.match(platform.platform()):
default_timeout_options = ClusterTimeoutOptions(
config_total_timeout=timedelta(seconds=30))
default_timeout_options.update(opts.get('timeout_options', {}))
opts['timeout_options'] = default_timeout_options
return self.try_n_times(10,
3,
cluster_class.connect,
connection_string=str(connstr_nobucket),
options=opts,
**mock_hack.kwargs)
def uploadDoc(self):
# connect to cb cluster
try:
connection = "couchbase://" + self.server
if "ip6" in self.server or self.server.startswith("["):
connection = connection + "?ipv6=allow"
cluster = Cluster(
connection,
ClusterOptions(
PasswordAuthenticator(self.username, self.password)))
# authenticator = PasswordAuthenticator(self.username, self.password)
# cluster.authenticate(authenticator)
cb = cluster.bucket(self.bucket)
cb_coll = cb.default_collection()
cb.timeout = 100
except Exception as e:
logging.error("Connection error\n" + traceback.format_exc())
json_docs = {}
for i in range(self.startseqnum, self.startseqnum + self.num_docs):
self.createContent()
dockey = self.keyprefix + str(i)
json_docs[dockey] = self.json_objs_dict
BYTES_PER_BATCH = 1024 * 256 # 256K
batches = []
cur_batch = {}
cur_size = 0
batches.append(cur_batch)
for key, value in list(json_docs.items()):
cur_batch[key] = value
cur_size += len(key) + len(value) + 24
if cur_size > BYTES_PER_BATCH:
cur_batch = {}
batches.append(cur_batch)
cur_size = 0
num_completed = 0
while batches:
batch = batches[-1]
try:
cb_coll.upsert_multi(batch)
num_completed += len(batch)
batches.pop()
except CouchbaseTransientError as e:
logging.error(e)
ok, fail = e.split_results()
new_batch = {}
for key in fail:
new_batch[key] = list(json_docs.items())[key]
batches.pop()
batches.append(new_batch)
num_completed += len(ok)
logging.info("Retrying {}/{} items".format(
len(new_batch), len(ok)))
logging.info("Completed {}/{} items".format(
num_completed, len(json_docs)))
if self.xattrs:
logging.info("Upserting xattrs")
self.add_xattrs(cb)
async def create_couchbase(self):
try:
auth = PasswordAuthenticator(self.bucket, self.user_password)
endpoint = 'couchbase://{0}'.format(self.active_hosts[0])
options = ClusterOptions(authenticator=auth)
cluster = ACluster(endpoint, options=options)
bucket = cluster.bucket(self.bucket)
b = cluster.bucket(self.bucket)
await bucket.on_connect()
for scope, _collections in self.collections.items():
for collection in _collections:
template = copy.deepcopy(self.template)
template['kv']['scope'] = scope
template['kv']["collection"] = collection
self.templates.append(template)
if collection == 'default':
cb = bucket.default_collection()
else:
cb = bucket.scope(scope).collection(collection)
self.cbs.append(cb)
except Exception as ex:
logging.error('Error trying to create CBS')
logging.error(ex)
def __init__(self, host=None, bucket=None, username=None, password=None):
config = os.environ
if not host or not bucket or not username or not password:
self.cb_host = config.get("health_cb_host", "172.23.104.180")
self.cb_bucket = config.get("health_cb_bucket",
"QE-staticserver-pool-health")
self.cb_username = config.get("health_cb_username",
"Administrator")
self.cb_userpassword = config.get("health_cb_password")
else:
self.cb_host = host
self.cb_bucket = bucket
self.cb_username = username
self.cb_userpassword = password
if not self.cb_userpassword:
print("Setting of env variable: heal_cb_password= is needed!")
return
try:
print("Connecting to {},{},{}".format(self.cb_host, self.cb_bucket,
self.cb_username))
self.cb_cluster = Cluster("couchbase://"+self.cb_host, ClusterOptions(PasswordAuthenticator(self.cb_username, self.cb_userpassword), \
timeout_options=ClusterTimeoutOptions(kv_timeout=timedelta(seconds=10))))
self.cb_b = self.cb_cluster.bucket(self.cb_bucket)
self.cb = self.cb_b.default_collection()
except Exception as e:
print('Connection Failed: %s ' % self.cb_host)
print(e)
def test_adb_cb4_connection_no_cert(self):
# noinspection PyBroadException
try:
_credentials_file = os.environ['HOME'] + "/adb-cb4-credentials"
# specify the cluster and specify an authenticator containing a
# username and password to be passed to the cluster.
if not Path(_credentials_file).is_file():
sys.exit("*** credentials_file file " + _credentials_file +
" can not be found!")
self.get_credentials(_credentials_file)
cluster = Cluster(
'couchbase://' + self.cb_host,
ClusterOptions(
PasswordAuthenticator(self.cb_user, self.cb_password)))
# following a successful authentication, a bucket can be opened.
# access a bucket in that cluster
bucket = cluster.bucket('mdata')
collection = bucket.default_collection()
ingest_document_result = collection.get(
"MD:V01:METAR:stations_ingest")
print("test_adb_cb4_connection_no_cert: successfully read ",
ingest_document_result.content)
except:
self.fail(
"test_adb_cb4_connection_no_cert: TestConnection.test_connection Exception failure: "
+ str(sys.exc_info()))
def connect(self):
db_config = get_db_config()
self.host = db_config['host']
self.bucket_name = db_config['bucket']
self.username = db_config['username']
self.password = db_config['password']
connection_str = 'couchbase://{0}'.format(self.host)
if db_config['secure'] == 'true':
connection_str = 'couchbases://{0}?ssl=no_verify'.format(self.host)
print(connection_str)
secure = db_config['secure']
print(secure)
try:
options = ClusterOptions(
PasswordAuthenticator(self.username, self.password))
self.__cluster = Cluster(connection_str, options)
self.__bucket = self.__cluster.bucket(self.bucket_name)
self.__collection = self.__bucket.default_collection()
output_message('repository')
if len(self.__bucket.name) > 0:
output_message(
self.__bucket.name,
'repository.py:connect() - connected to bucket:')
else:
output_message(
'repository.py:connect() - error connecting to bucket.')
except Exception as ex:
output_message(
ex, 'repository.py:connect() - error connecting to bucket.')
raise
def get_pool_data(servers):
servers_list = []
for server in servers.split(' '):
servers_list.append(server)
query = "SELECT ipaddr, os, state, origin, poolId FROM `QE-server-pool` WHERE ipaddr in ['" + "','".join(
servers_list) + "']"
pool_cb_host = os.environ.get('pool_cb_host')
if not pool_cb_host:
pool_cb_host = "172.23.104.162"
pool_cb_user = os.environ.get('pool_cb_user')
if not pool_cb_user:
pool_cb_user = "******"
pool_cb_user_p = os.environ.get('pool_cb_password')
if not cb_user_p:
print(
"Error: pool_cb_password environment variable setting is missing!")
exit(1)
data = ''
try:
pool_cluster = Cluster(
"couchbase://" + pool_cb_host,
ClusterOptions(PasswordAuthenticator(pool_cb_user, pool_cb_user_p),
timeout_options=ClusterTimeoutOptions(
kv_timeout=timedelta(seconds=10))))
result = pool_cluster.query(query)
for row in result:
data += ("{}=({} {} {} {}) ".format(row['ipaddr'], row['state'],
row['os'], row['poolId'],
row['origin'])).replace(
',', ' ')
except:
print("exception:", sys.exc_info()[0])
return data
def metrics():
metrics = []
clusters = {}
for [cluster_name, options] in settings['clusters'].items():
if cluster_name not in clusters:
try:
clusters[cluster_name] = Cluster(
'couchbase://' + options['host'],
ClusterOptions(
PasswordAuthenticator(options['username'],
options['password'])))
except Exception as e:
log.warning("Couldn't connect to cluster {}".format(e))
log.debug("Connected to {}".format(options['host']))
for options in settings["queries"] + settings["columns"]:
log.debug("Collecting metrics for {}".format(options["name"]))
try:
if "cluster" in options:
collect_cb(clusters, metrics, options)
elif "csv" in options:
collect_csv(metrics, options)
else:
raise Exception("Invalid type")
except Exception as e:
log.warning("Error while collecting {}: {}".format(
options["name"], e))
return Response("\n".join(metrics), mimetype="text/plain")
def __init__(self,
ssl_mode: str = 'none',
n1ql_timeout: int = None,
**kwargs):
connection_string = 'couchbase://{host}?password={password}&{params}'
connstr_params = parse.urlencode(kwargs["connstr_params"])
if ssl_mode == 'data':
connection_string = connection_string.replace(
'couchbase', 'couchbases')
connection_string += '&certpath=root.pem'
connection_string = connection_string.format(
host=kwargs['host'],
password=kwargs['password'],
params=connstr_params)
pass_auth = PasswordAuthenticator(kwargs['username'],
kwargs['password'])
timeout = ClusterTimeoutOptions(
kv_timeout=timedelta(seconds=self.TIMEOUT),
query_timeout=timedelta(
seconds=n1ql_timeout if n1ql_timeout else self.N1QL_TIMEOUT))
options = ClusterOptions(authenticator=pass_auth,
timeout_options=timeout)
self.cluster = Cluster(connection_string=connection_string,
options=options)
self.bucket_name = kwargs['bucket']
self.bucket = None
self.collections = dict()
self.collection = None
def check_mismatched_fcstValidEpoch_to_id(self):
"""This is a simple ultility test that can be used to see if there are
any missmatched fcstValidEpoch values among the observations i.e. the fcstValidEpoch in the id
does not match the fcstValidEpoch in the top level fcstValidEpoch field"""
try:
credentials_file = os.environ["HOME"] + "/adb-cb1-credentials"
self.assertTrue(
Path(credentials_file).is_file(),
"credentials_file Does not exist")
f = open(credentials_file)
yaml_data = yaml.load(f, yaml.SafeLoader)
host = yaml_data["cb_host"]
user = yaml_data["cb_user"]
password = yaml_data["cb_password"]
options = ClusterOptions(PasswordAuthenticator(user, password))
cluster = Cluster("couchbase://" + host, options)
result = cluster.query("""
select mdata.fcstValidEpoch, mdata.id
FROM mdata
WHERE
mdata.docType = "obs"
AND mdata.subset = "METAR"
AND mdata.type = "DD"
AND mdata.version = "V01"
AND NOT CONTAINS(mdata.id,to_string(mdata.fcstValidEpoch)) """)
for row in result:
self.fail(
"These do not have the same fcstValidEpoch: {0}".format(
str(row['fcstValidEpoch']) + row['id']))
except Exception as e:
self.fail("TestGsdIngestManager Exception failure: " + str(e))
def __init__(self):
self.cluster: Cluster = Cluster(
connection_string=f"couchbase://{STORAGE_HOST}",
options=ClusterOptions(authenticator=PasswordAuthenticator(
username=STORAGE_USER,
password=STORAGE_PASSWORD,
), ),
)
try:
self.bucket = self.cluster.bucket(name=STORAGE_BUCKET, )
except BucketNotFoundException:
print(
f"Bucket not found. Attempting to create new one.",
flush=True,
)
storage_management = Admin(
username=STORAGE_USER,
password=STORAGE_PASSWORD,
host=STORAGE_HOST,
)
storage_management.bucket_create(
name=STORAGE_BUCKET,
ram_quota=256,
)
initialized = self._init_bucket(storage_management)
if not initialized:
raise InitError
self.blocks: CBCollection = self.bucket.collection(
STORAGE_BLOCK_COLLECTION)
self.config: CBCollection = self.bucket.collection(
STORAGE_CONFIG_COLLECTION)
def get_bucket(conf):
cluster = Cluster("{host}:{port}".format(host=conf['host'],
port=conf['port']),
options=ClusterOptions(
PasswordAuthenticator(username=conf['user'],
password=conf['password'])))
return cluster.bucket(str(conf['bucket']))
def get_couchbase_db():
pa = PasswordAuthenticator('cosso_user', 'passw0rd')
try:
cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa))
return cluster
except:
print("exception:", sys.exc_info()[0])
return "Could not connect to CouchDB"
def test_async(self):
print("blockingtoasync")
#tag::blockingtoasync[]
cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")))
bucket = cluster.bucket("travel-sample")
# Same API as Bucket, but completely async with asyncio Futures
from acouchbase.bucket import Bucket
async_bucket=Bucket("couchbase://localhost/default")
cluster.disconnect()
#end::blockingtoasync[]
print("reactivecluster")
#tag::reactivecluster[]
from acouchbase.bucket import Bucket
cluster = Cluster("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")),bucket_class=Bucket)
bucket = cluster.bucket("travel-sample")
# A reactive cluster's disconnect methods returns a Mono<Void>.
# Nothing actually happens until you subscribe to the Mono.
# The simplest way to subscribe is to await completion by calling call `block()`.
cluster.disconnect()
#end::reactivecluster[]
print("asynccluster")
#tag::asynccluster[]
cluster = Cluster.connect("couchbase://localhost", ClusterOptions(PasswordAuthenticator("username", "password")))
bucket = cluster.bucket("travel-sample")
# An async cluster's disconnect methods returns a CompletableFuture<Void>.
# The disconnection starts as soon as you call disconnect().
# The simplest way to wait for the disconnect to complete is to call `join()`.
cluster.disconnect().join()
#end::asynccluster[]
print("tls")
#tag::tls[]
cluster = Cluster("couchbases://localhost",ClusterOptions(PasswordAuthenticator("username","password",cert_path="/path/to/cluster.crt")))
#end::tls[]
print("dnssrv")
#tag::dnssrv[]
env = ClusterEnvironment.builder() \
.ioConfig(IoConfig.enableDnsSrv(true)) \
.build()
async def get_couchbase():
cluster = Cluster(
"couchbase://localhost",
ClusterOptions(PasswordAuthenticator("Administrator", "password")))
bucket = cluster.bucket("travel-sample")
await bucket.on_connect()
return cluster, bucket
def connect_couchbase(bucket_name):
# get a reference to our cluster
cluster = Cluster(host_url, ClusterOptions(PasswordAuthenticator(user, password)))
# get a reference to our bucket
cb = cluster.bucket('default')
# get a reference to the default collection
collection = cb.default_collection()
return collection
def get_cb_bucket(cdbname):
pa = PasswordAuthenticator(cb_su, cb_pass)
try:
cluster = Cluster('couchbase://127.0.0.1', ClusterOptions(pa))
db = cluster.bucket(cdbname)
return db
except couchbase.exceptions.CouchbaseError as err:
print(err)
return "an error occurred"
def add_cluster(host: str, username: str, password: str):
try:
clusters_lock.acquire()
if host not in clusters:
clusters[host] = Cluster(
'couchbase://' + host,
ClusterOptions(PasswordAuthenticator(username, password)),
lockmode=LockMode.WAIT)
finally:
clusters_lock.release()
def _createConn(self):
try:
cluster = Cluster(self.connection_string, ClusterOptions(PasswordAuthenticator(self.bucket, 'password')))
#cluster.authenticate(PasswordAuthenticator(self.bucket, 'password'))
self.cb = cluster.bucket(self.bucket)
self.default_collection = self.cb.default_collection()
except BucketNotFoundError:
raise
except AuthError:
# Try using default user created by the tests, if any, in case there is no user with bucket name in the
# cluster.
try:
cluster = Cluster(self.connection_string,
ClusterOptions(PasswordAuthenticator("cbadminbucket", 'password')),
bucket_class=CouchbaseBucket)
self.cb = cluster.bucket(self.bucket)
self.default_collection = self.cb.default_collection()
except AuthError:
raise
def init_connection(self, args):
global couchbase_connection
connection_string = 'couchbase://%s' % (args['server'])
couchbase_cluster = Cluster(
connection_string,
ClusterOptions(
PasswordAuthenticator(args['username'], args['password'])))
couchbase_bucket = couchbase_cluster.bucket(args['bucket'])
self.bucket_name = args['bucket']
self.couchbase_connection = couchbase_bucket.default_collection()
def __init__(self):
# CONNECT DB
self.cluster = Cluster(
'couchbase://couchbase',
ClusterOptions(
PasswordAuthenticator(os.environ.get('DB_USER'),
os.environ.get('DB_PASSWORD'))))
# BUCKET
self.bucket_items = self.cluster.bucket('items').default_collection()
self.bucket_var = self.cluster.bucket('var').default_collection()
def __init__(self):
pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET,
self.COUCHBASE_PASSWORD)
options = ClusterOptions(authenticator=pass_auth)
self.cluster = Cluster(connection_string=self.connection_string,
options=options)
self.bucket = self.cluster.bucket(
self.COUCHBASE_BUCKET).default_collection()
self.jenkins = JenkinsScanner()
self.ps = PerfStore(host=CBMONITOR_HOST)
self.weekly = Weekly()
def connect_cb(self):
# get a reference to our cluster
# noinspection PyBroadException
try:
options = ClusterOptions(
PasswordAuthenticator(self.cb_credentials['user'], self.cb_credentials['password']))
self.cluster = Cluster('couchbase://' + self.cb_credentials['host'], options)
self.collection = self.cluster.bucket("mdata").default_collection()
except:
print("*** %s in connect_cb ***" + str(sys.exc_info()))
sys.exit("*** Error when connecting to mysql database: ")
def __init__(self):
pass_auth = PasswordAuthenticator(self.COUCHBASE_BUCKET,
self.COUCHBASE_PASSWORD)
options = ClusterOptions(authenticator=pass_auth)
self.cluster = Cluster(connection_string=self.connection_string,
options=options)
self.bucket = self.cluster.bucket(
self.COUCHBASE_BUCKET).default_collection()
self.jenkins = jenkins.Jenkins(self.JENKINS_URL)
self.weekly = Weekly()
self.jobs = set()
class CouchbaseHelper:
password_authenticator = PasswordAuthenticator(CouchbaseConfig.USERNAME,
CouchbaseConfig.PASSWORD)
cluster_options = ClusterOptions(password_authenticator)
cluster = Cluster(CouchbaseConfig.URL, cluster_options)
bucket = cluster.bucket(CouchbaseConfig.BUCKET_NANE)
collection = bucket.default_collection()
def __init__(self, *args, **kwargs):
self.id = str(uuid.uuid4())
def get_cluster():
try:
cluster = Cluster('couchbase://{}'.format(host))
authenticator = PasswordAuthenticator('Administrator', 'password')
cluster.authenticate(authenticator)
return cluster
except Exception:
cluster = Cluster(
'couchbase://{}'.format(host),
ClusterOptions(PasswordAuthenticator('Administrator', 'password')))
return cluster
def __init__(self, host, bucket, username, password, quiet=True, port=8091):
connection_string = 'couchbase://{}?password={}'.format(host, password)
pass_auth = PasswordAuthenticator(username, password)
timeout = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=self.TIMEOUT))
options = ClusterOptions(authenticator=pass_auth, timeout_options=timeout)
self.cluster = Cluster(connection_string=connection_string, options=options)
self.bucket = self.cluster.bucket(bucket)
self.client = self.bucket.default_collection()
self.use_count = 0
self.use_time = 0
self.last_use_time = 0
async def init_cb(app):
conf = app['config']['couchbase']
cluster = Cluster("{host}:{port}".format(host=conf['host'],
port=conf['port']),
options=ClusterOptions(
PasswordAuthenticator(username=conf['user'],
password=conf['password'])))
bucket = cluster.bucket(str(conf['bucket']))
bucket.on_connect()
collection = bucket.collection(conf['collection'])
app['cb'] = cluster
app['db'] = collection
def file_loc_processing():
global file_counter
# get a reference to our cluster
cluster = Cluster(
'couchbase://' + options.host,
ClusterOptions(PasswordAuthenticator(options.user, options.password)))
file_loc = options.agent_env['HVR_FILE_LOC']
tbl_map = table_file_name_map()
client = cluster.bucket(options.database)
db = cb_coll = client.default_collection()
for t in tbl_map:
base_name = table_name_normalize(t[0])
hvr_schema = base_name[0]
hvr_base_table = base_name[1]
collection = options.collection.format(hvr_schema=hvr_schema,
hvr_base_name=hvr_base_table,
hvr_tbl_name=t[1])
keys = t[3].split(",")
# if collection is absent - no any error
# if options.recreate :
# trace("Dropping collection '{0}'".format(collection))
# db.drop_collection(collection)
for name in tbl_map[t]:
full_name = file_loc + '/' + name
trace("Reading and parsing file '" + full_name + "' ... ")
try:
with open(full_name) as json_file:
for line in json_file:
json_obj = json.loads(line)
try:
#if options.mode == SetupMode.MODE_SOFTDELETE :
#added new pseudo column
column_id = ''
for column in keys:
column_id = column_id + "::" + str(
json_obj[column])
#trace("key will equal:" + str(column_id.replace('::', "", 1)) + " /n")
mykey = str(hvr_base_table) + str(column_id)
trace("key will equal:" + mykey + " /n")
result = db.upsert(mykey, str(json_obj))
#trace(str(result.cas) + " mykey:" + str(mykey) + " data:" + str(json_obj))
except Exception as e:
trace(e)
# remove successfully transmitted file
os.remove(full_name)
file_counter = file_counter + 1
except IOError as err:
raise Exception("Couldn't open file " + full_name)
def test_disconnect(self):
# for this test we need a new cluster...
if self.is_mock:
raise SkipTest("query not mocked")
cluster = Cluster.connect(self.cluster.connstr, ClusterOptions(
PasswordAuthenticator(self.cluster_info.admin_username, self.cluster_info.admin_password)))
# Temporarily, lets open a bucket to insure the admin object was created
b = cluster.bucket(self.bucket_name)
# verify that we can get a bucket manager
self.assertIsNotNone(cluster.buckets())
# disconnect cluster
cluster.disconnect()
self.assertRaises(AlreadyShutdownException, cluster.buckets)