def _createConn(self):
try:
print("-->connection_string:{}".format(self.connection_string))
cluster = Cluster(self.connection_string,
bucket_class=CouchbaseBucket)
#cluster = Cluster("couchbases://cb.5ff6f20a-1550-4d54-9174-8a01a32edd87.dp.cloud
# .couchbase.com?ssl"
# "=no_verify", bucket_class=CouchbaseBucket)
print("-->bucket={},username={},password={}".format(
self.bucket, self.username, self.password))
cluster.authenticate(
PasswordAuthenticator(self.username, self.password))
self.cb = cluster.open_bucket(self.bucket)
print("-->cb={}".format(self.cb))
self.cb.upsert('testkey', 'testvalue')
print(self.cb.get('testkey'))
except BucketNotFoundError:
raise
except AuthError:
# Try using default user created by the tests, if any, in case there is no user with bucket name in the
# cluster.
print("Auth Error!")
try:
cluster = Cluster(self.connection_string,
bucket_class=CouchbaseBucket)
cluster.authenticate(
PasswordAuthenticator(self.username, self.password))
self.cb = cluster.open_bucket(self.bucket)
except AuthError:
raise
def get_cluster():
try:
cluster = Cluster('couchbase://{}'.format(host))
authenticator = PasswordAuthenticator('Administrator', 'password')
cluster.authenticate(authenticator)
return cluster
except Exception:
cluster = Cluster(
'couchbase://{}'.format(host),
ClusterOptions(PasswordAuthenticator('Administrator', 'password')))
return cluster
def __init__(self,
queue_ID,
bucket='queue',
host='localhost',
username='******',
password='******',
timeout=50):
'''The priority queue implements a FIFO queue with optional priority
insertion. The "bucket", "host", "username", and "password" keywords
must match those specified in the database configuration. The "timeout"
keyword sets the amount of time before the queue becomes stale, upon
which the queue is emptied of all items. The queue object should be
threadsafe, each thread of execution enters in as a unique item.
'''
self.cluster = Cluster('couchbase://{:}'.format(host))
authenticator = PasswordAuthenticator(username, password)
self.cluster.authenticate(authenticator)
self.cb = self.cluster.open_bucket(bucket, lockmode=LOCKMODE_WAIT)
# Create id counter (if it does not exist)
self.c_ID = 'id_counter'
self.cb.counter(self.c_ID, initial=0)
# Create the queue (if it does not exist)
self.q_ID = queue_ID
try:
self.cb.insert(self.q_ID, [])
except (KeyExistsError, TemporaryFailError):
pass
self.last_id = {}
# Initialize queue timeout (s)
self.timeout = int(timeout)
try:
self.cb.touch(self.q_ID, ttl=self.timeout)
except TemporaryFailError:
pass
def create_clients(cb_cluster_ip, cb_username, cb_password):
"""
Creates couchbase server bucket clients and store it to CLIENT dict
Returns boolean to specify whether the creation of clients succeeded
"""
client_creation_success = True
try:
cb_cluster = Cluster("couchbase://{0}". format(cb_cluster_ip))
cb_cluster.authenticate(PasswordAuthenticator(cb_username, cb_password))
except Exception as cluster_err:
print("Error while connecting to couchbase cluster couchbase://{0} {1}".format(cb_cluster_ip, cluster_err))
client_creation_success = False
return client_creation_success
# Initialize the list of buckets to which client will be created
buckets_to_create_client = [qeTestSuitesBucketName]
for tem_view in VIEWS:
buckets_to_create_client.append(tem_view["bucket"])
for bucket in buckets_to_create_client:
try:
client = cb_cluster.open_bucket(bucket, lockmode=LOCKMODE_WAIT)
CLIENTS[bucket] = client
except Exception as cb_bucket_error:
print("Failed to open bucket {0} {1}".format(bucket, cb_bucket_error))
client_creation_success = False
break
return client_creation_success
def _createConn(self):
try:
cluster = Cluster(self.connection_string, bucket_class=CouchbaseBucket)
cluster.authenticate(PasswordAuthenticator(self.bucket, 'password'))
self.cb = cluster.open_bucket(self.bucket)
except BucketNotFoundError:
raise
def __init__(self):
self.cluster: Cluster = Cluster(
connection_string=f"couchbase://{STORAGE_HOST}",
options=ClusterOptions(authenticator=PasswordAuthenticator(
username=STORAGE_USER,
password=STORAGE_PASSWORD,
), ),
)
try:
self.bucket = self.cluster.bucket(name=STORAGE_BUCKET, )
except BucketNotFoundException:
print(
f"Bucket not found. Attempting to create new one.",
flush=True,
)
storage_management = Admin(
username=STORAGE_USER,
password=STORAGE_PASSWORD,
host=STORAGE_HOST,
)
storage_management.bucket_create(
name=STORAGE_BUCKET,
ram_quota=256,
)
initialized = self._init_bucket(storage_management)
if not initialized:
raise InitError
self.blocks: CBCollection = self.bucket.collection(
STORAGE_BLOCK_COLLECTION)
self.config: CBCollection = self.bucket.collection(
STORAGE_CONFIG_COLLECTION)
def create_xattr_data(self, type="system"):
cluster = Cluster('couchbase://'+str(self.master.ip))
authenticator = PasswordAuthenticator(self.username, self.password)
cluster.authenticate(authenticator)
cb = cluster.open_bucket('default')
docs = self.get_meta_ids()
self.log.info("Docs: " + str(docs[0:5]))
xattr_data = []
self.log.info("Adding xattrs to data")
val = 0
for doc in docs:
if type == "system":
rv = cb.mutate_in(doc["id"], SD.upsert('_system1', val, xattr=True, create_parents=True))
xattr_data.append({'_system1': val})
rv = cb.mutate_in(doc["id"], SD.upsert('_system2', {'field1': val, 'field2': val*val}, xattr=True, create_parents=True))
xattr_data.append({'_system2': {'field1': val, 'field2': val*val}})
rv = cb.mutate_in(doc["id"], SD.upsert('_system3', {'field1': {'sub_field1a': val, 'sub_field1b': val*val}, 'field2': {'sub_field2a': 2*val, 'sub_field2b': 2*val*val}}, xattr=True, create_parents=True))
xattr_data.append({'_system3': {'field1': {'sub_field1a': val, 'sub_field1b': val*val}, 'field2': {'sub_field2a': 2*val, 'sub_field2b': 2*val*val}}})
if type == "user":
rv = cb.mutate_in(doc["id"], SD.upsert('user1', val, xattr=True, create_parents=True))
xattr_data.append({'user1': val})
rv = cb.mutate_in(doc["id"], SD.upsert('user2', {'field1': val, 'field2': val*val}, xattr=True, create_parents=True))
xattr_data.append({'user2': {'field1': val, 'field2': val*val}})
rv = cb.mutate_in(doc["id"], SD.upsert('user3', {'field1': {'sub_field1a': val, 'sub_field1b': val*val}, 'field2': {'sub_field2a': 2*val, 'sub_field2b': 2*val*val}}, xattr=True, create_parents=True))
xattr_data.append({'user3': {'field1': {'sub_field1a': val, 'sub_field1b': val*val}, 'field2': {'sub_field2a': 2*val, 'sub_field2b': 2*val*val}}})
val = val + 1
self.log.info("Completed adding " + type + "xattrs to data to " + str(val) + " docs")
return xattr_data
def configure_cb():
cluster = Cluster('couchbase://localhost')
authenticator = PasswordAuthenticator(
os.environ['CB_USERNAME'], os.environ['CB_PASSWORD']
)
cluster.authenticate(authenticator)
return cluster
def start_database():
from couchbase.cluster import Cluster
from couchbase.cluster import PasswordAuthenticator
cluster = Cluster('couchbase://localhost')
authenticator = PasswordAuthenticator('couchbase', 'couchbase')
cluster.authenticate(authenticator)
return cluster
def __init__(self, username, password, ip):
try:
self.cluster = Cluster('couchbase://'+ip)
self.cluster.authenticate(PasswordAuthenticator(username, password))
except:
raise Exception('Unable to verify user')
def __init__(self):
# Constructor
_cluster = Cluster(app.config['DB_ADDRESS'])
_auth = PasswordAuthenticator(app.config['DB_USER'],
app.config['DB_PASSWORD'])
_cluster.authenticate(_auth)
self.db = _cluster.open_bucket(app.config['DB_BUCKET'])
def _get_cb_connection(self):
# instance_type = self.config['GLOBAL']['INSTANCE_TYPE']
cluster = Cluster(self.CB_URL)
authenticator = PasswordAuthenticator(self.CB_APPS_USER,
self.CB_APPS_PASSWORD)
cluster.authenticate(authenticator)
return cluster.open_bucket(self.CB_INSTANCE, lockmode=LOCKMODE_WAIT)
def get_bucket(bucket_id):
cluster = Cluster('couchbase://10.101.3.79:8091')
authenticator = PasswordAuthenticator('buckets', 'buckets')
cluster.authenticate(authenticator)
cb = cluster.open_bucket(bucket_id)
return cb
def action(self, includes: dict, variables: dict) -> any:
from couchbase.cluster import Cluster, PasswordAuthenticator
body = self.simple_input(variables)
in_data = body['request']
conf = in_data['conf']
cluster = Cluster('couchbase://' + conf['host'])
if 'user' in conf and 'password' in conf:
cluster.authenticate(
PasswordAuthenticator(conf['user'], conf['password']))
bucket = cluster.open_bucket(conf['bucket'])
if 'put' in in_data:
put = in_data['put']
result = bucket.upsert(put['key'], put['value'])
if result.success:
return variables, {}
else:
raise RuntimeError(result.errstr)
if 'get' in in_data:
get = in_data['get']
result = bucket.get(get['key'], quiet=True)
if result is None:
raise RuntimeError('no data found for key ' + get['key'])
return variables, result.value
if 'delete' in in_data:
delete = in_data['delete']
bucket.remove(delete['key'], quiet=True)
return variables, {}
if 'query' in in_data:
query = in_data['query']
res = [row for row in bucket.n1ql_query(query)]
if len(res) == 1:
return variables, res[0]
return variables, res
raise RuntimeError('nothing to do: ' + str(in_data))
def _createConn(self):
try:
cluster = Cluster(self.connection_string, bucket_class=CouchbaseBucket)
cluster.authenticate(PasswordAuthenticator(self.bucket, 'password'))
self.cb = cluster.open_bucket(self.bucket)
except BucketNotFoundError:
raise
except AuthError:
# Try using default user created by the tests, if any, in case there is no user with bucket name in the
# cluster.
try:
cluster = Cluster(self.connection_string, bucket_class=CouchbaseBucket)
cluster.authenticate(PasswordAuthenticator("cbadminbucket", 'password'))
self.cb = cluster.open_bucket(self.bucket)
except AuthError:
raise
def get_authenticated(self):
try:
self.user = self.default_user
self.passwrd = self.default_pw
self.url = self.default_url
except Exception as e:
self.user = "******"
self.passwrd = "password123"
self.url = "0.0.0.0"
self.version = self.get_version()
print(self.version)
if self.version >= 5:
self.cluster = Cluster('couchbase://{}'.format(self.url))
self.authenticator = PasswordAuthenticator(self.user, self.passwrd)
self.cluster.authenticate(self.authenticator)
self.main = self.cluster.open_bucket('main')
self.hxn = self.cluster.open_bucket('hxn')
self.main._cntlstr("operation_timeout", "5000")
else:
self.cluster = Cluster('couchbase://{}'.format(self.url))
self.cluster.authenticate(
ClassicAuthenticator(buckets={
'main': '',
'hxn': ''
}))
self.main = self.cluster.open_bucket('main')
self.hxn = self.cluster.open_bucket('hxn')
def __init__(self, bucket_name, host, username, password, creds=None):
self._host = host
self.username = username
self._userpass = password
# couch_bucket_url = 'couchbase://{0}/{1}'.format(host, bucket_name)
cluster = Cluster('couchbase://{0}'.format(host))
authenticator = PasswordAuthenticator(username, password)
cluster.authenticate(authenticator)
# print(bucket_name)
self._cb = cluster.open_bucket(bucket_name)
if self.bucket_name is None:
self.bucket_name = bucket_name
self._cb.n1ql_timeout = 5000 * 1000
self._cb.timeout = 5000 * 1000
self._cb.config_total_timeout = 5000 * 1000
# self._cb.cross_bucket = True
self._cb.operationTimeout = 5000 * 1000
if creds is not None:
self._creds = creds
def setup(self):
config = self.container.config[COUCHBASE_KEY]
uri = urlparse(config['URI'])
params = urlencode(config.get('CLIENT_CONFIG'), {})
self.authenticator = PasswordAuthenticator(uri.username, uri.password)
self.cluster = Cluster('{}://{}?{}'.format(uri.scheme, uri.hostname,
params))
def uploadDoc(self):
# connect to cb cluster
try:
connection = "couchbase://" + self.server
if "ip6" in self.server or self.server.startswith("["):
connection = connection + "?ipv6=allow"
cluster = Cluster(connection)
authenticator = PasswordAuthenticator(self.username, self.password)
cluster.authenticate(authenticator)
cb = cluster.open_bucket(self.bucket)
cb.timeout = 100
except Exception as e:
logging.error("Connection error\n" + traceback.format_exc())
json_docs = {}
for i in range(self.startseqnum, self.startseqnum + self.num_docs):
self.createContent()
dockey = self.keyprefix + str(i)
json_docs[dockey] = self.json_objs_dict
BYTES_PER_BATCH = 1024 * 256 # 256K
batches = []
cur_batch = {}
cur_size = 0
batches.append(cur_batch)
for key, value in json_docs.items():
cur_batch[key] = value
cur_size += len(key) + len(value) + 24
if cur_size > BYTES_PER_BATCH:
cur_batch = {}
batches.append(cur_batch)
cur_size = 0
num_completed = 0
while batches:
batch = batches[-1]
try:
cb.upsert_multi(batch)
num_completed += len(batch)
batches.pop()
except CouchbaseTransientError as e:
logging.error(e)
ok, fail = e.split_results()
new_batch = {}
for key in fail:
new_batch[key] = all_data[key]
batches.pop()
batches.append(new_batch)
num_completed += len(ok)
logging.info("Retrying {}/{} items".format(
len(new_batch), len(ok)))
logging.info("Completed {}/{} items".format(
num_completed, len(json_docs)))
if self.xattrs:
logging.info("Upserting xattrs")
self.add_xattrs(cb)
if self.filter:
logging.info("Upserting filter_expressions.json")
cb.upsert("filter_expressions", self.filters_json_objs_dict)
def check_service_up(self,service,nodes):
self.sleep(20)
if service == "nserv":
output = x509main()._execute_command_clientcert(nodes.ip, url='/pools/default', port=8091,
headers=' -u Administrator:password ', client_cert=False,
curl=True, plain_curl=True)
output = json.loads(output)
self.log.info("Print output of command is {0}".format(output))
self.assertEqual(output['rebalanceStatus'], 'none', "The Web request has failed on port 8091")
elif service == "index":
output = x509main()._execute_command_clientcert(nodes.ip, url='/getIndexStatus', port=9102,
headers='-u Administrator:password',
client_cert=False, curl=True, verb='GET', plain_curl=True)
output = json.loads(output)
self.log.info("Print output of command is {0}".format(output))
self.assertEqual(output["code"],'success',"The Index Service is not up")
elif service == "query":
output = x509main()._execute_command_clientcert(nodes.ip, url='/query/service', port=8093,
headers='-u Administrator:password ',
client_cert=False, curl=True, verb='GET', plain_curl=True,
data="statement='create index idx1 on default(name)'")
self.assertEqual(json.loads(output)['status'], "success", "Create Index Failed on port 8093")
elif service == "fts":
idx = {"sourceName": "default","sourceType": "couchbase","type": "fulltext-index"}
output = x509main()._execute_command_clientcert(nodes.ip, url='/api/index/default_idx', port=8094,
headers=" -XPUT -H \"Content-Type: application/json\" -u Administrator:password ",
client_cert=False, curl=True, verb='GET', plain_curl=True,
data="'" + json.dumps(idx) + "'")
self.assertEqual(json.loads(output)['status'], "ok", "Issue with creating FTS index with client Cert")
elif service == "cbas":
cmd = "curl -v " + \
" -s -u Administrator:password --data pretty=true --data-urlencode 'statement=create dataset on default' " + \
"http://{0}:{1}/_p/cbas/query/service ". \
format(nodes.ip, 8091)
self.log.info("Running command : {0}".format(cmd))
output = subprocess.check_output(cmd, shell=True)
self.assertEqual(json.loads(output)['status'], "success", "Create CBAS Index Failed")
elif service == "eventing":
cmd = "curl -v -X GET -u Administrator:password http://{0}:8096/api/v1/status" \
.format(nodes.ip)
self.log.info("Running command : {0}".format(cmd))
output = subprocess.check_output(cmd, shell=True)
self.assertEqual(json.loads(output)['num_eventing_nodes'], 1, "Eventing Node is not up")
elif service == "kv":
cluster = Cluster("couchbase://{0}".format(nodes.ip))
authenticator = PasswordAuthenticator("Administrator", "password")
cluster.authenticate(authenticator)
cb = cluster.open_bucket("default")
cb.upsert("key1", "value1")
self.assertEqual(cb.get("key1").value, "value1")
def __connect(self):
self.cluster = Cluster('couchbase://{host}:{port}'.format(
host=self.host, port=self.port))
self.cluster.authenticate(
PasswordAuthenticator(self.username, self.password))
self.__bucket_create(self.cluster, self.host, sync=True)
self.__xdcr_create()
self.bucket = self.cluster.open_bucket(self.name)
def get_cluster(username: str, password: str, host="couchbase", port="8091"):
# cluster_url="couchbase://couchbase"
# username = "******"
# password = "******"
cluster_url = get_cluster_couchbase_url(host=host, port=port)
cluster = Cluster(cluster_url)
authenticator = PasswordAuthenticator(username, password)
cluster.authenticate(authenticator)
return cluster
def connect():
host_name = main.get_host_name()
bucket = main.get_bucket_name()
password = main.get_bucket_password()
cluster = Cluster("couchbase://" + host_name + "?operation_timeout=60")
authenticator = PasswordAuthenticator(bucket, password)
cluster.authenticate(authenticator)
return cluster.open_bucket(bucket)
def get_bucket():
cluster = Cluster(
"couchbase://couchbasehost:8091?fetch_mutation_tokens=1&operation_timeout=30&n1ql_timeout=300"
)
authenticator = PasswordAuthenticator("username", "password")
cluster.authenticate(authenticator)
bucket: Bucket = cluster.open_bucket("bucket_name", lockmode=LOCKMODE_WAIT)
bucket.timeout = 30
bucket.n1ql_timeout = 300
return bucket
def couchbase_cluster_connect(self):
connection_str = 'couchbase://' + self.db_host
try:
# self.db_username = '******'
self.cluster = Cluster(connection_str)
authenticator = PasswordAuthenticator(self.db_username,
self.db_password)
self.cluster.authenticate(authenticator)
except CouchbaseError as err:
print("Problem connecting to couchbase ({})".format(err))
def create_connection(self, bucket_name):
"""
Create bucket connections. 5 bucket connections are created per instance.
:return: Nothing
"""
cluster = Cluster(self.spec)
auth = PasswordAuthenticator(self.user, self.password)
cluster.authenticate(auth)
self.bucket_connection = cluster.open_bucket(bucket_name,
lockmode=LOCKMODE_WAIT)
self.bucket_connection.timeout = self.timeout
def test_no_mixed_auth(self):
cluster, bucket_name = self._create_cluster()
auther = PasswordAuthenticator(bucket_name,
self.cluster_info.bucket_password)
cluster.authenticate(auther)
cb1 = cluster.open_bucket(bucket_name)
self.assertRaises(MixedAuthError, cluster.open_bucket, bucket_name,
password=self.cluster_info.bucket_password)
cluster2, bucket_name = self._create_cluster()
cb2 = cluster2.open_bucket(bucket_name,
password=self.cluster_info.bucket_password)
def __init__(self, username, password, bucket_name):
self._cluster = Cluster('couchbase://idc0.wiselight.kr:8091')
self._authenticator = PasswordAuthenticator(username, password)
self._cluster.authenticate(self._authenticator)
self._bucket = self._cluster.open_bucket(bucket_name)
print('connect')
self._userIndex=self._setIndex('user')
self._reviewIndex=self._setIndex('review')
self._restaurantIndex=self._setIndex('restaurant')
print(self._userIndex)
print(self._reviewIndex)
print(self._restaurantIndex)
print('index initialize')
def connect_to_bucket(host, username, password, bucket):
'''
This function takes four arguments to connect to bucket.
host, username, password and bucket name.
Returns connection instance of the specified bucket.
'''
if "couchbase" not in host:
host = "couchbase://" + str(host)
cluster = Cluster(host)
authenticator = PasswordAuthenticator(username, password)
cluster.authenticate(authenticator)
cb = cluster.open_bucket(bucket)
return cb
def data_ingestion(data, cb_user, cb_pwd, cb_host, cb_bucket):
'''
Ingests data into pipeline
'''
cluster = Cluster('couchbase://' + cb_host)
authenticator = PasswordAuthenticator(cb_user, cb_pwd)
cluster.authenticate(authenticator)
cb_conn = cluster.open_bucket(cb_bucket)
# Pushing entire data sequentially
# sequential_run(cb_conn, data)
# Pushing entire data parallely
parallel_run(cb_conn, data)
