def get_pool_data(servers):
servers_list = []
for server in servers.split(' '):
servers_list.append(server)
query = "SELECT ipaddr, os, state, origin, poolId FROM `QE-server-pool` WHERE ipaddr in ['" + "','".join(
servers_list) + "']"
pool_cb_host = os.environ.get('pool_cb_host')
if not pool_cb_host:
pool_cb_host = "172.23.104.162"
pool_cb_user = os.environ.get('pool_cb_user')
if not pool_cb_user:
pool_cb_user = "******"
pool_cb_user_p = os.environ.get('pool_cb_password')
if not cb_user_p:
print(
"Error: pool_cb_password environment variable setting is missing!")
exit(1)
data = ''
try:
pool_cluster = Cluster(
"couchbase://" + pool_cb_host,
ClusterOptions(PasswordAuthenticator(pool_cb_user, pool_cb_user_p),
timeout_options=ClusterTimeoutOptions(
kv_timeout=timedelta(seconds=10))))
result = pool_cluster.query(query)
for row in result:
data += ("{}=({} {} {} {}) ".format(row['ipaddr'], row['state'],
row['os'], row['poolId'],
row['origin'])).replace(
',', ' ')
except:
print("exception:", sys.exc_info()[0])
return data
def get_bucket(conf):
cluster = Cluster("{host}:{port}".format(host=conf['host'],
port=conf['port']),
options=ClusterOptions(
PasswordAuthenticator(username=conf['user'],
password=conf['password'])))
return cluster.bucket(str(conf['bucket']))
async def create_couchbase(self):
try:
auth = PasswordAuthenticator(self.bucket, self.user_password)
endpoint = 'couchbase://{0}'.format(self.active_hosts[0])
options = ClusterOptions(authenticator=auth)
cluster = ACluster(endpoint, options=options)
bucket = cluster.bucket(self.bucket)
b = cluster.bucket(self.bucket)
await bucket.on_connect()
for scope, _collections in self.collections.items():
for collection in _collections:
template = copy.deepcopy(self.template)
template['kv']['scope'] = scope
template['kv']["collection"] = collection
self.templates.append(template)
if collection == 'default':
cb = bucket.default_collection()
else:
cb = bucket.scope(scope).collection(collection)
self.cbs.append(cb)
except Exception as ex:
logging.error('Error trying to create CBS')
logging.error(ex)
def __init__(self, host=None, bucket=None, username=None, password=None):
config = os.environ
if not host or not bucket or not username or not password:
self.cb_host = config.get("health_cb_host", "172.23.104.180")
self.cb_bucket = config.get("health_cb_bucket",
"QE-staticserver-pool-health")
self.cb_username = config.get("health_cb_username",
"Administrator")
self.cb_userpassword = config.get("health_cb_password")
else:
self.cb_host = host
self.cb_bucket = bucket
self.cb_username = username
self.cb_userpassword = password
if not self.cb_userpassword:
print("Setting of env variable: heal_cb_password= is needed!")
return
try:
print("Connecting to {},{},{}".format(self.cb_host, self.cb_bucket,
self.cb_username))
self.cb_cluster = Cluster("couchbase://"+self.cb_host, ClusterOptions(PasswordAuthenticator(self.cb_username, self.cb_userpassword), \
timeout_options=ClusterTimeoutOptions(kv_timeout=timedelta(seconds=10))))
self.cb_b = self.cb_cluster.bucket(self.cb_bucket)
self.cb = self.cb_b.default_collection()
except Exception as e:
print('Connection Failed: %s ' % self.cb_host)
print(e)
def metrics():
metrics = []
clusters = {}
for [cluster_name, options] in settings['clusters'].items():
if cluster_name not in clusters:
try:
clusters[cluster_name] = Cluster(
'couchbase://' + options['host'],
ClusterOptions(
PasswordAuthenticator(options['username'],
options['password'])))
except Exception as e:
log.warning("Couldn't connect to cluster {}".format(e))
log.debug("Connected to {}".format(options['host']))
for options in settings["queries"] + settings["columns"]:
log.debug("Collecting metrics for {}".format(options["name"]))
try:
if "cluster" in options:
collect_cb(clusters, metrics, options)
elif "csv" in options:
collect_csv(metrics, options)
else:
raise Exception("Invalid type")
except Exception as e:
log.warning("Error while collecting {}: {}".format(
options["name"], e))
return Response("\n".join(metrics), mimetype="text/plain")
async def get_couchbase():
cluster = Cluster(
"couchbase://localhost",
ClusterOptions(PasswordAuthenticator("Administrator", "password")))
bucket = cluster.bucket("travel-sample")
await bucket.on_connect()
return cluster, bucket
def get_cluster():
opts = ClusterOptions(
authenticator=PasswordAuthenticator("Administrator", "password"),
transaction_config=TransactionConfig(
durability=ServerDurability(DurabilityLevel.PERSIST_TO_MAJORITY)))
example_cluster = Cluster.connect('couchbase://localhost', opts)
return example_cluster
def test_can_authenticate_with_cert_path_and_username_password_via_PasswordAuthenticator(
self):
cluster = Cluster('couchbases://{host}?certpath={certpath}'.format(
host=self.cluster_info.host, certpath=CERT_PATH))
authenticator = PasswordAuthenticator(self.cluster_info.admin_username,
self.cluster_info.admin_password)
cluster.authenticate(authenticator)
self._test_allow_cert_path_with_SSL_mock_errors(
cluster.open_bucket, self.cluster_info.bucket_name)
def add_cluster(host: str, username: str, password: str):
try:
clusters_lock.acquire()
if host not in clusters:
clusters[host] = Cluster(
'couchbase://' + host,
ClusterOptions(PasswordAuthenticator(username, password)),
lockmode=LockMode.WAIT)
finally:
clusters_lock.release()
async def init_cb(app):
conf = app['config']['couchbase']
cluster = Cluster("{host}:{port}".format(host=conf['host'],
port=conf['port']),
options=ClusterOptions(
PasswordAuthenticator(username=conf['user'],
password=conf['password'])))
bucket = cluster.bucket(str(conf['bucket']))
bucket.on_connect()
collection = bucket.collection(conf['collection'])
app['cb'] = cluster
app['db'] = collection
def test_disconnect(self):
# for this test we need a new cluster...
if self.is_mock:
raise SkipTest("query not mocked")
cluster = Cluster.connect(self.cluster.connstr, ClusterOptions(
PasswordAuthenticator(self.cluster_info.admin_username, self.cluster_info.admin_password)))
# Temporarily, lets open a bucket to insure the admin object was created
b = cluster.bucket(self.bucket_name)
# verify that we can get a bucket manager
self.assertIsNotNone(cluster.buckets())
# disconnect cluster
cluster.disconnect()
self.assertRaises(AlreadyShutdownException, cluster.buckets)
def connect(self, **kwargs):
# note: kwargs would be how one could pass in
# more info for client config
conn_str = 'couchbase://{0}'.format(self.host)
try:
cluster_opts = ClusterOptions(authenticator=PasswordAuthenticator(
self.username, self.password))
self._cluster = Cluster(conn_str, options=cluster_opts)
self._bucket = self._cluster.bucket(self.bucket_name)
self._collection = self._bucket.default_collection()
except CouchbaseException as error:
print('Could not connect to cluster. Error: {}'.format(error))
raise
def apply(self, backup_base, backup):
uid = backup.get_collection_uid(self.collection_string)
cluster = Cluster("couchbase://" + backup_base.master.ip,
authenticator=PasswordAuthenticator(
"Administrator", "password"))
bucket = cluster.bucket(self.collection_string.bucket_name)
scope = bucket.scope(self.collection_string.scope_name)
collection = scope.collection(self.collection_string.collection_name)
set_result = collection.mutate_in(self.key, [
SD.upsert(self.path, self.value, xattr=True, create_parents=True)
])
backup.set_subdoc(self.collection_string, self.key, self.path,
self.value, Tag.XATTR_CHANGED)
def test_no_mixed_auth(self):
cluster, bucket_name = self._create_cluster()
auther = PasswordAuthenticator(bucket_name,
self.cluster_info.bucket_password)
cluster.authenticate(auther)
cb1 = cluster.open_bucket(bucket_name)
self.assertRaises(MixedAuthException,
cluster.open_bucket,
bucket_name,
password=self.cluster_info.bucket_password)
cluster2, bucket_name = self._create_cluster()
cb2 = cluster2.open_bucket(bucket_name,
password=self.cluster_info.bucket_password)
def authenticate(self, authenticator=None, username=None, password=None):
"""
Set the type of authenticator to use when opening buckets or performing
cluster management operations
:param authenticator: The new authenticator to use
:param username: The username to authenticate with
:param password: The password to authenticate with
"""
if authenticator is None:
if not username:
raise ValueError('username must not be empty.')
if not password:
raise ValueError('password must not be empty.')
authenticator = PasswordAuthenticator(username, password)
self.authenticator = authenticator
def setUp(self,
trace_all=True,
flushcount=0,
enable_logging=False,
use_parent_tracer=False,
*args,
**kwargs):
self.timeout = None
# self.enable_logging = enable_logging or os.environ.get("PYCBC_ENABLE_LOGGING")
self.use_parent_tracer = use_parent_tracer
self.using_jaeger = (os.environ.get("PYCBC_USE_JAEGER") == "TRUE")
self.flushdict = {
k: v
for k, v in zip(map(str, range(1, 100)), map(str, range(1, 100)))
}
self.trace_all = os.environ.get("PYCBC_TRACE_ALL") or trace_all
self.flushcount = flushcount
if self.using_jaeger and self.flushcount > 5:
raise SkipTest("too slow when using jaeger")
enable_logging |= bool(self.trace_all)
if enable_logging:
couchbase.enable_logging()
if self.use_parent_tracer:
kwargs['init_tracer'] = self.init_tracer
kwargs['enable_tracing'] = "true"
if self.trace_all:
tracing_options = ClusterTracingOptions(
tracing_orphaned_queue_flush_interval=timedelta(
milliseconds=1),
tracing_orphaned_queue_size=9,
tracing_threshold_queue_flush_interval=timedelta(
milliseconds=1),
tracing_threshold_queue_size=9,
tracing_threshold_kv=timedelta(milliseconds=1),
#tracing_threshold_query=timedelta(milliseconds=1),
tracing_threshold_view=timedelta(milliseconds=1),
tracing_threshold_search=timedelta(milliseconds=1),
tracing_threshold_analytics=timedelta(milliseconds=1))
dummy_auth = PasswordAuthenticator("default", "password")
# the dummy_auth isn't really used, the base class decides between classic
# and password dependng on mock or not.
opts = ClusterOptions(authenticator=dummy_auth,
tracing_options=tracing_options)
kwargs["cluster_options"] = opts
super(TracedCase, self).setUp(**kwargs)
def _get_connection(self):
"""Connect to the Couchbase server."""
if self._connection is None:
if self.host and self.port:
uri = f"couchbase://{self.host}:{self.port}"
else:
uri = f"couchbase://{self.host}"
if self.username and self.password:
opt = PasswordAuthenticator(self.username, self.password)
else:
opt = None
cluster = Cluster(uri, opt)
bucket = cluster.bucket(self.bucket)
self._connection = bucket.default_collection()
return self._connection
import uuid
from couchbase.mutation_state import MutationState
from couchbase.cluster import QueryScanConsistency
# tag::n1ql_basic_example[]
from couchbase.cluster import Cluster, ClusterOptions, QueryOptions
from couchbase.auth import PasswordAuthenticator
from couchbase.exceptions import CouchbaseException
cluster = Cluster.connect(
"couchbase://localhost",
ClusterOptions(PasswordAuthenticator("Administrator", "password")))
bucket = cluster.bucket("travel-sample")
collection = bucket.default_collection()
try:
result = cluster.query(
"SELECT * FROM `travel-sample`.inventory.airport LIMIT 10",
QueryOptions(metrics=True))
for row in result.rows():
print("Found row: {}".format(row))
print("Report execution time: {}".format(
result.metadata().metrics().execution_time()))
except CouchbaseException as ex:
import traceback
traceback.print_exc()
# end::n1ql_basic_example[]
from couchbase.cluster import Cluster
from couchbase.auth import PasswordAuthenticator
from couchbase.management.views import (View, DesignDocument,
DesignDocumentNamespace,
DesignDocumentNotFoundException)
# tag::create_view_mgr[]
cluster = Cluster("couchbase://localhost",
authenticator=PasswordAuthenticator("Administrator",
"password"))
# For Server versions 6.5 or later you do not need to open a bucket here
bucket = cluster.bucket("travel-sample")
view_manager = bucket.view_indexes()
# end::create_view_mgr[]
# tag::create_view[]
design_doc = DesignDocument(
name="landmarks",
views={
"by_country":
View(
map=
"function (doc, meta) { if (doc.type == 'landmark') { emit([doc.country, doc.city], null); } }"
),
"by_activity":
View(
map=
"function (doc, meta) { if (doc.type == 'landmark') { emit(doc.activity, null); } }",
reduce="_count")
if options.components and options.exclude_components:
logger.error("both include and exclude components specified")
sys.exit(1)
if options.subcomponents and len(options.components) > 1:
logger.error("Can't supply multiple components with subcomponents")
sys.exit(1)
if __name__ == "__main__":
options = parse_arguments()
cluster = Cluster(
'couchbase://{}'.format(options.server),
ClusterOptions(
PasswordAuthenticator(options.username, options.password)))
validate_options(options, cluster)
logger.debug(options)
setup_logs(options)
server = connect_to_jenkins(options.jenkins_url)
if options.wait_for_main_run:
wait_for_main_run(options, cluster, server)
pool_thresholds_hit = []
queue = {}
already_rerun = set()
# timeout after 20 hours
timeout = time.time() + (options.timeout * 60 * 60)
def _authenticator(self):
if self.is_mock:
return ClassicAuthenticator(self.cluster_info.admin_username,
self.cluster_info.admin_password)
return PasswordAuthenticator(self.cluster_info.admin_username,
self.cluster_info.admin_password)
from couchbase.cluster import Cluster, ClusterOptions
from couchbase.auth import PasswordAuthenticator
from uuid import uuid4
from datetime import datetime
import os
import traceback
from configparser import ConfigParser
from requests.auth import HTTPBasicAuth
CB_BUCKET = os.environ.get("CB_BUCKET") or "system_test_dashboard"
CB_USERNAME = os.environ.get("CB_USERNAME") or "Administrator"
CB_PASSWORD = os.environ.get("CB_PASSWORD") or "password"
cluster = Cluster(
"couchbase://{}".format(os.environ["CB_SERVER"]),
ClusterOptions(PasswordAuthenticator(CB_USERNAME, CB_PASSWORD),
lockmode=LockMode.WAIT))
bucket = cluster.bucket(CB_BUCKET)
collection = bucket.default_collection()
requests_cache.install_cache(expire_after=60, backend="memory")
app = Flask("systen_testing_dashbord")
app.secret_key = b'\xe0\xac#\x06\xe3\xc5\x19\xd6\xfd\xaf+e\xb9\xd0\xb0\x1f'
LAUNCHER_TO_PARSER_CACHE = {}
JENKINS_PREFIX = "http://qa.sc.couchbase.com/job/"
def fetch_launchers():
from couchbase.cluster import Cluster
from couchbase.auth import PasswordAuthenticator
from couchbase.management.queries import CreatePrimaryQueryIndexOptions
from couchbase.management.users import User, Role
bucket_name = "travel-sample"
username = "******"
pw = "test-passw0rd!"
adm_cluster = Cluster("couchbase://localhost",
authenticator=PasswordAuthenticator(
"Administrator", "password"))
# For Server versions 6.5 or later you do not need to open a bucket here
adm_bucket = adm_cluster.bucket(bucket_name)
# tag::create_user[]
user_manager = adm_cluster.users()
user = User(
username=username,
display_name="Test User",
roles=[
# Roles required for reading data from bucket
Role(name="data_reader", bucket="*"),
Role(name="query_select", bucket="*"),
# Roles require for writing data to bucket
Role(name="data_writer", bucket=bucket_name),
Role(name="query_insert", bucket=bucket_name),
Role(name="query_delete", bucket=bucket_name),
# Role required for idx creation on bucket
Role(name="query_manage_index", bucket=bucket_name),
],
level=logging.DEBUG)
with open("queries.json") as json_file:
settings = json.load(json_file)
log.info("Loaded queries.json")
log.info("Connecting to clusters")
clusters = {}
for [cluster_name, options] in settings['clusters'].items():
if cluster_name not in clusters:
clusters[cluster_name] = Cluster('couchbase://' + options['host'],
ClusterOptions(
PasswordAuthenticator(
options['username'],
options['password'])),
lockmode=LockMode.WAIT)
log.info("Connected to %s", options['host'])
log.info("Connected to clusters")
class CouchbaseQueryCollector():
def __init__(self,
cluster,
name,
description,
query,
value_key,
labels=[]):
from datetime import timedelta
from couchbase.auth import PasswordAuthenticator
from couchbase.cluster import Cluster, ClusterOptions
from couchbase.management.queries import (CreatePrimaryQueryIndexOptions,
CreateQueryIndexOptions,
DropPrimaryQueryIndexOptions,
WatchQueryIndexOptions)
cluster = Cluster(
'couchbase://localhost',
ClusterOptions(PasswordAuthenticator('Administrator', 'password')))
print("[primary]")
# tag::primary[]
cluster.query_indexes().create_primary_index(
"travel-sample",
# Don't error if the primary index already exists.
CreatePrimaryQueryIndexOptions(ignore_if_exists=True))
# end::primary[]
print("Index creation complete")
print("\n[named-primary]")
# tag::named-primary[]
cluster.query_indexes().create_primary_index(
"travel-sample",
CreatePrimaryQueryIndexOptions(index_name="named_primary_index"))
# end::named-primary[]
print("Named primary index creation complete")
print("\n[secondary]")
# needed for options -- cluster, timeout, SQL++ (N1QL) query, etc.
from couchbase.options import (ClusterOptions, ClusterTimeoutOptions,
QueryOptions)
# end::imports[]
# tag::connect[]
# Update this to your cluster
endpoint = "--your-instance--.dp.cloud.couchbase.com"
username = "******"
password = "******"
bucket_name = "travel-sample"
# User Input ends here.
# Connect options - authentication
auth = PasswordAuthenticator(username, password)
# Connect options - global timeout opts
timeout_opts = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=10))
# get a reference to our cluster
cluster = Cluster('couchbases://{}'.format(endpoint),
ClusterOptions(auth, timeout_options=timeout_opts))
# Wait until the cluster is ready for use.
cluster.wait_until_ready(timedelta(seconds=5))
# end::connect[]
# tag::bucket[]
# get a reference to our bucket
cb = cluster.bucket(bucket_name)
cb_user_p = os.environ.get('cb_password')
if not cb_user_p:
print("Error: cb_password environment variable setting is missing!")
exit(1)
cb_bucket = os.environ.get('cb_bucket')
if not cb_bucket:
cb_bucket = "greenboard"
is_include_unstable = os.environ.get('is_include_unstable')
if not is_include_unstable:
print("No result=UNSTABLE jobs included while getting the IPs list")
is_include_unstable = False
#print("Connecting to the greenboard couchbase nosql...")
cluster = Cluster(
"couchbase://" + cb_host,
ClusterOptions(PasswordAuthenticator(cb_user, cb_user_p),
timeout_options=ClusterTimeoutOptions(kv_timeout=timedelta(
seconds=10))))
bucket = cluster.bucket(cb_bucket)
doc = bucket.get(cb_build + "_server").value
index = 0
success_count = 0
failure_count = 0
aborted_count = 0
unstable_count = 0
unknown_count = 0
xen_hosts_map = {}
if xen_hosts_file:
hosts_file = open(xen_hosts_file)
#print("Please wait while loading the xenhosts information...")
lines = hosts_file.readlines()
ch.setFormatter(formatter)
logger.addHandler(ch)
ap = argparse.ArgumentParser()
ap.add_argument("--cb_server", default="172.23.121.84")
ap.add_argument("--cb_username", default="Administrator")
ap.add_argument("--cb_password", default="password")
ap.add_argument("versions")
args = vars(ap.parse_args())
cluster = Cluster(
"couchbase://" + args["cb_server"],
ClusterOptions(
PasswordAuthenticator(args["cb_username"], args["cb_password"])))
server_bucket = cluster.bucket("server")
greenboard_bucket = cluster.bucket("greenboard")
greenboard_collection = greenboard_bucket.default_collection()
supplied_versions = args["versions"].split(",")
versions = set()
for v in supplied_versions:
for version in list(
server_bucket.query(
"select raw `build` from server where `build` like '%{}%' group by `build`"
.format(v))):
versions.add(version)
from couchbase import enable_logging
# NOTE: for simple test to see output, drop the threshold
# ex: tracing_threshold_kv=timedelta(microseconds=1)
# tag::threshold_logging_config[]
# configure logging
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
# setup couchbase logging
enable_logging()
tracing_opts = ClusterTracingOptions(
tracing_threshold_queue_size=10,
tracing_threshold_kv=timedelta(milliseconds=500))
cluster_opts = ClusterOptions(authenticator=PasswordAuthenticator(
"Administrator",
"password"),
tracing_options=tracing_opts)
cluster = Cluster(
"couchbase://localhost",
options=cluster_opts
)
# end::threshold_logging_config[]
collection = cluster.bucket("beer-sample").default_collection()
for _ in range(100):
collection.get("21st_amendment_brewery_cafe")
ch.setFormatter(formatter)
logger.addHandler(ch)
ap = argparse.ArgumentParser()
ap.add_argument("--cb_server", default="172.23.121.84")
ap.add_argument("--cb_username", default="Administrator")
ap.add_argument("--cb_password", default="password")
ap.add_argument("--update", default=False, action="store_true")
ap.add_argument("versions")
args = vars(ap.parse_args())
cluster = Cluster(
"couchbase://" + args["cb_server"],
ClusterOptions(PasswordAuthenticator(args["cb_username"],
args["cb_password"]),
timeout_options=ClusterTimeoutOptions(kv_timeout=timedelta(
seconds=10))))
server_bucket = cluster.bucket("server")
greenboard_bucket = cluster.bucket("greenboard")
greenboard_collection = greenboard_bucket.default_collection()
server_collection = server_bucket.default_collection()
supplied_versions = args["versions"].split(",")
versions = set()
for v in supplied_versions:
for version in list(
server_bucket.query(
"select raw `build` from server where `build` like '%{}%' group by `build`"
