def setUp(self):
super(N1qlBase, self).setUp()
self.scan_consistency = self.input.param("scan_consistency",
'REQUEST_PLUS')
self.path = testconstants.LINUX_COUCHBASE_BIN_PATH
self.use_rest = self.input.param("use_rest", True)
self.hint_index = self.input.param("hint", None)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics", False)
self.named_prepare = self.input.param("named_prepare", None)
self.version = self.input.param("cbq_version", "sherlock")
self.isprepared = False
self.ipv6 = self.input.param("ipv6", False)
self.trans = self.input.param("n1ql_txn", True)
self.commit = self.input.param("commit", True)
self.rollback_to_savepoint = self.input.param("rollback_to_savepoint", False)
self.skip_index = self.input.param("skip_index", False)
self.primary_indx_type = self.input.param("primary_indx_type", 'GSI')
self.index_type = self.input.param("index_type", 'GSI')
self.skip_primary_index = self.input.param("skip_primary_index", False)
self.flat_json = self.input.param("flat_json", False)
self.dataset = self.input.param("dataset", "sabre")
self.array_indexing = self.input.param("array_indexing", False)
self.max_verify = self.input.param("max_verify", None)
self.num_stmt_txn = self.input.param("num_stmt_txn", 5)
self.num_collection = self.input.param("num_collection", 1)
self.num_savepoints = self.input.param("num_savepoints", 0)
self.override_savepoint = self.input.param("override_savepoint", 0)
self.num_buckets = self.input.param("num_buckets", 1)
self.prepare = self.input.param("prepare_stmt", False)
self.num_txn = self.input.param("num_txn", 3)
self.clause = WhereClause()
self.buckets = self.bucket_util.get_all_buckets(self.cluster)
self.collection_map = {}
self.txtimeout = self.input.param("txntimeout", 0)
self.atrcollection = self.input.param("atrcollection", False)
self.num_commit = self.input.param("num_commit", 3)
self.num_rollback_to_savepoint = \
self.input.param("num_rollback_to_savepoint", 0)
self.num_conflict = self.input.param("num_conflict", 0)
self.write_conflict = self.input.param("write_conflict", False)
self.Kvtimeout = self.input.param("Kvtimeout", None)
self.memory_quota = self.input.param("memory_quota", 1)
self.n1ql_server = self.cluster_util.get_nodes_from_services_map(
cluster=self.cluster,
service_type=CbServer.Services.N1QL,
get_all_nodes=True)
self.n1ql_helper = N1QLHelper(server=self.n1ql_server,
use_rest=True,
buckets=self.buckets,
log=self.log,
scan_consistency='REQUEST_PLUS',
num_collection=self.num_collection,
num_buckets=self.num_buckets,
num_savepoints=self.num_savepoints,
override_savepoint=self.override_savepoint,
num_stmt=self.num_stmt_txn,
load_spec=self.data_spec_name)
self.num_insert, self.num_update, self.num_delete, self.num_merge = \
self.n1ql_helper.get_random_number_stmt(self.num_stmt_txn)
def setUp(self):
self.array_indexing = False
super(UpgradeN1QLRBAC, self).setUp()
self.initial_version = self.input.param('initial_version',
'4.6.0-3653')
self.upgrade_to = self.input.param("upgrade_to")
self.n1ql_helper = N1QLHelper(version=self.version,
shell=self.shell,
use_rest=self.use_rest,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=[],
log=self.log,
input=self.input,
master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
log.info(self.n1ql_node)
if self.ddocs_num:
self.create_ddocs_and_views()
gen_load = BlobGenerator('pre-upgrade',
'preupgrade-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master,
gen_load,
"create",
self.expire_time,
flag=self.item_flag)
self.query_standard_bucket = self.query_buckets[1]
def setUp(self):
super(CollectionsDropRecreateRebalance, self).setUp()
self.known_nodes = self.cluster.servers[:self.nodes_init]
self.nodes_failover = self.input.param("nodes_failover", 1)
self.nodes_swap = self.input.param("nodes_swap", 0)
self.recovery_type = self.input.param("recovery_type", "delta")
self.rebalance_moves_per_node = self.input.param(
"rebalance_moves_per_node", 2)
self.cluster_util.set_rebalance_moves_per_nodes(
rebalanceMovesPerNode=self.rebalance_moves_per_node)
self.data_load_flag = False # When to start/stop drop/recreate
self.data_loading_thread = None
self.data_load_exception = None # Object variable to assign data load thread's exception
self.N1qltxn = self.input.param("N1ql_txn", False)
if self.N1qltxn:
self.n1ql_server = self.cluster_util.get_nodes_from_services_map(
service_type="n1ql", get_all_nodes=True)
self.n1ql_helper = N1QLHelper(server=self.n1ql_server,
use_rest=True,
buckets=self.bucket_util.buckets,
log=self.log,
scan_consistency='REQUEST_PLUS',
num_collection=3,
num_buckets=1,
num_savepoints=1,
override_savepoint=False,
num_stmt=10,
load_spec=self.data_spec_name)
self.bucket_col = self.n1ql_helper.get_collections()
self.stmts = self.n1ql_helper.get_stmt(self.bucket_col)
self.stmts = self.n1ql_helper.create_full_stmts(self.stmts)
def _create_primary_index(self):
if self.n1ql_server:
if self.doc_ops:
self.ops_dist_map = self.calculate_data_change_distribution(
create_per=self.create_ops_per,
update_per=self.update_ops_per,
delete_per=self.delete_ops_per,
expiry_per=self.expiry_ops_per,
start=0,
end=self.docs_per_day)
log.info(self.ops_dist_map)
self.docs_gen_map = self.generate_ops_docs(
self.docs_per_day, 0)
self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(
self.docs_gen_map)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
log.info(self.n1ql_server)
if self.create_primary_index:
self.n1ql_helper.create_primary_index(
using_gsi=self.use_gsi_for_primary,
server=self.n1ql_server)
def setUp(self):
if self._testMethodDoc:
log.info("\n\nStarting Test: %s \n%s" % (self._testMethodName, self._testMethodDoc))
else:
log.info("\n\nStarting Test: %s" % (self._testMethodName))
self.input = TestInputSingleton.input
self.input.test_params.update({"default_bucket": False})
super(EventingBaseTest, self).setUp()
self.master = self.servers[0]
self.server = self.master
self.restServer = self.get_nodes_from_services_map(service_type="eventing")
self.rest = RestConnection(self.restServer)
self.rest.set_indexer_storage_mode()
self.log.info(
"Setting the min possible memory quota so that adding mode nodes to the cluster wouldn't be a problem.")
self.rest.set_service_memoryQuota(service='memoryQuota', memoryQuota=330)
self.rest.set_service_memoryQuota(service='indexMemoryQuota', memoryQuota=INDEX_QUOTA)
# self.rest.set_service_memoryQuota(service='eventingMemoryQuota', memoryQuota=EVENTING_QUOTA)
self.src_bucket_name = self.input.param('src_bucket_name', 'src_bucket')
self.eventing_log_level = self.input.param('eventing_log_level', 'INFO')
self.dst_bucket_name = self.input.param('dst_bucket_name', 'dst_bucket')
self.dst_bucket_name1 = self.input.param('dst_bucket_name1', 'dst_bucket1')
self.metadata_bucket_name = self.input.param('metadata_bucket_name', 'metadata')
self.create_functions_buckets = self.input.param('create_functions_buckets', True)
self.docs_per_day = self.input.param("doc-per-day", 1)
self.use_memory_manager = self.input.param('use_memory_manager', True)
self.print_eventing_handler_code_in_logs = self.input.param('print_eventing_handler_code_in_logs', True)
random.seed(datetime.time)
function_name = "Function_{0}_{1}".format(random.randint(1, 1000000000), self._testMethodName)
# See MB-28447, From now function name can only be max of 100 chars
self.function_name = function_name[0:90]
self.timer_storage_chan_size = self.input.param('timer_storage_chan_size', 10000)
self.dcp_gen_chan_size = self.input.param('dcp_gen_chan_size', 10000)
self.is_sbm=self.input.param('source_bucket_mutation', False)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.n1ql_helper = N1QLHelper(shell=self.shell, max_verify=self.max_verify, buckets=self.buckets,
item_flag=self.item_flag, n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list, log=self.log, input=self.input,
master=self.master, use_rest=True)
self.pause_resume = self.input.param('pause_resume', False)
self.pause_resume_number = self.input.param('pause_resume_number', 1)
self.is_curl=self.input.param('curl', False)
self.hostname = self.input.param('host', 'https://postman-echo.com/')
self.curl_username = self.input.param('curl_user', None)
self.curl_password = self.input.param('curl_password', None)
self.auth_type = self.input.param('auth_type', 'no-auth')
self.bearer_key=self.input.param('bearer_key', None)
self.url = self.input.param('path', None)
self.cookies = self.input.param('cookies', False)
self.bearer_key = self.input.param('bearer_key', '')
if self.hostname=='local':
self.insall_dependencies()
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip = s.getsockname()[0]
s.close()
self.hostname= "http://"+ip+":1080/"
self.log.info("local ip address:{}".format(self.hostname))
self.setup_curl()
def test_gsi_with_crud_with_redaction_enabled(self):
# load bucket and do some ops
self.set_indexer_logLevel("trace")
json_generator = JsonGenerator()
gen_docs = json_generator.generate_all_type_documents_for_gsi(
docs_per_day=self.doc_per_day, start=0)
full_docs_list = self.generate_full_docs_list(gen_docs)
n1ql_helper = N1QLHelper(use_rest=True,
buckets=self.buckets,
full_docs_list=full_docs_list,
log=log,
input=self.input,
master=self.master)
self.load(gen_docs)
n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
query_definition_generator = SQLDefinitionGenerator()
query_definitions = query_definition_generator.generate_airlines_data_query_definitions(
)
query_definitions = query_definition_generator.filter_by_group(
"all", query_definitions)
# set log redaction level, collect logs, verify log files exist and verify them for redaction
self.set_redaction_level()
self.start_logs_collection()
# Create partial Index
for query_definition in query_definitions:
for bucket in self.buckets:
create_query = query_definition.generate_index_create_query(
bucket.name)
n1ql_helper.run_cbq_query(query=create_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
scan_query = query_definition.generate_query(
bucket=bucket.name)
n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
drop_query = query_definition.generate_index_drop_query(
bucket=bucket.name)
n1ql_helper.run_cbq_query(query=drop_query, server=n1ql_node)
result = self.monitor_logs_collection()
log.info(result)
try:
logs_path = result["perNode"]["ns_1@" +
str(self.master.ip)]["path"]
except KeyError:
logs_path = result["perNode"]["[email protected]"]["path"]
redactFileName = logs_path.split('/')[-1]
nonredactFileName = logs_path.split('/')[-1].replace('-redacted', '')
remotepath = logs_path[0:logs_path.rfind('/') + 1]
log_file = self.input.param("log_file_name", "indexer.log")
self.verify_log_files_exist(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName)
self.verify_log_redaction(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName,
logFileName="ns_server.{0}".format(log_file))
def setUp(self):
super(AggregatePushdownClass, self).setUp()
self.n1ql_helper = N1QLHelper(master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.aggr_distinct = self.input.param("aggr_distinct", False)
self.order_by = self.input.param("order_by", False)
self.having = self.input.param("having", False)
self.offset_limit = self.input.param("offset_limit", False)
def _create_primary_index(self):
n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
n1ql_helper = N1QLHelper(shell=self.shell, max_verify=self.max_verify, buckets=self.buckets,
item_flag=self.item_flag, n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list, log=self.log, input=self.input,
master=self.master, use_rest=True)
# primary index is required as we run some queries from handler code
n1ql_helper.create_primary_index(using_gsi=True, server=n1ql_node)
def setUp(self):
super(EventingMultiHandler, self).setUp()
self.num_src_buckets = self.input.param('num_src_buckets', 1)
self.num_dst_buckets = self.input.param('num_dst_buckets', 1)
self.num_handlers = self.input.param('num_handlers', 1)
self.deploy_handler = self.input.param('deploy_handler', 1)
self.sequential = self.input.param('sequential', True)
self.num_pause = self.input.param('num_pause', 0)
self.worker_count = self.input.param('worker_count', 1)
self.handler_code = self.input.param('handler_code',
'handler_code/ABO/insert.js')
self.gens_load = self.generate_docs(self.docs_per_day)
quota = (self.num_src_buckets + self.num_dst_buckets) * 100 + 300
self.rest.set_service_memoryQuota(service='memoryQuota',
memoryQuota=quota)
self.metadata_bucket_size = 300
bucket_params_meta = self._create_bucket_params(
server=self.server,
size=self.metadata_bucket_size,
replicas=self.num_replicas)
self.create_n_buckets(self.src_bucket_name, self.num_src_buckets)
self.buckets = RestConnection(self.master).get_buckets()
if self.num_dst_buckets > 0:
self.create_n_buckets(self.dst_bucket_name, self.num_dst_buckets)
self.cluster.create_standard_bucket(name=self.metadata_bucket_name,
port=STANDARD_BUCKET_PORT + 1,
bucket_params=bucket_params_meta)
self.create_scope_collection(bucket=self.metadata_bucket_name,
scope=self.metadata_bucket_name,
collection=self.metadata_bucket_name)
self.deploying = []
self.pausing = []
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
self.n1ql_helper.create_primary_index(using_gsi=True,
server=self.n1ql_node)
self.binding_map = {}
def test_n1ql_max_connection(self):
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
# primary index is required as we run some queries from handler code
self.n1ql_helper.create_primary_index(using_gsi=True,
server=self.n1ql_node)
self.load_sample_buckets(self.server, "travel-sample")
self.load(self.gens_load,
buckets=self.src_bucket,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size)
body = self.create_save_function_body(
self.function_name,
'handler_code/n1ql_op_connection.js',
worker_count=1)
self.deploy_function(body)
self.verify_eventing_results(self.function_name,
self.docs_per_day * 2016,
skip_stats_validation=True)
# delete all documents
self.load(self.gens_load,
buckets=self.src_bucket,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size,
op_type='delete')
# Wait for eventing to catch up with all the delete mutations and verify results
self.verify_eventing_results(self.function_name,
0,
on_delete=True,
skip_stats_validation=True)
self.undeploy_and_delete_function(body)
def create_index(self, bucket, query_definition, deploy_node_info=None):
defer_build = True
query = query_definition.generate_index_create_query(
bucket=bucket,
use_gsi_for_secondary=self.use_gsi_for_secondary,
deploy_node_info=deploy_node_info,
defer_build=defer_build,
num_replica=self.num_index_replicas)
log.info(query)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
create_index_task = self.cluster.async_create_index(
server=self.n1ql_server,
bucket=bucket,
query=query,
n1ql_helper=self.n1ql_helper,
index_name=query_definition.index_name,
defer_build=defer_build)
create_index_task.result()
query = self.n1ql_helper.gen_build_index_query(
bucket=bucket, index_list=[query_definition.index_name])
build_index_task = self.cluster.async_build_index(
server=self.n1ql_server,
bucket=bucket,
query=query,
n1ql_helper=self.n1ql_helper)
build_index_task.result()
check = self.n1ql_helper.is_index_ready_and_in_list(
bucket, query_definition.index_name, server=self.n1ql_server)
self.assertTrue(
check, "index {0} failed to be created".format(
query_definition.index_name))
def setUp(self):
super(AggregatePushdownRecoveryClass, self).setUp()
self.n1ql_helper = N1QLHelper(master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.aggr_distinct = self.input.param("aggr_distinct", False)
self.graceful = self.input.param("graceful", False)
def setUp(self):
super(CrashTest, self).setUp()
self.doc_ops = self.input.param("doc_ops", None)
self.process_name = self.input.param("process", None)
self.service_name = self.input.param("service", "data")
self.sig_type = self.input.param("sig_type", "SIGKILL").upper()
self.target_node = self.input.param("target_node", "active")
self.client_type = self.input.param("client_type", "sdk").lower()
self.N1qltxn = self.input.param("N1qltxn", False)
self.pre_warmup_stats = dict()
self.timeout = 120
self.new_docs_to_add = 10000
if self.doc_ops is not None:
self.doc_ops = self.doc_ops.split(";")
if not self.atomicity:
self.durability_helper = DurabilityHelper(
self.log, self.nodes_init,
durability=self.durability_level,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
verification_dict = dict()
verification_dict["ops_create"] = \
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items
verification_dict["sync_write_aborted_count"] = 0
verification_dict["rollback_item_count"] = 0
verification_dict["pending_writes"] = 0
if self.durability_level:
verification_dict["sync_write_committed_count"] = \
verification_dict["ops_create"]
# Load initial documents into the buckets
transaction_gen_create = doc_generator(
"transaction_key", 0, self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
gen_create = doc_generator(
self.key, 0, self.num_items,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets)
if self.atomicity:
transaction_task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.cluster.buckets,
transaction_gen_create, DocLoading.Bucket.DocOps.CREATE,
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
self.task.jython_task_manager.get_task_result(transaction_task)
for bucket in self.cluster.buckets:
task = self.task.async_load_gen_docs(
self.cluster, bucket, gen_create,
DocLoading.Bucket.DocOps.CREATE, self.maxttl,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
batch_size=10, process_concurrency=8)
self.task.jython_task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets(self.cluster.buckets)
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items += self.num_items
verification_dict["ops_create"] += self.num_items
if self.durability_level:
verification_dict["sync_write_committed_count"] += \
self.num_items
# Verify cbstats vbucket-details
stats_failed = self.durability_helper.verify_vbucket_details_stats(
bucket, self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=verification_dict)
if self.atomicity is False:
if stats_failed:
self.fail("Cbstats verification failed")
self.bucket_util.verify_stats_all_buckets(
self.cluster,
self.cluster.buckets[0].scopes[
CbServer.default_scope].collections[
CbServer.default_collection].num_items)
self.bucket = self.cluster.buckets[0]
if self.N1qltxn:
self.n1ql_server = self.cluster_util.get_nodes_from_services_map(
service_type="n1ql",
get_all_nodes=True)
self.n1ql_helper = N1QLHelper(server=self.n1ql_server,
use_rest=True,
buckets=self.cluster.buckets,
log=self.log,
scan_consistency='REQUEST_PLUS',
num_collection=3,
num_buckets=1,
num_savepoints=1,
override_savepoint=False,
num_stmt=10,
load_spec=self.data_spec_name)
self.bucket_col = self.n1ql_helper.get_collections()
self.stmts = self.n1ql_helper.get_stmt(self.bucket_col)
self.stmts = self.n1ql_helper.create_full_stmts(self.stmts)
self.log.info("==========Finished CrashTest setup========")
def setUp(self):
super(AggregatePushdownClass, self).setUp()
self.n1ql_helper = N1QLHelper(master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
def setUp(self):
super(QueryTests, self).setUp()
self.expiry = self.input.param("expiry", 0)
self.batch_size = self.input.param("batch_size", 1)
self.scan_consistency = self.input.param("scan_consistency",
"request_plus")
self.skip_cleanup = self.input.param("skip_cleanup", False)
self.run_async = self.input.param("run_async", True)
self.version = self.input.param("cbq_version", "git_repo")
for server in self.servers:
rest = RestConnection(server)
temp = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(
server.ip, temp['nodes'][0]['status']))
while (temp['nodes'][0]['status'] == 'warmup'):
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
temp = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(
server.ip, temp['nodes'][0]['status']))
indexer_node = self.get_nodes_from_services_map(service_type="index",
get_all_nodes=True)
# Set indexer storage mode
indexer_rest = RestConnection(indexer_node[0])
doc = {"indexer.settings.storage_mode": self.gsi_type}
indexer_rest.set_index_settings_internal(doc)
doc = {"indexer.api.enableTestServer": True}
indexer_rest.set_index_settings_internal(doc)
self.indexer_scanTimeout = self.input.param("indexer_scanTimeout",
None)
if self.indexer_scanTimeout is not None:
for server in indexer_node:
rest = RestConnection(server)
rest.set_index_settings({
"indexer.settings.scan_timeout":
self.indexer_scanTimeout
})
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
self.use_gsi_for_primary = self.input.param("use_gsi_for_primary",
True)
self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary",
True)
self.create_primary_index = self.input.param("create_primary_index",
True)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.value_size = self.input.param("value_size", 1024)
self.doc_ops = self.input.param("doc_ops", False)
self.create_ops_per = self.input.param("create_ops_per", 0)
self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
self.delete_ops_per = self.input.param("delete_ops_per", 0)
self.update_ops_per = self.input.param("update_ops_per", 0)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.n1ql_helper.configure_gomaxprocs()
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
verify_data = False
if self.scan_consistency != "request_plus":
verify_data = True
self.load(self.gens_load,
flag=self.item_flag,
verify_data=verify_data,
batch_size=self.batch_size)
if self.doc_ops:
self.ops_dist_map = self.calculate_data_change_distribution(
create_per=self.create_ops_per,
update_per=self.update_ops_per,
delete_per=self.delete_ops_per,
expiry_per=self.expiry_ops_per,
start=0,
end=self.docs_per_day)
self.log.info(self.ops_dist_map)
self.docs_gen_map = self.generate_ops_docs(self.docs_per_day, 0)
self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(
self.docs_gen_map)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(version=self.version,
shell=self.shell,
use_rest=self.use_rest,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.log.info(self.n1ql_node)
#self.n1ql_helper._start_command_line_query(self.n1ql_node)
# sleep to avoid race condition during bootstrap
if self.create_primary_index:
try:
self.n1ql_helper.create_primary_index(
using_gsi=self.use_gsi_for_primary, server=self.n1ql_node)
except Exception as ex:
self.log.info(ex)
raise ex
def __init__(self, node):
self.log = logger.Logger.get_logger()
self.node = node
self.use_rest = True
self.n1ql_helper = N1QLHelper(use_rest=True, log=self.log)
self.cli_helper = CollectionsCLI(node)
def setUp(self):
super(AggregatePushdownClass, self).setUp()
self.n1ql_helper = N1QLHelper(master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.array_type = self.input.param("array_type", "all")
self.aggr_distinct = self.input.param("aggr_distinct", False)
def setUp(self):
if self._testMethodDoc:
log.info("\n\nStarting Test: %s \n%s" %
(self._testMethodName, self._testMethodDoc))
else:
log.info("\n\nStarting Test: %s" % (self._testMethodName))
self.input = TestInputSingleton.input
self.input.test_params.update({"default_bucket": False})
super(EventingBaseTest, self).setUp()
self.master = self.servers[0]
self.server = self.master
self.restServer = self.get_nodes_from_services_map(
service_type="eventing")
self.rest = RestConnection(self.restServer)
self.rest.set_indexer_storage_mode()
self.log.info(
"Setting the min possible memory quota so that adding mode nodes to the cluster wouldn't be a problem."
)
self.rest.set_service_memoryQuota(service='memoryQuota',
memoryQuota=330)
self.rest.set_service_memoryQuota(service='indexMemoryQuota',
memoryQuota=INDEX_QUOTA)
# self.rest.set_service_memoryQuota(service='eventingMemoryQuota', memoryQuota=EVENTING_QUOTA)
self.src_bucket_name = self.input.param('src_bucket_name',
'src_bucket')
self.eventing_log_level = self.input.param('eventing_log_level',
'INFO')
self.dst_bucket_name = self.input.param('dst_bucket_name',
'dst_bucket')
self.dst_bucket_name1 = self.input.param('dst_bucket_name1',
'dst_bucket1')
self.metadata_bucket_name = self.input.param('metadata_bucket_name',
'metadata')
self.create_functions_buckets = self.input.param(
'create_functions_buckets', True)
self.docs_per_day = self.input.param("doc-per-day", 1)
self.use_memory_manager = self.input.param('use_memory_manager', True)
self.print_eventing_handler_code_in_logs = self.input.param(
'print_eventing_handler_code_in_logs', True)
random.seed(datetime.time)
function_name = "Function_{0}_{1}".format(
random.randint(1, 1000000000), self._testMethodName)
# See MB-28447, From now function name can only be max of 100 chars
self.function_name = function_name[0:90]
self.timer_storage_chan_size = self.input.param(
'timer_storage_chan_size', 10000)
self.dcp_gen_chan_size = self.input.param('dcp_gen_chan_size', 10000)
self.is_sbm = self.input.param('source_bucket_mutation', False)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
self.pause_resume = self.input.param('pause_resume', False)
self.pause_resume_number = self.input.param('pause_resume_number', 1)
def load_gsi_fts_indexes(self):
def create_gsi_index(b_name, t_index):
query = gsi_create_template % (b_name.replace(
".", ""), t_index, b_name, self.index_replica)
self.log.debug("Executing query: %s" % query)
try:
n1ql_helper.run_cbq_query(query)
except Exception as e:
self.log.critical(e)
def drop_gsi_index(b_name, t_index):
query = gsi_drop_template % (b_name, b_name.replace(".",
""), t_index)
self.log.debug("Executing query: %s" % query)
try:
n1ql_helper.run_cbq_query(query)
except Exception as e:
self.log.critical(e)
def create_fts_index(b_name, t_index):
fts_index_name = "%s_fts_%d" % (b_name.replace(".", ""), t_index)
status, content = fts_helper.create_fts_index_from_json(
fts_index_name, fts_param_template %
(fts_index_name, b_name, self.fts_index_partitions))
if status is False:
self.fail("Failed to create fts index %s: %s" %
(fts_index_name, content))
def drop_fts_index(b_name, t_index):
fts_index_name = "%s_fts_%d" % (b_name.replace(".", ""), t_index)
status, content = fts_helper.delete_fts_index(fts_index_name)
if status is False:
self.fail("Failed to drop fts index %s: %s" %
(fts_index_name, content))
n1ql_node = self.cluster.query_nodes[0]
fts_helper = FtsHelper(self.cluster.fts_nodes[0])
n1ql_helper = N1QLHelper(server=n1ql_node, use_rest=True, log=self.log)
gsi_index_name_pattern = "%s_primary_%d"
gsi_create_template = "CREATE PRIMARY INDEX `" \
+ gsi_index_name_pattern \
+ "` on `%s` USING GSI " \
+ "WITH {\"num_replica\": %s}"
gsi_drop_template = "DROP INDEX `%s`.`" + gsi_index_name_pattern \
+ "` USING GSI"
fts_param_template = '{ \
"type": "fulltext-index", \
"name": "%s", \
"sourceType": "couchbase", \
"sourceName": "%s", \
"planParams": { \
"maxPartitionsPerPIndex": 171, \
"indexPartitions": %d \
}, \
"params": { \
"doc_config": { \
"docid_prefix_delim": "", \
"docid_regexp": "", \
"mode": "type_field", \
"type_field": "type" \
}, \
"mapping": { \
"analysis": {}, \
"default_analyzer": "standard", \
"default_datetime_parser": "dateTimeOptional", \
"default_field": "_all", \
"default_mapping": { \
"dynamic": false, \
"enabled": true, \
"properties": { \
"rsx": { \
"dynamic": false, \
"enabled": true, \
"fields": [ \
{ \
"index": true, \
"name": "rsx", \
"type": "text" \
} \
] \
} \
} \
}, \
"default_type": "_default", \
"docvalues_dynamic": true, \
"index_dynamic": true, \
"store_dynamic": false, \
"type_field": "_type" \
}, \
"store": { \
"indexType": "scorch" \
} \
}, \
"sourceParams": {} \
}'
# Open SDK for connection for running n1ql queries
client = SDKClient([self.cluster.master], self.cluster.buckets[0])
for bucket in self.cluster.buckets[:4]:
self.log.info("Creating GSI indexes %d::%d for %s" %
(0, self.num_gsi_index, bucket.name))
for index in range(0, self.num_gsi_index):
create_gsi_index(bucket.name, index)
self.log.info("Done creating GSI indexes for %s" % bucket.name)
for bucket in self.cluster.buckets[:3]:
self.log.info("Creating FTS indexes %d::%d for %s" %
(0, self.num_fts_index, bucket.name))
for index in range(0, self.num_fts_index):
create_fts_index(bucket.name, index)
self.log.info("Done creating FTS indexes for %s" % bucket.name)
for bucket in self.cluster.buckets[:4]:
self.log.info("Create and drop %s GSI indexes on %s" %
(self.gsi_indexes_to_create_drop, bucket.name))
for index in range(
self.num_gsi_index,
self.num_gsi_index + self.gsi_indexes_to_create_drop):
create_gsi_index(bucket.name, index)
drop_gsi_index(bucket.name, index)
for bucket in self.cluster.buckets[:3]:
self.log.info("Create and drop %s FTS indexes on %s" %
(self.fts_indexes_to_create_drop, bucket.name))
for index in range(
self.num_fts_index,
self.num_fts_index + self.fts_indexes_to_create_drop):
create_fts_index(bucket.name, index)
drop_fts_index(bucket.name, index)
# Close the SDK connection
client.close()
def setUp(self):
super(ConfigPurging, self).setUp()
is_windows = False
for node in self.cluster.servers:
shell = RemoteMachineShellConnection(node)
if shell.info.type.lower() == OS.WINDOWS:
is_windows = True
shell.enable_diag_eval_on_non_local_hosts()
shell.disconnect()
self.cluster_util.update_cluster_nodes_service_list(self.cluster)
# Default purger values
self.default_run_interval = 60
self.default_purge_age = 300
self.time_stamp = time()
self.num_index = self.input.param("num_index", 0)
self.index_type = self.input.param("index_type", CbServer.Services.FTS)
self.index_name_len = self.input.param("index_name_len", 10)
self.fts_index_partition = self.input.param("fts_index_partition", 1)
self.index_replicas = self.input.param("gsi_index_replicas", 1)
self.fts_helper = FtsHelper(self.cluster.fts_nodes[0]) \
if self.cluster.fts_nodes else None
self.n1ql_helper = N1QLHelper(server=self.cluster.query_nodes[0],
use_rest=True, log=self.log) \
if self.cluster.query_nodes else None
self.spare_node = self.servers[-1]
self.couchbase_base_dir = "/opt/couchbase"
if is_windows:
self.couchbase_base_dir = \
"/cygdrive/c/Program\\ Files/Couchbase/Server"
# Param order:
# fts_name, bucket_name, index_partitions, scope_name, collection_name
self.fts_param_template = '{ \
"type": "fulltext-index", \
"name": "%s", \
"sourceType": "gocbcore", \
"sourceName": "%s", \
"sourceUUID": "%s", \
"planParams": { \
"maxPartitionsPerPIndex": 1024, \
"indexPartitions": %d \
}, \
"params": { \
"doc_config": { \
"docid_prefix_delim": "", \
"docid_regexp": "", \
"mode": "scope.collection.type_field", \
"type_field": "type" \
}, \
"mapping": { \
"analysis": {}, \
"default_analyzer": "standard", \
"default_datetime_parser": "dateTimeOptional", \
"default_field": "_all", \
"default_mapping": { \
"dynamic": true, \
"enabled": false \
}, \
"default_type": "_default", \
"docvalues_dynamic": false, \
"index_dynamic": true, \
"store_dynamic": false, \
"type_field": "_type", \
"types": { \
"%s.%s": { \
"dynamic": true, \
"enabled": true \
} \
} \
}, \
"store": { \
"indexType": "scorch", \
"segmentVersion": 15 \
} \
}, \
"sourceParams": {} \
}'
self.gsi_index_name_template = "%s_%s_%s_%d"
self.gsi_create_template = "CREATE PRIMARY INDEX `%s` " \
"ON `%s`.`%s`.`%s` USING GSI " \
"WITH {\"num_replica\": %d}"
self.gsi_drop_template = "DROP INDEX `%s`.`%s` USING GSI"
self.op_create = "key_create"
self.op_remove = "key_delete"
self.ts_during_start = self.__get_current_timestamps_from_debug_log()
self.initial_tombstones = \
self.cluster_util.get_metakv_dicts(self.cluster.master)
self.log.info(self.ts_during_start)
def setUp(self):
super(USINGFTS, self).setUp()
self.n1ql_helper = N1QLHelper(use_rest=True, log=self.log)
self.use_multiindex = TestInputSingleton.input.param("use_multiindex", False)
self.using_fts_and_gsi = TestInputSingleton.input.param("using_fts_and_gsi", False)
self.specific_index = TestInputSingleton.input.param("specific_index", False)
