def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "git_repo")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(
self.log, self.generate_full_docs_list(self.gens_load))
# temporary for MB-12848
self.create_primary_index_for_3_0_and_greater()
def __init__(self,
version=None,
master=None,
shell=None,
use_rest=None,
max_verify=0,
buckets=[],
item_flag=0,
analytics_port=8095,
n1ql_port=8093,
full_docs_list=[],
log=None,
input=None,
database=None):
self.version = version
self.shell = shell
self.n1ql_port = n1ql_port
self.max_verify = max_verify
self.buckets = buckets
self.item_flag = item_flag
self.analytics_port = analytics_port
self.input = input
self.log = log
self.use_rest = True
self.full_docs_list = full_docs_list
self.master = master
self.database = database
if self.full_docs_list and len(self.full_docs_list) > 0:
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def test_change_doc_size(self):
self.iterations = self.input.param("num_iterations", 5)
buckets = self._create_plasma_buckets()
if self.plasma_dgm:
self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer])
query_definition = QueryDefinition(
index_name="index_name_big_values",
index_fields=["bigValues"],
query_template="SELECT * FROM %s WHERE bigValues IS NOT NULL",
groups=["simple"],
index_where_clause=" bigValues IS NOT NULL ")
self.multi_create_index(buckets=buckets,
query_definitions=[query_definition])
template = '{{"name":"{0}", "age":{1}, "bigValues": "{2}" }}'
generators = []
for j in range(self.iterations):
for i in range(10):
name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))]
id = "{0}-{1}".format(name, str(i))
age = random.choice(range(4, 19))
bigValue_size = random.choice(range(10, 15))
bigValues = "".join(
random.choice(lowercase) for k in range(bigValue_size))
generators.append(
DocumentGenerator(id,
template, [name], [age], [bigValues],
start=0,
end=10))
self.load(generators,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size)
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
self.multi_query_using_index(buckets=buckets,
query_definitions=[query_definition])
for i in range(10):
name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))]
id = "{0}-{1}".format(name, str(i))
age = random.choice(range(4, 19))
bigValue_size = random.choice(range(1000, 5000))
bigValues = "".join(
random.choice(lowercase) for k in range(bigValue_size))
generators.append(
DocumentGenerator(id,
template, [name], [age], [bigValues],
start=0,
end=10))
self.load(generators,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size)
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
self.multi_query_using_index(buckets=buckets,
query_definitions=[query_definition])
self.sleep(30)
self.multi_drop_index(buckets=buckets,
query_definitions=[query_definition])
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics",False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'GSI')
self.index_type = self.input.param("index_type", 'GSI')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring",False)
self.isprepared = False
self.named_prepare = self.input.param("named_prepare", None)
self.skip_primary_index = self.input.param("skip_primary_index",False)
self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS')
shell = RemoteMachineShellConnection(self.master)
type = shell.extract_remote_info().distribution_type
self.path = testconstants.LINUX_COUCHBASE_BIN_PATH
if type.lower() == 'windows':
self.path = testconstants.WIN_COUCHBASE_BIN_PATH
elif type.lower() == "mac":
self.path = testconstants.MAC_COUCHBASE_BIN_PATH
self.threadFailure = False
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", 'plasma')
else:
self.gsi_type = None
if self.input.param("reload_data", False):
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas'])
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket,
timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas'])
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
if (self.analytics == False):
self.create_primary_index_for_3_0_and_greater()
if (self.analytics):
self.setup_analytics()
self.sleep(30,'wait for analytics setup')
def async_run_doc_ops(self):
if self.doc_ops:
tasks = self.async_ops_all_buckets(self.docs_gen_map,
batch_size=self.batch_size)
self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops
self.gen_results = TuqGenerators(
full_set=self.n1ql_helper.full_docs_list)
return tasks
return []
def setUp(self):
super(QueryHelperTests, self).setUp()
self.create_primary_index = self.input.param("create_primary_index",
True)
self.use_gsi_for_primary = self.input.param("use_gsi_for_primary",
True)
self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary",
True)
self.scan_consistency = self.input.param("scan_consistency",
"request_plus")
self.skip_host_login = self.input.param("skip_host_login", False)
if not self.skip_host_login:
self.shell = RemoteMachineShellConnection(self.master)
else:
self.shell = None
if not self.skip_init_check_cbserver: # for upgrade tests
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.item_flag = self.input.param("item_flag", 0)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.groups = self.input.param("groups", "all").split(":")
self.doc_ops = self.input.param("doc_ops", False)
self.batch_size = self.input.param("batch_size", 1)
self.create_ops_per = self.input.param("create_ops_per", 0)
self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
self.delete_ops_per = self.input.param("delete_ops_per", 0)
self.update_ops_per = self.input.param("update_ops_per", 0)
self.gens_load = self.generate_docs(self.docs_per_day)
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
self.gen_results = TuqGenerators(self.log,
full_set=self.full_docs_list)
if not self.skip_init_check_cbserver: # for upgrade tests
self.n1ql_server = self.get_nodes_from_services_map(
service_type="n1ql")
query_definition_generator = SQLDefinitionGenerator()
if self.dataset == "default" or self.dataset == "employee":
self.query_definitions = query_definition_generator.generate_employee_data_query_definitions(
)
if self.dataset == "simple":
self.query_definitions = query_definition_generator.generate_simple_data_query_definitions(
)
if self.dataset == "sabre":
self.query_definitions = query_definition_generator.generate_sabre_data_query_definitions(
)
if self.dataset == "bigdata":
self.query_definitions = query_definition_generator.generate_big_data_query_definitions(
)
if self.dataset == "array":
self.query_definitions = query_definition_generator.generate_airlines_data_query_definitions(
)
self.query_definitions = query_definition_generator.filter_by_group(
self.groups, self.query_definitions)
self.num_index_replicas = self.input.param("num_index_replica", 0)
def run_doc_ops(self):
verify_data = True
if self.scan_consistency == "request_plus":
verify_data = False
if self.doc_ops:
self.sync_ops_all_buckets(docs_gen_map=self.docs_gen_map,
batch_size=self.batch_size,
verify_data=verify_data)
self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops
self.gen_results = TuqGenerators(
full_set=self.n1ql_helper.full_docs_list)
log.info("------ KV OPS Done ------")
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r".*\$subquery\(", "", query_template)
subquery_template = subquery_template[: subquery_template.rfind(")")]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template))
alias = re.sub(r".*as ", "", alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r"\$subquery\(.*\).*%s" % alias, " %s" % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as ', '', alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def test_array_item_limit(self):
query_definition = QueryDefinition(index_name="index_name_big_values",
index_fields=["DISTINCT ARRAY t FOR t in bigValues END"],
query_template="SELECT {0} FROM %s WHERE bigValues IS NOT NULL",
groups=["array"], index_where_clause=" bigValues IS NOT NULL ")
self.rest.flush_bucket(self.buckets[0])
generators = []
template = '{{"name":"{0}", "age":{1}, "bigValues":{2} }}'
for i in range(10):
name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))]
id = "{0}-{1}".format(name, str(i))
age = random.choice(range(4, 19))
bigValues = []
arrLen = random.choice(range(10, 15))
indiSize = (4096 * 4)
for j in range(arrLen):
longStr = "".join(random.choice(lowercase) for k in range(indiSize))
bigValues.append(longStr)
generators.append(DocumentGenerator(id, template, [name], [age], [bigValues],
start=0, end=1))
self.load(generators, flag=self.item_flag, verify_data=False, batch_size=self.batch_size)
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
self.multi_create_index_using_rest(buckets=self.buckets,
query_definitions=[query_definition])
for bucket in self.buckets:
self.run_full_table_scan_using_rest(bucket, query_definition)
self.multi_drop_index_using_rest(buckets=self.buckets,
query_definitions=[query_definition])
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'VIEW')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS')
if self.input.param("reload_data", False):
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket,
timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
# temporary for MB-12848
self.create_primary_index_for_3_0_and_greater()
def _load_aggregate_function_dataset(self, buckets=[]):
generators = []
document_template = '{{ "name":"{0}" , "str_arr" : {1} , ' \
'"int_num": {2} , "int_arr": {3} , ' \
'"float_num" : {4} , "float_arr" : {5} }}'
num_items = 1000
for i in range(num_items):
name = random.choice(FIRST_NAMES)
str_arr = [random.choice(COUNTRIES) for i in range(10)]
int_num = random.randint(0, 100)
int_arr = [random.randint(30, 100) for i in range(100)]
float_num = random.uniform(0.0, 100.0)
float_arr = [random.uniform(30.0, 100.0) for i in range(100)]
doc_id = "student_" + str(random.random() * 100000)
generators.append(
DocumentGenerator(doc_id,
document_template, [name], [str_arr],
[int_num], [int_arr], [float_num],
[float_arr],
start=0,
end=1))
self.load(generators,
buckets=buckets,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size)
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def test_create_query_drop_index_on_missing_empty_null_field(self):
data_types = ["empty", "null"]
index_field, data_type = self._find_datatype(self.query_definitions[0])
doc_list = self.full_docs_list[:len(self.full_docs_list)/2]
for data_type in data_types:
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
definitions_list = []
query_definition = QueryDefinition(index_name="index_name_{0}_distinct".format(data_type),
index_fields=["DISTINCT ARRAY t FOR t in `{0}` END".format(index_field)],
query_template="SELECT {0} FROM %s WHERE `{0}` IS NOT NULL".format(index_field),
groups=["array"], index_where_clause=" `{0}` IS NOT NULL ".format(index_field))
definitions_list.append(query_definition)
query_definition = QueryDefinition(index_name="index_name_{0}_duplicate".format(data_type),
index_fields=["ALL ARRAY t FOR t in `{0}` END".format(index_field)],
query_template="SELECT {0} FROM %s WHERE `{0}` IS NOT NULL".format(index_field),
groups=["array"], index_where_clause=" `{0}` IS NOT NULL ".format(index_field))
definitions_list.append(query_definition)
for bucket in self.buckets:
for query_definition in definitions_list:
self.change_index_field_type(bucket.name, index_field, doc_list, data_type, query_definition)
self.multi_create_index_using_rest(buckets=self.buckets, query_definitions=definitions_list)
for bucket in self.buckets:
for query_definition in definitions_list:
self.run_full_table_scan_using_rest(bucket, query_definition)
self.multi_drop_index_using_rest(buckets=self.buckets, query_definitions=definitions_list)
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
def _create_int64_dataset(self):
generators = []
document_template = '{{"name":"{0}", "int_num": {1}, "long_num":{2}, "long_num_partial":{3}, "long_arr": {4},' \
'"int_arr": {5}}}'
num_items = len(self.full_docs_list)
for i in range(num_items):
name = random.choice(FIRST_NAMES)
int_num = random.randint(-100, 100)
long_num = random.choice(INT64_VALUES)
long_arr = [random.choice(INT64_VALUES) for i in range(10)]
int_arr = [random.randint(-100, 100) for i in range(10)]
doc_id = "int64_" + str(random.random() * 100000)
generators.append(
DocumentGenerator(doc_id,
document_template, [name], [int_num],
[long_num], [long_num], [long_arr],
[int_arr],
start=0,
end=1))
self.load(generators,
buckets=self.buckets,
flag=self.item_flag,
verify_data=False,
batch_size=self.batch_size)
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def kv_mutations(self, docs=1):
if not docs:
docs = self.docs_per_day
gens_load = self.generate_docs(docs)
self.full_docs_list = self.generate_full_docs_list(gens_load)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
self.load(gens_load, buckets=self.buckets, flag=self.item_flag,
verify_data=False, batch_size=self.batch_size)
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics",False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'GSI')
self.index_type = self.input.param("index_type", 'GSI')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring",False)
self.isprepared = False
self.named_prepare = self.input.param("named_prepare", None)
self.skip_primary_index = self.input.param("skip_primary_index",False)
self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS')
shell = RemoteMachineShellConnection(self.master)
type = shell.extract_remote_info().distribution_type
self.path = testconstants.LINUX_COUCHBASE_BIN_PATH
if type.lower() == 'windows':
self.path = testconstants.WIN_COUCHBASE_BIN_PATH
elif type.lower() == "mac":
self.path = testconstants.MAC_COUCHBASE_BIN_PATH
self.threadFailure = False
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", 'plasma')
else:
self.gsi_type = None
if self.input.param("reload_data", False):
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas'])
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket,
timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas'])
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
if (self.analytics == False):
self.create_primary_index_for_3_0_and_greater()
if (self.analytics):
self.setup_analytics()
self.sleep(30,'wait for analytics setup')
def test_sorted_removed_items_indexed(self):
if self.plasma_dgm:
self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer])
generators = self._upload_documents_in_sorted()
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
query_definition = QueryDefinition(
index_name="index_range_shrink_name",
index_fields=["name"],
query_template="SELECT * FROM %s WHERE name IS NOT NULL",
groups=["simple"],
index_where_clause=" name IS NOT NULL ")
buckets = []
for bucket in self.buckets:
if bucket.name.startswith("plasma_dgm"):
buckets.append(bucket)
self.multi_create_index(buckets=buckets,
query_definitions=[query_definition])
self.sleep(30)
intervals = [["d", "e", "f"], ["j", "k", "l", "m"],
["p", "q", "r", "s"]]
temp_list = []
for doc_gen in self.full_docs_list:
for interval in intervals:
for character in interval:
if doc_gen["name"].lower().startswith(character):
for bucket in buckets:
url = "couchbase://{0}/{1}".format(
self.master.ip, bucket.name)
cb = Bucket(url,
username=bucket.name,
password="******")
cb.remove(doc_gen["_id"])
temp_list.append(doc_gen)
if not self.multi_intervals:
break
self.full_docs_list = [
doc for doc in self.full_docs_list if doc not in temp_list
]
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
self.multi_query_using_index(buckets=buckets,
query_definitions=[query_definition])
self.sleep(30)
self.multi_drop_index(buckets=buckets,
query_definitions=[query_definition])
def kv_mutations(self, docs=1):
if not docs:
docs = self.docs_per_day
gens_load = self.generate_docs(docs)
self.full_docs_list = self.generate_full_docs_list(gens_load)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
tasks = self.async_load(generators_load=gens_load,
op_type="create",
batch_size=self.batch_size)
return tasks
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', ''))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num))
subquery_template = re.sub(r'USE KEYS.*', '', subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip()
if not alias:
alias = '$1'
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', ''))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num))
subquery_template = re.sub(r'USE KEYS.*', '', subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip()
if not alias:
alias = '$1'
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r".*\$subquery\(", "", query_template)
subquery_template = subquery_template[: subquery_template.rfind(")")]
keys_num = int(re.sub(r".*KEYS \$", "", subquery_template).replace("KEYS $", ""))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load, keys=self._get_keys(keys_num))
subquery_template = re.sub(r"USE KEYS.*", "", subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template))
alias = re.sub(r".*as", "", re.sub(r"FROM.*", "", alias)).strip()
if not alias:
alias = "$1"
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r",.*\$subquery\(.*\).*%s" % alias, ",%s" % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def __init__(self,
version=None,
master=None,
shell=None,
use_rest=None,
max_verify=0,
buckets=[],
item_flag=0,
n1ql_port=8093,
full_docs_list=[],
log=None,
input=None):
self.version = version
self.shell = shell
self.use_rest = use_rest
self.max_verify = max_verify
self.buckets = buckets
self.item_flag = item_flag
self.n1ql_port = n1ql_port
self.input = input
self.log = log
self.full_docs_list = full_docs_list
self.master = master
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def setUp(self):
if not self._testMethodName == "suite_setUp":
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == "suite_setUp" and self.input.param("cbq_version", "sherlock") != "sherlock":
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics", False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", "GSI")
self.index_type = self.input.param("index_type", "GSI")
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring", False)
self.isprepared = False
self.skip_primary_index = self.input.param("skip_primary_index", False)
self.scan_consistency = self.input.param("scan_consistency", "REQUEST_PLUS")
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", None)
else:
self.gsi_type = None
if self.input.param("reload_data", False):
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
if self.analytics == False:
self.create_primary_index_for_3_0_and_greater()
if self.analytics:
self.setup_analytics()
self.sleep(30, "wait for analytics setup")
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as ', '', alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def test_sorted_items_indexed(self):
if self.plasma_dgm:
self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer])
generators = self._upload_documents_in_sorted()
self.full_docs_list = self.generate_full_docs_list(generators)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
query_definition = QueryDefinition(index_name="index_range_shrink_name", index_fields=["name"],
query_template="SELECT * FROM %s WHERE name IS NOT NULL", groups=["simple"],
index_where_clause=" name IS NOT NULL ")
buckets = []
for bucket in self.buckets:
if bucket.name.startswith("plasma_dgm"):
buckets.append(bucket)
self.multi_create_index(buckets=buckets,
query_definitions=[query_definition])
self.sleep(30)
self.multi_query_using_index(buckets=buckets,
query_definitions=[query_definition])
self.sleep(30)
self.multi_drop_index(buckets=buckets,
query_definitions=[query_definition])
def test_create_query_drop_index_on_mixed_datatypes(self):
query_definition = QueryDefinition(
index_name="index_name_travel_history",
index_fields=["DISTINCT ARRAY t FOR t in `travel_history` END"],
query_template="SELECT {0} FROM %s WHERE `travel_history` IS NOT NULL",
groups=["array"], index_where_clause=" `travel_history` IS NOT NULL ")
end = 0
for bucket in self.buckets:
for data in DATATYPES:
start = end
end = end + len(self.full_docs_list)/len(DATATYPES)
doc_list = self.full_docs_list[start:end]
self.change_index_field_type(bucket.name,
"travel_history",
doc_list, data, query_definition)
self.multi_create_index_using_rest(buckets=self.buckets,
query_definitions=[query_definition])
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
for bucket in self.buckets:
self.run_full_table_scan_using_rest(bucket, query_definition)
self.multi_drop_index_using_rest(buckets=self.buckets,
query_definitions=[query_definition])
class QueryHelperTests(BaseTestCase):
def setUp(self):
super(QueryHelperTests, self).setUp()
self.create_primary_index = self.input.param("create_primary_index",
True)
self.use_gsi_for_primary = self.input.param("use_gsi_for_primary",
True)
self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary",
True)
self.scan_consistency = self.input.param("scan_consistency",
"request_plus")
self.shell = RemoteMachineShellConnection(self.master)
if not self.skip_init_check_cbserver: # for upgrade tests
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.item_flag = self.input.param("item_flag", 0)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.groups = self.input.param("groups", "all").split(":")
self.doc_ops = self.input.param("doc_ops", False)
self.batch_size = self.input.param("batch_size", 1)
self.create_ops_per = self.input.param("create_ops_per", 0)
self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
self.delete_ops_per = self.input.param("delete_ops_per", 0)
self.update_ops_per = self.input.param("update_ops_per", 0)
self.gens_load = self.generate_docs(self.docs_per_day)
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
self.gen_results = TuqGenerators(self.log,
full_set=self.full_docs_list)
if not self.skip_init_check_cbserver: # for upgrade tests
self.n1ql_server = self.get_nodes_from_services_map(
service_type="n1ql")
query_definition_generator = SQLDefinitionGenerator()
if self.dataset == "default" or self.dataset == "employee":
self.query_definitions = query_definition_generator.generate_employee_data_query_definitions(
)
if self.dataset == "simple":
self.query_definitions = query_definition_generator.generate_simple_data_query_definitions(
)
if self.dataset == "sabre":
self.query_definitions = query_definition_generator.generate_sabre_data_query_definitions(
)
if self.dataset == "bigdata":
self.query_definitions = query_definition_generator.generate_big_data_query_definitions(
)
if self.dataset == "array":
self.query_definitions = query_definition_generator.generate_airlines_data_query_definitions(
)
self.query_definitions = query_definition_generator.filter_by_group(
self.groups, self.query_definitions)
self.num_index_replicas = self.input.param("num_index_replica", 0)
def tearDown(self):
super(QueryHelperTests, self).tearDown()
def generate_docs(self, num_items, start=0):
try:
if self.dataset == "simple":
return self.generate_docs_simple(num_items, start)
if self.dataset == "array":
return self.generate_docs_array(num_items, start)
return getattr(self, 'generate_docs_' + self.dataset)(num_items,
start)
except Exception as ex:
log.info(str(ex))
self.fail("There is no dataset %s, please enter a valid one" %
self.dataset)
def generate_ops_docs(self, num_items, start=0):
try:
json_generator = JsonGenerator()
if self.dataset == "simple":
return self.generate_ops(num_items, start,
json_generator.generate_docs_simple)
if self.dataset == "array":
return self.generate_ops(
num_items, start,
json_generator.generate_all_type_documents_for_gsi)
except Exception as ex:
log.info(ex)
self.fail("There is no dataset %s, please enter a valid one" %
self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_simple(start=start,
docs_per_day=docs_per_day)
def generate_docs_array(self, num_items=10, start=0):
json_generator = JsonGenerator()
return json_generator.generate_all_type_documents_for_gsi(
start=start, docs_per_day=num_items)
def generate_ops(self, docs_per_day, start=0, method=None):
gen_docs_map = {}
for key in list(self.ops_dist_map.keys()):
isShuffle = False
if key == "update":
isShuffle = True
if self.dataset != "bigdata":
gen_docs_map[key] = method(
docs_per_day=self.ops_dist_map[key]["end"],
start=self.ops_dist_map[key]["start"])
else:
gen_docs_map[key] = method(
value_size=self.value_size,
end=self.ops_dist_map[key]["end"],
start=self.ops_dist_map[key]["start"])
return gen_docs_map
def generate_full_docs_list_after_ops(self, gen_docs_map):
docs = []
for key in list(gen_docs_map.keys()):
if key != "delete" and key != "expiry":
update = False
if key == "update":
update = True
gen_docs = self.generate_full_docs_list(
gens_load=gen_docs_map[key], update=update)
for doc in gen_docs:
docs.append(doc)
return docs
def async_run_doc_ops(self):
if self.doc_ops:
tasks = self.async_ops_all_buckets(self.docs_gen_map,
batch_size=self.batch_size)
self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops
self.gen_results = TuqGenerators(
full_set=self.n1ql_helper.full_docs_list)
return tasks
return []
def run_doc_ops(self):
verify_data = True
if self.scan_consistency == "request_plus":
verify_data = False
if self.doc_ops:
self.sync_ops_all_buckets(docs_gen_map=self.docs_gen_map,
batch_size=self.batch_size,
verify_data=verify_data)
self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops
self.gen_results = TuqGenerators(
full_set=self.n1ql_helper.full_docs_list)
log.info("------ KV OPS Done ------")
def create_index(self, bucket, query_definition, deploy_node_info=None):
defer_build = True
query = query_definition.generate_index_create_query(
bucket=bucket,
use_gsi_for_secondary=self.use_gsi_for_secondary,
deploy_node_info=deploy_node_info,
defer_build=defer_build,
num_replica=self.num_index_replicas)
log.info(query)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
create_index_task = self.cluster.async_create_index(
server=self.n1ql_server,
bucket=bucket,
query=query,
n1ql_helper=self.n1ql_helper,
index_name=query_definition.index_name,
defer_build=defer_build)
create_index_task.result()
query = self.n1ql_helper.gen_build_index_query(
bucket=bucket, index_list=[query_definition.index_name])
build_index_task = self.cluster.async_build_index(
server=self.n1ql_server,
bucket=bucket,
query=query,
n1ql_helper=self.n1ql_helper)
build_index_task.result()
check = self.n1ql_helper.is_index_ready_and_in_list(
bucket, query_definition.index_name, server=self.n1ql_server)
self.assertTrue(
check, "index {0} failed to be created".format(
query_definition.index_name))
def _create_primary_index(self):
if self.n1ql_server:
if self.doc_ops:
self.ops_dist_map = self.calculate_data_change_distribution(
create_per=self.create_ops_per,
update_per=self.update_ops_per,
delete_per=self.delete_ops_per,
expiry_per=self.expiry_ops_per,
start=0,
end=self.docs_per_day)
log.info(self.ops_dist_map)
self.docs_gen_map = self.generate_ops_docs(
self.docs_per_day, 0)
self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(
self.docs_gen_map)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
log.info(self.n1ql_server)
if self.create_primary_index:
self.n1ql_helper.create_primary_index(
using_gsi=self.use_gsi_for_primary,
server=self.n1ql_server)
def query_using_index(self,
bucket,
query_definition,
expected_result=None,
scan_consistency=None,
scan_vector=None,
verify_results=True):
if not scan_consistency:
scan_consistency = self.scan_consistency
self.gen_results.query = query_definition.generate_query(bucket=bucket)
if expected_result == None:
expected_result = self.gen_results.generate_expected_result(
print_expected_result=False)
query = self.gen_results.query
log.info("Query : {0}".format(query))
msg, check = self.n1ql_helper.run_query_and_verify_result(
query=query,
server=self.n1ql_server,
expected_result=expected_result,
scan_consistency=scan_consistency,
scan_vector=scan_vector,
verify_results=verify_results)
self.assertTrue(check, msg)
def drop_index(self, bucket, query_definition, verify_drop=True):
try:
query = query_definition.generate_index_drop_query(
bucket=bucket,
use_gsi_for_secondary=self.use_gsi_for_secondary,
use_gsi_for_primary=self.use_gsi_for_primary)
log.info(query)
actual_result = self.n1ql_helper.run_cbq_query(
query=query, server=self.n1ql_server)
if verify_drop:
check = self.n1ql_helper._is_index_in_list(
bucket,
query_definition.index_name,
server=self.n1ql_server)
self.assertFalse(
check, "index {0} failed to be "
"deleted".format(query_definition.index_name))
except Exception as ex:
log.info(ex)
query = "select * from system:indexes"
actual_result = self.n1ql_helper.run_cbq_query(
query=query, server=self.n1ql_server)
log.info(actual_result)
def run_async_index_operations(self, operation_type):
if operation_type == "create_index":
self._create_primary_index()
for bucket in self.buckets:
for query_definition in self.query_definitions:
self.create_index(bucket=bucket,
query_definition=query_definition)
if operation_type == "query":
for bucket in self.buckets:
for query_definition in self.query_definitions:
self.query_using_index(bucket=bucket,
query_definition=query_definition)
if operation_type == "drop_index":
for bucket in self.buckets:
for query_definition in self.query_definitions:
self.drop_index(bucket=bucket,
query_definition=query_definition)
if operation_type == "generate_docs":
for bucket in self.buckets:
self.generate_docs(self.docs_per_day,
start=randint(0, 1000000000))
class QueryTests(BaseTestCase):
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'VIEW')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS')
if self.input.param("reload_data", False):
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket,
timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
# temporary for MB-12848
self.create_primary_index_for_3_0_and_greater()
def suite_setUp(self):
try:
self.load(self.gens_load, flag=self.item_flag)
# self.create_primary_index_for_3_0_and_greater()
if not self.input.param("skip_build_tuq", True):
self._build_tuq(self.master)
self.skip_buckets_handle = True
except:
self.log.error('SUITE SETUP FAILED')
self.tearDown()
def tearDown(self):
if self._testMethodName == 'suite_tearDown':
self.skip_buckets_handle = False
super(QueryTests, self).tearDown()
def suite_tearDown(self):
if not self.input.param("skip_build_tuq", False):
if hasattr(self, 'shell'):
self.shell.execute_command("killall /tmp/tuq/cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.disconnect()
##############################################################################################
#
# SIMPLE CHECKS
##############################################################################################
def test_simple_check(self):
for bucket in self.buckets:
query_template = 'FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_simple_negative_check(self):
queries_errors = {'SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3' :
'Aggregates not allowed in WHERE',
'SELECT *.$str0 FROM {0}' : 'syntax error',
'SELECT *.* FROM {0} ... ERROR' : 'syntax error',
'FROM %s SELECT $str0 WHERE id=null' : 'syntax error',}
self.negative_common_body(queries_errors)
def test_unnest(self):
for bucket in self.buckets:
query_template = 'SELECT emp.$int0, task FROM %s emp UNNEST emp.$nested_list_3l0 task' % bucket.name
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(sorted(actual_result['results']), sorted(expected_result))
def test_subquery_select(self):
for bucket in self.buckets:
self.query = 'SELECT $str0, $subquery(SELECT COUNT($str0) cn FROM %s d USE KEYS $5) as names FROM %s' % (bucket.name,
bucket.name)
actual_result, expected_result = self.run_query_with_subquery_select_from_template(self.query)
self._verify_results(actual_result['results'], expected_result)
def test_subquery_from(self):
for bucket in self.buckets:
self.query = 'SELECT tasks.$str0 FROM $subquery(SELECT $str0, $int0 FROM %s) as tasks' % (bucket.name)
actual_result, expected_result = self.run_query_with_subquery_from_template(self.query)
self._verify_results(actual_result['results'], expected_result)
def test_consistent_simple_check(self):
queries = [self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\
'WHERE $str0 IS NOT NULL AND $int0<10 ' +\
'OR $int1 = 6 ORDER BY $int0, $int1'),
self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\
'WHERE $int1 = 6 OR $str0 IS NOT NULL AND ' +\
'$int0<10 ORDER BY $int0, $int1')]
for bucket in self.buckets:
actual_result1 = self.run_cbq_query(queries[0] % bucket.name)
actual_result2 = self.run_cbq_query(queries[1] % bucket.name)
self.assertTrue(actual_result1['results'] == actual_result2['results'],
"Results are inconsistent.Difference: %s %s %s %s" %(
len(actual_result1['results']), len(actual_result2['results']),
actual_result1['results'][:100], actual_result2['results'][:100]))
def test_simple_nulls(self):
queries = ['SELECT id FROM %s WHERE id=NULL or id="null"']
for bucket in self.buckets:
for query in queries:
actual_result = self.run_cbq_query(query % (bucket.name))
self._verify_results(actual_result['results'], [])
##############################################################################################
#
# LIMIT OFFSET CHECKS
##############################################################################################
def test_limit_negative(self):
#queries_errors = {'SELECT * FROM default LIMIT {0}' : ('Invalid LIMIT value 2.5', 5030)}
queries_errors = {'SELECT ALL * FROM %s' : ('syntax error', 3000)}
self.negative_common_body(queries_errors)
def test_limit_offset(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
def test_limit_offset_zero(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0' % (bucket.name)
self.query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query()
self.assertEquals(actual_result['results'], [],
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (
actual_result['results'], []))
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self.assertEquals(actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (
actual_result['results'], expected_result))
def test_limit_offset_negative_check(self):
queries_errors = {'SELECT DISTINCT $str0 FROM {0} LIMIT 1.1' :
'Invalid LIMIT value 1.1',
'SELECT DISTINCT $str0 FROM {0} OFFSET 1.1' :
'Invalid OFFSET value 1.1'}
self.negative_common_body(queries_errors)
def test_limit_offset_sp_char_check(self):
queries_errors = {'SELECT DISTINCT $str0 FROM {0} LIMIT ~' :
'syntax erro',
'SELECT DISTINCT $str0 FROM {0} OFFSET ~' :
'syntax erro'}
self.negative_common_body(queries_errors)
##############################################################################################
#
# ALIAS CHECKS
##############################################################################################
def test_simple_alias(self):
for bucket in self.buckets:
query_template = 'SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self.assertEquals(actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (
actual_result['results'], expected_result))
query_template = 'SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
expected_result = [ { "COUNT_EMPLOYEE": expected_result[0]['COUNT_EMPLOYEE'] + 1 } ]
self.assertEquals(actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (
actual_result['results'], expected_result))
def test_simple_negative_alias(self):
queries_errors = {'SELECT $str0._last_name as *' : 'syntax error',
'SELECT $str0._last_name as DATABASE ?' : 'syntax error',
'SELECT $str0 AS NULL FROM {0}' : 'syntax error',
'SELECT $str1 as $str0, $str0 FROM {0}' :
'Duplicate result alias name',
'SELECT test.$obj0 as points FROM {0} AS TEST ' +
'GROUP BY $obj0 AS GROUP_POINT' :
'syntax error'}
self.negative_common_body(queries_errors)
def test_alias_from_clause(self):
queries_templates = ['SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points',
'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\
' ORDER BY points',
'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' +\
'GROUP BY test.$obj0.$_obj0_int0 ORDER BY points']
for bucket in self.buckets:
for query_template in queries_templates:
actual_result, expected_result = self.run_query_from_template(query_template % (bucket.name))
self._verify_results(actual_result['results'], expected_result)
def test_alias_from_clause_group(self):
for bucket in self.buckets:
query_template = 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' %(bucket.name) +\
'GROUP BY $obj0.$_obj0_int0 ORDER BY points'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_order_desc(self):
for bucket in self.buckets:
query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC' %(
bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_order_asc(self):
for bucket in self.buckets:
query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC' %(
bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT COUNT(TEST.$str0) from %s AS TEST' %(bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_unnest(self):
for bucket in self.buckets:
query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill' %(
bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill' %(
bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# ORDER BY CHECKS
##############################################################################################
def test_order_by_check(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s' % (bucket.name) +\
' ORDER BY $str1, $str0, $obj0.$_obj0_int0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $obj0.$_obj0_int0, $str0, $str1'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias(self):
for bucket in self.buckets:
query_template = 'SELECT $str1, $obj0 AS points FROM %s' % (bucket.name) +\
' AS test ORDER BY $str1 DESC, points DESC'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias_arrays(self):
for bucket in self.buckets:
query_template = 'SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s' % (
bucket.name) +\
' AS TEST ORDER BY SKILL, $str1, TEST.$obj0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT $int0, $int1, count(*) AS emp_per_month from %s'% (
bucket.name) +\
' WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT $str1 AS TITLE, min($int1) day FROM %s GROUP' % (bucket.name) +\
' BY $str1 ORDER BY MIN($int1), $str1'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_precedence(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $str0, $str1'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $str1, $str0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_satisfy(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $list_obj0 FROM %s AS employee ' % (bucket.name) +\
'WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND' +\
' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# DISTINCT
##############################################################################################
def test_distinct(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str1 FROM %s ORDER BY $str1' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_distinct_nested(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s ' % (bucket.name) +\
'ORDER BY $obj0.$_obj0_int0'
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT DISTINCT $list_str0[0] as skill' +\
' FROM %s ORDER BY $list_str0[0]' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
self.query = 'SELECT DISTINCT $obj0.* FROM %s' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# COMPLEX PATHS
##############################################################################################
def test_simple_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 FROM %s.$obj0' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 as new_attribute FROM %s.$obj0' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
def test_where_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# COMMON FUNCTIONS
##############################################################################################
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', ''))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num))
subquery_template = re.sub(r'USE KEYS.*', '', subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip()
if not alias:
alias = '$1'
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as ', '', alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def negative_common_body(self, queries_errors={}):
if not queries_errors:
self.fail("No queries to run!")
for bucket in self.buckets:
for query_template, error in queries_errors.iteritems():
try:
query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query(query.format(bucket.name))
except CBQError as ex:
self.log.error(ex)
self.assertTrue(str(ex).find(error) != -1,
"Error is incorrect.Actual %s.\n Expected: %s.\n" %(
str(ex).split(':')[-1], error))
else:
self.fail("There were no errors. Error expected: %s" % error)
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.n1ql_port == None or self.n1ql_port == '':
self.n1ql_port = self.input.param("n1ql_port", 8093)
if not self.n1ql_port:
self.log.info(" n1ql_port is not defined, processing will not proceed further")
raise Exception("n1ql_port is not defined, processing will not proceed further")
query_params = {}
cred_params = {'creds': []}
for bucket in self.buckets:
if bucket.saslPassword:
cred_params['creds'].append({'user': '******' % bucket.name, 'pass': bucket.saslPassword})
query_params.update(cred_params)
if self.use_rest:
query_params.update({'scan_consistency': self.scan_consistency})
self.log.info('RUN QUERY %s' % query)
result = RestConnection(server).query_tool(query, self.n1ql_port, query_params=query_params)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\
"tuq_client/tuq_client " +\
"-engine=http://%s:8093/" % server.ip,
subcommands=[query,],
min_output_size=20,
end_msg='tuq_client>')
else:
output = self.shell.execute_commands_inside("/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip,
subcommands=[query,],
min_output_size=20,
end_msg='cbq>')
result = self._parse_query_output(output)
if isinstance(result, str) or 'errors' in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"])
return result
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" %(
version, info.architecture_type)
#TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'go get github.com/couchbaselabs/tuqtng;' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\
'go get -d -v ./...; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .'
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
#TODO for windows
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
if self.services_init.find('n1ql') != -1:
return
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"]
print "PATH TO SHERLOCK: %s" % couchbase_path
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def generate_docs(self, num_items, start=0):
try:
return getattr(self, 'generate_docs_' + self.dataset)(num_items, start)
except:
self.fail("There is no dataset %s, please enter a valid one" % self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_sabre(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_sabre(docs_per_day, start)
def generate_docs_employee(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_data(docs_per_day = docs_per_day, start = start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_simple_data(docs_per_day = docs_per_day, start = start)
def generate_docs_sales(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_sales_data(docs_per_day = docs_per_day, start = start)
def generate_docs_bigdata(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_bigdata(end=(1000*docs_per_day), start=start, value_size=self.value_size)
def _verify_results(self, actual_result, expected_result, missing_count = 1, extra_count = 1):
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count]))
self.fail("Results are incorrect.Actual num %s. Expected num: %s.\n" % (
len(actual_result), len(expected_result)))
if self.max_verify is not None:
actual_result = actual_result[:self.max_verify]
expected_result = expected_result[:self.max_verify]
msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\
"Expected first and last 100: %s.\n ... \n %s"
self.assertTrue(actual_result == expected_result,
msg % (actual_result[:100],actual_result[-100:],
expected_result[:100],expected_result[-100:]))
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.iteritems():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self):
self.log.info("CREATE PRIMARY INDEX")
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
if versions[0].startswith("4") or versions[0].startswith("3"):
for bucket in self.buckets:
if self.primary_indx_drop:
self.log.info("Dropping primary index for %s ..." % bucket.name)
self.query = "DROP PRIMARY INDEX ON %s" % (bucket.name)
self.sleep(3, 'Sleep for some time after index drop')
self.log.info("Creating primary index for %s ..." % bucket.name)
self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
try:
self.run_cbq_query()
if self.primary_indx_type.lower() == 'gsi':
self._wait_for_index_online(bucket, '#primary')
except Exception, ex:
self.log.info(str(ex))
class AnalyticsHelper():
def __init__(self,
version=None,
master=None,
shell=None,
use_rest=None,
max_verify=0,
buckets=[],
item_flag=0,
analytics_port=8095,
n1ql_port=8093,
full_docs_list=[],
log=None,
input=None,
database=None):
self.version = version
self.shell = shell
self.n1ql_port = n1ql_port
self.max_verify = max_verify
self.buckets = buckets
self.item_flag = item_flag
self.analytics_port = analytics_port
self.input = input
self.log = log
self.use_rest = True
self.full_docs_list = full_docs_list
self.master = master
self.database = database
if self.full_docs_list and len(self.full_docs_list) > 0:
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def killall_tuq_process(self):
self.shell.execute_command("killall cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.execute_command("killall indexer")
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_analytics_query()
return actual_result, expected_result
def run_analytics_query(self,
query=None,
min_output_size=10,
server=None,
query_params={},
is_prepared=False,
scan_consistency=None,
scan_vector=None,
verbose=False):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.analytics_port = server.analytics_port
else:
if server.ip == "127.0.0.1":
self.analytics_port = server.analytics_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.analytics_port == None or self.analytics_port == '':
self.analytics_port = self.input.param("analytics_port", 8095)
if not self.analytics_port:
self.log.info(
" analytics_port is not defined, processing will not proceed further"
)
raise Exception(
"analytics_port is not defined, processing will not proceed further"
)
if self.use_rest:
query_params = {}
if scan_consistency:
query_params['scan_consistency'] = scan_consistency
if scan_vector:
query_params['scan_vector'] = str(scan_vector).replace(
"'", '"')
if verbose:
self.log.info('RUN QUERY %s' % query)
query = query + ";"
if "USE INDEX" in query:
query = query.replace("USE INDEX(`#primary` USING GSI)", " ")
for bucket in self.buckets:
query = query.replace(bucket.name + " ",
bucket.name + "_shadow ")
self.log.info(" CBAS QUERY :: {0}".format(query))
result = RestConnection(server).analytics_tool(
query,
self.analytics_port,
query_params=query_params,
verbose=verbose)
if isinstance(result, str) or 'errors' in result:
error_result = str(result)
length_display = len(error_result)
if length_display > 500:
error_result = error_result[:500]
raise CBQError(error_result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" %
result["metrics"]["elapsedTime"])
return result
def _verify_results(self,
actual_result,
expected_result,
missing_count=1,
extra_count=1):
self.log.info(" Analyzing Actual Result")
actual_result = self._gen_dict(actual_result)
self.log.info(" Analyzing Expected Result")
expected_result = self._gen_dict(expected_result)
if len(actual_result) != len(expected_result):
raise Exception(
"Results are incorrect.Actual num %s. Expected num: %s.\n" %
(len(actual_result), len(expected_result)))
msg = "The number of rows match but the results mismatch, please check"
if actual_result != expected_result:
raise Exception(msg)
def _verify_results_rqg_new(self,
n1ql_result=[],
sql_result=[],
hints=["a1"]):
new_n1ql_result = []
for result in n1ql_result:
if result != {}:
for key in list(result.keys()):
if key.find('_shadow') != -1:
new_n1ql_result.append(result[key])
else:
new_n1ql_result.append(result)
break
n1ql_result = new_n1ql_result
if self._is_function_in_result(hints):
return self._verify_results_rqg_for_function(
n1ql_result, sql_result)
check = self._check_sample(n1ql_result, hints)
actual_result = n1ql_result
if actual_result == [{}]:
actual_result = []
if check:
actual_result = self._gen_dict(n1ql_result)
actual_result = sorted(actual_result)
expected_result = sorted(sql_result)
if len(actual_result) != len(expected_result):
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(
"Results are incorrect.Actual num %s. Expected num: %s.:: %s \n"
% (len(actual_result), len(expected_result), extra_msg))
msg = "The number of rows match but the results mismatch, please check"
if self._sort_data(actual_result) != self._sort_data(expected_result):
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(msg + "\n " + extra_msg)
def _verify_results_rqg(self, n1ql_result=[], sql_result=[], hints=["a1"]):
new_n1ql_result = []
for result in n1ql_result:
if result != {}:
new_n1ql_result.append(result)
n1ql_result = new_n1ql_result
if self._is_function_in_result(hints):
return self._verify_results_rqg_for_function(
n1ql_result, sql_result)
check = self._check_sample(n1ql_result, hints)
actual_result = n1ql_result
if actual_result == [{}]:
actual_result = []
if check:
actual_result = self._gen_dict(n1ql_result)
actual_result = sorted(actual_result)
expected_result = sorted(sql_result)
if len(actual_result) != len(expected_result):
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(
"Results are incorrect.Actual num %s. Expected num: %s.:: %s \n"
% (len(actual_result), len(expected_result), extra_msg))
msg = "The number of rows match but the results mismatch, please check"
if self._sort_data(actual_result) != self._sort_data(expected_result):
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(msg + "\n " + extra_msg)
def _sort_data(self, result):
new_data = []
for data in result:
new_data.append(sorted(data))
return new_data
def _verify_results_crud_rqg(self,
n1ql_result=[],
sql_result=[],
hints=["primary_key_id"]):
new_n1ql_result = []
for result in n1ql_result:
if result != {}:
new_n1ql_result.append(result)
n1ql_result = new_n1ql_result
if self._is_function_in_result(hints):
return self._verify_results_rqg_for_function(
n1ql_result, sql_result)
check = self._check_sample(n1ql_result, hints)
actual_result = n1ql_result
if actual_result == [{}]:
actual_result = []
if check:
actual_result = self._gen_dict(n1ql_result)
actual_result = sorted(actual_result)
expected_result = sorted(sql_result)
if len(actual_result) != len(expected_result):
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(
"Results are incorrect.Actual num %s. Expected num: %s.:: %s \n"
% (len(actual_result), len(expected_result), extra_msg))
if not self._result_comparison_analysis(actual_result,
expected_result):
msg = "The number of rows match but the results mismatch, please check"
extra_msg = self._get_failure_message(expected_result,
actual_result)
raise Exception(msg + "\n " + extra_msg)
def _get_failure_message(self, expected_result, actual_result):
if expected_result == None:
expected_result = []
if actual_result == None:
actual_result = []
len_expected_result = len(expected_result)
len_actual_result = len(actual_result)
len_expected_result = min(5, len_expected_result)
len_actual_result = min(5, len_actual_result)
extra_msg = "mismatch in results :: expected :: {0}, actual :: {1} ".format(
expected_result[0:len_expected_result],
actual_result[0:len_actual_result])
return extra_msg
def _result_comparison_analysis(self, expected_result, actual_result):
expected_map = {}
actual_map = {}
for data in expected_result:
primary = None
for key in list(data.keys()):
keys = key
if keys.encode('ascii') == "primary_key_id":
primary = keys
expected_map[data[primary]] = data
for data in actual_result:
primary = None
for key in list(data.keys()):
keys = key
if keys.encode('ascii') == "primary_key_id":
primary = keys
actual_map[data[primary]] = data
check = True
for key in list(expected_map.keys()):
if sorted(actual_map[key]) != sorted(expected_map[key]):
check = False
return check
def _analyze_for_special_case_using_func(self, expected_result,
actual_result):
if expected_result == None:
expected_result = []
if actual_result == None:
actual_result = []
if len(expected_result) == 1:
value = list(expected_result[0].values())[0]
if value == None or value == 0:
expected_result = []
if len(actual_result) == 1:
value = list(actual_result[0].values())[0]
if value == None or value == 0:
actual_result = []
return expected_result, actual_result
def _is_function_in_result(self, result):
if result == "FUN":
return True
return False
def _verify_results_rqg_for_function(self,
n1ql_result=[],
sql_result=[],
hints=["a1"]):
actual_count = -1
expected_count = -1
actual_result = n1ql_result
sql_result, actual_result = self._analyze_for_special_case_using_func(
sql_result, actual_result)
if len(sql_result) != len(actual_result):
msg = "the number of results do not match :: expected = {0}, actual = {1}".format(
len(n1ql_result), len(sql_result))
extra_msg = self._get_failure_message(sql_result, actual_result)
raise Exception(msg + "\n" + extra_msg)
n1ql_result = self._gen_dict_n1ql_func_result(n1ql_result)
n1ql_result = sorted(n1ql_result)
sql_result = self._gen_dict_n1ql_func_result(sql_result)
sql_result = sorted(sql_result)
if len(sql_result) == 0 and len(actual_result) == 0:
return
if sql_result != n1ql_result:
max = 2
if len(sql_result) < 5:
max = len(sql_result)
msg = "mismatch in results :: expected [0:{0}]:: {1}, actual [0:{0}]:: {2} ".format(
max, sql_result[0:max], n1ql_result[0:max])
raise Exception(msg)
def _convert_to_number(self, val):
if not isinstance(val, str):
return val
value = -1
try:
if value == '':
return 0
value = int(val.split("(")[1].split(")")[0])
except Exception as ex:
self.log.info(ex)
finally:
return value
def analyze_failure(self, actual, expected):
missing_keys = []
different_values = []
for key in list(expected.keys()):
if key not in list(actual.keys()):
missing_keys.append(key)
if expected[key] != actual[key]:
different_values.append(
"for key {0}, expected {1} \n actual {2}".format(
key, expected[key], actual[key]))
self.log.info(missing_keys)
if (len(different_values) > 0):
self.log.info(" number of such cases {0}".format(
len(different_values)))
self.log.info(" example key {0}".format(different_values[0]))
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" % (
version, info.architecture_type)
#TODO for windows
return url
def _restart_indexer(self):
couchbase_path = "/opt/couchbase/var/lib/couchbase"
cmd = "rm -f {0}/meta;rm -f /tmp/log_upr_client.sock".format(
couchbase_path)
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
self.shell = RemoteMachineShellConnection(server)
self._set_env_variable(server)
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client[
"sherlock_path"]
print("PATH TO SHERLOCK: %s" % couchbase_path)
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.items():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def drop_primary_index(self, using_gsi=True, server=None):
if server == None:
server = self.master
self.log.info("CHECK FOR PRIMARY INDEXES")
for bucket in self.buckets:
self.query = "DROP PRIMARY INDEX ON {0}".format(bucket.name)
if using_gsi:
self.query += " USING GSI"
if not using_gsi:
self.query += " USING VIEW "
self.log.info(self.query)
try:
check = self._is_index_in_list(bucket.name,
"#primary",
server=server)
if check:
self.run_analytics_query(server=server)
except Exception as ex:
self.log.error('ERROR during index creation %s' % str(ex))
def create_primary_index(self, using_gsi=True, server=None):
if server == None:
server = self.master
for bucket in self.buckets:
self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name)
if using_gsi:
self.query += " USING GSI"
# if gsi_type == "memdb":
# self.query += " WITH {'index_type': 'memdb'}"
if not using_gsi:
self.query += " USING VIEW "
try:
check = self._is_index_in_list(bucket.name,
"#primary",
server=server)
if not check:
self.run_analytics_query(server=server)
check = self.is_index_online_and_in_list(bucket.name,
"#primary",
server=server)
if not check:
raise Exception(
" Timed-out Exception while building primary index for bucket {0} !!!"
.format(bucket.name))
else:
raise Exception(
" Primary Index Already present, This looks like a bug !!!"
)
except Exception as ex:
self.log.error('ERROR during index creation %s' % str(ex))
raise ex
def verify_index_with_explain(self,
actual_result,
index_name,
check_covering_index=False):
check = True
if check_covering_index:
if "covering" in str(actual_result):
check = True
else:
check = False
if index_name in str(actual_result):
return True and check
return False
def run_query_and_verify_result(self,
server=None,
query=None,
timeout=120.0,
max_try=1,
expected_result=None,
scan_consistency=None,
scan_vector=None,
verify_results=True):
check = False
init_time = time.time()
try_count = 0
while not check:
next_time = time.time()
try:
actual_result = self.run_analytics_query(
query=query,
server=server,
scan_consistency=scan_consistency,
scan_vector=scan_vector)
if verify_results:
self._verify_results(actual_result['results'],
expected_result)
else:
return "ran query with success and validated results", True
check = True
except Exception as ex:
if (next_time - init_time > timeout or try_count >= max_try):
return ex, False
finally:
try_count += 1
return "ran query with success and validated results", check
def run_cbq_query(self,
query=None,
min_output_size=10,
server=None,
query_params={},
is_prepared=False,
scan_consistency=None,
scan_vector=None,
verbose=True):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.n1ql_port == None or self.n1ql_port == '':
self.n1ql_port = self.input.param("n1ql_port", 90)
if not self.n1ql_port:
self.log.info(
" n1ql_port is not defined, processing will not proceed further"
)
raise Exception(
"n1ql_port is not defined, processing will not proceed further"
)
if self.use_rest:
query_params = {}
if scan_consistency:
query_params['scan_consistency'] = scan_consistency
if scan_vector:
query_params['scan_vector'] = str(scan_vector).replace(
"'", '"')
if verbose:
self.log.info('RUN QUERY %s' % query)
result = RestConnection(server).query_tool(
query,
self.n1ql_port,
query_params=query_params,
is_prepared=is_prepared,
verbose=verbose)
else:
# if self.version == "git_repo":
# output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\
# "tuq_client/tuq_client " +\
# "-engine=http://%s:8093/" % server.ip,
# subcommands=[query,],
# min_output_size=20,
# end_msg='tuq_client>')
# else:
#os = self.shell.extract_remote_info().type.lower()
shell = RemoteMachineShellConnection(server)
#query = query.replace('"', '\\"')
#query = query.replace('`', '\\`')
#if os == "linux":
cmd = "%s/cbq -engine=http://%s:8093/" % (
testconstants.LINUX_COUCHBASE_BIN_PATH, server.ip)
output = shell.execute_commands_inside(cmd, query, "", "", "", "",
"")
print(
"--------------------------------------------------------------------------------------------------------------------------------"
)
print(output)
result = json.loads(output)
print(result)
result = self._parse_query_output(output)
if isinstance(result, str) or 'errors' in result:
error_result = str(result)
length_display = len(error_result)
if length_display > 500:
error_result = error_result[:500]
raise CBQError(error_result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" %
result["metrics"]["elapsedTime"])
return result
# def is_index_online_and_in_list(self, bucket, index_name, server=None, timeout=600.0):
# check = self._is_index_in_list(bucket, index_name, server = server)
# init_time = time.time()
# while not check:
# time.sleep(1)
# check = self._is_index_in_list(bucket, index_name, server = server)
# next_time = time.time()
# if check or (next_time - init_time > timeout):
# return check
# return check
#
# def is_index_ready_and_in_list(self, bucket, index_name, server=None, timeout=600.0):
# query = "SELECT * FROM system:indexes where name = \'{0}\'".format(index_name)
# if server == None:
# server = self.master
# init_time = time.time()
# check = False
# while not check:
# res = self.run_analytics_query(query=query, server=server)
# for item in res['results']:
# if 'keyspace_id' not in item['indexes']:
# check = False
# elif item['indexes']['keyspace_id'] == str(bucket) \
# and item['indexes']['name'] == index_name \
# and item['indexes']['state'] == "online":
# check = True
# time.sleep(1)
# next_time = time.time()
# check = check or (next_time - init_time > timeout)
# return check
# def is_index_online_and_in_list_bulk(self, bucket, index_names = [], server = None, index_state = "online", timeout = 600.0):
# check, index_names = self._is_index_in_list_bulk(bucket, index_names, server = server, index_state = index_state)
# init_time = time.time()
# while not check:
# check, index_names = self._is_index_in_list_bulk(bucket, index_names, server = server, index_state = index_state)
# next_time = time.time()
# if check or (next_time - init_time > timeout):
# return check
# return check
#
# def gen_build_index_query(self, bucket = "default", index_list = []):
# return "BUILD INDEX on {0}({1}) USING GSI".format(bucket,",".join(index_list))
#
# def gen_query_parameter(self, scan_vector = None, scan_consistency = None):
# query_params = {}
# if scan_vector:
# query_params.update("scan_vector", scan_vector)
# if scan_consistency:
# query_params.update("scan_consistency", scan_consistency)
# return query_params
# def _is_index_in_list(self, bucket, index_name, server = None, index_state = ["pending", "building", "deferred"]):
# query = "SELECT * FROM system:indexes where name = \'{0}\'".format(index_name)
# if server == None:
# server = self.master
# res = self.run_cbq_query(query = query, server = server)
# for item in res['results']:
# if 'keyspace_id' not in item['indexes']:
# return False
# if item['indexes']['keyspace_id'] == str(bucket) and item['indexes']['name'] == index_name and item['indexes']['state'] not in index_state:
# return True
# return False
#
# def _is_index_in_list_bulk(self, bucket, index_names = [], server = None, index_state = ["pending","building"]):
# query = "SELECT * FROM system:indexes"
# if server == None:
# server = self.master
# res = self.run_cbq_query(query = query, server = server)
# index_count=0
# found_index_list = []
# for item in res['results']:
# if 'keyspace_id' not in item['indexes']:
# return False
# for index_name in index_names:
# if item['indexes']['keyspace_id'] == str(bucket) and item['indexes']['name'] == index_name and item['indexes']['state'] not in index_state:
# found_index_list.append(index_name)
# if len(found_index_list) == len(index_names):
# return True, []
# return False, list(set(index_names) - set(found_index_list))
#
# def gen_index_map(self, server = None):
# query = "SELECT * FROM system:indexes"
# if server == None:
# server = self.master
# res = self.run_cbq_query(query = query, server = server)
# index_map = {}
# for item in res['results']:
# bucket_name = item['indexes']['keyspace_id'].encode('ascii','ignore')
# if bucket_name not in index_map.keys():
# index_map[bucket_name] = {}
# index_name = str(item['indexes']['name'])
# index_map[bucket_name][index_name] = {}
# index_map[bucket_name][index_name]['state'] = item['indexes']['state']
# return index_map
#
# def get_index_count_using_primary_index(self, buckets, server = None):
# query = "SELECT COUNT(*) FROM {0}"
# map= {}
# if server == None:
# server = self.master
# for bucket in buckets:
# res = self.run_cbq_query(query = query.format(bucket.name), server = server)
# map[bucket.name] = int(res["results"][0]["$1"])
# return map
#
# def get_index_count_using_index(self, bucket, index_name,server=None):
# query = 'SELECT COUNT(*) FROM {0} USE INDEX ({1})'.format(bucket.name, index_name)
# if not server:
# server = self.master
# res = self.run_cbq_query(query=query, server=server)
# return int(res['results'][0]['$1'])
def _gen_dict(self, result):
result_set = []
if result != None and len(result) > 0:
for val in result:
for key in list(val.keys()):
result_set.append(val[key])
return result_set
def _gen_dict_n1ql_func_result(self, result):
result_set = [val[key] for val in result for key in list(val.keys())]
new_result_set = []
if len(result_set) > 0:
for value in result_set:
if isinstance(value, float):
new_result_set.append(round(value, 0))
else:
new_result_set.append(value)
else:
new_result_set = result_set
return new_result_set
def _check_sample(self, result, expected_in_key=None):
if expected_in_key == "FUN":
return False
if expected_in_key == None or len(expected_in_key) == 0:
return False
if result != None and len(result) > 0:
sample = result[0]
for key in list(sample.keys()):
for sample in expected_in_key:
if key in sample:
return True
return False
def old_gen_dict(self, result):
result_set = []
map = {}
duplicate_keys = []
try:
if result != None and len(result) > 0:
for val in result:
for key in list(val.keys()):
result_set.append(val[key])
for val in result_set:
if val["_id"] in list(map.keys()):
duplicate_keys.append(val["_id"])
map[val["_id"]] = val
keys = list(map.keys())
keys.sort()
except Exception as ex:
self.log.info(ex)
raise
if len(duplicate_keys) > 0:
raise Exception(" duplicate_keys {0}".format(duplicate_keys))
return map
class QueryTests(BaseTestCase):
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param(
"cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics", False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'GSI')
self.index_type = self.input.param("index_type", 'GSI')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring", False)
self.isprepared = False
self.named_prepare = self.input.param("named_prepare", None)
self.skip_primary_index = self.input.param("skip_primary_index", False)
self.scan_consistency = self.input.param("scan_consistency",
'REQUEST_PLUS')
shell = RemoteMachineShellConnection(self.master)
type = shell.extract_remote_info().distribution_type
self.path = testconstants.LINUX_COUCHBASE_BIN_PATH
if type.lower() == 'windows':
self.path = testconstants.WIN_COUCHBASE_BIN_PATH
elif type.lower() == "mac":
self.path = testconstants.MAC_COUCHBASE_BIN_PATH
self.threadFailure = False
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", 'plasma')
else:
self.gsi_type = None
if self.input.param("reload_data", False):
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [],
[self.cbas_node],
services=['cbas'])
for bucket in self.buckets:
self.cluster.bucket_flush(self.master,
bucket=bucket,
timeout=self.wait_timeout * 5)
# Adding sleep after flushing buckets (see CBQE-5838)
self.sleep(210)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node],
[self.cbas_node], [],
services=['cbas'])
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(
self.log, self.generate_full_docs_list(self.gens_load))
if (self.analytics == False):
self.create_primary_index_for_3_0_and_greater()
if (self.analytics):
self.setup_analytics()
self.sleep(30, 'wait for analytics setup')
def suite_setUp(self):
try:
self.load(self.gens_load, flag=self.item_flag)
if not self.input.param("skip_build_tuq", True):
self._build_tuq(self.master)
self.skip_buckets_handle = True
if (self.analytics):
self.cluster.rebalance([self.master, self.cbas_node],
[self.cbas_node], [],
services=['cbas'])
self.setup_analytics()
self.sleep(30, 'wait for analytics setup')
except:
self.log.error('SUITE SETUP FAILED')
self.tearDown()
def tearDown(self):
if self._testMethodName == 'suite_tearDown':
self.skip_buckets_handle = False
if self.analytics:
bucket_username = "******"
bucket_password = "******"
data = 'use Default ;'
for bucket in self.buckets:
data += 'disconnect bucket {0} if connected;'.format(
bucket.name)
data += 'drop dataset {0} if exists;'.format(bucket.name +
"_shadow")
data += 'drop bucket {0} if exists;'.format(bucket.name)
filename = "file.txt"
f = open(filename, 'w')
f.write(data)
f.close()
url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password
os.system(cmd)
os.remove(filename)
super(QueryTests, self).tearDown()
def suite_tearDown(self):
if not self.input.param("skip_build_tuq", False):
if hasattr(self, 'shell'):
self.shell.execute_command("killall /tmp/tuq/cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.disconnect()
##############################################################################################
#
# Setup Helpers
##############################################################################################
def setup_analytics(self):
data = 'use Default;'
bucket_username = "******"
bucket_password = "******"
for bucket in self.buckets:
data += 'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format(
bucket.name, self.master.ip)
data += 'create shadow dataset {1} on {0}; '.format(
bucket.name, bucket.name + "_shadow")
data += 'connect bucket {0} with {{"username":"******","password":"******"}};'.format(
bucket.name, bucket_username, bucket_password)
filename = "file.txt"
f = open(filename, 'w')
f.write(data)
f.close()
url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password
os.system(cmd)
os.remove(filename)
def run_active_requests(self, e, t):
while not e.isSet():
logging.debug('wait_for_event_timeout starting')
event_is_set = e.wait(t)
logging.debug('event set: %s', event_is_set)
if event_is_set:
result = self.run_cbq_query(
"select * from system:active_requests")
self.assertTrue(result['metrics']['resultCount'] == 1)
requestId = result['requestID']
result = self.run_cbq_query(
'delete from system:active_requests where requestId = "%s"'
% requestId)
time.sleep(20)
result = self.run_cbq_query(
'select * from system:active_requests where requestId = "%s"'
% requestId)
self.assertTrue(result['metrics']['resultCount'] == 0)
result = self.run_cbq_query(
"select * from system:completed_requests")
requestId = result['requestID']
result = self.run_cbq_query(
'delete from system:completed_requests where requestId = "%s"'
% requestId)
time.sleep(10)
result = self.run_cbq_query(
'select * from system:completed_requests where requestId = "%s"'
% requestId)
self.assertTrue(result['metrics']['resultCount'] == 0)
##############################################################################################
#
# COMMON FUNCTIONS
##############################################################################################
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
keys_num = int(
re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', ''))
subquery_full_list = self.generate_full_docs_list(
gens_load=self.gens_load, keys=self._get_keys(keys_num))
subquery_template = re.sub(r'USE KEYS.*', '', subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '',
re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as', '', re.sub(r'FROM.*', '', alias)).strip()
if not alias:
alias = '$1'
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias,
',%s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
subquery_full_list = self.generate_full_docs_list(
gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '',
re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as ', '', alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias,
query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def negative_common_body(self, queries_errors={}):
if not queries_errors:
self.fail("No queries to run!")
for bucket in self.buckets:
for query_template, error in queries_errors.items():
try:
query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query(
query.format(bucket.name))
except CBQError as ex:
self.log.error(ex)
self.assertTrue(
str(ex).find(error) != -1,
"Error is incorrect.Actual %s.\n Expected: %s.\n" %
(str(ex).split(':')[-1], error))
else:
self.fail("There were no errors. Error expected: %s" %
error)
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
query_params = {}
cred_params = {'creds': []}
rest = RestConnection(server)
username = rest.username
password = rest.password
cred_params['creds'].append({'user': username, 'pass': password})
query_params.update(cred_params)
if self.use_rest:
query_params.update({'scan_consistency': self.scan_consistency})
self.log.info('RUN QUERY %s' % query)
if self.analytics:
query = query + ";"
for bucket in self.buckets:
query = query.replace(bucket.name, bucket.name + "_shadow")
result = RestConnection(
self.cbas_node).execute_statement_on_cbas(
query, "immediate")
result = json.loads(result)
else:
result = rest.query_tool(query,
self.n1ql_port,
query_params=query_params)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbase/query/" +\
"shell/cbq/cbq ", "", "", "", "", "", "")
else:
os = self.shell.extract_remote_info().type.lower()
if not (self.isprepared):
query = query.replace('"', '\\"')
query = query.replace('`', '\\`')
cmd = "%s/cbq -engine=http://%s:%s/ -q -u %s -p %s" % (
self.path, server.ip, server.port, username, password)
output = self.shell.execute_commands_inside(
cmd, query, "", "", "", "", "")
if not (output[0] == '{'):
output1 = '{' + str(output)
else:
output1 = output
result = json.loads(output1)
if isinstance(result, str) or 'errors' in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" %
result["metrics"]["elapsedTime"])
return result
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" % (
version, info.architecture_type)
#TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'go get github.com/couchbaselabs/tuqtng;' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\
'go get -d -v ./...; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .'
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
#TODO for windows
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(
cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
if self.services_init.find('n1ql') != -1:
return
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client[
"sherlock_path"]
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def generate_docs(self, num_items, start=0):
try:
return getattr(self, 'generate_docs_' + self.dataset)(num_items,
start)
except:
self.fail("There is no dataset %s, please enter a valid one" %
self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_sabre(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_sabre(docs_per_day, start)
def generate_docs_employee(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_simple_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_sales(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_sales_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_bigdata(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_bigdata(end=(1000 * docs_per_day),
start=start,
value_size=self.value_size)
def _verify_results(self,
actual_result,
expected_result,
missing_count=1,
extra_count=1):
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(
actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" %
(missing[:missing_count], extra[:extra_count]))
self.fail(
"Results are incorrect.Actual num %s. Expected num: %s.\n" %
(len(actual_result), len(expected_result)))
if self.max_verify is not None:
actual_result = actual_result[:self.max_verify]
expected_result = expected_result[:self.max_verify]
msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\
"Expected first and last 100: %s.\n ... \n %s"
self.assertTrue(
actual_result == expected_result,
msg % (actual_result[:100], actual_result[-100:],
expected_result[:100], expected_result[-100:]))
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.items():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self):
self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type)
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
if versions[0].startswith("4") or versions[0].startswith(
"3") or versions[0].startswith("5"):
for bucket in self.buckets:
if self.primary_indx_drop:
self.log.info(
"Dropping primary index for %s using %s ..." %
(bucket.name, self.primary_indx_type))
self.query = "DROP PRIMARY INDEX ON %s USING %s" % (
bucket.name, self.primary_indx_type)
#self.run_cbq_query()
self.sleep(3, 'Sleep for some time after index drop')
self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name
res = self.run_cbq_query()
self.sleep(10)
if self.monitoring:
self.query = "delete from system:completed_requests"
self.run_cbq_query()
if not self.skip_primary_index:
if (res['metrics']['resultCount'] == 0):
self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (
bucket.name, self.primary_indx_type)
self.log.info("Creating primary index for %s ..." %
bucket.name)
try:
self.run_cbq_query()
self.primary_index_created = True
if self.primary_indx_type.lower() == 'gsi':
self._wait_for_index_online(bucket, '#primary')
except Exception as ex:
self.log.info(str(ex))
def _wait_for_index_online(self, bucket, index_name, timeout=6000):
end_time = time.time() + timeout
while time.time() < end_time:
query = "SELECT * FROM system:indexes where name='%s'" % index_name
res = self.run_cbq_query(query)
for item in res['results']:
if 'keyspace_id' not in item['indexes']:
self.log.error(item)
continue
if item['indexes']['keyspace_id'] == bucket.name:
if item['indexes']['state'] == "online":
return
self.sleep(
5, 'index is pending or not in the list. sleeping... (%s)' %
[item['indexes'] for item in res['results']])
raise Exception('index %s is not online. last response is %s' %
(index_name, res))
def _get_keys(self, key_num):
keys = []
for gen in self.gens_load:
gen_copy = copy.deepcopy(gen)
for i in range(gen_copy.end):
key, _ = next(gen_copy)
keys.append(key)
if len(keys) == key_num:
return keys
return keys
def setUp(self):
super(QueryTests, self).setUp()
self.expiry = self.input.param("expiry", 0)
self.batch_size = self.input.param("batch_size", 1)
self.scan_consistency = self.input.param("scan_consistency",
"request_plus")
self.skip_cleanup = self.input.param("skip_cleanup", False)
self.run_async = self.input.param("run_async", True)
self.version = self.input.param("cbq_version", "git_repo")
for server in self.servers:
rest = RestConnection(server)
temp = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(
server.ip, temp['nodes'][0]['status']))
while (temp['nodes'][0]['status'] == 'warmup'):
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
temp = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(
server.ip, temp['nodes'][0]['status']))
indexer_node = self.get_nodes_from_services_map(service_type="index",
get_all_nodes=True)
# Set indexer storage mode
indexer_rest = RestConnection(indexer_node[0])
doc = {"indexer.settings.storage_mode": self.gsi_type}
indexer_rest.set_index_settings_internal(doc)
doc = {"indexer.api.enableTestServer": True}
indexer_rest.set_index_settings_internal(doc)
self.indexer_scanTimeout = self.input.param("indexer_scanTimeout",
None)
if self.indexer_scanTimeout is not None:
for server in indexer_node:
rest = RestConnection(server)
rest.set_index_settings({
"indexer.settings.scan_timeout":
self.indexer_scanTimeout
})
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
self.use_gsi_for_primary = self.input.param("use_gsi_for_primary",
True)
self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary",
True)
self.create_primary_index = self.input.param("create_primary_index",
True)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.value_size = self.input.param("value_size", 1024)
self.doc_ops = self.input.param("doc_ops", False)
self.create_ops_per = self.input.param("create_ops_per", 0)
self.expiry_ops_per = self.input.param("expiry_ops_per", 0)
self.delete_ops_per = self.input.param("delete_ops_per", 0)
self.update_ops_per = self.input.param("update_ops_per", 0)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.n1ql_helper.configure_gomaxprocs()
self.full_docs_list = self.generate_full_docs_list(self.gens_load)
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
verify_data = False
if self.scan_consistency != "request_plus":
verify_data = True
self.load(self.gens_load,
flag=self.item_flag,
verify_data=verify_data,
batch_size=self.batch_size)
if self.doc_ops:
self.ops_dist_map = self.calculate_data_change_distribution(
create_per=self.create_ops_per,
update_per=self.update_ops_per,
delete_per=self.delete_ops_per,
expiry_per=self.expiry_ops_per,
start=0,
end=self.docs_per_day)
self.log.info(self.ops_dist_map)
self.docs_gen_map = self.generate_ops_docs(self.docs_per_day, 0)
self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops(
self.docs_gen_map)
# Define Helper Method which will be used for running n1ql queries, create index, drop index
self.n1ql_helper = N1QLHelper(version=self.version,
shell=self.shell,
use_rest=self.use_rest,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master)
self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
self.log.info(self.n1ql_node)
#self.n1ql_helper._start_command_line_query(self.n1ql_node)
# sleep to avoid race condition during bootstrap
if self.create_primary_index:
try:
self.n1ql_helper.create_primary_index(
using_gsi=self.use_gsi_for_primary, server=self.n1ql_node)
except Exception as ex:
self.log.info(ex)
raise ex
class N1QLHelper():
def __init__(self,
version=None,
master=None,
shell=None,
use_rest=None,
max_verify=0,
buckets=[],
item_flag=0,
n1ql_port=8093,
full_docs_list=[],
log=None,
input=None):
self.version = version
self.shell = shell
self.use_rest = use_rest
self.max_verify = max_verify
self.buckets = buckets
self.item_flag = item_flag
self.n1ql_port = n1ql_port
self.input = input
self.log = log
self.full_docs_list = full_docs_list
self.master = master
self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def killall_tuq_process(self):
self.shell.execute_command("killall cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.execute_command("killall indexer")
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.n1ql_port == None or self.n1ql_port == '':
self.n1ql_port = self.input.param("n1ql_port", 8093)
if not self.n1ql_port:
self.log.info(
" n1ql_port is not defined, processing will not proceed further"
)
raise Exception(
"n1ql_port is not defined, processing will not proceed further"
)
if self.use_rest:
result = RestConnection(server).query_tool(query, self.n1ql_port)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\
"tuq_client/tuq_client " +\
"-engine=http://%s:8093/" % server.ip,
subcommands=[query,],
min_output_size=20,
end_msg='tuq_client>')
else:
output = self.shell.execute_commands_inside(
"/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip,
subcommands=[
query,
],
min_output_size=20,
end_msg='cbq>')
result = self._parse_query_output(output)
if isinstance(result, str) or 'errors' in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" %
result["metrics"]["elapsedTime"])
return result
def _verify_results(self,
actual_result,
expected_result,
missing_count=1,
extra_count=1):
actual_result = self._gen_dict(actual_result)
expected_result = self._gen_dict(expected_result)
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(
actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" %
(missing[:missing_count], extra[:extra_count]))
raise Exception(
"Results are incorrect.Actual num %s. Expected num: %s.\n" %
(len(actual_result), len(expected_result)))
if self.max_verify is not None:
actual_result = actual_result[:self.max_verify]
expected_result = expected_result[:self.max_verify]
msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\
"Expected first and last 100: %s.\n ... \n %s"
error_message = msg % (actual_result[:10], actual_result[-10:],
expected_result[:10], expected_result[-10:])
if actual_result != expected_result:
raise Exception(error_message)
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" % (
version, info.architecture_type)
#TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'go get github.com/couchbaselabs/tuqtng;' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\
'go get -d -v ./...; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .'
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
#TODO for windowsself.shell.execute_command(cmd)
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(
cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _restart_indexer(self):
couchbase_path = "/opt/couchbase/var/lib/couchbase"
cmd = "rm -f {0}/meta;rm -f /tmp/log_upr_client.sock".format(
couchbase_path)
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
self._set_env_variable(server)
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbaselabs/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbaselabs/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client[
"sherlock_path"]
print "PATH TO SHERLOCK: %s" % couchbase_path
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.iteritems():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self, using_gsi=True):
self.log.info("CHECK FOR PRIMARY INDEXES")
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
ddoc_name = 'ddl_#primary'
if versions[0].startswith("3"):
try:
rest.get_ddoc(self.buckets[0], ddoc_name)
except ReadDocumentException:
for bucket in self.buckets:
self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name)
if using_gsi:
self.query += " USING GSI"
self.log.info(self.query)
try:
self.run_cbq_query()
except Exception, ex:
self.log.error('ERROR during index creation %s' %
str(ex))
class QueryTests(BaseTestCase):
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics",False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", 'GSI')
self.index_type = self.input.param("index_type", 'GSI')
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring",False)
self.isprepared = False
self.named_prepare = self.input.param("named_prepare", None)
self.skip_primary_index = self.input.param("skip_primary_index",False)
self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS')
shell = RemoteMachineShellConnection(self.master)
type = shell.extract_remote_info().distribution_type
self.path = testconstants.LINUX_COUCHBASE_BIN_PATH
if type.lower() == 'windows':
self.path = testconstants.WIN_COUCHBASE_BIN_PATH
elif type.lower() == "mac":
self.path = testconstants.MAC_COUCHBASE_BIN_PATH
self.threadFailure = False
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", 'plasma')
else:
self.gsi_type = None
if self.input.param("reload_data", False):
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas'])
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket,
timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
if self.analytics:
self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas'])
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
if (self.analytics == False):
self.create_primary_index_for_3_0_and_greater()
if (self.analytics):
self.setup_analytics()
self.sleep(30,'wait for analytics setup')
def suite_setUp(self):
try:
self.load(self.gens_load, flag=self.item_flag)
if not self.input.param("skip_build_tuq", True):
self._build_tuq(self.master)
self.skip_buckets_handle = True
if (self.analytics):
self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas'])
self.setup_analytics()
self.sleep(30,'wait for analytics setup')
except:
self.log.error('SUITE SETUP FAILED')
self.tearDown()
def tearDown(self):
if self._testMethodName == 'suite_tearDown':
self.skip_buckets_handle = False
if self.analytics:
bucket_username = "******"
bucket_password = "******"
data = 'use Default ;'
for bucket in self.buckets:
data += 'disconnect bucket {0} if connected;'.format(bucket.name)
data += 'drop dataset {0} if exists;'.format(bucket.name+ "_shadow")
data += 'drop bucket {0} if exists;'.format(bucket.name)
filename = "file.txt"
f = open(filename,'w')
f.write(data)
f.close()
url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password
os.system(cmd)
os.remove(filename)
super(QueryTests, self).tearDown()
def suite_tearDown(self):
if not self.input.param("skip_build_tuq", False):
if hasattr(self, 'shell'):
self.shell.execute_command("killall /tmp/tuq/cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.disconnect()
##############################################################################################
#
# Setup Helpers
##############################################################################################
def setup_analytics(self):
data = 'use Default;'
bucket_username = "******"
bucket_password = "******"
for bucket in self.buckets:
data += 'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format(
bucket.name, self.master.ip)
data += 'create shadow dataset {1} on {0}; '.format(bucket.name,
bucket.name + "_shadow")
data += 'connect bucket {0} with {{"username":"******","password":"******"}};'.format(
bucket.name, bucket_username, bucket_password)
filename = "file.txt"
f = open(filename,'w')
f.write(data)
f.close()
url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password
os.system(cmd)
os.remove(filename)
def run_active_requests(self, e, t):
while not e.isSet():
logging.debug('wait_for_event_timeout starting')
event_is_set = e.wait(t)
logging.debug('event set: %s', event_is_set)
if event_is_set:
result = self.run_cbq_query("select * from system:active_requests")
self.assertTrue(result['metrics']['resultCount'] == 1)
requestId = result['requestID']
result = self.run_cbq_query(
'delete from system:active_requests where requestId = "%s"' % requestId)
time.sleep(20)
result = self.run_cbq_query(
'select * from system:active_requests where requestId = "%s"' % requestId)
self.assertTrue(result['metrics']['resultCount'] == 0)
result = self.run_cbq_query("select * from system:completed_requests")
requestId = result['requestID']
result = self.run_cbq_query(
'delete from system:completed_requests where requestId = "%s"' % requestId)
time.sleep(10)
result = self.run_cbq_query(
'select * from system:completed_requests where requestId = "%s"' % requestId)
self.assertTrue(result['metrics']['resultCount'] == 0)
##############################################################################################
#
# COMMON FUNCTIONS
##############################################################################################
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', ''))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num))
subquery_template = re.sub(r'USE KEYS.*', '', subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip()
if not alias:
alias = '$1'
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r'.*\$subquery\(', '', query_template)
subquery_template = subquery_template[:subquery_template.rfind(')')]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template))
alias = re.sub(r'.*as ', '', alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def negative_common_body(self, queries_errors={}):
if not queries_errors:
self.fail("No queries to run!")
for bucket in self.buckets:
for query_template, error in queries_errors.iteritems():
try:
query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query(query.format(bucket.name))
except CBQError as ex:
self.log.error(ex)
self.assertTrue(str(ex).find(error) != -1,
"Error is incorrect.Actual %s.\n Expected: %s.\n" %(
str(ex).split(':')[-1], error))
else:
self.fail("There were no errors. Error expected: %s" % error)
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
query_params = {}
cred_params = {'creds': []}
rest = RestConnection(server)
username = rest.username
password = rest.password
cred_params['creds'].append({'user': username, 'pass': password})
for bucket in self.buckets:
if bucket.saslPassword:
cred_params['creds'].append({'user': '******' % bucket.name, 'pass': bucket.saslPassword})
query_params.update(cred_params)
if self.use_rest:
query_params.update({'scan_consistency': self.scan_consistency})
self.log.info('RUN QUERY %s' % query)
if self.analytics:
query = query + ";"
for bucket in self.buckets:
query = query.replace(bucket.name,bucket.name+"_shadow")
result = RestConnection(self.cbas_node).execute_statement_on_cbas(query, "immediate")
result = json.loads(result)
else :
result = rest.query_tool(query, self.n1ql_port, query_params=query_params)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbase/query/" +\
"shell/cbq/cbq ","","","","","","")
else:
os = self.shell.extract_remote_info().type.lower()
if not(self.isprepared):
query = query.replace('"', '\\"')
query = query.replace('`', '\\`')
cmd = "%s/cbq -engine=http://%s:%s/ -q -u %s -p %s" % (self.path, server.ip, server.port, username, password)
output = self.shell.execute_commands_inside(cmd,query,"","","","","")
if not(output[0] == '{'):
output1 = '{'+str(output)
else:
output1 = output
result = json.loads(output1)
if isinstance(result, str) or 'errors' in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"])
return result
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" %(
version, info.architecture_type)
#TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'go get github.com/couchbaselabs/tuqtng;' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\
'go get -d -v ./...; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .'
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
#TODO for windows
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
if self.services_init.find('n1ql') != -1:
return
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"]
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def generate_docs(self, num_items, start=0):
try:
return getattr(self, 'generate_docs_' + self.dataset)(num_items, start)
except:
self.fail("There is no dataset %s, please enter a valid one" % self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_sabre(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_sabre(docs_per_day, start)
def generate_docs_employee(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_data(docs_per_day = docs_per_day, start = start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_simple_data(docs_per_day = docs_per_day, start = start)
def generate_docs_sales(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_sales_data(docs_per_day = docs_per_day, start = start)
def generate_docs_bigdata(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_bigdata(end=(1000*docs_per_day), start=start, value_size=self.value_size)
def _verify_results(self, actual_result, expected_result, missing_count = 1, extra_count = 1):
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count]))
self.fail("Results are incorrect.Actual num %s. Expected num: %s.\n" % (
len(actual_result), len(expected_result)))
if self.max_verify is not None:
actual_result = actual_result[:self.max_verify]
expected_result = expected_result[:self.max_verify]
msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\
"Expected first and last 100: %s.\n ... \n %s"
self.assertTrue(actual_result == expected_result,
msg % (actual_result[:100],actual_result[-100:],
expected_result[:100],expected_result[-100:]))
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.iteritems():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self):
self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type)
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
if versions[0].startswith("4") or versions[0].startswith("3") or versions[0].startswith("5"):
for bucket in self.buckets:
if self.primary_indx_drop:
self.log.info("Dropping primary index for %s using %s ..." % (bucket.name,self.primary_indx_type))
self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name,self.primary_indx_type)
#self.run_cbq_query()
self.sleep(3, 'Sleep for some time after index drop')
self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name
res = self.run_cbq_query()
self.sleep(10)
if self.monitoring:
self.query = "delete from system:completed_requests"
self.run_cbq_query()
if not self.skip_primary_index:
if (res['metrics']['resultCount'] == 0):
self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
self.log.info("Creating primary index for %s ..." % bucket.name)
try:
self.run_cbq_query()
self.primary_index_created = True
if self.primary_indx_type.lower() == 'gsi':
self._wait_for_index_online(bucket, '#primary')
except Exception, ex:
self.log.info(str(ex))
class QueryTests(BaseTestCase):
def setUp(self):
if not self._testMethodName == "suite_setUp":
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "sherlock")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == "suite_setUp" and self.input.param("cbq_version", "sherlock") != "sherlock":
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.analytics = self.input.param("analytics", False)
self.dataset = self.input.param("dataset", "default")
self.primary_indx_type = self.input.param("primary_indx_type", "GSI")
self.index_type = self.input.param("index_type", "GSI")
self.primary_indx_drop = self.input.param("primary_indx_drop", False)
self.monitoring = self.input.param("monitoring", False)
self.isprepared = False
self.skip_primary_index = self.input.param("skip_primary_index", False)
self.scan_consistency = self.input.param("scan_consistency", "REQUEST_PLUS")
if self.primary_indx_type.lower() == "gsi":
self.gsi_type = self.input.param("gsi_type", None)
else:
self.gsi_type = None
if self.input.param("reload_data", False):
for bucket in self.buckets:
self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5)
self.gens_load = self.generate_docs(self.docs_per_day)
self.load(self.gens_load, flag=self.item_flag)
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load))
if self.analytics == False:
self.create_primary_index_for_3_0_and_greater()
if self.analytics:
self.setup_analytics()
self.sleep(30, "wait for analytics setup")
def suite_setUp(self):
try:
self.load(self.gens_load, flag=self.item_flag)
if not self.input.param("skip_build_tuq", True):
self._build_tuq(self.master)
self.skip_buckets_handle = True
except:
self.log.error("SUITE SETUP FAILED")
self.tearDown()
def tearDown(self):
if self._testMethodName == "suite_tearDown":
self.skip_buckets_handle = False
if self.analytics:
data = "use Default ;" + "\n"
for bucket in self.buckets:
data += "disconnect bucket {0} if connected;".format(bucket.name) + "\n"
data += "drop dataset {0} if exists;".format(bucket.name + "_shadow") + "\n"
data += "drop bucket {0} if exists;".format(bucket.name) + "\n"
filename = "file.txt"
f = open(filename, "w")
f.write(data)
f.close()
url = "http://{0}:8095/analytics/service".format(self.master.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url
os.system(cmd)
os.remove(filename)
super(QueryTests, self).tearDown()
def suite_tearDown(self):
if not self.input.param("skip_build_tuq", False):
if hasattr(self, "shell"):
self.shell.execute_command("killall /tmp/tuq/cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.disconnect()
def setup_analytics(self):
# data = ""
# for bucket in self.buckets:
# data += 'disconnect bucket {0} ;'.format(bucket.name) + "\n"
# data += 'connect bucket {0};'.format(bucket.name) + "\n"
# filename = "file.txt"
# f = open(filename,'w')
# f.write(data)
# f.close()
# url = 'http://{0}:8095/analytics/service'.format(self.master.ip)
# cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url
# os.system(cmd)
# os.remove(filename)
data = "use Default;" + "\n"
for bucket in self.buckets:
data += (
'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format(bucket.name, self.master.ip) + "\n"
)
data += "create shadow dataset {1} on {0}; ".format(bucket.name, bucket.name + "_shadow") + "\n"
data += "connect bucket {0} ;".format(bucket.name) + "\n"
filename = "file.txt"
f = open(filename, "w")
f.write(data)
f.close()
url = "http://{0}:8095/analytics/service".format(self.master.ip)
cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url
os.system(cmd)
os.remove(filename)
##############################################################################################
#
# SIMPLE CHECKS
##############################################################################################
def test_simple_check(self):
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t1 = threading.Thread(name="run_simple", target=self.run_active_requests, args=(e, 2))
t1.start()
query = "select * from %s" % (bucket.name)
self.run_cbq_query(query)
logging.debug("event is set")
if self.monitoring:
e.set()
t1.join(100)
query_template = "FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_joins_monitoring(self):
for bucket in self.buckets:
e = threading.Event()
if self.monitoring:
e = threading.Event()
t2 = threading.Thread(name="run_joins", target=self.run_active_requests, args=(e, 2))
t2.start()
query = (
"select * from %s b1 inner join %s b2 on keys b1.CurrencyCode inner join %s b3 on keys b1.CurrencyCode left outer join %s b4 on keys b1.CurrencyCode"
% (bucket.name, bucket.name, bucket.name, bucket.name)
)
actual_result = self.run_cbq_query(query)
logging.debug("event is set")
if self.monitoring:
e.set()
t2.join(100)
def run_active_requests(self, e, t):
while not e.isSet():
logging.debug("wait_for_event_timeout starting")
event_is_set = e.wait(t)
logging.debug("event set: %s", event_is_set)
if event_is_set:
result = self.run_cbq_query("select * from system:active_requests")
print result
self.assertTrue(result["metrics"]["resultCount"] == 1)
requestId = result["requestID"]
result = self.run_cbq_query('delete from system:active_requests where RequestId = "%s"' % requestId)
time.sleep(20)
result = self.run_cbq_query(
'select * from system:active_requests where RequestId = "%s"' % requestId
)
self.assertTrue(result["metrics"]["resultCount"] == 0)
result = self.run_cbq_query("select * from system:completed_requests")
print result
requestId = result["requestID"]
result = self.run_cbq_query(
'delete from system:completed_requests where RequestId = "%s"' % requestId
)
time.sleep(10)
result = self.run_cbq_query(
'select * from system:completed_requests where RequestId = "%s"' % requestId
)
print result
self.assertTrue(result["metrics"]["resultCount"] == 0)
def test_simple_negative_check(self):
queries_errors = {
"SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3": "Aggregates not allowed in WHERE",
"SELECT *.$str0 FROM {0}": "syntax error",
"SELECT *.* FROM {0} ... ERROR": "syntax error",
"FROM %s SELECT $str0 WHERE id=null": "syntax error",
}
self.negative_common_body(queries_errors)
def test_unnest(self):
for bucket in self.buckets:
query_template = "SELECT emp.$int0, task FROM %s emp UNNEST emp.$nested_list_3l0 task" % bucket.name
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(sorted(actual_result["results"]), sorted(expected_result))
def test_subquery_select(self):
for bucket in self.buckets:
self.query = "SELECT $str0, $subquery(SELECT COUNT($str0) cn FROM %s d USE KEYS $5) as names FROM %s" % (
bucket.name,
bucket.name,
)
actual_result, expected_result = self.run_query_with_subquery_select_from_template(self.query)
self._verify_results(actual_result["results"], expected_result)
def test_subquery_from(self):
for bucket in self.buckets:
self.query = "SELECT tasks.$str0 FROM $subquery(SELECT $str0, $int0 FROM %s) as tasks" % (bucket.name)
actual_result, expected_result = self.run_query_with_subquery_from_template(self.query)
self._verify_results(actual_result["results"], expected_result)
def test_consistent_simple_check(self):
queries = [
self.gen_results.generate_query(
"SELECT $str0, $int0, $int1 FROM %s "
+ "WHERE $str0 IS NOT NULL AND $int0<10 "
+ "OR $int1 = 6 ORDER BY $int0, $int1"
),
self.gen_results.generate_query(
"SELECT $str0, $int0, $int1 FROM %s "
+ "WHERE $int1 = 6 OR $str0 IS NOT NULL AND "
+ "$int0<10 ORDER BY $int0, $int1"
),
]
for bucket in self.buckets:
actual_result1 = self.run_cbq_query(queries[0] % bucket.name)
actual_result2 = self.run_cbq_query(queries[1] % bucket.name)
self.assertTrue(
actual_result1["results"] == actual_result2["results"],
"Results are inconsistent.Difference: %s %s %s %s"
% (
len(actual_result1["results"]),
len(actual_result2["results"]),
actual_result1["results"][:100],
actual_result2["results"][:100],
),
)
def test_simple_nulls(self):
queries = ['SELECT id FROM %s WHERE id=NULL or id="null"']
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t3 = threading.Thread(name="run_simple_nulls", target=self.run_active_requests, args=(e, 2))
t3.start()
for query in queries:
actual_result = self.run_cbq_query(query % (bucket.name))
logging.debug("event is set")
if self.monitoring:
e.set()
t3.join(100)
self._verify_results(actual_result["results"], [])
##############################################################################################
#
# LIMIT OFFSET CHECKS
##############################################################################################
def test_limit_negative(self):
# queries_errors = {'SELECT * FROM default LIMIT {0}' : ('Invalid LIMIT value 2.5', 5030)}
queries_errors = {"SELECT ALL * FROM %s": ("syntax error", 3000)}
self.negative_common_body(queries_errors)
def test_limit_offset(self):
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t4 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2))
t4.start()
query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
if self.monitoring:
e.set()
t4.join(100)
self._verify_results(actual_result["results"], expected_result)
query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
def test_limit_offset_zero(self):
for bucket in self.buckets:
query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0" % (bucket.name)
self.query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query()
self.assertEquals(
actual_result["results"],
[],
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], []),
)
query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self.assertEquals(
actual_result["results"],
expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result),
)
def test_limit_offset_negative_check(self):
queries_errors = {
"SELECT DISTINCT $str0 FROM {0} LIMIT 1.1": "Invalid LIMIT value 1.1",
"SELECT DISTINCT $str0 FROM {0} OFFSET 1.1": "Invalid OFFSET value 1.1",
}
self.negative_common_body(queries_errors)
def test_limit_offset_sp_char_check(self):
queries_errors = {
"SELECT DISTINCT $str0 FROM {0} LIMIT ~": "syntax erro",
"SELECT DISTINCT $str0 FROM {0} OFFSET ~": "syntax erro",
}
self.negative_common_body(queries_errors)
##############################################################################################
#
# ALIAS CHECKS
##############################################################################################
def test_simple_alias(self):
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t5 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2))
t5.start()
query_template = "SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s" % (bucket.name)
if self.analytics:
query_template = "SELECT COUNT(`$str0`) AS COUNT_EMPLOYEE FROM %s" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self.assertEquals(
actual_result["results"],
expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result),
)
query_template = "SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
if self.monitoring:
e.set()
t5.join(100)
expected_result = [{"COUNT_EMPLOYEE": expected_result[0]["COUNT_EMPLOYEE"] + 1}]
self.assertEquals(
actual_result["results"],
expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result),
)
def test_simple_negative_alias(self):
queries_errors = {
"SELECT $str0._last_name as *": "syntax error",
"SELECT $str0._last_name as DATABASE ?": "syntax error",
"SELECT $str0 AS NULL FROM {0}": "syntax error",
"SELECT $str1 as $str0, $str0 FROM {0}": "Duplicate result alias name",
"SELECT test.$obj0 as points FROM {0} AS TEST " + "GROUP BY $obj0 AS GROUP_POINT": "syntax error",
}
self.negative_common_body(queries_errors)
def test_alias_from_clause(self):
queries_templates = [
"SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points",
"SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0" + " ORDER BY points",
"SELECT $obj0.$_obj0_int0 AS points FROM %s AS test " + "GROUP BY test.$obj0.$_obj0_int0 ORDER BY points",
]
# if self.analytics:
# queries_templates = ['SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY test.points',
# 'SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\
# ' ORDER BY test.points',
# 'SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test ' +\
# 'GROUP BY test.$obj0.$_obj0_int0 ORDER BY test.points']
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t6 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2))
t6.start()
for query_template in queries_templates:
actual_result, expected_result = self.run_query_from_template(query_template % (bucket.name))
if self.monitoring:
e.set()
t6.join(100)
self._verify_results(actual_result["results"], expected_result)
def test_alias_from_clause_group(self):
for bucket in self.buckets:
query_template = (
"SELECT $obj0.$_obj0_int0 AS points FROM %s AS test " % (bucket.name)
+ "GROUP BY $obj0.$_obj0_int0 ORDER BY points"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_alias_order_desc(self):
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t7 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2))
t7.start()
query_template = "SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
if self.monitoring:
e.set()
t7.join(100)
self._verify_results(actual_result["results"], expected_result)
def test_alias_order_asc(self):
for bucket in self.buckets:
query_template = "SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC" % (bucket.name)
if self.analytics:
query_template = "SELECT `$str0` AS name_new FROM %s AS test ORDER BY name_new ASC" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_alias_aggr_fn(self):
for bucket in self.buckets:
if self.monitoring:
e = threading.Event()
t8 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2))
t8.start()
query_template = "SELECT COUNT(TEST.$str0) from %s AS TEST" % (bucket.name)
if self.analytics:
query_template = "SELECT COUNT(TEST.`$str0`) from %s AS TEST" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
if self.monitoring:
e.set()
t8.join(100)
self._verify_results(actual_result["results"], expected_result)
def test_alias_unnest(self):
for bucket in self.buckets:
query_template = "SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
query_template = "SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
##############################################################################################
#
# ORDER BY CHECKS
##############################################################################################
def test_order_by_check(self):
for bucket in self.buckets:
query_template = (
"SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s" % (bucket.name)
+ " ORDER BY $str1, $str0, $obj0.$_obj0_int0"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $obj0.$_obj0_int0, $str0, $str1"
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_order_by_alias(self):
for bucket in self.buckets:
query_template = (
"SELECT $str1, $obj0 AS points FROM %s" % (bucket.name) + " AS test ORDER BY $str1 DESC, points DESC"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_order_by_alias_arrays(self):
for bucket in self.buckets:
query_template = (
"SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s" % (bucket.name)
+ " AS TEST ORDER BY SKILL, $str1, TEST.$obj0"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_order_by_alias_aggr_fn(self):
for bucket in self.buckets:
query_template = (
"SELECT $int0, $int1, count(*) AS emp_per_month from %s" % (bucket.name)
+ " WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_order_by_aggr_fn(self):
for bucket in self.buckets:
query_template = (
"SELECT $str1 AS TITLE, min($int1) day FROM %s GROUP" % (bucket.name)
+ " BY $str1 ORDER BY MIN($int1), $str1"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
if self.analytics:
self.query = (
"SELECT d.email AS TITLE, min(d.join_day) day FROM %s d GROUP" % (bucket.name)
+ " BY d.$str1 ORDER BY MIN(d.join_day), d.$str1"
)
actual_result1 = self.run_cbq_query()
self._verify_results(actual_result1["results"], actual_result["results"])
def test_order_by_precedence(self):
for bucket in self.buckets:
query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $str0, $str1"
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $str1, $str0"
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_order_by_satisfy(self):
for bucket in self.buckets:
query_template = (
"SELECT $str0, $list_obj0 FROM %s AS employee " % (bucket.name)
+ "WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND"
+ ' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0'
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
##############################################################################################
#
# DISTINCT
##############################################################################################
def test_distinct(self):
for bucket in self.buckets:
query_template = "SELECT DISTINCT $str1 FROM %s ORDER BY $str1" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_distinct_nested(self):
for bucket in self.buckets:
query_template = (
"SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s " % (bucket.name) + "ORDER BY $obj0.$_obj0_int0"
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
query_template = "SELECT DISTINCT $list_str0[0] as skill" + " FROM %s ORDER BY $list_str0[0]" % (
bucket.name
)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
self.query = "SELECT DISTINCT $obj0.* FROM %s" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
##############################################################################################
#
# COMPLEX PATHS
##############################################################################################
def test_simple_complex_paths(self):
for bucket in self.buckets:
query_template = "SELECT $_obj0_int0 FROM %s.$obj0" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_alias_complex_paths(self):
for bucket in self.buckets:
query_template = "SELECT $_obj0_int0 as new_attribute FROM %s.$obj0" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
def test_where_complex_paths(self):
for bucket in self.buckets:
query_template = "SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1" % (bucket.name)
actual_result, expected_result = self.run_query_from_template(query_template)
self._verify_results(actual_result["results"], expected_result)
##############################################################################################
#
# COMMON FUNCTIONS
##############################################################################################
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template):
subquery_template = re.sub(r".*\$subquery\(", "", query_template)
subquery_template = subquery_template[: subquery_template.rfind(")")]
keys_num = int(re.sub(r".*KEYS \$", "", subquery_template).replace("KEYS $", ""))
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load, keys=self._get_keys(keys_num))
subquery_template = re.sub(r"USE KEYS.*", "", subquery_template)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template))
alias = re.sub(r".*as", "", re.sub(r"FROM.*", "", alias)).strip()
if not alias:
alias = "$1"
for item in self.gen_results.full_set:
item[alias] = expected_sub[0]
query_template = re.sub(r",.*\$subquery\(.*\).*%s" % alias, ",%s" % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template):
subquery_template = re.sub(r".*\$subquery\(", "", query_template)
subquery_template = subquery_template[: subquery_template.rfind(")")]
subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load)
sub_results = TuqGenerators(self.log, subquery_full_list)
self.query = sub_results.generate_query(subquery_template)
expected_sub = sub_results.generate_expected_result()
alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template))
alias = re.sub(r".*as ", "", alias).strip()
self.gen_results = TuqGenerators(self.log, expected_sub)
query_template = re.sub(r"\$subquery\(.*\).*%s" % alias, " %s" % alias, query_template)
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def negative_common_body(self, queries_errors={}):
if not queries_errors:
self.fail("No queries to run!")
for bucket in self.buckets:
for query_template, error in queries_errors.iteritems():
try:
query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query(query.format(bucket.name))
except CBQError as ex:
self.log.error(ex)
self.assertTrue(
str(ex).find(error) != -1,
"Error is incorrect.Actual %s.\n Expected: %s.\n" % (str(ex).split(":")[-1], error),
)
else:
self.fail("There were no errors. Error expected: %s" % error)
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.n1ql_port == None or self.n1ql_port == "":
self.n1ql_port = self.input.param("n1ql_port", 8093)
if not self.n1ql_port:
self.log.info(" n1ql_port is not defined, processing will not proceed further")
raise Exception("n1ql_port is not defined, processing will not proceed further")
query_params = {}
cred_params = {"creds": []}
for bucket in self.buckets:
if bucket.saslPassword:
cred_params["creds"].append({"user": "******" % bucket.name, "pass": bucket.saslPassword})
query_params.update(cred_params)
if self.use_rest:
query_params.update({"scan_consistency": self.scan_consistency})
self.log.info("RUN QUERY %s" % query)
if self.analytics:
query = query + ";"
for bucket in self.buckets:
query = query.replace(bucket.name, bucket.name + "_shadow")
result = RestConnection(server).analytics_tool(query, 8095, query_params=query_params)
else:
result = RestConnection(server).query_tool(query, self.n1ql_port, query_params=query_params)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside(
"$GOPATH/src/github.com/couchbase/query/" + "shell/cbq/cbq ", "", "", "", "", "", ""
)
else:
os = self.shell.extract_remote_info().type.lower()
# if (query.find("VALUES") > 0):
if not (self.isprepared):
query = query.replace('"', '\\"')
query = query.replace("`", "\\`")
if os == "linux":
cmd = "%s/cbq -engine=http://%s:8091/ -q" % (testconstants.LINUX_COUCHBASE_BIN_PATH, server.ip)
elif os == "windows":
cmd = "%s/cbq -q" % (testconstants.WIN_COUCHBASE_BIN_PATH)
output = self.shell.execute_commands_inside(cmd, query, "", "", "", "", "")
if not (output[0] == "{"):
output1 = "{" + str(output)
else:
output1 = output
result = json.loads(output1)
# result = self._parse_query_output(output)
if isinstance(result, str) or "errors" in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"])
return result
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" % (version, info.architecture_type)
# TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != "windows":
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd = (
"export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath)
+ " export PATH=$PATH:$GOROOT/bin && "
+ "go get github.com/couchbaselabs/tuqtng;"
+ "cd $GOPATH/src/github.com/couchbaselabs/tuqtng; "
+ "go get -d -v ./...; cd ."
)
self.shell.execute_command(cmd)
cmd = (
"export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath)
+ " export PATH=$PATH:$GOROOT/bin && "
+ "cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd ."
)
self.shell.execute_command(cmd)
cmd = (
"export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath)
+ " export PATH=$PATH:$GOROOT/bin && "
+ "cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd ."
)
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
# TODO for windows
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != "windows":
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == "windows":
cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (
gopath
) + "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" % (server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (
gopath
) + "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" % (server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
if self.services_init.find("n1ql") != -1:
return
os = self.shell.extract_remote_info().type.lower()
if os != "windows":
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"]
print "PATH TO SHERLOCK: %s" % couchbase_path
if os == "windows":
cmd = "cd %s; " % (couchbase_path) + "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip,
server.port,
)
else:
cmd = "cd %s; " % (couchbase_path) + "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" % (
server.ip,
server.port,
)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (
couchbase_path
) + './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' % (
server.ip,
server.port,
n1ql_port,
)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != "windows":
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip,
server.port,
)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4 :].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11 :].strip()
if output.find("cbq>") != -1:
output = output[: output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[: output.find("tuq_client>")].strip()
return json.loads(output)
def generate_docs(self, num_items, start=0):
try:
return getattr(self, "generate_docs_" + self.dataset)(num_items, start)
except:
self.fail("There is no dataset %s, please enter a valid one" % self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_sabre(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_sabre(docs_per_day, start)
def generate_docs_employee(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_data(docs_per_day=docs_per_day, start=start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_simple_data(docs_per_day=docs_per_day, start=start)
def generate_docs_sales(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_sales_data(docs_per_day=docs_per_day, start=start)
def generate_docs_bigdata(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_bigdata(end=(1000 * docs_per_day), start=start, value_size=self.value_size)
def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1):
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count]))
self.fail(
"Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result))
)
if self.max_verify is not None:
actual_result = actual_result[: self.max_verify]
expected_result = expected_result[: self.max_verify]
msg = (
"Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s"
+ "Expected first and last 100: %s.\n ... \n %s"
)
self.assertTrue(
actual_result == expected_result,
msg % (actual_result[:100], actual_result[-100:], expected_result[:100], expected_result[-100:]),
)
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.iteritems():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self):
self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type)
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
if versions[0].startswith("4") or versions[0].startswith("3"):
for bucket in self.buckets:
if self.primary_indx_drop:
self.log.info("Dropping primary index for %s using %s ..." % (bucket.name, self.primary_indx_type))
self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
# self.run_cbq_query()
self.sleep(3, "Sleep for some time after index drop")
self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name
res = self.run_cbq_query()
self.sleep(10)
print res
if self.monitoring:
self.query = "delete from system:completed_requests"
self.run_cbq_query()
if not self.skip_primary_index:
if res["metrics"]["resultCount"] == 0:
self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type)
self.log.info("Creating primary index for %s ..." % bucket.name)
# if self.gsi_type:
# self.query += " WITH {'index_type': 'memdb'}"
try:
self.run_cbq_query()
self.primary_index_created = True
if self.primary_indx_type.lower() == "gsi":
self._wait_for_index_online(bucket, "#primary")
except Exception, ex:
self.log.info(str(ex))
if self.monitoring:
self.query = "select * from system:active_requests"
result = self.run_cbq_query()
print result
self.assertTrue(result["metrics"]["resultCount"] == 1)
self.query = "select * from system:completed_requests"
time.sleep(20)
result = self.run_cbq_query()
print result
class QueryTests(BaseTestCase):
def setUp(self):
if not self._testMethodName == 'suite_setUp':
self.skip_buckets_handle = True
super(QueryTests, self).setUp()
self.version = self.input.param("cbq_version", "git_repo")
if self.input.tuq_client and "client" in self.input.tuq_client:
self.shell = RemoteMachineShellConnection(
self.input.tuq_client["client"])
else:
self.shell = RemoteMachineShellConnection(self.master)
if not self._testMethodName == 'suite_setUp':
self._start_command_line_query(self.master)
self.use_rest = self.input.param("use_rest", True)
self.max_verify = self.input.param("max_verify", None)
self.buckets = RestConnection(self.master).get_buckets()
self.docs_per_day = self.input.param("doc-per-day", 49)
self.item_flag = self.input.param("item_flag", 4042322160)
self.n1ql_port = self.input.param("n1ql_port", 8093)
self.dataset = self.input.param("dataset", "default")
self.gens_load = self.generate_docs(self.docs_per_day)
if self.input.param("gomaxprocs", None):
self.configure_gomaxprocs()
self.gen_results = TuqGenerators(
self.log, self.generate_full_docs_list(self.gens_load))
# temporary for MB-12848
self.create_primary_index_for_3_0_and_greater()
def suite_setUp(self):
try:
self.load(self.gens_load, flag=self.item_flag)
self.create_primary_index_for_3_0_and_greater()
if not self.input.param("skip_build_tuq", True):
self._build_tuq(self.master)
self.skip_buckets_handle = True
except:
self.log.error('SUITE SETUP FAILED')
self.tearDown()
def tearDown(self):
if self._testMethodName == 'suite_tearDown':
self.skip_buckets_handle = False
super(QueryTests, self).tearDown()
def suite_tearDown(self):
if not self.input.param("skip_build_tuq", False):
if hasattr(self, 'shell'):
self.shell.execute_command("killall /tmp/tuq/cbq-engine")
self.shell.execute_command("killall tuqtng")
self.shell.disconnect()
##############################################################################################
#
# SIMPLE CHECKS
##############################################################################################
def test_simple_check(self):
for bucket in self.buckets:
query_template = 'FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_simple_negative_check(self):
queries_errors = {
'SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3':
'Aggregates not allowed in WHERE',
'SELECT *.$str0 FROM {0}': 'syntax error',
'SELECT *.* FROM {0} ... ERROR': 'syntax error',
'FROM %s SELECT $str0 WHERE id=null': 'syntax error',
}
self.negative_common_body(queries_errors)
def test_consistent_simple_check(self):
queries = [self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\
'WHERE $str0 IS NOT NULL AND $int0<10 ' +\
'OR $int1 = 6 ORDER BY $int0, $int1'),
self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\
'WHERE $int1 = 6 OR $str0 IS NOT NULL AND ' +\
'$int0<10 ORDER BY $int0, $int1')]
for bucket in self.buckets:
actual_result1 = self.run_cbq_query(queries[0] % bucket.name)
actual_result2 = self.run_cbq_query(queries[1] % bucket.name)
self.assertTrue(
actual_result1['results'] == actual_result2['results'],
"Results are inconsistent.Difference: %s %s %s %s" %
(len(actual_result1['results']), len(
actual_result2['results']),
actual_result1['results'][:100],
actual_result2['results'][:100]))
def test_simple_nulls(self):
queries = ['SELECT id FROM %s WHERE id=NULL or id="null"']
for bucket in self.buckets:
for query in queries:
actual_result = self.run_cbq_query(query % (bucket.name))
self._verify_results(actual_result['results'], [])
##############################################################################################
#
# LIMIT OFFSET CHECKS
##############################################################################################
def test_limit_offset(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
def test_limit_offset_zero(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0' % (
bucket.name)
self.query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query()
self.assertEquals(
actual_result['results'], [],
"Results are incorrect.Actual %s.\n Expected: %s.\n" %
(actual_result['results'], []))
query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self.assertEquals(
actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" %
(actual_result['results'], expected_result))
def test_limit_offset_negative_check(self):
queries_errors = {
'SELECT DISTINCT $str0 FROM {0} LIMIT -1':
'Parse Error - syntax error',
'SELECT DISTINCT $str0 FROM {0} LIMIT 1.1':
'Parse Error - syntax error',
'SELECT DISTINCT $str0 FROM {0} OFFSET -1':
'Parse Error - syntax error',
'SELECT DISTINCT $str0 FROM {0} OFFSET 1.1':
'Parse Error - syntax error'
}
self.negative_common_body(queries_errors)
##############################################################################################
#
# ALIAS CHECKS
##############################################################################################
def test_simple_alias(self):
for bucket in self.buckets:
query_template = 'SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self.assertEquals(
actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" %
(actual_result['results'], expected_result))
query_template = 'SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
expected_result = [{
"COUNT_EMPLOYEE":
expected_result[0]['COUNT_EMPLOYEE'] + 1
}]
self.assertEquals(
actual_result['results'], expected_result,
"Results are incorrect.Actual %s.\n Expected: %s.\n" %
(actual_result['results'], expected_result))
def test_simple_negative_alias(self):
queries_errors = {
'SELECT $str0._last_name as *':
'syntax error',
'SELECT $str0._last_name as DATABASE ?':
'syntax error',
'SELECT $str0 AS NULL FROM {0}':
'syntax error',
'SELECT $str1 as $str0, $str0 FROM {0}':
'Duplicate result alias name',
'SELECT test.$obj0 as points FROM {0} AS TEST ' + 'GROUP BY $obj0 AS GROUP_POINT':
'syntax error'
}
self.negative_common_body(queries_errors)
def test_alias_from_clause(self):
queries_templates = ['SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points',
'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\
' ORDER BY points',
'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' +\
'GROUP BY test.$obj0.$_obj0_int0 ORDER BY points']
for bucket in self.buckets:
for query_template in queries_templates:
actual_result, expected_result = self.run_query_from_template(
query_template % (bucket.name))
self._verify_results(actual_result['results'], expected_result)
def test_alias_from_clause_group(self):
for bucket in self.buckets:
query_template = 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' %(bucket.name) +\
'GROUP BY $obj0.$_obj0_int0 ORDER BY points'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_order_desc(self):
for bucket in self.buckets:
query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_order_asc(self):
for bucket in self.buckets:
query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT COUNT(TEST.$str0) from %s AS TEST' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_unnest(self):
for bucket in self.buckets:
query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# ORDER BY CHECKS
##############################################################################################
def test_order_by_check(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s' % (bucket.name) +\
' ORDER BY $str1, $str0, $obj0.$_obj0_int0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $obj0.$_obj0_int0, $str0, $str1'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias(self):
for bucket in self.buckets:
query_template = 'SELECT $str1, $obj0 AS points FROM %s' % (bucket.name) +\
' AS test ORDER BY $str1 DESC, points DESC'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias_arrays(self):
for bucket in self.buckets:
query_template = 'SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s' % (
bucket.name) +\
' AS TEST ORDER BY SKILL, $str1, TEST.$obj0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_alias_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT $int0, $int1, count(*) AS emp_per_month from %s'% (
bucket.name) +\
' WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_aggr_fn(self):
for bucket in self.buckets:
query_template = 'SELECT $str1 AS TITLE FROM %s GROUP' % (bucket.name) +\
' BY $str1 ORDER BY MIN($int1), $str1'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_precedence(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $str0, $str1'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\
' ORDER BY $str1, $str0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_order_by_satisfy(self):
for bucket in self.buckets:
query_template = 'SELECT $str0, $list_obj0 FROM %s AS employee ' % (bucket.name) +\
'WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND' +\
' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# DISTINCT
##############################################################################################
def test_distinct(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $str1 FROM %s ORDER BY $str1' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_distinct_nested(self):
for bucket in self.buckets:
query_template = 'SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s ' % (bucket.name) +\
'ORDER BY $obj0.$_obj0_int0'
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
query_template = 'SELECT DISTINCT $list_str0[0] as skill' +\
' FROM %s ORDER BY $list_str0[0]' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
self.query = 'SELECT DISTINCT $obj0.* FROM %s' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# COMPLEX PATHS
##############################################################################################
def test_simple_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 FROM %s.$obj0' % (bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_alias_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 as new_attribute FROM %s.$obj0' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
def test_where_complex_paths(self):
for bucket in self.buckets:
query_template = 'SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1' % (
bucket.name)
actual_result, expected_result = self.run_query_from_template(
query_template)
self._verify_results(actual_result['results'], expected_result)
##############################################################################################
#
# COMMON FUNCTIONS
##############################################################################################
def run_query_from_template(self, query_template):
self.query = self.gen_results.generate_query(query_template)
expected_result = self.gen_results.generate_expected_result()
actual_result = self.run_cbq_query()
return actual_result, expected_result
def negative_common_body(self, queries_errors={}):
if not queries_errors:
self.fail("No queries to run!")
for bucket in self.buckets:
for query_template, error in queries_errors.iteritems():
try:
query = self.gen_results.generate_query(query_template)
actual_result = self.run_cbq_query(
query.format(bucket.name))
except CBQError as ex:
self.log.error(ex)
self.assertTrue(
str(ex).find(error) != -1,
"Error is incorrect.Actual %s.\n Expected: %s.\n" %
(str(ex).split(':')[-1], error))
else:
self.fail("There was no errors. Error expected: %s" %
error)
def run_cbq_query(self, query=None, min_output_size=10, server=None):
if query is None:
query = self.query
if server is None:
server = self.master
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
else:
if server.ip == "127.0.0.1":
self.n1ql_port = server.n1ql_port
if self.input.tuq_client and "client" in self.input.tuq_client:
server = self.tuq_client
if self.n1ql_port == None or self.n1ql_port == '':
self.n1ql_port = self.input.param("n1ql_port", 8093)
if not self.n1ql_port:
self.log.info(
" n1ql_port is not defined, processing will not proceed further"
)
raise Exception(
"n1ql_port is not defined, processing will not proceed further"
)
if self.use_rest:
result = RestConnection(server).query_tool(query, self.n1ql_port)
else:
if self.version == "git_repo":
output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\
"tuq_client/tuq_client " +\
"-engine=http://%s:8093/" % server.ip,
subcommands=[query,],
min_output_size=20,
end_msg='tuq_client>')
else:
output = self.shell.execute_commands_inside(
"/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip,
subcommands=[
query,
],
min_output_size=20,
end_msg='cbq>')
result = self._parse_query_output(output)
if isinstance(result, str) or 'errors' in result:
raise CBQError(result, server.ip)
self.log.info("TOTAL ELAPSED TIME: %s" %
result["metrics"]["elapsedTime"])
return result
def build_url(self, version):
info = self.shell.extract_remote_info()
type = info.distribution_type.lower()
if type in ["ubuntu", "centos", "red hat"]:
url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/"
url += "couchbase-query_%s_%s_linux.tar.gz" % (
version, info.architecture_type)
#TODO for windows
return url
def _build_tuq(self, server):
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
goroot = testconstants.LINUX_GOROOT
gopath = testconstants.LINUX_GOPATH
else:
goroot = testconstants.WINDOWS_GOROOT
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if self.input.tuq_client and "goroot" in self.input.tuq_client:
goroot = self.input.tuq_client["goroot"]
cmd = "rm -rf {0}/src/github.com".format(gopath)
self.shell.execute_command(cmd)
cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'go get github.com/couchbaselabs/tuqtng;' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\
'go get -d -v ./...; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .'
self.shell.execute_command(cmd)
cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\
' export PATH=$PATH:$GOROOT/bin && ' +\
'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .'
self.shell.execute_command(cmd)
else:
cbq_url = self.build_url(self.version)
#TODO for windows
cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(
cbq_url)
cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz"
self.shell.execute_command(cmd)
def _start_command_line_query(self, server):
self.shell.execute_command(
"export NS_SERVER_CBAUTH_URL=\"http://{0}:{1}/_cbauth\"".format(
server.ip, server.port))
self.shell.execute_command(
"export NS_SERVER_CBAUTH_USER=\"{0}\"".format(
server.rest_username))
self.shell.execute_command(
"export NS_SERVER_CBAUTH_PWD=\"{0}\"".format(server.rest_password))
self.shell.execute_command(
"export NS_SERVER_CBAUTH_RPC_URL=\"http://{0}:{1}/cbauth-demo\"".
format(server.ip, server.port))
if self.version == "git_repo":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
gopath = testconstants.LINUX_GOPATH
else:
gopath = testconstants.WINDOWS_GOPATH
if self.input.tuq_client and "gopath" in self.input.tuq_client:
gopath = self.input.tuq_client["gopath"]
if os == 'windows':
cmd = "cd %s/src/github.com/couchbaselabs/query/server/main; " % (gopath) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s/src/github.com/couchbaselabs/query//server/main; " % (gopath) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
self.shell.execute_command(cmd)
elif self.version == "sherlock":
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH
else:
couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH
if self.input.tuq_client and "sherlock_path" in self.input.tuq_client:
couchbase_path = "%s/bin" % self.input.tuq_client[
"sherlock_path"]
print "PATH TO SHERLOCK: %s" % couchbase_path
if os == 'windows':
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %(
server.ip, server.port)
else:
cmd = "cd %s; " % (couchbase_path) +\
"./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %(
server.ip, server.port)
n1ql_port = self.input.param("n1ql_port", None)
if server.ip == "127.0.0.1" and server.n1ql_port:
n1ql_port = server.n1ql_port
if n1ql_port:
cmd = "cd %s; " % (couchbase_path) +\
'./cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %(
server.ip, server.port, n1ql_port)
self.shell.execute_command(cmd)
else:
os = self.shell.extract_remote_info().type.lower()
if os != 'windows':
cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
else:
cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (
server.ip, server.port)
self.shell.execute_command(cmd)
def _parse_query_output(self, output):
if output.find("cbq>") == 0:
output = output[output.find("cbq>") + 4:].strip()
if output.find("tuq_client>") == 0:
output = output[output.find("tuq_client>") + 11:].strip()
if output.find("cbq>") != -1:
output = output[:output.find("cbq>")].strip()
if output.find("tuq_client>") != -1:
output = output[:output.find("tuq_client>")].strip()
return json.loads(output)
def generate_docs(self, num_items, start=0):
try:
return getattr(self, 'generate_docs_' + self.dataset)(num_items,
start)
except:
self.fail("There is no dataset %s, please enter a valid one" %
self.dataset)
def generate_docs_default(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee(docs_per_day, start)
def generate_docs_sabre(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_sabre(docs_per_day, start)
def generate_docs_employee(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_simple(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_simple_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_sales(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_employee_sales_data(
docs_per_day=docs_per_day, start=start)
def generate_docs_bigdata(self, docs_per_day, start=0):
json_generator = JsonGenerator()
return json_generator.generate_docs_bigdata(docs_per_day=docs_per_day *
1000,
start=start,
value_size=self.value_size)
def _verify_results(self,
actual_result,
expected_result,
missing_count=1,
extra_count=1):
if len(actual_result) != len(expected_result):
missing, extra = self.check_missing_and_extra(
actual_result, expected_result)
self.log.error("Missing items: %s.\n Extra items: %s" %
(missing[:missing_count], extra[:extra_count]))
self.fail(
"Results are incorrect.Actual num %s. Expected num: %s.\n" %
(len(actual_result), len(expected_result)))
if self.max_verify is not None:
actual_result = actual_result[:self.max_verify]
expected_result = expected_result[:self.max_verify]
msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\
"Expected first and last 100: %s.\n ... \n %s"
self.assertTrue(
actual_result == expected_result,
msg % (actual_result[:100], actual_result[-100:],
expected_result[:100], expected_result[-100:]))
def check_missing_and_extra(self, actual, expected):
missing = []
extra = []
for item in actual:
if not (item in expected):
extra.append(item)
for item in expected:
if not (item in actual):
missing.append(item)
return missing, extra
def sort_nested_list(self, result):
actual_result = []
for item in result:
curr_item = {}
for key, value in item.iteritems():
if isinstance(value, list) or isinstance(value, set):
curr_item[key] = sorted(value)
else:
curr_item[key] = value
actual_result.append(curr_item)
return actual_result
def configure_gomaxprocs(self):
max_proc = self.input.param("gomaxprocs", None)
cmd = "export GOMAXPROCS=%s" % max_proc
for server in self.servers:
shell_connection = RemoteMachineShellConnection(self.master)
shell_connection.execute_command(cmd)
def create_primary_index_for_3_0_and_greater(self):
self.log.info("CHECK FOR PRIMARY INDEXES")
rest = RestConnection(self.master)
versions = rest.get_nodes_versions()
ddoc_name = 'ddl_#primary'
if versions[0].startswith("3"):
try:
rest.get_ddoc(self.buckets[0], ddoc_name)
except ReadDocumentException:
for bucket in self.buckets:
self.log.info("Creating primary index for %s ..." %
bucket.name)
self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name)
try:
self.run_cbq_query()
except Exception, ex:
self.log.error('ERROR during index creation %s' %
str(ex))
