def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "git_repo") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection( self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators( self.log, self.generate_full_docs_list(self.gens_load)) # temporary for MB-12848 self.create_primary_index_for_3_0_and_greater()
def __init__(self, version=None, master=None, shell=None, use_rest=None, max_verify=0, buckets=[], item_flag=0, analytics_port=8095, n1ql_port=8093, full_docs_list=[], log=None, input=None, database=None): self.version = version self.shell = shell self.n1ql_port = n1ql_port self.max_verify = max_verify self.buckets = buckets self.item_flag = item_flag self.analytics_port = analytics_port self.input = input self.log = log self.use_rest = True self.full_docs_list = full_docs_list self.master = master self.database = database if self.full_docs_list and len(self.full_docs_list) > 0: self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def test_change_doc_size(self): self.iterations = self.input.param("num_iterations", 5) buckets = self._create_plasma_buckets() if self.plasma_dgm: self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer]) query_definition = QueryDefinition( index_name="index_name_big_values", index_fields=["bigValues"], query_template="SELECT * FROM %s WHERE bigValues IS NOT NULL", groups=["simple"], index_where_clause=" bigValues IS NOT NULL ") self.multi_create_index(buckets=buckets, query_definitions=[query_definition]) template = '{{"name":"{0}", "age":{1}, "bigValues": "{2}" }}' generators = [] for j in range(self.iterations): for i in range(10): name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))] id = "{0}-{1}".format(name, str(i)) age = random.choice(range(4, 19)) bigValue_size = random.choice(range(10, 15)) bigValues = "".join( random.choice(lowercase) for k in range(bigValue_size)) generators.append( DocumentGenerator(id, template, [name], [age], [bigValues], start=0, end=10)) self.load(generators, flag=self.item_flag, verify_data=False, batch_size=self.batch_size) self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list) self.multi_query_using_index(buckets=buckets, query_definitions=[query_definition]) for i in range(10): name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))] id = "{0}-{1}".format(name, str(i)) age = random.choice(range(4, 19)) bigValue_size = random.choice(range(1000, 5000)) bigValues = "".join( random.choice(lowercase) for k in range(bigValue_size)) generators.append( DocumentGenerator(id, template, [name], [age], [bigValues], start=0, end=10)) self.load(generators, flag=self.item_flag, verify_data=False, batch_size=self.batch_size) self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list) self.multi_query_using_index(buckets=buckets, query_definitions=[query_definition]) self.sleep(30) self.multi_drop_index(buckets=buckets, query_definitions=[query_definition])
def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.analytics = self.input.param("analytics",False) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", 'GSI') self.index_type = self.input.param("index_type", 'GSI') self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.monitoring = self.input.param("monitoring",False) self.isprepared = False self.named_prepare = self.input.param("named_prepare", None) self.skip_primary_index = self.input.param("skip_primary_index",False) self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS') shell = RemoteMachineShellConnection(self.master) type = shell.extract_remote_info().distribution_type self.path = testconstants.LINUX_COUCHBASE_BIN_PATH if type.lower() == 'windows': self.path = testconstants.WIN_COUCHBASE_BIN_PATH elif type.lower() == "mac": self.path = testconstants.MAC_COUCHBASE_BIN_PATH self.threadFailure = False if self.primary_indx_type.lower() == "gsi": self.gsi_type = self.input.param("gsi_type", 'plasma') else: self.gsi_type = None if self.input.param("reload_data", False): if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas']) for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas']) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) if (self.analytics == False): self.create_primary_index_for_3_0_and_greater() if (self.analytics): self.setup_analytics() self.sleep(30,'wait for analytics setup')
def async_run_doc_ops(self): if self.doc_ops: tasks = self.async_ops_all_buckets(self.docs_gen_map, batch_size=self.batch_size) self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops self.gen_results = TuqGenerators( full_set=self.n1ql_helper.full_docs_list) return tasks return []
def setUp(self): super(QueryHelperTests, self).setUp() self.create_primary_index = self.input.param("create_primary_index", True) self.use_gsi_for_primary = self.input.param("use_gsi_for_primary", True) self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary", True) self.scan_consistency = self.input.param("scan_consistency", "request_plus") self.skip_host_login = self.input.param("skip_host_login", False) if not self.skip_host_login: self.shell = RemoteMachineShellConnection(self.master) else: self.shell = None if not self.skip_init_check_cbserver: # for upgrade tests self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.item_flag = self.input.param("item_flag", 0) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.groups = self.input.param("groups", "all").split(":") self.doc_ops = self.input.param("doc_ops", False) self.batch_size = self.input.param("batch_size", 1) self.create_ops_per = self.input.param("create_ops_per", 0) self.expiry_ops_per = self.input.param("expiry_ops_per", 0) self.delete_ops_per = self.input.param("delete_ops_per", 0) self.update_ops_per = self.input.param("update_ops_per", 0) self.gens_load = self.generate_docs(self.docs_per_day) self.full_docs_list = self.generate_full_docs_list(self.gens_load) self.gen_results = TuqGenerators(self.log, full_set=self.full_docs_list) if not self.skip_init_check_cbserver: # for upgrade tests self.n1ql_server = self.get_nodes_from_services_map( service_type="n1ql") query_definition_generator = SQLDefinitionGenerator() if self.dataset == "default" or self.dataset == "employee": self.query_definitions = query_definition_generator.generate_employee_data_query_definitions( ) if self.dataset == "simple": self.query_definitions = query_definition_generator.generate_simple_data_query_definitions( ) if self.dataset == "sabre": self.query_definitions = query_definition_generator.generate_sabre_data_query_definitions( ) if self.dataset == "bigdata": self.query_definitions = query_definition_generator.generate_big_data_query_definitions( ) if self.dataset == "array": self.query_definitions = query_definition_generator.generate_airlines_data_query_definitions( ) self.query_definitions = query_definition_generator.filter_by_group( self.groups, self.query_definitions) self.num_index_replicas = self.input.param("num_index_replica", 0)
def run_doc_ops(self): verify_data = True if self.scan_consistency == "request_plus": verify_data = False if self.doc_ops: self.sync_ops_all_buckets(docs_gen_map=self.docs_gen_map, batch_size=self.batch_size, verify_data=verify_data) self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops self.gen_results = TuqGenerators( full_set=self.n1ql_helper.full_docs_list) log.info("------ KV OPS Done ------")
def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r".*\$subquery\(", "", query_template) subquery_template = subquery_template[: subquery_template.rfind(")")] subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template)) alias = re.sub(r".*as ", "", alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r"\$subquery\(.*\).*%s" % alias, " %s" % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result
def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as ', '', alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result
def test_array_item_limit(self): query_definition = QueryDefinition(index_name="index_name_big_values", index_fields=["DISTINCT ARRAY t FOR t in bigValues END"], query_template="SELECT {0} FROM %s WHERE bigValues IS NOT NULL", groups=["array"], index_where_clause=" bigValues IS NOT NULL ") self.rest.flush_bucket(self.buckets[0]) generators = [] template = '{{"name":"{0}", "age":{1}, "bigValues":{2} }}' for i in range(10): name = FIRST_NAMES[random.choice(range(len(FIRST_NAMES)))] id = "{0}-{1}".format(name, str(i)) age = random.choice(range(4, 19)) bigValues = [] arrLen = random.choice(range(10, 15)) indiSize = (4096 * 4) for j in range(arrLen): longStr = "".join(random.choice(lowercase) for k in range(indiSize)) bigValues.append(longStr) generators.append(DocumentGenerator(id, template, [name], [age], [bigValues], start=0, end=1)) self.load(generators, flag=self.item_flag, verify_data=False, batch_size=self.batch_size) self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list) self.multi_create_index_using_rest(buckets=self.buckets, query_definitions=[query_definition]) for bucket in self.buckets: self.run_full_table_scan_using_rest(bucket, query_definition) self.multi_drop_index_using_rest(buckets=self.buckets, query_definitions=[query_definition])
def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", 'VIEW') self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS') if self.input.param("reload_data", False): for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) # temporary for MB-12848 self.create_primary_index_for_3_0_and_greater()
def _load_aggregate_function_dataset(self, buckets=[]): generators = [] document_template = '{{ "name":"{0}" , "str_arr" : {1} , ' \ '"int_num": {2} , "int_arr": {3} , ' \ '"float_num" : {4} , "float_arr" : {5} }}' num_items = 1000 for i in range(num_items): name = random.choice(FIRST_NAMES) str_arr = [random.choice(COUNTRIES) for i in range(10)] int_num = random.randint(0, 100) int_arr = [random.randint(30, 100) for i in range(100)] float_num = random.uniform(0.0, 100.0) float_arr = [random.uniform(30.0, 100.0) for i in range(100)] doc_id = "student_" + str(random.random() * 100000) generators.append( DocumentGenerator(doc_id, document_template, [name], [str_arr], [int_num], [int_arr], [float_num], [float_arr], start=0, end=1)) self.load(generators, buckets=buckets, flag=self.item_flag, verify_data=False, batch_size=self.batch_size) self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def test_create_query_drop_index_on_missing_empty_null_field(self): data_types = ["empty", "null"] index_field, data_type = self._find_datatype(self.query_definitions[0]) doc_list = self.full_docs_list[:len(self.full_docs_list)/2] for data_type in data_types: self.gen_results = TuqGenerators(self.log, self.full_docs_list) definitions_list = [] query_definition = QueryDefinition(index_name="index_name_{0}_distinct".format(data_type), index_fields=["DISTINCT ARRAY t FOR t in `{0}` END".format(index_field)], query_template="SELECT {0} FROM %s WHERE `{0}` IS NOT NULL".format(index_field), groups=["array"], index_where_clause=" `{0}` IS NOT NULL ".format(index_field)) definitions_list.append(query_definition) query_definition = QueryDefinition(index_name="index_name_{0}_duplicate".format(data_type), index_fields=["ALL ARRAY t FOR t in `{0}` END".format(index_field)], query_template="SELECT {0} FROM %s WHERE `{0}` IS NOT NULL".format(index_field), groups=["array"], index_where_clause=" `{0}` IS NOT NULL ".format(index_field)) definitions_list.append(query_definition) for bucket in self.buckets: for query_definition in definitions_list: self.change_index_field_type(bucket.name, index_field, doc_list, data_type, query_definition) self.multi_create_index_using_rest(buckets=self.buckets, query_definitions=definitions_list) for bucket in self.buckets: for query_definition in definitions_list: self.run_full_table_scan_using_rest(bucket, query_definition) self.multi_drop_index_using_rest(buckets=self.buckets, query_definitions=definitions_list) self.full_docs_list = self.generate_full_docs_list(self.gens_load)
def _create_int64_dataset(self): generators = [] document_template = '{{"name":"{0}", "int_num": {1}, "long_num":{2}, "long_num_partial":{3}, "long_arr": {4},' \ '"int_arr": {5}}}' num_items = len(self.full_docs_list) for i in range(num_items): name = random.choice(FIRST_NAMES) int_num = random.randint(-100, 100) long_num = random.choice(INT64_VALUES) long_arr = [random.choice(INT64_VALUES) for i in range(10)] int_arr = [random.randint(-100, 100) for i in range(10)] doc_id = "int64_" + str(random.random() * 100000) generators.append( DocumentGenerator(doc_id, document_template, [name], [int_num], [long_num], [long_num], [long_arr], [int_arr], start=0, end=1)) self.load(generators, buckets=self.buckets, flag=self.item_flag, verify_data=False, batch_size=self.batch_size) self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def kv_mutations(self, docs=1): if not docs: docs = self.docs_per_day gens_load = self.generate_docs(docs) self.full_docs_list = self.generate_full_docs_list(gens_load) self.gen_results = TuqGenerators(self.log, self.full_docs_list) self.load(gens_load, buckets=self.buckets, flag=self.item_flag, verify_data=False, batch_size=self.batch_size)
def test_sorted_removed_items_indexed(self): if self.plasma_dgm: self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer]) generators = self._upload_documents_in_sorted() self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list) query_definition = QueryDefinition( index_name="index_range_shrink_name", index_fields=["name"], query_template="SELECT * FROM %s WHERE name IS NOT NULL", groups=["simple"], index_where_clause=" name IS NOT NULL ") buckets = [] for bucket in self.buckets: if bucket.name.startswith("plasma_dgm"): buckets.append(bucket) self.multi_create_index(buckets=buckets, query_definitions=[query_definition]) self.sleep(30) intervals = [["d", "e", "f"], ["j", "k", "l", "m"], ["p", "q", "r", "s"]] temp_list = [] for doc_gen in self.full_docs_list: for interval in intervals: for character in interval: if doc_gen["name"].lower().startswith(character): for bucket in buckets: url = "couchbase://{0}/{1}".format( self.master.ip, bucket.name) cb = Bucket(url, username=bucket.name, password="******") cb.remove(doc_gen["_id"]) temp_list.append(doc_gen) if not self.multi_intervals: break self.full_docs_list = [ doc for doc in self.full_docs_list if doc not in temp_list ] self.gen_results = TuqGenerators(self.log, self.full_docs_list) self.multi_query_using_index(buckets=buckets, query_definitions=[query_definition]) self.sleep(30) self.multi_drop_index(buckets=buckets, query_definitions=[query_definition])
def kv_mutations(self, docs=1): if not docs: docs = self.docs_per_day gens_load = self.generate_docs(docs) self.full_docs_list = self.generate_full_docs_list(gens_load) self.gen_results = TuqGenerators(self.log, self.full_docs_list) tasks = self.async_load(generators_load=gens_load, op_type="create", batch_size=self.batch_size) return tasks
def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', '')) subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num)) subquery_template = re.sub(r'USE KEYS.*', '', subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip() if not alias: alias = '$1' for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result
def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r".*\$subquery\(", "", query_template) subquery_template = subquery_template[: subquery_template.rfind(")")] keys_num = int(re.sub(r".*KEYS \$", "", subquery_template).replace("KEYS $", "")) subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load, keys=self._get_keys(keys_num)) subquery_template = re.sub(r"USE KEYS.*", "", subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template)) alias = re.sub(r".*as", "", re.sub(r"FROM.*", "", alias)).strip() if not alias: alias = "$1" for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r",.*\$subquery\(.*\).*%s" % alias, ",%s" % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result
def __init__(self, version=None, master=None, shell=None, use_rest=None, max_verify=0, buckets=[], item_flag=0, n1ql_port=8093, full_docs_list=[], log=None, input=None): self.version = version self.shell = shell self.use_rest = use_rest self.max_verify = max_verify self.buckets = buckets self.item_flag = item_flag self.n1ql_port = n1ql_port self.input = input self.log = log self.full_docs_list = full_docs_list self.master = master self.gen_results = TuqGenerators(self.log, self.full_docs_list)
def setUp(self): if not self._testMethodName == "suite_setUp": self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == "suite_setUp" and self.input.param("cbq_version", "sherlock") != "sherlock": self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.analytics = self.input.param("analytics", False) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", "GSI") self.index_type = self.input.param("index_type", "GSI") self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.monitoring = self.input.param("monitoring", False) self.isprepared = False self.skip_primary_index = self.input.param("skip_primary_index", False) self.scan_consistency = self.input.param("scan_consistency", "REQUEST_PLUS") if self.primary_indx_type.lower() == "gsi": self.gsi_type = self.input.param("gsi_type", None) else: self.gsi_type = None if self.input.param("reload_data", False): for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) if self.analytics == False: self.create_primary_index_for_3_0_and_greater() if self.analytics: self.setup_analytics() self.sleep(30, "wait for analytics setup")
def test_sorted_items_indexed(self): if self.plasma_dgm: self.get_dgm_for_plasma(indexer_nodes=[self.dgmServer]) generators = self._upload_documents_in_sorted() self.full_docs_list = self.generate_full_docs_list(generators) self.gen_results = TuqGenerators(self.log, self.full_docs_list) query_definition = QueryDefinition(index_name="index_range_shrink_name", index_fields=["name"], query_template="SELECT * FROM %s WHERE name IS NOT NULL", groups=["simple"], index_where_clause=" name IS NOT NULL ") buckets = [] for bucket in self.buckets: if bucket.name.startswith("plasma_dgm"): buckets.append(bucket) self.multi_create_index(buckets=buckets, query_definitions=[query_definition]) self.sleep(30) self.multi_query_using_index(buckets=buckets, query_definitions=[query_definition]) self.sleep(30) self.multi_drop_index(buckets=buckets, query_definitions=[query_definition])
def test_create_query_drop_index_on_mixed_datatypes(self): query_definition = QueryDefinition( index_name="index_name_travel_history", index_fields=["DISTINCT ARRAY t FOR t in `travel_history` END"], query_template="SELECT {0} FROM %s WHERE `travel_history` IS NOT NULL", groups=["array"], index_where_clause=" `travel_history` IS NOT NULL ") end = 0 for bucket in self.buckets: for data in DATATYPES: start = end end = end + len(self.full_docs_list)/len(DATATYPES) doc_list = self.full_docs_list[start:end] self.change_index_field_type(bucket.name, "travel_history", doc_list, data, query_definition) self.multi_create_index_using_rest(buckets=self.buckets, query_definitions=[query_definition]) self.gen_results = TuqGenerators(self.log, self.full_docs_list) for bucket in self.buckets: self.run_full_table_scan_using_rest(bucket, query_definition) self.multi_drop_index_using_rest(buckets=self.buckets, query_definitions=[query_definition])
class QueryHelperTests(BaseTestCase): def setUp(self): super(QueryHelperTests, self).setUp() self.create_primary_index = self.input.param("create_primary_index", True) self.use_gsi_for_primary = self.input.param("use_gsi_for_primary", True) self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary", True) self.scan_consistency = self.input.param("scan_consistency", "request_plus") self.shell = RemoteMachineShellConnection(self.master) if not self.skip_init_check_cbserver: # for upgrade tests self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.item_flag = self.input.param("item_flag", 0) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.groups = self.input.param("groups", "all").split(":") self.doc_ops = self.input.param("doc_ops", False) self.batch_size = self.input.param("batch_size", 1) self.create_ops_per = self.input.param("create_ops_per", 0) self.expiry_ops_per = self.input.param("expiry_ops_per", 0) self.delete_ops_per = self.input.param("delete_ops_per", 0) self.update_ops_per = self.input.param("update_ops_per", 0) self.gens_load = self.generate_docs(self.docs_per_day) self.full_docs_list = self.generate_full_docs_list(self.gens_load) self.gen_results = TuqGenerators(self.log, full_set=self.full_docs_list) if not self.skip_init_check_cbserver: # for upgrade tests self.n1ql_server = self.get_nodes_from_services_map( service_type="n1ql") query_definition_generator = SQLDefinitionGenerator() if self.dataset == "default" or self.dataset == "employee": self.query_definitions = query_definition_generator.generate_employee_data_query_definitions( ) if self.dataset == "simple": self.query_definitions = query_definition_generator.generate_simple_data_query_definitions( ) if self.dataset == "sabre": self.query_definitions = query_definition_generator.generate_sabre_data_query_definitions( ) if self.dataset == "bigdata": self.query_definitions = query_definition_generator.generate_big_data_query_definitions( ) if self.dataset == "array": self.query_definitions = query_definition_generator.generate_airlines_data_query_definitions( ) self.query_definitions = query_definition_generator.filter_by_group( self.groups, self.query_definitions) self.num_index_replicas = self.input.param("num_index_replica", 0) def tearDown(self): super(QueryHelperTests, self).tearDown() def generate_docs(self, num_items, start=0): try: if self.dataset == "simple": return self.generate_docs_simple(num_items, start) if self.dataset == "array": return self.generate_docs_array(num_items, start) return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except Exception as ex: log.info(str(ex)) self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_ops_docs(self, num_items, start=0): try: json_generator = JsonGenerator() if self.dataset == "simple": return self.generate_ops(num_items, start, json_generator.generate_docs_simple) if self.dataset == "array": return self.generate_ops( num_items, start, json_generator.generate_all_type_documents_for_gsi) except Exception as ex: log.info(ex) self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_simple(start=start, docs_per_day=docs_per_day) def generate_docs_array(self, num_items=10, start=0): json_generator = JsonGenerator() return json_generator.generate_all_type_documents_for_gsi( start=start, docs_per_day=num_items) def generate_ops(self, docs_per_day, start=0, method=None): gen_docs_map = {} for key in list(self.ops_dist_map.keys()): isShuffle = False if key == "update": isShuffle = True if self.dataset != "bigdata": gen_docs_map[key] = method( docs_per_day=self.ops_dist_map[key]["end"], start=self.ops_dist_map[key]["start"]) else: gen_docs_map[key] = method( value_size=self.value_size, end=self.ops_dist_map[key]["end"], start=self.ops_dist_map[key]["start"]) return gen_docs_map def generate_full_docs_list_after_ops(self, gen_docs_map): docs = [] for key in list(gen_docs_map.keys()): if key != "delete" and key != "expiry": update = False if key == "update": update = True gen_docs = self.generate_full_docs_list( gens_load=gen_docs_map[key], update=update) for doc in gen_docs: docs.append(doc) return docs def async_run_doc_ops(self): if self.doc_ops: tasks = self.async_ops_all_buckets(self.docs_gen_map, batch_size=self.batch_size) self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops self.gen_results = TuqGenerators( full_set=self.n1ql_helper.full_docs_list) return tasks return [] def run_doc_ops(self): verify_data = True if self.scan_consistency == "request_plus": verify_data = False if self.doc_ops: self.sync_ops_all_buckets(docs_gen_map=self.docs_gen_map, batch_size=self.batch_size, verify_data=verify_data) self.n1ql_helper.full_docs_list = self.full_docs_list_after_ops self.gen_results = TuqGenerators( full_set=self.n1ql_helper.full_docs_list) log.info("------ KV OPS Done ------") def create_index(self, bucket, query_definition, deploy_node_info=None): defer_build = True query = query_definition.generate_index_create_query( bucket=bucket, use_gsi_for_secondary=self.use_gsi_for_secondary, deploy_node_info=deploy_node_info, defer_build=defer_build, num_replica=self.num_index_replicas) log.info(query) # Define Helper Method which will be used for running n1ql queries, create index, drop index self.n1ql_helper = N1QLHelper(shell=self.shell, max_verify=self.max_verify, buckets=self.buckets, item_flag=self.item_flag, n1ql_port=self.n1ql_port, full_docs_list=self.full_docs_list, log=self.log, input=self.input, master=self.master, use_rest=True) create_index_task = self.cluster.async_create_index( server=self.n1ql_server, bucket=bucket, query=query, n1ql_helper=self.n1ql_helper, index_name=query_definition.index_name, defer_build=defer_build) create_index_task.result() query = self.n1ql_helper.gen_build_index_query( bucket=bucket, index_list=[query_definition.index_name]) build_index_task = self.cluster.async_build_index( server=self.n1ql_server, bucket=bucket, query=query, n1ql_helper=self.n1ql_helper) build_index_task.result() check = self.n1ql_helper.is_index_ready_and_in_list( bucket, query_definition.index_name, server=self.n1ql_server) self.assertTrue( check, "index {0} failed to be created".format( query_definition.index_name)) def _create_primary_index(self): if self.n1ql_server: if self.doc_ops: self.ops_dist_map = self.calculate_data_change_distribution( create_per=self.create_ops_per, update_per=self.update_ops_per, delete_per=self.delete_ops_per, expiry_per=self.expiry_ops_per, start=0, end=self.docs_per_day) log.info(self.ops_dist_map) self.docs_gen_map = self.generate_ops_docs( self.docs_per_day, 0) self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops( self.docs_gen_map) # Define Helper Method which will be used for running n1ql queries, create index, drop index self.n1ql_helper = N1QLHelper(shell=self.shell, max_verify=self.max_verify, buckets=self.buckets, item_flag=self.item_flag, n1ql_port=self.n1ql_port, full_docs_list=self.full_docs_list, log=self.log, input=self.input, master=self.master, use_rest=True) log.info(self.n1ql_server) if self.create_primary_index: self.n1ql_helper.create_primary_index( using_gsi=self.use_gsi_for_primary, server=self.n1ql_server) def query_using_index(self, bucket, query_definition, expected_result=None, scan_consistency=None, scan_vector=None, verify_results=True): if not scan_consistency: scan_consistency = self.scan_consistency self.gen_results.query = query_definition.generate_query(bucket=bucket) if expected_result == None: expected_result = self.gen_results.generate_expected_result( print_expected_result=False) query = self.gen_results.query log.info("Query : {0}".format(query)) msg, check = self.n1ql_helper.run_query_and_verify_result( query=query, server=self.n1ql_server, expected_result=expected_result, scan_consistency=scan_consistency, scan_vector=scan_vector, verify_results=verify_results) self.assertTrue(check, msg) def drop_index(self, bucket, query_definition, verify_drop=True): try: query = query_definition.generate_index_drop_query( bucket=bucket, use_gsi_for_secondary=self.use_gsi_for_secondary, use_gsi_for_primary=self.use_gsi_for_primary) log.info(query) actual_result = self.n1ql_helper.run_cbq_query( query=query, server=self.n1ql_server) if verify_drop: check = self.n1ql_helper._is_index_in_list( bucket, query_definition.index_name, server=self.n1ql_server) self.assertFalse( check, "index {0} failed to be " "deleted".format(query_definition.index_name)) except Exception as ex: log.info(ex) query = "select * from system:indexes" actual_result = self.n1ql_helper.run_cbq_query( query=query, server=self.n1ql_server) log.info(actual_result) def run_async_index_operations(self, operation_type): if operation_type == "create_index": self._create_primary_index() for bucket in self.buckets: for query_definition in self.query_definitions: self.create_index(bucket=bucket, query_definition=query_definition) if operation_type == "query": for bucket in self.buckets: for query_definition in self.query_definitions: self.query_using_index(bucket=bucket, query_definition=query_definition) if operation_type == "drop_index": for bucket in self.buckets: for query_definition in self.query_definitions: self.drop_index(bucket=bucket, query_definition=query_definition) if operation_type == "generate_docs": for bucket in self.buckets: self.generate_docs(self.docs_per_day, start=randint(0, 1000000000))
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", 'VIEW') self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS') if self.input.param("reload_data", False): for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) # temporary for MB-12848 self.create_primary_index_for_3_0_and_greater() def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) # self.create_primary_index_for_3_0_and_greater() if not self.input.param("skip_build_tuq", True): self._build_tuq(self.master) self.skip_buckets_handle = True except: self.log.error('SUITE SETUP FAILED') self.tearDown() def tearDown(self): if self._testMethodName == 'suite_tearDown': self.skip_buckets_handle = False super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, 'shell'): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() ############################################################################################## # # SIMPLE CHECKS ############################################################################################## def test_simple_check(self): for bucket in self.buckets: query_template = 'FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_simple_negative_check(self): queries_errors = {'SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3' : 'Aggregates not allowed in WHERE', 'SELECT *.$str0 FROM {0}' : 'syntax error', 'SELECT *.* FROM {0} ... ERROR' : 'syntax error', 'FROM %s SELECT $str0 WHERE id=null' : 'syntax error',} self.negative_common_body(queries_errors) def test_unnest(self): for bucket in self.buckets: query_template = 'SELECT emp.$int0, task FROM %s emp UNNEST emp.$nested_list_3l0 task' % bucket.name actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(sorted(actual_result['results']), sorted(expected_result)) def test_subquery_select(self): for bucket in self.buckets: self.query = 'SELECT $str0, $subquery(SELECT COUNT($str0) cn FROM %s d USE KEYS $5) as names FROM %s' % (bucket.name, bucket.name) actual_result, expected_result = self.run_query_with_subquery_select_from_template(self.query) self._verify_results(actual_result['results'], expected_result) def test_subquery_from(self): for bucket in self.buckets: self.query = 'SELECT tasks.$str0 FROM $subquery(SELECT $str0, $int0 FROM %s) as tasks' % (bucket.name) actual_result, expected_result = self.run_query_with_subquery_from_template(self.query) self._verify_results(actual_result['results'], expected_result) def test_consistent_simple_check(self): queries = [self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $str0 IS NOT NULL AND $int0<10 ' +\ 'OR $int1 = 6 ORDER BY $int0, $int1'), self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $int1 = 6 OR $str0 IS NOT NULL AND ' +\ '$int0<10 ORDER BY $int0, $int1')] for bucket in self.buckets: actual_result1 = self.run_cbq_query(queries[0] % bucket.name) actual_result2 = self.run_cbq_query(queries[1] % bucket.name) self.assertTrue(actual_result1['results'] == actual_result2['results'], "Results are inconsistent.Difference: %s %s %s %s" %( len(actual_result1['results']), len(actual_result2['results']), actual_result1['results'][:100], actual_result2['results'][:100])) def test_simple_nulls(self): queries = ['SELECT id FROM %s WHERE id=NULL or id="null"'] for bucket in self.buckets: for query in queries: actual_result = self.run_cbq_query(query % (bucket.name)) self._verify_results(actual_result['results'], []) ############################################################################################## # # LIMIT OFFSET CHECKS ############################################################################################## def test_limit_negative(self): #queries_errors = {'SELECT * FROM default LIMIT {0}' : ('Invalid LIMIT value 2.5', 5030)} queries_errors = {'SELECT ALL * FROM %s' : ('syntax error', 3000)} self.negative_common_body(queries_errors) def test_limit_offset(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) def test_limit_offset_zero(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0' % (bucket.name) self.query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query() self.assertEquals(actual_result['results'], [], "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['results'], [])) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals(actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['results'], expected_result)) def test_limit_offset_negative_check(self): queries_errors = {'SELECT DISTINCT $str0 FROM {0} LIMIT 1.1' : 'Invalid LIMIT value 1.1', 'SELECT DISTINCT $str0 FROM {0} OFFSET 1.1' : 'Invalid OFFSET value 1.1'} self.negative_common_body(queries_errors) def test_limit_offset_sp_char_check(self): queries_errors = {'SELECT DISTINCT $str0 FROM {0} LIMIT ~' : 'syntax erro', 'SELECT DISTINCT $str0 FROM {0} OFFSET ~' : 'syntax erro'} self.negative_common_body(queries_errors) ############################################################################################## # # ALIAS CHECKS ############################################################################################## def test_simple_alias(self): for bucket in self.buckets: query_template = 'SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals(actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['results'], expected_result)) query_template = 'SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) expected_result = [ { "COUNT_EMPLOYEE": expected_result[0]['COUNT_EMPLOYEE'] + 1 } ] self.assertEquals(actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % ( actual_result['results'], expected_result)) def test_simple_negative_alias(self): queries_errors = {'SELECT $str0._last_name as *' : 'syntax error', 'SELECT $str0._last_name as DATABASE ?' : 'syntax error', 'SELECT $str0 AS NULL FROM {0}' : 'syntax error', 'SELECT $str1 as $str0, $str0 FROM {0}' : 'Duplicate result alias name', 'SELECT test.$obj0 as points FROM {0} AS TEST ' + 'GROUP BY $obj0 AS GROUP_POINT' : 'syntax error'} self.negative_common_body(queries_errors) def test_alias_from_clause(self): queries_templates = ['SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points', 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\ ' ORDER BY points', 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' +\ 'GROUP BY test.$obj0.$_obj0_int0 ORDER BY points'] for bucket in self.buckets: for query_template in queries_templates: actual_result, expected_result = self.run_query_from_template(query_template % (bucket.name)) self._verify_results(actual_result['results'], expected_result) def test_alias_from_clause_group(self): for bucket in self.buckets: query_template = 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' %(bucket.name) +\ 'GROUP BY $obj0.$_obj0_int0 ORDER BY points' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_order_desc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_order_asc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT COUNT(TEST.$str0) from %s AS TEST' %(bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_unnest(self): for bucket in self.buckets: query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill' %( bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # ORDER BY CHECKS ############################################################################################## def test_order_by_check(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0, $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $obj0.$_obj0_int0, $str0, $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0 AS points FROM %s' % (bucket.name) +\ ' AS test ORDER BY $str1 DESC, points DESC' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias_arrays(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s' % ( bucket.name) +\ ' AS TEST ORDER BY SKILL, $str1, TEST.$obj0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $int0, $int1, count(*) AS emp_per_month from %s'% ( bucket.name) +\ ' WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $str1 AS TITLE, min($int1) day FROM %s GROUP' % (bucket.name) +\ ' BY $str1 ORDER BY MIN($int1), $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_precedence(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str0, $str1' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_satisfy(self): for bucket in self.buckets: query_template = 'SELECT $str0, $list_obj0 FROM %s AS employee ' % (bucket.name) +\ 'WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND' +\ ' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # DISTINCT ############################################################################################## def test_distinct(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str1 FROM %s ORDER BY $str1' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_distinct_nested(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s ' % (bucket.name) +\ 'ORDER BY $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT DISTINCT $list_str0[0] as skill' +\ ' FROM %s ORDER BY $list_str0[0]' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) self.query = 'SELECT DISTINCT $obj0.* FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # COMPLEX PATHS ############################################################################################## def test_simple_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 as new_attribute FROM %s.$obj0' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) def test_where_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1' % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', '')) subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num)) subquery_template = re.sub(r'USE KEYS.*', '', subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip() if not alias: alias = '$1' for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as ', '', alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.iteritems(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query(query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue(str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" %( str(ex).split(':')[-1], error)) else: self.fail("There were no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.n1ql_port == None or self.n1ql_port == '': self.n1ql_port = self.input.param("n1ql_port", 8093) if not self.n1ql_port: self.log.info(" n1ql_port is not defined, processing will not proceed further") raise Exception("n1ql_port is not defined, processing will not proceed further") query_params = {} cred_params = {'creds': []} for bucket in self.buckets: if bucket.saslPassword: cred_params['creds'].append({'user': '******' % bucket.name, 'pass': bucket.saslPassword}) query_params.update(cred_params) if self.use_rest: query_params.update({'scan_consistency': self.scan_consistency}) self.log.info('RUN QUERY %s' % query) result = RestConnection(server).query_tool(query, self.n1ql_port, query_params=query_params) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\ "tuq_client/tuq_client " +\ "-engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='tuq_client>') else: output = self.shell.execute_commands_inside("/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='cbq>') result = self._parse_query_output(output) if isinstance(result, str) or 'errors' in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" %( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": if self.services_init.find('n1ql') != -1: return os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"] print "PATH TO SHERLOCK: %s" % couchbase_path if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_sabre(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_sabre(docs_per_day, start) def generate_docs_employee(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_data(docs_per_day = docs_per_day, start = start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_simple_data(docs_per_day = docs_per_day, start = start) def generate_docs_sales(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_sales_data(docs_per_day = docs_per_day, start = start) def generate_docs_bigdata(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_bigdata(end=(1000*docs_per_day), start=start, value_size=self.value_size) def _verify_results(self, actual_result, expected_result, missing_count = 1, extra_count = 1): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra(actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) self.fail("Results are incorrect.Actual num %s. Expected num: %s.\n" % ( len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" self.assertTrue(actual_result == expected_result, msg % (actual_result[:100],actual_result[-100:], expected_result[:100],expected_result[-100:])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self): self.log.info("CREATE PRIMARY INDEX") rest = RestConnection(self.master) versions = rest.get_nodes_versions() if versions[0].startswith("4") or versions[0].startswith("3"): for bucket in self.buckets: if self.primary_indx_drop: self.log.info("Dropping primary index for %s ..." % bucket.name) self.query = "DROP PRIMARY INDEX ON %s" % (bucket.name) self.sleep(3, 'Sleep for some time after index drop') self.log.info("Creating primary index for %s ..." % bucket.name) self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type) try: self.run_cbq_query() if self.primary_indx_type.lower() == 'gsi': self._wait_for_index_online(bucket, '#primary') except Exception, ex: self.log.info(str(ex))
class AnalyticsHelper(): def __init__(self, version=None, master=None, shell=None, use_rest=None, max_verify=0, buckets=[], item_flag=0, analytics_port=8095, n1ql_port=8093, full_docs_list=[], log=None, input=None, database=None): self.version = version self.shell = shell self.n1ql_port = n1ql_port self.max_verify = max_verify self.buckets = buckets self.item_flag = item_flag self.analytics_port = analytics_port self.input = input self.log = log self.use_rest = True self.full_docs_list = full_docs_list self.master = master self.database = database if self.full_docs_list and len(self.full_docs_list) > 0: self.gen_results = TuqGenerators(self.log, self.full_docs_list) def killall_tuq_process(self): self.shell.execute_command("killall cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.execute_command("killall indexer") def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_analytics_query() return actual_result, expected_result def run_analytics_query(self, query=None, min_output_size=10, server=None, query_params={}, is_prepared=False, scan_consistency=None, scan_vector=None, verbose=False): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.analytics_port = server.analytics_port else: if server.ip == "127.0.0.1": self.analytics_port = server.analytics_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.analytics_port == None or self.analytics_port == '': self.analytics_port = self.input.param("analytics_port", 8095) if not self.analytics_port: self.log.info( " analytics_port is not defined, processing will not proceed further" ) raise Exception( "analytics_port is not defined, processing will not proceed further" ) if self.use_rest: query_params = {} if scan_consistency: query_params['scan_consistency'] = scan_consistency if scan_vector: query_params['scan_vector'] = str(scan_vector).replace( "'", '"') if verbose: self.log.info('RUN QUERY %s' % query) query = query + ";" if "USE INDEX" in query: query = query.replace("USE INDEX(`#primary` USING GSI)", " ") for bucket in self.buckets: query = query.replace(bucket.name + " ", bucket.name + "_shadow ") self.log.info(" CBAS QUERY :: {0}".format(query)) result = RestConnection(server).analytics_tool( query, self.analytics_port, query_params=query_params, verbose=verbose) if isinstance(result, str) or 'errors' in result: error_result = str(result) length_display = len(error_result) if length_display > 500: error_result = error_result[:500] raise CBQError(error_result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1): self.log.info(" Analyzing Actual Result") actual_result = self._gen_dict(actual_result) self.log.info(" Analyzing Expected Result") expected_result = self._gen_dict(expected_result) if len(actual_result) != len(expected_result): raise Exception( "Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result))) msg = "The number of rows match but the results mismatch, please check" if actual_result != expected_result: raise Exception(msg) def _verify_results_rqg_new(self, n1ql_result=[], sql_result=[], hints=["a1"]): new_n1ql_result = [] for result in n1ql_result: if result != {}: for key in list(result.keys()): if key.find('_shadow') != -1: new_n1ql_result.append(result[key]) else: new_n1ql_result.append(result) break n1ql_result = new_n1ql_result if self._is_function_in_result(hints): return self._verify_results_rqg_for_function( n1ql_result, sql_result) check = self._check_sample(n1ql_result, hints) actual_result = n1ql_result if actual_result == [{}]: actual_result = [] if check: actual_result = self._gen_dict(n1ql_result) actual_result = sorted(actual_result) expected_result = sorted(sql_result) if len(actual_result) != len(expected_result): extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception( "Results are incorrect.Actual num %s. Expected num: %s.:: %s \n" % (len(actual_result), len(expected_result), extra_msg)) msg = "The number of rows match but the results mismatch, please check" if self._sort_data(actual_result) != self._sort_data(expected_result): extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception(msg + "\n " + extra_msg) def _verify_results_rqg(self, n1ql_result=[], sql_result=[], hints=["a1"]): new_n1ql_result = [] for result in n1ql_result: if result != {}: new_n1ql_result.append(result) n1ql_result = new_n1ql_result if self._is_function_in_result(hints): return self._verify_results_rqg_for_function( n1ql_result, sql_result) check = self._check_sample(n1ql_result, hints) actual_result = n1ql_result if actual_result == [{}]: actual_result = [] if check: actual_result = self._gen_dict(n1ql_result) actual_result = sorted(actual_result) expected_result = sorted(sql_result) if len(actual_result) != len(expected_result): extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception( "Results are incorrect.Actual num %s. Expected num: %s.:: %s \n" % (len(actual_result), len(expected_result), extra_msg)) msg = "The number of rows match but the results mismatch, please check" if self._sort_data(actual_result) != self._sort_data(expected_result): extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception(msg + "\n " + extra_msg) def _sort_data(self, result): new_data = [] for data in result: new_data.append(sorted(data)) return new_data def _verify_results_crud_rqg(self, n1ql_result=[], sql_result=[], hints=["primary_key_id"]): new_n1ql_result = [] for result in n1ql_result: if result != {}: new_n1ql_result.append(result) n1ql_result = new_n1ql_result if self._is_function_in_result(hints): return self._verify_results_rqg_for_function( n1ql_result, sql_result) check = self._check_sample(n1ql_result, hints) actual_result = n1ql_result if actual_result == [{}]: actual_result = [] if check: actual_result = self._gen_dict(n1ql_result) actual_result = sorted(actual_result) expected_result = sorted(sql_result) if len(actual_result) != len(expected_result): extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception( "Results are incorrect.Actual num %s. Expected num: %s.:: %s \n" % (len(actual_result), len(expected_result), extra_msg)) if not self._result_comparison_analysis(actual_result, expected_result): msg = "The number of rows match but the results mismatch, please check" extra_msg = self._get_failure_message(expected_result, actual_result) raise Exception(msg + "\n " + extra_msg) def _get_failure_message(self, expected_result, actual_result): if expected_result == None: expected_result = [] if actual_result == None: actual_result = [] len_expected_result = len(expected_result) len_actual_result = len(actual_result) len_expected_result = min(5, len_expected_result) len_actual_result = min(5, len_actual_result) extra_msg = "mismatch in results :: expected :: {0}, actual :: {1} ".format( expected_result[0:len_expected_result], actual_result[0:len_actual_result]) return extra_msg def _result_comparison_analysis(self, expected_result, actual_result): expected_map = {} actual_map = {} for data in expected_result: primary = None for key in list(data.keys()): keys = key if keys.encode('ascii') == "primary_key_id": primary = keys expected_map[data[primary]] = data for data in actual_result: primary = None for key in list(data.keys()): keys = key if keys.encode('ascii') == "primary_key_id": primary = keys actual_map[data[primary]] = data check = True for key in list(expected_map.keys()): if sorted(actual_map[key]) != sorted(expected_map[key]): check = False return check def _analyze_for_special_case_using_func(self, expected_result, actual_result): if expected_result == None: expected_result = [] if actual_result == None: actual_result = [] if len(expected_result) == 1: value = list(expected_result[0].values())[0] if value == None or value == 0: expected_result = [] if len(actual_result) == 1: value = list(actual_result[0].values())[0] if value == None or value == 0: actual_result = [] return expected_result, actual_result def _is_function_in_result(self, result): if result == "FUN": return True return False def _verify_results_rqg_for_function(self, n1ql_result=[], sql_result=[], hints=["a1"]): actual_count = -1 expected_count = -1 actual_result = n1ql_result sql_result, actual_result = self._analyze_for_special_case_using_func( sql_result, actual_result) if len(sql_result) != len(actual_result): msg = "the number of results do not match :: expected = {0}, actual = {1}".format( len(n1ql_result), len(sql_result)) extra_msg = self._get_failure_message(sql_result, actual_result) raise Exception(msg + "\n" + extra_msg) n1ql_result = self._gen_dict_n1ql_func_result(n1ql_result) n1ql_result = sorted(n1ql_result) sql_result = self._gen_dict_n1ql_func_result(sql_result) sql_result = sorted(sql_result) if len(sql_result) == 0 and len(actual_result) == 0: return if sql_result != n1ql_result: max = 2 if len(sql_result) < 5: max = len(sql_result) msg = "mismatch in results :: expected [0:{0}]:: {1}, actual [0:{0}]:: {2} ".format( max, sql_result[0:max], n1ql_result[0:max]) raise Exception(msg) def _convert_to_number(self, val): if not isinstance(val, str): return val value = -1 try: if value == '': return 0 value = int(val.split("(")[1].split(")")[0]) except Exception as ex: self.log.info(ex) finally: return value def analyze_failure(self, actual, expected): missing_keys = [] different_values = [] for key in list(expected.keys()): if key not in list(actual.keys()): missing_keys.append(key) if expected[key] != actual[key]: different_values.append( "for key {0}, expected {1} \n actual {2}".format( key, expected[key], actual[key])) self.log.info(missing_keys) if (len(different_values) > 0): self.log.info(" number of such cases {0}".format( len(different_values))) self.log.info(" example key {0}".format(different_values[0])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" % ( version, info.architecture_type) #TODO for windows return url def _restart_indexer(self): couchbase_path = "/opt/couchbase/var/lib/couchbase" cmd = "rm -f {0}/meta;rm -f /tmp/log_upr_client.sock".format( couchbase_path) self.shell.execute_command(cmd) def _start_command_line_query(self, server): self.shell = RemoteMachineShellConnection(server) self._set_env_variable(server) if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client[ "sherlock_path"] print("PATH TO SHERLOCK: %s" % couchbase_path) if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.items(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def drop_primary_index(self, using_gsi=True, server=None): if server == None: server = self.master self.log.info("CHECK FOR PRIMARY INDEXES") for bucket in self.buckets: self.query = "DROP PRIMARY INDEX ON {0}".format(bucket.name) if using_gsi: self.query += " USING GSI" if not using_gsi: self.query += " USING VIEW " self.log.info(self.query) try: check = self._is_index_in_list(bucket.name, "#primary", server=server) if check: self.run_analytics_query(server=server) except Exception as ex: self.log.error('ERROR during index creation %s' % str(ex)) def create_primary_index(self, using_gsi=True, server=None): if server == None: server = self.master for bucket in self.buckets: self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name) if using_gsi: self.query += " USING GSI" # if gsi_type == "memdb": # self.query += " WITH {'index_type': 'memdb'}" if not using_gsi: self.query += " USING VIEW " try: check = self._is_index_in_list(bucket.name, "#primary", server=server) if not check: self.run_analytics_query(server=server) check = self.is_index_online_and_in_list(bucket.name, "#primary", server=server) if not check: raise Exception( " Timed-out Exception while building primary index for bucket {0} !!!" .format(bucket.name)) else: raise Exception( " Primary Index Already present, This looks like a bug !!!" ) except Exception as ex: self.log.error('ERROR during index creation %s' % str(ex)) raise ex def verify_index_with_explain(self, actual_result, index_name, check_covering_index=False): check = True if check_covering_index: if "covering" in str(actual_result): check = True else: check = False if index_name in str(actual_result): return True and check return False def run_query_and_verify_result(self, server=None, query=None, timeout=120.0, max_try=1, expected_result=None, scan_consistency=None, scan_vector=None, verify_results=True): check = False init_time = time.time() try_count = 0 while not check: next_time = time.time() try: actual_result = self.run_analytics_query( query=query, server=server, scan_consistency=scan_consistency, scan_vector=scan_vector) if verify_results: self._verify_results(actual_result['results'], expected_result) else: return "ran query with success and validated results", True check = True except Exception as ex: if (next_time - init_time > timeout or try_count >= max_try): return ex, False finally: try_count += 1 return "ran query with success and validated results", check def run_cbq_query(self, query=None, min_output_size=10, server=None, query_params={}, is_prepared=False, scan_consistency=None, scan_vector=None, verbose=True): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.n1ql_port == None or self.n1ql_port == '': self.n1ql_port = self.input.param("n1ql_port", 90) if not self.n1ql_port: self.log.info( " n1ql_port is not defined, processing will not proceed further" ) raise Exception( "n1ql_port is not defined, processing will not proceed further" ) if self.use_rest: query_params = {} if scan_consistency: query_params['scan_consistency'] = scan_consistency if scan_vector: query_params['scan_vector'] = str(scan_vector).replace( "'", '"') if verbose: self.log.info('RUN QUERY %s' % query) result = RestConnection(server).query_tool( query, self.n1ql_port, query_params=query_params, is_prepared=is_prepared, verbose=verbose) else: # if self.version == "git_repo": # output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\ # "tuq_client/tuq_client " +\ # "-engine=http://%s:8093/" % server.ip, # subcommands=[query,], # min_output_size=20, # end_msg='tuq_client>') # else: #os = self.shell.extract_remote_info().type.lower() shell = RemoteMachineShellConnection(server) #query = query.replace('"', '\\"') #query = query.replace('`', '\\`') #if os == "linux": cmd = "%s/cbq -engine=http://%s:8093/" % ( testconstants.LINUX_COUCHBASE_BIN_PATH, server.ip) output = shell.execute_commands_inside(cmd, query, "", "", "", "", "") print( "--------------------------------------------------------------------------------------------------------------------------------" ) print(output) result = json.loads(output) print(result) result = self._parse_query_output(output) if isinstance(result, str) or 'errors' in result: error_result = str(result) length_display = len(error_result) if length_display > 500: error_result = error_result[:500] raise CBQError(error_result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result # def is_index_online_and_in_list(self, bucket, index_name, server=None, timeout=600.0): # check = self._is_index_in_list(bucket, index_name, server = server) # init_time = time.time() # while not check: # time.sleep(1) # check = self._is_index_in_list(bucket, index_name, server = server) # next_time = time.time() # if check or (next_time - init_time > timeout): # return check # return check # # def is_index_ready_and_in_list(self, bucket, index_name, server=None, timeout=600.0): # query = "SELECT * FROM system:indexes where name = \'{0}\'".format(index_name) # if server == None: # server = self.master # init_time = time.time() # check = False # while not check: # res = self.run_analytics_query(query=query, server=server) # for item in res['results']: # if 'keyspace_id' not in item['indexes']: # check = False # elif item['indexes']['keyspace_id'] == str(bucket) \ # and item['indexes']['name'] == index_name \ # and item['indexes']['state'] == "online": # check = True # time.sleep(1) # next_time = time.time() # check = check or (next_time - init_time > timeout) # return check # def is_index_online_and_in_list_bulk(self, bucket, index_names = [], server = None, index_state = "online", timeout = 600.0): # check, index_names = self._is_index_in_list_bulk(bucket, index_names, server = server, index_state = index_state) # init_time = time.time() # while not check: # check, index_names = self._is_index_in_list_bulk(bucket, index_names, server = server, index_state = index_state) # next_time = time.time() # if check or (next_time - init_time > timeout): # return check # return check # # def gen_build_index_query(self, bucket = "default", index_list = []): # return "BUILD INDEX on {0}({1}) USING GSI".format(bucket,",".join(index_list)) # # def gen_query_parameter(self, scan_vector = None, scan_consistency = None): # query_params = {} # if scan_vector: # query_params.update("scan_vector", scan_vector) # if scan_consistency: # query_params.update("scan_consistency", scan_consistency) # return query_params # def _is_index_in_list(self, bucket, index_name, server = None, index_state = ["pending", "building", "deferred"]): # query = "SELECT * FROM system:indexes where name = \'{0}\'".format(index_name) # if server == None: # server = self.master # res = self.run_cbq_query(query = query, server = server) # for item in res['results']: # if 'keyspace_id' not in item['indexes']: # return False # if item['indexes']['keyspace_id'] == str(bucket) and item['indexes']['name'] == index_name and item['indexes']['state'] not in index_state: # return True # return False # # def _is_index_in_list_bulk(self, bucket, index_names = [], server = None, index_state = ["pending","building"]): # query = "SELECT * FROM system:indexes" # if server == None: # server = self.master # res = self.run_cbq_query(query = query, server = server) # index_count=0 # found_index_list = [] # for item in res['results']: # if 'keyspace_id' not in item['indexes']: # return False # for index_name in index_names: # if item['indexes']['keyspace_id'] == str(bucket) and item['indexes']['name'] == index_name and item['indexes']['state'] not in index_state: # found_index_list.append(index_name) # if len(found_index_list) == len(index_names): # return True, [] # return False, list(set(index_names) - set(found_index_list)) # # def gen_index_map(self, server = None): # query = "SELECT * FROM system:indexes" # if server == None: # server = self.master # res = self.run_cbq_query(query = query, server = server) # index_map = {} # for item in res['results']: # bucket_name = item['indexes']['keyspace_id'].encode('ascii','ignore') # if bucket_name not in index_map.keys(): # index_map[bucket_name] = {} # index_name = str(item['indexes']['name']) # index_map[bucket_name][index_name] = {} # index_map[bucket_name][index_name]['state'] = item['indexes']['state'] # return index_map # # def get_index_count_using_primary_index(self, buckets, server = None): # query = "SELECT COUNT(*) FROM {0}" # map= {} # if server == None: # server = self.master # for bucket in buckets: # res = self.run_cbq_query(query = query.format(bucket.name), server = server) # map[bucket.name] = int(res["results"][0]["$1"]) # return map # # def get_index_count_using_index(self, bucket, index_name,server=None): # query = 'SELECT COUNT(*) FROM {0} USE INDEX ({1})'.format(bucket.name, index_name) # if not server: # server = self.master # res = self.run_cbq_query(query=query, server=server) # return int(res['results'][0]['$1']) def _gen_dict(self, result): result_set = [] if result != None and len(result) > 0: for val in result: for key in list(val.keys()): result_set.append(val[key]) return result_set def _gen_dict_n1ql_func_result(self, result): result_set = [val[key] for val in result for key in list(val.keys())] new_result_set = [] if len(result_set) > 0: for value in result_set: if isinstance(value, float): new_result_set.append(round(value, 0)) else: new_result_set.append(value) else: new_result_set = result_set return new_result_set def _check_sample(self, result, expected_in_key=None): if expected_in_key == "FUN": return False if expected_in_key == None or len(expected_in_key) == 0: return False if result != None and len(result) > 0: sample = result[0] for key in list(sample.keys()): for sample in expected_in_key: if key in sample: return True return False def old_gen_dict(self, result): result_set = [] map = {} duplicate_keys = [] try: if result != None and len(result) > 0: for val in result: for key in list(val.keys()): result_set.append(val[key]) for val in result_set: if val["_id"] in list(map.keys()): duplicate_keys.append(val["_id"]) map[val["_id"]] = val keys = list(map.keys()) keys.sort() except Exception as ex: self.log.info(ex) raise if len(duplicate_keys) > 0: raise Exception(" duplicate_keys {0}".format(duplicate_keys)) return map
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection( self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp' and self.input.param( "cbq_version", "sherlock") != 'sherlock': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.analytics = self.input.param("analytics", False) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", 'GSI') self.index_type = self.input.param("index_type", 'GSI') self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.monitoring = self.input.param("monitoring", False) self.isprepared = False self.named_prepare = self.input.param("named_prepare", None) self.skip_primary_index = self.input.param("skip_primary_index", False) self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS') shell = RemoteMachineShellConnection(self.master) type = shell.extract_remote_info().distribution_type self.path = testconstants.LINUX_COUCHBASE_BIN_PATH if type.lower() == 'windows': self.path = testconstants.WIN_COUCHBASE_BIN_PATH elif type.lower() == "mac": self.path = testconstants.MAC_COUCHBASE_BIN_PATH self.threadFailure = False if self.primary_indx_type.lower() == "gsi": self.gsi_type = self.input.param("gsi_type", 'plasma') else: self.gsi_type = None if self.input.param("reload_data", False): if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas']) for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) # Adding sleep after flushing buckets (see CBQE-5838) self.sleep(210) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas']) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators( self.log, self.generate_full_docs_list(self.gens_load)) if (self.analytics == False): self.create_primary_index_for_3_0_and_greater() if (self.analytics): self.setup_analytics() self.sleep(30, 'wait for analytics setup') def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) if not self.input.param("skip_build_tuq", True): self._build_tuq(self.master) self.skip_buckets_handle = True if (self.analytics): self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas']) self.setup_analytics() self.sleep(30, 'wait for analytics setup') except: self.log.error('SUITE SETUP FAILED') self.tearDown() def tearDown(self): if self._testMethodName == 'suite_tearDown': self.skip_buckets_handle = False if self.analytics: bucket_username = "******" bucket_password = "******" data = 'use Default ;' for bucket in self.buckets: data += 'disconnect bucket {0} if connected;'.format( bucket.name) data += 'drop dataset {0} if exists;'.format(bucket.name + "_shadow") data += 'drop bucket {0} if exists;'.format(bucket.name) filename = "file.txt" f = open(filename, 'w') f.write(data) f.close() url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password os.system(cmd) os.remove(filename) super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, 'shell'): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() ############################################################################################## # # Setup Helpers ############################################################################################## def setup_analytics(self): data = 'use Default;' bucket_username = "******" bucket_password = "******" for bucket in self.buckets: data += 'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format( bucket.name, self.master.ip) data += 'create shadow dataset {1} on {0}; '.format( bucket.name, bucket.name + "_shadow") data += 'connect bucket {0} with {{"username":"******","password":"******"}};'.format( bucket.name, bucket_username, bucket_password) filename = "file.txt" f = open(filename, 'w') f.write(data) f.close() url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password os.system(cmd) os.remove(filename) def run_active_requests(self, e, t): while not e.isSet(): logging.debug('wait_for_event_timeout starting') event_is_set = e.wait(t) logging.debug('event set: %s', event_is_set) if event_is_set: result = self.run_cbq_query( "select * from system:active_requests") self.assertTrue(result['metrics']['resultCount'] == 1) requestId = result['requestID'] result = self.run_cbq_query( 'delete from system:active_requests where requestId = "%s"' % requestId) time.sleep(20) result = self.run_cbq_query( 'select * from system:active_requests where requestId = "%s"' % requestId) self.assertTrue(result['metrics']['resultCount'] == 0) result = self.run_cbq_query( "select * from system:completed_requests") requestId = result['requestID'] result = self.run_cbq_query( 'delete from system:completed_requests where requestId = "%s"' % requestId) time.sleep(10) result = self.run_cbq_query( 'select * from system:completed_requests where requestId = "%s"' % requestId) self.assertTrue(result['metrics']['resultCount'] == 0) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] keys_num = int( re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', '')) subquery_full_list = self.generate_full_docs_list( gens_load=self.gens_load, keys=self._get_keys(keys_num)) subquery_template = re.sub(r'USE KEYS.*', '', subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as', '', re.sub(r'FROM.*', '', alias)).strip() if not alias: alias = '$1' for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] subquery_full_list = self.generate_full_docs_list( gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as ', '', alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.items(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query( query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue( str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" % (str(ex).split(':')[-1], error)) else: self.fail("There were no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client query_params = {} cred_params = {'creds': []} rest = RestConnection(server) username = rest.username password = rest.password cred_params['creds'].append({'user': username, 'pass': password}) query_params.update(cred_params) if self.use_rest: query_params.update({'scan_consistency': self.scan_consistency}) self.log.info('RUN QUERY %s' % query) if self.analytics: query = query + ";" for bucket in self.buckets: query = query.replace(bucket.name, bucket.name + "_shadow") result = RestConnection( self.cbas_node).execute_statement_on_cbas( query, "immediate") result = json.loads(result) else: result = rest.query_tool(query, self.n1ql_port, query_params=query_params) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbase/query/" +\ "shell/cbq/cbq ", "", "", "", "", "", "") else: os = self.shell.extract_remote_info().type.lower() if not (self.isprepared): query = query.replace('"', '\\"') query = query.replace('`', '\\`') cmd = "%s/cbq -engine=http://%s:%s/ -q -u %s -p %s" % ( self.path, server.ip, server.port, username, password) output = self.shell.execute_commands_inside( cmd, query, "", "", "", "", "") if not (output[0] == '{'): output1 = '{' + str(output) else: output1 = output result = json.loads(output1) if isinstance(result, str) or 'errors' in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" % ( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format( cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": if self.services_init.find('n1ql') != -1: return os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client[ "sherlock_path"] if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_sabre(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_sabre(docs_per_day, start) def generate_docs_employee(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_data( docs_per_day=docs_per_day, start=start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_simple_data( docs_per_day=docs_per_day, start=start) def generate_docs_sales(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_sales_data( docs_per_day=docs_per_day, start=start) def generate_docs_bigdata(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_bigdata(end=(1000 * docs_per_day), start=start, value_size=self.value_size) def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra( actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) self.fail( "Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" self.assertTrue( actual_result == expected_result, msg % (actual_result[:100], actual_result[-100:], expected_result[:100], expected_result[-100:])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.items(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self): self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type) rest = RestConnection(self.master) versions = rest.get_nodes_versions() if versions[0].startswith("4") or versions[0].startswith( "3") or versions[0].startswith("5"): for bucket in self.buckets: if self.primary_indx_drop: self.log.info( "Dropping primary index for %s using %s ..." % (bucket.name, self.primary_indx_type)) self.query = "DROP PRIMARY INDEX ON %s USING %s" % ( bucket.name, self.primary_indx_type) #self.run_cbq_query() self.sleep(3, 'Sleep for some time after index drop') self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name res = self.run_cbq_query() self.sleep(10) if self.monitoring: self.query = "delete from system:completed_requests" self.run_cbq_query() if not self.skip_primary_index: if (res['metrics']['resultCount'] == 0): self.query = "CREATE PRIMARY INDEX ON %s USING %s" % ( bucket.name, self.primary_indx_type) self.log.info("Creating primary index for %s ..." % bucket.name) try: self.run_cbq_query() self.primary_index_created = True if self.primary_indx_type.lower() == 'gsi': self._wait_for_index_online(bucket, '#primary') except Exception as ex: self.log.info(str(ex)) def _wait_for_index_online(self, bucket, index_name, timeout=6000): end_time = time.time() + timeout while time.time() < end_time: query = "SELECT * FROM system:indexes where name='%s'" % index_name res = self.run_cbq_query(query) for item in res['results']: if 'keyspace_id' not in item['indexes']: self.log.error(item) continue if item['indexes']['keyspace_id'] == bucket.name: if item['indexes']['state'] == "online": return self.sleep( 5, 'index is pending or not in the list. sleeping... (%s)' % [item['indexes'] for item in res['results']]) raise Exception('index %s is not online. last response is %s' % (index_name, res)) def _get_keys(self, key_num): keys = [] for gen in self.gens_load: gen_copy = copy.deepcopy(gen) for i in range(gen_copy.end): key, _ = next(gen_copy) keys.append(key) if len(keys) == key_num: return keys return keys
def setUp(self): super(QueryTests, self).setUp() self.expiry = self.input.param("expiry", 0) self.batch_size = self.input.param("batch_size", 1) self.scan_consistency = self.input.param("scan_consistency", "request_plus") self.skip_cleanup = self.input.param("skip_cleanup", False) self.run_async = self.input.param("run_async", True) self.version = self.input.param("cbq_version", "git_repo") for server in self.servers: rest = RestConnection(server) temp = rest.cluster_status() self.log.info("Initial status of {0} cluster is {1}".format( server.ip, temp['nodes'][0]['status'])) while (temp['nodes'][0]['status'] == 'warmup'): self.log.info("Waiting for cluster to become healthy") self.sleep(5) temp = rest.cluster_status() self.log.info("current status of {0} is {1}".format( server.ip, temp['nodes'][0]['status'])) indexer_node = self.get_nodes_from_services_map(service_type="index", get_all_nodes=True) # Set indexer storage mode indexer_rest = RestConnection(indexer_node[0]) doc = {"indexer.settings.storage_mode": self.gsi_type} indexer_rest.set_index_settings_internal(doc) doc = {"indexer.api.enableTestServer": True} indexer_rest.set_index_settings_internal(doc) self.indexer_scanTimeout = self.input.param("indexer_scanTimeout", None) if self.indexer_scanTimeout is not None: for server in indexer_node: rest = RestConnection(server) rest.set_index_settings({ "indexer.settings.scan_timeout": self.indexer_scanTimeout }) if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection( self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) self.use_gsi_for_primary = self.input.param("use_gsi_for_primary", True) self.use_gsi_for_secondary = self.input.param("use_gsi_for_secondary", True) self.create_primary_index = self.input.param("create_primary_index", True) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.value_size = self.input.param("value_size", 1024) self.doc_ops = self.input.param("doc_ops", False) self.create_ops_per = self.input.param("create_ops_per", 0) self.expiry_ops_per = self.input.param("expiry_ops_per", 0) self.delete_ops_per = self.input.param("delete_ops_per", 0) self.update_ops_per = self.input.param("update_ops_per", 0) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.n1ql_helper.configure_gomaxprocs() self.full_docs_list = self.generate_full_docs_list(self.gens_load) self.gen_results = TuqGenerators(self.log, self.full_docs_list) verify_data = False if self.scan_consistency != "request_plus": verify_data = True self.load(self.gens_load, flag=self.item_flag, verify_data=verify_data, batch_size=self.batch_size) if self.doc_ops: self.ops_dist_map = self.calculate_data_change_distribution( create_per=self.create_ops_per, update_per=self.update_ops_per, delete_per=self.delete_ops_per, expiry_per=self.expiry_ops_per, start=0, end=self.docs_per_day) self.log.info(self.ops_dist_map) self.docs_gen_map = self.generate_ops_docs(self.docs_per_day, 0) self.full_docs_list_after_ops = self.generate_full_docs_list_after_ops( self.docs_gen_map) # Define Helper Method which will be used for running n1ql queries, create index, drop index self.n1ql_helper = N1QLHelper(version=self.version, shell=self.shell, use_rest=self.use_rest, max_verify=self.max_verify, buckets=self.buckets, item_flag=self.item_flag, n1ql_port=self.n1ql_port, full_docs_list=self.full_docs_list, log=self.log, input=self.input, master=self.master) self.n1ql_node = self.get_nodes_from_services_map(service_type="n1ql") self.log.info(self.n1ql_node) #self.n1ql_helper._start_command_line_query(self.n1ql_node) # sleep to avoid race condition during bootstrap if self.create_primary_index: try: self.n1ql_helper.create_primary_index( using_gsi=self.use_gsi_for_primary, server=self.n1ql_node) except Exception as ex: self.log.info(ex) raise ex
class N1QLHelper(): def __init__(self, version=None, master=None, shell=None, use_rest=None, max_verify=0, buckets=[], item_flag=0, n1ql_port=8093, full_docs_list=[], log=None, input=None): self.version = version self.shell = shell self.use_rest = use_rest self.max_verify = max_verify self.buckets = buckets self.item_flag = item_flag self.n1ql_port = n1ql_port self.input = input self.log = log self.full_docs_list = full_docs_list self.master = master self.gen_results = TuqGenerators(self.log, self.full_docs_list) def killall_tuq_process(self): self.shell.execute_command("killall cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.execute_command("killall indexer") def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.n1ql_port == None or self.n1ql_port == '': self.n1ql_port = self.input.param("n1ql_port", 8093) if not self.n1ql_port: self.log.info( " n1ql_port is not defined, processing will not proceed further" ) raise Exception( "n1ql_port is not defined, processing will not proceed further" ) if self.use_rest: result = RestConnection(server).query_tool(query, self.n1ql_port) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\ "tuq_client/tuq_client " +\ "-engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='tuq_client>') else: output = self.shell.execute_commands_inside( "/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip, subcommands=[ query, ], min_output_size=20, end_msg='cbq>') result = self._parse_query_output(output) if isinstance(result, str) or 'errors' in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1): actual_result = self._gen_dict(actual_result) expected_result = self._gen_dict(expected_result) if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra( actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) raise Exception( "Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" error_message = msg % (actual_result[:10], actual_result[-10:], expected_result[:10], expected_result[-10:]) if actual_result != expected_result: raise Exception(error_message) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" % ( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windowsself.shell.execute_command(cmd) cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format( cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _restart_indexer(self): couchbase_path = "/opt/couchbase/var/lib/couchbase" cmd = "rm -f {0}/meta;rm -f /tmp/log_upr_client.sock".format( couchbase_path) self.shell.execute_command(cmd) def _start_command_line_query(self, server): self._set_env_variable(server) if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbaselabs/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbaselabs/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client[ "sherlock_path"] print "PATH TO SHERLOCK: %s" % couchbase_path if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self, using_gsi=True): self.log.info("CHECK FOR PRIMARY INDEXES") rest = RestConnection(self.master) versions = rest.get_nodes_versions() ddoc_name = 'ddl_#primary' if versions[0].startswith("3"): try: rest.get_ddoc(self.buckets[0], ddoc_name) except ReadDocumentException: for bucket in self.buckets: self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name) if using_gsi: self.query += " USING GSI" self.log.info(self.query) try: self.run_cbq_query() except Exception, ex: self.log.error('ERROR during index creation %s' % str(ex))
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp' and self.input.param("cbq_version", "sherlock") != 'sherlock': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.analytics = self.input.param("analytics",False) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", 'GSI') self.index_type = self.input.param("index_type", 'GSI') self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.monitoring = self.input.param("monitoring",False) self.isprepared = False self.named_prepare = self.input.param("named_prepare", None) self.skip_primary_index = self.input.param("skip_primary_index",False) self.scan_consistency = self.input.param("scan_consistency", 'REQUEST_PLUS') shell = RemoteMachineShellConnection(self.master) type = shell.extract_remote_info().distribution_type self.path = testconstants.LINUX_COUCHBASE_BIN_PATH if type.lower() == 'windows': self.path = testconstants.WIN_COUCHBASE_BIN_PATH elif type.lower() == "mac": self.path = testconstants.MAC_COUCHBASE_BIN_PATH self.threadFailure = False if self.primary_indx_type.lower() == "gsi": self.gsi_type = self.input.param("gsi_type", 'plasma') else: self.gsi_type = None if self.input.param("reload_data", False): if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [], [self.cbas_node], services=['cbas']) for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) if self.analytics: self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas']) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) if (self.analytics == False): self.create_primary_index_for_3_0_and_greater() if (self.analytics): self.setup_analytics() self.sleep(30,'wait for analytics setup') def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) if not self.input.param("skip_build_tuq", True): self._build_tuq(self.master) self.skip_buckets_handle = True if (self.analytics): self.cluster.rebalance([self.master, self.cbas_node], [self.cbas_node], [], services=['cbas']) self.setup_analytics() self.sleep(30,'wait for analytics setup') except: self.log.error('SUITE SETUP FAILED') self.tearDown() def tearDown(self): if self._testMethodName == 'suite_tearDown': self.skip_buckets_handle = False if self.analytics: bucket_username = "******" bucket_password = "******" data = 'use Default ;' for bucket in self.buckets: data += 'disconnect bucket {0} if connected;'.format(bucket.name) data += 'drop dataset {0} if exists;'.format(bucket.name+ "_shadow") data += 'drop bucket {0} if exists;'.format(bucket.name) filename = "file.txt" f = open(filename,'w') f.write(data) f.close() url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password os.system(cmd) os.remove(filename) super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, 'shell'): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() ############################################################################################## # # Setup Helpers ############################################################################################## def setup_analytics(self): data = 'use Default;' bucket_username = "******" bucket_password = "******" for bucket in self.buckets: data += 'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format( bucket.name, self.master.ip) data += 'create shadow dataset {1} on {0}; '.format(bucket.name, bucket.name + "_shadow") data += 'connect bucket {0} with {{"username":"******","password":"******"}};'.format( bucket.name, bucket_username, bucket_password) filename = "file.txt" f = open(filename,'w') f.write(data) f.close() url = 'http://{0}:8095/analytics/service'.format(self.cbas_node.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url + " -u " + bucket_username + ":" + bucket_password os.system(cmd) os.remove(filename) def run_active_requests(self, e, t): while not e.isSet(): logging.debug('wait_for_event_timeout starting') event_is_set = e.wait(t) logging.debug('event set: %s', event_is_set) if event_is_set: result = self.run_cbq_query("select * from system:active_requests") self.assertTrue(result['metrics']['resultCount'] == 1) requestId = result['requestID'] result = self.run_cbq_query( 'delete from system:active_requests where requestId = "%s"' % requestId) time.sleep(20) result = self.run_cbq_query( 'select * from system:active_requests where requestId = "%s"' % requestId) self.assertTrue(result['metrics']['resultCount'] == 0) result = self.run_cbq_query("select * from system:completed_requests") requestId = result['requestID'] result = self.run_cbq_query( 'delete from system:completed_requests where requestId = "%s"' % requestId) time.sleep(10) result = self.run_cbq_query( 'select * from system:completed_requests where requestId = "%s"' % requestId) self.assertTrue(result['metrics']['resultCount'] == 0) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] keys_num = int(re.sub(r'.*KEYS \$', '', subquery_template).replace('KEYS $', '')) subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load,keys=self._get_keys(keys_num)) subquery_template = re.sub(r'USE KEYS.*', '', subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as','', re.sub(r'FROM.*', '', alias)).strip() if not alias: alias = '$1' for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r',.*\$subquery\(.*\).*%s' % alias, ',%s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r'.*\$subquery\(', '', query_template) subquery_template = subquery_template[:subquery_template.rfind(')')] subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r',.*', '', re.sub(r'.*\$subquery\(.*\)', '', query_template)) alias = re.sub(r'.*as ', '', alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r'\$subquery\(.*\).*%s' % alias, ' %s' % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.iteritems(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query(query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue(str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" %( str(ex).split(':')[-1], error)) else: self.fail("There were no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client query_params = {} cred_params = {'creds': []} rest = RestConnection(server) username = rest.username password = rest.password cred_params['creds'].append({'user': username, 'pass': password}) for bucket in self.buckets: if bucket.saslPassword: cred_params['creds'].append({'user': '******' % bucket.name, 'pass': bucket.saslPassword}) query_params.update(cred_params) if self.use_rest: query_params.update({'scan_consistency': self.scan_consistency}) self.log.info('RUN QUERY %s' % query) if self.analytics: query = query + ";" for bucket in self.buckets: query = query.replace(bucket.name,bucket.name+"_shadow") result = RestConnection(self.cbas_node).execute_statement_on_cbas(query, "immediate") result = json.loads(result) else : result = rest.query_tool(query, self.n1ql_port, query_params=query_params) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbase/query/" +\ "shell/cbq/cbq ","","","","","","") else: os = self.shell.extract_remote_info().type.lower() if not(self.isprepared): query = query.replace('"', '\\"') query = query.replace('`', '\\`') cmd = "%s/cbq -engine=http://%s:%s/ -q -u %s -p %s" % (self.path, server.ip, server.port, username, password) output = self.shell.execute_commands_inside(cmd,query,"","","","","") if not(output[0] == '{'): output1 = '{'+str(output) else: output1 = output result = json.loads(output1) if isinstance(result, str) or 'errors' in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" %( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbase/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbase/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": if self.services_init.find('n1ql') != -1: return os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"] if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_sabre(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_sabre(docs_per_day, start) def generate_docs_employee(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_data(docs_per_day = docs_per_day, start = start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_simple_data(docs_per_day = docs_per_day, start = start) def generate_docs_sales(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_sales_data(docs_per_day = docs_per_day, start = start) def generate_docs_bigdata(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_bigdata(end=(1000*docs_per_day), start=start, value_size=self.value_size) def _verify_results(self, actual_result, expected_result, missing_count = 1, extra_count = 1): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra(actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) self.fail("Results are incorrect.Actual num %s. Expected num: %s.\n" % ( len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" self.assertTrue(actual_result == expected_result, msg % (actual_result[:100],actual_result[-100:], expected_result[:100],expected_result[-100:])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self): self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type) rest = RestConnection(self.master) versions = rest.get_nodes_versions() if versions[0].startswith("4") or versions[0].startswith("3") or versions[0].startswith("5"): for bucket in self.buckets: if self.primary_indx_drop: self.log.info("Dropping primary index for %s using %s ..." % (bucket.name,self.primary_indx_type)) self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name,self.primary_indx_type) #self.run_cbq_query() self.sleep(3, 'Sleep for some time after index drop') self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name res = self.run_cbq_query() self.sleep(10) if self.monitoring: self.query = "delete from system:completed_requests" self.run_cbq_query() if not self.skip_primary_index: if (res['metrics']['resultCount'] == 0): self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type) self.log.info("Creating primary index for %s ..." % bucket.name) try: self.run_cbq_query() self.primary_index_created = True if self.primary_indx_type.lower() == 'gsi': self._wait_for_index_online(bucket, '#primary') except Exception, ex: self.log.info(str(ex))
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == "suite_setUp": self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "sherlock") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection(self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == "suite_setUp" and self.input.param("cbq_version", "sherlock") != "sherlock": self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.analytics = self.input.param("analytics", False) self.dataset = self.input.param("dataset", "default") self.primary_indx_type = self.input.param("primary_indx_type", "GSI") self.index_type = self.input.param("index_type", "GSI") self.primary_indx_drop = self.input.param("primary_indx_drop", False) self.monitoring = self.input.param("monitoring", False) self.isprepared = False self.skip_primary_index = self.input.param("skip_primary_index", False) self.scan_consistency = self.input.param("scan_consistency", "REQUEST_PLUS") if self.primary_indx_type.lower() == "gsi": self.gsi_type = self.input.param("gsi_type", None) else: self.gsi_type = None if self.input.param("reload_data", False): for bucket in self.buckets: self.cluster.bucket_flush(self.master, bucket=bucket, timeout=self.wait_timeout * 5) self.gens_load = self.generate_docs(self.docs_per_day) self.load(self.gens_load, flag=self.item_flag) self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators(self.log, self.generate_full_docs_list(self.gens_load)) if self.analytics == False: self.create_primary_index_for_3_0_and_greater() if self.analytics: self.setup_analytics() self.sleep(30, "wait for analytics setup") def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) if not self.input.param("skip_build_tuq", True): self._build_tuq(self.master) self.skip_buckets_handle = True except: self.log.error("SUITE SETUP FAILED") self.tearDown() def tearDown(self): if self._testMethodName == "suite_tearDown": self.skip_buckets_handle = False if self.analytics: data = "use Default ;" + "\n" for bucket in self.buckets: data += "disconnect bucket {0} if connected;".format(bucket.name) + "\n" data += "drop dataset {0} if exists;".format(bucket.name + "_shadow") + "\n" data += "drop bucket {0} if exists;".format(bucket.name) + "\n" filename = "file.txt" f = open(filename, "w") f.write(data) f.close() url = "http://{0}:8095/analytics/service".format(self.master.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url os.system(cmd) os.remove(filename) super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, "shell"): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() def setup_analytics(self): # data = "" # for bucket in self.buckets: # data += 'disconnect bucket {0} ;'.format(bucket.name) + "\n" # data += 'connect bucket {0};'.format(bucket.name) + "\n" # filename = "file.txt" # f = open(filename,'w') # f.write(data) # f.close() # url = 'http://{0}:8095/analytics/service'.format(self.master.ip) # cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url # os.system(cmd) # os.remove(filename) data = "use Default;" + "\n" for bucket in self.buckets: data += ( 'create bucket {0} with {{"bucket":"{0}","nodes":"{1}"}} ;'.format(bucket.name, self.master.ip) + "\n" ) data += "create shadow dataset {1} on {0}; ".format(bucket.name, bucket.name + "_shadow") + "\n" data += "connect bucket {0} ;".format(bucket.name) + "\n" filename = "file.txt" f = open(filename, "w") f.write(data) f.close() url = "http://{0}:8095/analytics/service".format(self.master.ip) cmd = 'curl -s --data pretty=true --data-urlencode "*****@*****.**" ' + url os.system(cmd) os.remove(filename) ############################################################################################## # # SIMPLE CHECKS ############################################################################################## def test_simple_check(self): for bucket in self.buckets: if self.monitoring: e = threading.Event() t1 = threading.Thread(name="run_simple", target=self.run_active_requests, args=(e, 2)) t1.start() query = "select * from %s" % (bucket.name) self.run_cbq_query(query) logging.debug("event is set") if self.monitoring: e.set() t1.join(100) query_template = "FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_joins_monitoring(self): for bucket in self.buckets: e = threading.Event() if self.monitoring: e = threading.Event() t2 = threading.Thread(name="run_joins", target=self.run_active_requests, args=(e, 2)) t2.start() query = ( "select * from %s b1 inner join %s b2 on keys b1.CurrencyCode inner join %s b3 on keys b1.CurrencyCode left outer join %s b4 on keys b1.CurrencyCode" % (bucket.name, bucket.name, bucket.name, bucket.name) ) actual_result = self.run_cbq_query(query) logging.debug("event is set") if self.monitoring: e.set() t2.join(100) def run_active_requests(self, e, t): while not e.isSet(): logging.debug("wait_for_event_timeout starting") event_is_set = e.wait(t) logging.debug("event set: %s", event_is_set) if event_is_set: result = self.run_cbq_query("select * from system:active_requests") print result self.assertTrue(result["metrics"]["resultCount"] == 1) requestId = result["requestID"] result = self.run_cbq_query('delete from system:active_requests where RequestId = "%s"' % requestId) time.sleep(20) result = self.run_cbq_query( 'select * from system:active_requests where RequestId = "%s"' % requestId ) self.assertTrue(result["metrics"]["resultCount"] == 0) result = self.run_cbq_query("select * from system:completed_requests") print result requestId = result["requestID"] result = self.run_cbq_query( 'delete from system:completed_requests where RequestId = "%s"' % requestId ) time.sleep(10) result = self.run_cbq_query( 'select * from system:completed_requests where RequestId = "%s"' % requestId ) print result self.assertTrue(result["metrics"]["resultCount"] == 0) def test_simple_negative_check(self): queries_errors = { "SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3": "Aggregates not allowed in WHERE", "SELECT *.$str0 FROM {0}": "syntax error", "SELECT *.* FROM {0} ... ERROR": "syntax error", "FROM %s SELECT $str0 WHERE id=null": "syntax error", } self.negative_common_body(queries_errors) def test_unnest(self): for bucket in self.buckets: query_template = "SELECT emp.$int0, task FROM %s emp UNNEST emp.$nested_list_3l0 task" % bucket.name actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(sorted(actual_result["results"]), sorted(expected_result)) def test_subquery_select(self): for bucket in self.buckets: self.query = "SELECT $str0, $subquery(SELECT COUNT($str0) cn FROM %s d USE KEYS $5) as names FROM %s" % ( bucket.name, bucket.name, ) actual_result, expected_result = self.run_query_with_subquery_select_from_template(self.query) self._verify_results(actual_result["results"], expected_result) def test_subquery_from(self): for bucket in self.buckets: self.query = "SELECT tasks.$str0 FROM $subquery(SELECT $str0, $int0 FROM %s) as tasks" % (bucket.name) actual_result, expected_result = self.run_query_with_subquery_from_template(self.query) self._verify_results(actual_result["results"], expected_result) def test_consistent_simple_check(self): queries = [ self.gen_results.generate_query( "SELECT $str0, $int0, $int1 FROM %s " + "WHERE $str0 IS NOT NULL AND $int0<10 " + "OR $int1 = 6 ORDER BY $int0, $int1" ), self.gen_results.generate_query( "SELECT $str0, $int0, $int1 FROM %s " + "WHERE $int1 = 6 OR $str0 IS NOT NULL AND " + "$int0<10 ORDER BY $int0, $int1" ), ] for bucket in self.buckets: actual_result1 = self.run_cbq_query(queries[0] % bucket.name) actual_result2 = self.run_cbq_query(queries[1] % bucket.name) self.assertTrue( actual_result1["results"] == actual_result2["results"], "Results are inconsistent.Difference: %s %s %s %s" % ( len(actual_result1["results"]), len(actual_result2["results"]), actual_result1["results"][:100], actual_result2["results"][:100], ), ) def test_simple_nulls(self): queries = ['SELECT id FROM %s WHERE id=NULL or id="null"'] for bucket in self.buckets: if self.monitoring: e = threading.Event() t3 = threading.Thread(name="run_simple_nulls", target=self.run_active_requests, args=(e, 2)) t3.start() for query in queries: actual_result = self.run_cbq_query(query % (bucket.name)) logging.debug("event is set") if self.monitoring: e.set() t3.join(100) self._verify_results(actual_result["results"], []) ############################################################################################## # # LIMIT OFFSET CHECKS ############################################################################################## def test_limit_negative(self): # queries_errors = {'SELECT * FROM default LIMIT {0}' : ('Invalid LIMIT value 2.5', 5030)} queries_errors = {"SELECT ALL * FROM %s": ("syntax error", 3000)} self.negative_common_body(queries_errors) def test_limit_offset(self): for bucket in self.buckets: if self.monitoring: e = threading.Event() t4 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2)) t4.start() query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) if self.monitoring: e.set() t4.join(100) self._verify_results(actual_result["results"], expected_result) query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) def test_limit_offset_zero(self): for bucket in self.buckets: query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0" % (bucket.name) self.query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query() self.assertEquals( actual_result["results"], [], "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], []), ) query_template = "SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals( actual_result["results"], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result), ) def test_limit_offset_negative_check(self): queries_errors = { "SELECT DISTINCT $str0 FROM {0} LIMIT 1.1": "Invalid LIMIT value 1.1", "SELECT DISTINCT $str0 FROM {0} OFFSET 1.1": "Invalid OFFSET value 1.1", } self.negative_common_body(queries_errors) def test_limit_offset_sp_char_check(self): queries_errors = { "SELECT DISTINCT $str0 FROM {0} LIMIT ~": "syntax erro", "SELECT DISTINCT $str0 FROM {0} OFFSET ~": "syntax erro", } self.negative_common_body(queries_errors) ############################################################################################## # # ALIAS CHECKS ############################################################################################## def test_simple_alias(self): for bucket in self.buckets: if self.monitoring: e = threading.Event() t5 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2)) t5.start() query_template = "SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s" % (bucket.name) if self.analytics: query_template = "SELECT COUNT(`$str0`) AS COUNT_EMPLOYEE FROM %s" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self.assertEquals( actual_result["results"], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result), ) query_template = "SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) if self.monitoring: e.set() t5.join(100) expected_result = [{"COUNT_EMPLOYEE": expected_result[0]["COUNT_EMPLOYEE"] + 1}] self.assertEquals( actual_result["results"], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result["results"], expected_result), ) def test_simple_negative_alias(self): queries_errors = { "SELECT $str0._last_name as *": "syntax error", "SELECT $str0._last_name as DATABASE ?": "syntax error", "SELECT $str0 AS NULL FROM {0}": "syntax error", "SELECT $str1 as $str0, $str0 FROM {0}": "Duplicate result alias name", "SELECT test.$obj0 as points FROM {0} AS TEST " + "GROUP BY $obj0 AS GROUP_POINT": "syntax error", } self.negative_common_body(queries_errors) def test_alias_from_clause(self): queries_templates = [ "SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points", "SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0" + " ORDER BY points", "SELECT $obj0.$_obj0_int0 AS points FROM %s AS test " + "GROUP BY test.$obj0.$_obj0_int0 ORDER BY points", ] # if self.analytics: # queries_templates = ['SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY test.points', # 'SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\ # ' ORDER BY test.points', # 'SELECT test.$obj0.$_obj0_int0 AS points FROM %s AS test ' +\ # 'GROUP BY test.$obj0.$_obj0_int0 ORDER BY test.points'] for bucket in self.buckets: if self.monitoring: e = threading.Event() t6 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2)) t6.start() for query_template in queries_templates: actual_result, expected_result = self.run_query_from_template(query_template % (bucket.name)) if self.monitoring: e.set() t6.join(100) self._verify_results(actual_result["results"], expected_result) def test_alias_from_clause_group(self): for bucket in self.buckets: query_template = ( "SELECT $obj0.$_obj0_int0 AS points FROM %s AS test " % (bucket.name) + "GROUP BY $obj0.$_obj0_int0 ORDER BY points" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_alias_order_desc(self): for bucket in self.buckets: if self.monitoring: e = threading.Event() t7 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2)) t7.start() query_template = "SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) if self.monitoring: e.set() t7.join(100) self._verify_results(actual_result["results"], expected_result) def test_alias_order_asc(self): for bucket in self.buckets: query_template = "SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC" % (bucket.name) if self.analytics: query_template = "SELECT `$str0` AS name_new FROM %s AS test ORDER BY name_new ASC" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_alias_aggr_fn(self): for bucket in self.buckets: if self.monitoring: e = threading.Event() t8 = threading.Thread(name="run_limit_offset", target=self.run_active_requests, args=(e, 2)) t8.start() query_template = "SELECT COUNT(TEST.$str0) from %s AS TEST" % (bucket.name) if self.analytics: query_template = "SELECT COUNT(TEST.`$str0`) from %s AS TEST" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) if self.monitoring: e.set() t8.join(100) self._verify_results(actual_result["results"], expected_result) def test_alias_unnest(self): for bucket in self.buckets: query_template = "SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) query_template = "SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) ############################################################################################## # # ORDER BY CHECKS ############################################################################################## def test_order_by_check(self): for bucket in self.buckets: query_template = ( "SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s" % (bucket.name) + " ORDER BY $str1, $str0, $obj0.$_obj0_int0" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $obj0.$_obj0_int0, $str0, $str1" actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_order_by_alias(self): for bucket in self.buckets: query_template = ( "SELECT $str1, $obj0 AS points FROM %s" % (bucket.name) + " AS test ORDER BY $str1 DESC, points DESC" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_order_by_alias_arrays(self): for bucket in self.buckets: query_template = ( "SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s" % (bucket.name) + " AS TEST ORDER BY SKILL, $str1, TEST.$obj0" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_order_by_alias_aggr_fn(self): for bucket in self.buckets: query_template = ( "SELECT $int0, $int1, count(*) AS emp_per_month from %s" % (bucket.name) + " WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_order_by_aggr_fn(self): for bucket in self.buckets: query_template = ( "SELECT $str1 AS TITLE, min($int1) day FROM %s GROUP" % (bucket.name) + " BY $str1 ORDER BY MIN($int1), $str1" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) if self.analytics: self.query = ( "SELECT d.email AS TITLE, min(d.join_day) day FROM %s d GROUP" % (bucket.name) + " BY d.$str1 ORDER BY MIN(d.join_day), d.$str1" ) actual_result1 = self.run_cbq_query() self._verify_results(actual_result1["results"], actual_result["results"]) def test_order_by_precedence(self): for bucket in self.buckets: query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $str0, $str1" actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) query_template = "SELECT $str0, $str1 FROM %s" % (bucket.name) + " ORDER BY $str1, $str0" actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_order_by_satisfy(self): for bucket in self.buckets: query_template = ( "SELECT $str0, $list_obj0 FROM %s AS employee " % (bucket.name) + "WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND" + ' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0' ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) ############################################################################################## # # DISTINCT ############################################################################################## def test_distinct(self): for bucket in self.buckets: query_template = "SELECT DISTINCT $str1 FROM %s ORDER BY $str1" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_distinct_nested(self): for bucket in self.buckets: query_template = ( "SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s " % (bucket.name) + "ORDER BY $obj0.$_obj0_int0" ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) query_template = "SELECT DISTINCT $list_str0[0] as skill" + " FROM %s ORDER BY $list_str0[0]" % ( bucket.name ) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) self.query = "SELECT DISTINCT $obj0.* FROM %s" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) ############################################################################################## # # COMPLEX PATHS ############################################################################################## def test_simple_complex_paths(self): for bucket in self.buckets: query_template = "SELECT $_obj0_int0 FROM %s.$obj0" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_alias_complex_paths(self): for bucket in self.buckets: query_template = "SELECT $_obj0_int0 as new_attribute FROM %s.$obj0" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) def test_where_complex_paths(self): for bucket in self.buckets: query_template = "SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1" % (bucket.name) actual_result, expected_result = self.run_query_from_template(query_template) self._verify_results(actual_result["results"], expected_result) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_select_from_template(self, query_template): subquery_template = re.sub(r".*\$subquery\(", "", query_template) subquery_template = subquery_template[: subquery_template.rfind(")")] keys_num = int(re.sub(r".*KEYS \$", "", subquery_template).replace("KEYS $", "")) subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load, keys=self._get_keys(keys_num)) subquery_template = re.sub(r"USE KEYS.*", "", subquery_template) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template)) alias = re.sub(r".*as", "", re.sub(r"FROM.*", "", alias)).strip() if not alias: alias = "$1" for item in self.gen_results.full_set: item[alias] = expected_sub[0] query_template = re.sub(r",.*\$subquery\(.*\).*%s" % alias, ",%s" % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def run_query_with_subquery_from_template(self, query_template): subquery_template = re.sub(r".*\$subquery\(", "", query_template) subquery_template = subquery_template[: subquery_template.rfind(")")] subquery_full_list = self.generate_full_docs_list(gens_load=self.gens_load) sub_results = TuqGenerators(self.log, subquery_full_list) self.query = sub_results.generate_query(subquery_template) expected_sub = sub_results.generate_expected_result() alias = re.sub(r",.*", "", re.sub(r".*\$subquery\(.*\)", "", query_template)) alias = re.sub(r".*as ", "", alias).strip() self.gen_results = TuqGenerators(self.log, expected_sub) query_template = re.sub(r"\$subquery\(.*\).*%s" % alias, " %s" % alias, query_template) self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.iteritems(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query(query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue( str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" % (str(ex).split(":")[-1], error), ) else: self.fail("There were no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.n1ql_port == None or self.n1ql_port == "": self.n1ql_port = self.input.param("n1ql_port", 8093) if not self.n1ql_port: self.log.info(" n1ql_port is not defined, processing will not proceed further") raise Exception("n1ql_port is not defined, processing will not proceed further") query_params = {} cred_params = {"creds": []} for bucket in self.buckets: if bucket.saslPassword: cred_params["creds"].append({"user": "******" % bucket.name, "pass": bucket.saslPassword}) query_params.update(cred_params) if self.use_rest: query_params.update({"scan_consistency": self.scan_consistency}) self.log.info("RUN QUERY %s" % query) if self.analytics: query = query + ";" for bucket in self.buckets: query = query.replace(bucket.name, bucket.name + "_shadow") result = RestConnection(server).analytics_tool(query, 8095, query_params=query_params) else: result = RestConnection(server).query_tool(query, self.n1ql_port, query_params=query_params) else: if self.version == "git_repo": output = self.shell.execute_commands_inside( "$GOPATH/src/github.com/couchbase/query/" + "shell/cbq/cbq ", "", "", "", "", "", "" ) else: os = self.shell.extract_remote_info().type.lower() # if (query.find("VALUES") > 0): if not (self.isprepared): query = query.replace('"', '\\"') query = query.replace("`", "\\`") if os == "linux": cmd = "%s/cbq -engine=http://%s:8091/ -q" % (testconstants.LINUX_COUCHBASE_BIN_PATH, server.ip) elif os == "windows": cmd = "%s/cbq -q" % (testconstants.WIN_COUCHBASE_BIN_PATH) output = self.shell.execute_commands_inside(cmd, query, "", "", "", "", "") if not (output[0] == "{"): output1 = "{" + str(output) else: output1 = output result = json.loads(output1) # result = self._parse_query_output(output) if isinstance(result, str) or "errors" in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" % (version, info.architecture_type) # TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != "windows": goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd = ( "export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath) + " export PATH=$PATH:$GOROOT/bin && " + "go get github.com/couchbaselabs/tuqtng;" + "cd $GOPATH/src/github.com/couchbaselabs/tuqtng; " + "go get -d -v ./...; cd ." ) self.shell.execute_command(cmd) cmd = ( "export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath) + " export PATH=$PATH:$GOROOT/bin && " + "cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd ." ) self.shell.execute_command(cmd) cmd = ( "export GOROOT={0} && export GOPATH={1} &&".format(goroot, gopath) + " export PATH=$PATH:$GOROOT/bin && " + "cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd ." ) self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) # TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format(cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != "windows": gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == "windows": cmd = "cd %s/src/github.com/couchbase/query/server/main; " % ( gopath ) + "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" % (server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbase/query//server/main; " % ( gopath ) + "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" % (server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": if self.services_init.find("n1ql") != -1: return os = self.shell.extract_remote_info().type.lower() if os != "windows": couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client["sherlock_path"] print "PATH TO SHERLOCK: %s" % couchbase_path if os == "windows": cmd = "cd %s; " % (couchbase_path) + "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port, ) else: cmd = "cd %s; " % (couchbase_path) + "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" % ( server.ip, server.port, ) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % ( couchbase_path ) + './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' % ( server.ip, server.port, n1ql_port, ) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != "windows": cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % (server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port, ) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4 :].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11 :].strip() if output.find("cbq>") != -1: output = output[: output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[: output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, "generate_docs_" + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_sabre(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_sabre(docs_per_day, start) def generate_docs_employee(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_data(docs_per_day=docs_per_day, start=start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_simple_data(docs_per_day=docs_per_day, start=start) def generate_docs_sales(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_sales_data(docs_per_day=docs_per_day, start=start) def generate_docs_bigdata(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_bigdata(end=(1000 * docs_per_day), start=start, value_size=self.value_size) def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra(actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) self.fail( "Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result)) ) if self.max_verify is not None: actual_result = actual_result[: self.max_verify] expected_result = expected_result[: self.max_verify] msg = ( "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" + "Expected first and last 100: %s.\n ... \n %s" ) self.assertTrue( actual_result == expected_result, msg % (actual_result[:100], actual_result[-100:], expected_result[:100], expected_result[-100:]), ) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self): self.log.info("CREATE PRIMARY INDEX using %s" % self.primary_indx_type) rest = RestConnection(self.master) versions = rest.get_nodes_versions() if versions[0].startswith("4") or versions[0].startswith("3"): for bucket in self.buckets: if self.primary_indx_drop: self.log.info("Dropping primary index for %s using %s ..." % (bucket.name, self.primary_indx_type)) self.query = "DROP PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type) # self.run_cbq_query() self.sleep(3, "Sleep for some time after index drop") self.query = 'select * from system:indexes where name="#primary" and keyspace_id = "%s"' % bucket.name res = self.run_cbq_query() self.sleep(10) print res if self.monitoring: self.query = "delete from system:completed_requests" self.run_cbq_query() if not self.skip_primary_index: if res["metrics"]["resultCount"] == 0: self.query = "CREATE PRIMARY INDEX ON %s USING %s" % (bucket.name, self.primary_indx_type) self.log.info("Creating primary index for %s ..." % bucket.name) # if self.gsi_type: # self.query += " WITH {'index_type': 'memdb'}" try: self.run_cbq_query() self.primary_index_created = True if self.primary_indx_type.lower() == "gsi": self._wait_for_index_online(bucket, "#primary") except Exception, ex: self.log.info(str(ex)) if self.monitoring: self.query = "select * from system:active_requests" result = self.run_cbq_query() print result self.assertTrue(result["metrics"]["resultCount"] == 1) self.query = "select * from system:completed_requests" time.sleep(20) result = self.run_cbq_query() print result
class QueryTests(BaseTestCase): def setUp(self): if not self._testMethodName == 'suite_setUp': self.skip_buckets_handle = True super(QueryTests, self).setUp() self.version = self.input.param("cbq_version", "git_repo") if self.input.tuq_client and "client" in self.input.tuq_client: self.shell = RemoteMachineShellConnection( self.input.tuq_client["client"]) else: self.shell = RemoteMachineShellConnection(self.master) if not self._testMethodName == 'suite_setUp': self._start_command_line_query(self.master) self.use_rest = self.input.param("use_rest", True) self.max_verify = self.input.param("max_verify", None) self.buckets = RestConnection(self.master).get_buckets() self.docs_per_day = self.input.param("doc-per-day", 49) self.item_flag = self.input.param("item_flag", 4042322160) self.n1ql_port = self.input.param("n1ql_port", 8093) self.dataset = self.input.param("dataset", "default") self.gens_load = self.generate_docs(self.docs_per_day) if self.input.param("gomaxprocs", None): self.configure_gomaxprocs() self.gen_results = TuqGenerators( self.log, self.generate_full_docs_list(self.gens_load)) # temporary for MB-12848 self.create_primary_index_for_3_0_and_greater() def suite_setUp(self): try: self.load(self.gens_load, flag=self.item_flag) self.create_primary_index_for_3_0_and_greater() if not self.input.param("skip_build_tuq", True): self._build_tuq(self.master) self.skip_buckets_handle = True except: self.log.error('SUITE SETUP FAILED') self.tearDown() def tearDown(self): if self._testMethodName == 'suite_tearDown': self.skip_buckets_handle = False super(QueryTests, self).tearDown() def suite_tearDown(self): if not self.input.param("skip_build_tuq", False): if hasattr(self, 'shell'): self.shell.execute_command("killall /tmp/tuq/cbq-engine") self.shell.execute_command("killall tuqtng") self.shell.disconnect() ############################################################################################## # # SIMPLE CHECKS ############################################################################################## def test_simple_check(self): for bucket in self.buckets: query_template = 'FROM %s select $str0, $str1 ORDER BY $str0,$str1 ASC' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_simple_negative_check(self): queries_errors = { 'SELECT $str0 FROM {0} WHERE COUNT({0}.$str0)>3': 'Aggregates not allowed in WHERE', 'SELECT *.$str0 FROM {0}': 'syntax error', 'SELECT *.* FROM {0} ... ERROR': 'syntax error', 'FROM %s SELECT $str0 WHERE id=null': 'syntax error', } self.negative_common_body(queries_errors) def test_consistent_simple_check(self): queries = [self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $str0 IS NOT NULL AND $int0<10 ' +\ 'OR $int1 = 6 ORDER BY $int0, $int1'), self.gen_results.generate_query('SELECT $str0, $int0, $int1 FROM %s ' +\ 'WHERE $int1 = 6 OR $str0 IS NOT NULL AND ' +\ '$int0<10 ORDER BY $int0, $int1')] for bucket in self.buckets: actual_result1 = self.run_cbq_query(queries[0] % bucket.name) actual_result2 = self.run_cbq_query(queries[1] % bucket.name) self.assertTrue( actual_result1['results'] == actual_result2['results'], "Results are inconsistent.Difference: %s %s %s %s" % (len(actual_result1['results']), len( actual_result2['results']), actual_result1['results'][:100], actual_result2['results'][:100])) def test_simple_nulls(self): queries = ['SELECT id FROM %s WHERE id=NULL or id="null"'] for bucket in self.buckets: for query in queries: actual_result = self.run_cbq_query(query % (bucket.name)) self._verify_results(actual_result['results'], []) ############################################################################################## # # LIMIT OFFSET CHECKS ############################################################################################## def test_limit_offset(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 10' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) def test_limit_offset_zero(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 0' % ( bucket.name) self.query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query() self.assertEquals( actual_result['results'], [], "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result['results'], [])) query_template = 'SELECT DISTINCT $str0 FROM %s ORDER BY $str0 LIMIT 10 OFFSET 0' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self.assertEquals( actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result['results'], expected_result)) def test_limit_offset_negative_check(self): queries_errors = { 'SELECT DISTINCT $str0 FROM {0} LIMIT -1': 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} LIMIT 1.1': 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} OFFSET -1': 'Parse Error - syntax error', 'SELECT DISTINCT $str0 FROM {0} OFFSET 1.1': 'Parse Error - syntax error' } self.negative_common_body(queries_errors) ############################################################################################## # # ALIAS CHECKS ############################################################################################## def test_simple_alias(self): for bucket in self.buckets: query_template = 'SELECT COUNT($str0) AS COUNT_EMPLOYEE FROM %s' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self.assertEquals( actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result['results'], expected_result)) query_template = 'SELECT COUNT(*) + 1 AS COUNT_EMPLOYEE FROM %s' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) expected_result = [{ "COUNT_EMPLOYEE": expected_result[0]['COUNT_EMPLOYEE'] + 1 }] self.assertEquals( actual_result['results'], expected_result, "Results are incorrect.Actual %s.\n Expected: %s.\n" % (actual_result['results'], expected_result)) def test_simple_negative_alias(self): queries_errors = { 'SELECT $str0._last_name as *': 'syntax error', 'SELECT $str0._last_name as DATABASE ?': 'syntax error', 'SELECT $str0 AS NULL FROM {0}': 'syntax error', 'SELECT $str1 as $str0, $str0 FROM {0}': 'Duplicate result alias name', 'SELECT test.$obj0 as points FROM {0} AS TEST ' + 'GROUP BY $obj0 AS GROUP_POINT': 'syntax error' } self.negative_common_body(queries_errors) def test_alias_from_clause(self): queries_templates = ['SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ORDER BY points', 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test WHERE test.$int0 >0' +\ ' ORDER BY points', 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' +\ 'GROUP BY test.$obj0.$_obj0_int0 ORDER BY points'] for bucket in self.buckets: for query_template in queries_templates: actual_result, expected_result = self.run_query_from_template( query_template % (bucket.name)) self._verify_results(actual_result['results'], expected_result) def test_alias_from_clause_group(self): for bucket in self.buckets: query_template = 'SELECT $obj0.$_obj0_int0 AS points FROM %s AS test ' %(bucket.name) +\ 'GROUP BY $obj0.$_obj0_int0 ORDER BY points' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_order_desc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new DESC' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_order_asc(self): for bucket in self.buckets: query_template = 'SELECT $str0 AS name_new FROM %s AS test ORDER BY name_new ASC' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT COUNT(TEST.$str0) from %s AS TEST' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_unnest(self): for bucket in self.buckets: query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 AS skill' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT count(skill) FROM %s AS emp UNNEST emp.$list_str0 skill' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # ORDER BY CHECKS ############################################################################################## def test_order_by_check(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1, $obj0.$_obj0_int0 points FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0, $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $obj0.$_obj0_int0, $str0, $str1' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0 AS points FROM %s' % (bucket.name) +\ ' AS test ORDER BY $str1 DESC, points DESC' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias_arrays(self): for bucket in self.buckets: query_template = 'SELECT $str1, $obj0, $list_str0[0] AS SKILL FROM %s' % ( bucket.name) +\ ' AS TEST ORDER BY SKILL, $str1, TEST.$obj0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_alias_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $int0, $int1, count(*) AS emp_per_month from %s'% ( bucket.name) +\ ' WHERE $int1 >7 GROUP BY $int0, $int1 ORDER BY emp_per_month, $int1, $int0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_aggr_fn(self): for bucket in self.buckets: query_template = 'SELECT $str1 AS TITLE FROM %s GROUP' % (bucket.name) +\ ' BY $str1 ORDER BY MIN($int1), $str1' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_precedence(self): for bucket in self.buckets: query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str0, $str1' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT $str0, $str1 FROM %s' % (bucket.name) +\ ' ORDER BY $str1, $str0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_order_by_satisfy(self): for bucket in self.buckets: query_template = 'SELECT $str0, $list_obj0 FROM %s AS employee ' % (bucket.name) +\ 'WHERE ANY vm IN employee.$list_obj0 SATISFIES vm.$_list_obj0_int0 > 5 AND' +\ ' vm.$_list_obj0_str0 = "ubuntu" END ORDER BY $str0, $list_obj0[0].$_list_obj0_int0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # DISTINCT ############################################################################################## def test_distinct(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $str1 FROM %s ORDER BY $str1' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_distinct_nested(self): for bucket in self.buckets: query_template = 'SELECT DISTINCT $obj0.$_obj0_int0 as VAR FROM %s ' % (bucket.name) +\ 'ORDER BY $obj0.$_obj0_int0' actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) query_template = 'SELECT DISTINCT $list_str0[0] as skill' +\ ' FROM %s ORDER BY $list_str0[0]' % (bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) self.query = 'SELECT DISTINCT $obj0.* FROM %s' % (bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # COMPLEX PATHS ############################################################################################## def test_simple_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0' % (bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_alias_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 as new_attribute FROM %s.$obj0' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) def test_where_complex_paths(self): for bucket in self.buckets: query_template = 'SELECT $_obj0_int0 FROM %s.$obj0 WHERE $_obj0_int0 = 1' % ( bucket.name) actual_result, expected_result = self.run_query_from_template( query_template) self._verify_results(actual_result['results'], expected_result) ############################################################################################## # # COMMON FUNCTIONS ############################################################################################## def run_query_from_template(self, query_template): self.query = self.gen_results.generate_query(query_template) expected_result = self.gen_results.generate_expected_result() actual_result = self.run_cbq_query() return actual_result, expected_result def negative_common_body(self, queries_errors={}): if not queries_errors: self.fail("No queries to run!") for bucket in self.buckets: for query_template, error in queries_errors.iteritems(): try: query = self.gen_results.generate_query(query_template) actual_result = self.run_cbq_query( query.format(bucket.name)) except CBQError as ex: self.log.error(ex) self.assertTrue( str(ex).find(error) != -1, "Error is incorrect.Actual %s.\n Expected: %s.\n" % (str(ex).split(':')[-1], error)) else: self.fail("There was no errors. Error expected: %s" % error) def run_cbq_query(self, query=None, min_output_size=10, server=None): if query is None: query = self.query if server is None: server = self.master if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port else: if server.ip == "127.0.0.1": self.n1ql_port = server.n1ql_port if self.input.tuq_client and "client" in self.input.tuq_client: server = self.tuq_client if self.n1ql_port == None or self.n1ql_port == '': self.n1ql_port = self.input.param("n1ql_port", 8093) if not self.n1ql_port: self.log.info( " n1ql_port is not defined, processing will not proceed further" ) raise Exception( "n1ql_port is not defined, processing will not proceed further" ) if self.use_rest: result = RestConnection(server).query_tool(query, self.n1ql_port) else: if self.version == "git_repo": output = self.shell.execute_commands_inside("$GOPATH/src/github.com/couchbaselabs/tuqtng/" +\ "tuq_client/tuq_client " +\ "-engine=http://%s:8093/" % server.ip, subcommands=[query,], min_output_size=20, end_msg='tuq_client>') else: output = self.shell.execute_commands_inside( "/tmp/tuq/cbq -engine=http://%s:8093/" % server.ip, subcommands=[ query, ], min_output_size=20, end_msg='cbq>') result = self._parse_query_output(output) if isinstance(result, str) or 'errors' in result: raise CBQError(result, server.ip) self.log.info("TOTAL ELAPSED TIME: %s" % result["metrics"]["elapsedTime"]) return result def build_url(self, version): info = self.shell.extract_remote_info() type = info.distribution_type.lower() if type in ["ubuntu", "centos", "red hat"]: url = "https://s3.amazonaws.com/packages.couchbase.com/releases/couchbase-query/dp1/" url += "couchbase-query_%s_%s_linux.tar.gz" % ( version, info.architecture_type) #TODO for windows return url def _build_tuq(self, server): if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': goroot = testconstants.LINUX_GOROOT gopath = testconstants.LINUX_GOPATH else: goroot = testconstants.WINDOWS_GOROOT gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if self.input.tuq_client and "goroot" in self.input.tuq_client: goroot = self.input.tuq_client["goroot"] cmd = "rm -rf {0}/src/github.com".format(gopath) self.shell.execute_command(cmd) cmd= 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'go get github.com/couchbaselabs/tuqtng;' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; ' +\ 'go get -d -v ./...; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng; go build; cd .' self.shell.execute_command(cmd) cmd = 'export GOROOT={0} && export GOPATH={1} &&'.format(goroot, gopath) +\ ' export PATH=$PATH:$GOROOT/bin && ' +\ 'cd $GOPATH/src/github.com/couchbaselabs/tuqtng/tuq_client; go build; cd .' self.shell.execute_command(cmd) else: cbq_url = self.build_url(self.version) #TODO for windows cmd = "cd /tmp; mkdir tuq;cd tuq; wget {0} -O tuq.tar.gz;".format( cbq_url) cmd += "tar -xvf tuq.tar.gz;rm -rf tuq.tar.gz" self.shell.execute_command(cmd) def _start_command_line_query(self, server): self.shell.execute_command( "export NS_SERVER_CBAUTH_URL=\"http://{0}:{1}/_cbauth\"".format( server.ip, server.port)) self.shell.execute_command( "export NS_SERVER_CBAUTH_USER=\"{0}\"".format( server.rest_username)) self.shell.execute_command( "export NS_SERVER_CBAUTH_PWD=\"{0}\"".format(server.rest_password)) self.shell.execute_command( "export NS_SERVER_CBAUTH_RPC_URL=\"http://{0}:{1}/cbauth-demo\"". format(server.ip, server.port)) if self.version == "git_repo": os = self.shell.extract_remote_info().type.lower() if os != 'windows': gopath = testconstants.LINUX_GOPATH else: gopath = testconstants.WINDOWS_GOPATH if self.input.tuq_client and "gopath" in self.input.tuq_client: gopath = self.input.tuq_client["gopath"] if os == 'windows': cmd = "cd %s/src/github.com/couchbaselabs/query/server/main; " % (gopath) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s/src/github.com/couchbaselabs/query//server/main; " % (gopath) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) self.shell.execute_command(cmd) elif self.version == "sherlock": os = self.shell.extract_remote_info().type.lower() if os != 'windows': couchbase_path = testconstants.LINUX_COUCHBASE_BIN_PATH else: couchbase_path = testconstants.WIN_COUCHBASE_BIN_PATH if self.input.tuq_client and "sherlock_path" in self.input.tuq_client: couchbase_path = "%s/bin" % self.input.tuq_client[ "sherlock_path"] print "PATH TO SHERLOCK: %s" % couchbase_path if os == 'windows': cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine.exe -datastore http://%s:%s/ >/dev/null 2>&1 &" %( server.ip, server.port) else: cmd = "cd %s; " % (couchbase_path) +\ "./cbq-engine -datastore http://%s:%s/ >n1ql.log 2>&1 &" %( server.ip, server.port) n1ql_port = self.input.param("n1ql_port", None) if server.ip == "127.0.0.1" and server.n1ql_port: n1ql_port = server.n1ql_port if n1ql_port: cmd = "cd %s; " % (couchbase_path) +\ './cbq-engine -datastore http://%s:%s/ -http=":%s">n1ql.log 2>&1 &' %( server.ip, server.port, n1ql_port) self.shell.execute_command(cmd) else: os = self.shell.extract_remote_info().type.lower() if os != 'windows': cmd = "cd /tmp/tuq;./cbq-engine -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) else: cmd = "cd /cygdrive/c/tuq;./cbq-engine.exe -couchbase http://%s:%s/ >/dev/null 2>&1 &" % ( server.ip, server.port) self.shell.execute_command(cmd) def _parse_query_output(self, output): if output.find("cbq>") == 0: output = output[output.find("cbq>") + 4:].strip() if output.find("tuq_client>") == 0: output = output[output.find("tuq_client>") + 11:].strip() if output.find("cbq>") != -1: output = output[:output.find("cbq>")].strip() if output.find("tuq_client>") != -1: output = output[:output.find("tuq_client>")].strip() return json.loads(output) def generate_docs(self, num_items, start=0): try: return getattr(self, 'generate_docs_' + self.dataset)(num_items, start) except: self.fail("There is no dataset %s, please enter a valid one" % self.dataset) def generate_docs_default(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee(docs_per_day, start) def generate_docs_sabre(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_sabre(docs_per_day, start) def generate_docs_employee(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_data( docs_per_day=docs_per_day, start=start) def generate_docs_simple(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_simple_data( docs_per_day=docs_per_day, start=start) def generate_docs_sales(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_employee_sales_data( docs_per_day=docs_per_day, start=start) def generate_docs_bigdata(self, docs_per_day, start=0): json_generator = JsonGenerator() return json_generator.generate_docs_bigdata(docs_per_day=docs_per_day * 1000, start=start, value_size=self.value_size) def _verify_results(self, actual_result, expected_result, missing_count=1, extra_count=1): if len(actual_result) != len(expected_result): missing, extra = self.check_missing_and_extra( actual_result, expected_result) self.log.error("Missing items: %s.\n Extra items: %s" % (missing[:missing_count], extra[:extra_count])) self.fail( "Results are incorrect.Actual num %s. Expected num: %s.\n" % (len(actual_result), len(expected_result))) if self.max_verify is not None: actual_result = actual_result[:self.max_verify] expected_result = expected_result[:self.max_verify] msg = "Results are incorrect.\n Actual first and last 100: %s.\n ... \n %s" +\ "Expected first and last 100: %s.\n ... \n %s" self.assertTrue( actual_result == expected_result, msg % (actual_result[:100], actual_result[-100:], expected_result[:100], expected_result[-100:])) def check_missing_and_extra(self, actual, expected): missing = [] extra = [] for item in actual: if not (item in expected): extra.append(item) for item in expected: if not (item in actual): missing.append(item) return missing, extra def sort_nested_list(self, result): actual_result = [] for item in result: curr_item = {} for key, value in item.iteritems(): if isinstance(value, list) or isinstance(value, set): curr_item[key] = sorted(value) else: curr_item[key] = value actual_result.append(curr_item) return actual_result def configure_gomaxprocs(self): max_proc = self.input.param("gomaxprocs", None) cmd = "export GOMAXPROCS=%s" % max_proc for server in self.servers: shell_connection = RemoteMachineShellConnection(self.master) shell_connection.execute_command(cmd) def create_primary_index_for_3_0_and_greater(self): self.log.info("CHECK FOR PRIMARY INDEXES") rest = RestConnection(self.master) versions = rest.get_nodes_versions() ddoc_name = 'ddl_#primary' if versions[0].startswith("3"): try: rest.get_ddoc(self.buckets[0], ddoc_name) except ReadDocumentException: for bucket in self.buckets: self.log.info("Creating primary index for %s ..." % bucket.name) self.query = "CREATE PRIMARY INDEX ON %s " % (bucket.name) try: self.run_cbq_query() except Exception, ex: self.log.error('ERROR during index creation %s' % str(ex))