def test_gsi_with_crud_with_redaction_enabled(self): # load bucket and do some ops self.set_indexer_logLevel("trace") json_generator = JsonGenerator() gen_docs = json_generator.generate_all_type_documents_for_gsi( docs_per_day=self.doc_per_day, start=0) full_docs_list = self.generate_full_docs_list(gen_docs) n1ql_helper = N1QLHelper(use_rest=True, buckets=self.buckets, full_docs_list=full_docs_list, log=log, input=self.input, master=self.master) self.load(gen_docs) n1ql_node = self.get_nodes_from_services_map(service_type="n1ql") query_definition_generator = SQLDefinitionGenerator() query_definitions = query_definition_generator.generate_airlines_data_query_definitions( ) query_definitions = query_definition_generator.filter_by_group( "all", query_definitions) # set log redaction level, collect logs, verify log files exist and verify them for redaction self.set_redaction_level() self.start_logs_collection() # Create partial Index for query_definition in query_definitions: for bucket in self.buckets: create_query = query_definition.generate_index_create_query( bucket.name) n1ql_helper.run_cbq_query(query=create_query, server=n1ql_node) for query_definition in query_definitions: for bucket in self.buckets: scan_query = query_definition.generate_query( bucket=bucket.name) n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node) for query_definition in query_definitions: for bucket in self.buckets: drop_query = query_definition.generate_index_drop_query( bucket=bucket.name) n1ql_helper.run_cbq_query(query=drop_query, server=n1ql_node) result = self.monitor_logs_collection() log.info(result) try: logs_path = result["perNode"]["ns_1@" + str(self.master.ip)]["path"] except KeyError: logs_path = result["perNode"]["[email protected]"]["path"] redactFileName = logs_path.split('/')[-1] nonredactFileName = logs_path.split('/')[-1].replace('-redacted', '') remotepath = logs_path[0:logs_path.rfind('/') + 1] log_file = self.input.param("log_file_name", "indexer.log") self.verify_log_files_exist(remotepath=remotepath, redactFileName=redactFileName, nonredactFileName=nonredactFileName) self.verify_log_redaction(remotepath=remotepath, redactFileName=redactFileName, nonredactFileName=nonredactFileName, logFileName="ns_server.{0}".format(log_file))
def test_gsi_with_index_rebalance_redaction_enabled(self): # load bucket and do some ops self.set_indexer_logLevel("trace") json_generator = JsonGenerator() gen_docs = json_generator.generate_all_type_documents_for_gsi(docs_per_day=self.doc_per_day, start=0) full_docs_list = self.generate_full_docs_list(gen_docs) n1ql_helper = N1QLHelper(use_rest=True, buckets=self.buckets, full_docs_list=full_docs_list, log=log, input=self.input, master=self.master) self.load(gen_docs) n1ql_node = self.get_nodes_from_services_map(service_type="n1ql") query_definition_generator = SQLDefinitionGenerator() query_definitions = query_definition_generator.generate_airlines_data_query_definitions() query_definitions = query_definition_generator.filter_by_group("all", query_definitions) # set log redaction level, collect logs, verify log files exist and verify them for redaction self.set_redaction_level() self.start_logs_collection() # Create partial Index for query_definition in query_definitions: for bucket in self.buckets: create_query = query_definition.generate_index_create_query(bucket.name) n1ql_helper.run_cbq_query(query=create_query, server=n1ql_node) for query_definition in query_definitions: for bucket in self.buckets: scan_query = query_definition.generate_query(bucket=bucket.name) n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node) rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init], self.nodes_in_list, [], services=self.services_in) rebalance.result() self.sleep(30) for query_definition in query_definitions: for bucket in self.buckets: scan_query = query_definition.generate_query(bucket=bucket.name) n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node) for query_definition in query_definitions: for bucket in self.buckets: drop_query = query_definition.generate_index_drop_query(bucket=bucket.name) n1ql_helper.run_cbq_query(query=drop_query, server=n1ql_node) result = self.monitor_logs_collection() log.info(result) try: logs_path = result["perNode"]["ns_1@" + str(self.master.ip)]["path"] except KeyError: logs_path = result["perNode"]["[email protected]"]["path"] redactFileName = logs_path.split('/')[-1] nonredactFileName = logs_path.split('/')[-1].replace('-redacted', '') remotepath = logs_path[0:logs_path.rfind('/') + 1] log_file = self.input.param("log_file_name", "indexer.log") self.verify_log_files_exist(remotepath=remotepath, redactFileName=redactFileName, nonredactFileName=nonredactFileName) self.verify_log_redaction(remotepath=remotepath, redactFileName=redactFileName, nonredactFileName=nonredactFileName, logFileName="ns_server.{0}".format(log_file))
def generate_docs_array(self, num_items=10, start=0): json_generator = JsonGenerator() return json_generator.generate_all_type_documents_for_gsi( start=start, docs_per_day=num_items)
def setupForTest(self): # self.create_default_bucket() self.cbas_util.createConn("default") json_generator = JsonGenerator() generators = json_generator.generate_all_type_documents_for_gsi( docs_per_day=10, start=0) tasks = self._async_load_all_buckets(self.master, generators, "create", 0) for task in tasks: task.get_result() # # Create bucket on CBAS self.cbas_util.create_bucket_on_cbas( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_name="default", cb_server_ip=self.cb_server_ip) # Create dataset on the CBAS bucket self.cbas_util.create_dataset_on_bucket( cbas_bucket_name=self.cb_bucket_name, cbas_dataset_name=self.cbas_dataset_name) # Connect to Bucket self.cbas_util.connect_to_bucket( cbas_bucket_name=self.cbas_bucket_name, cb_bucket_password=self.cb_bucket_password) # Allow ingestion to complete self.cbas_util.wait_for_ingestion_complete([self.cbas_dataset_name], 10, 300) #load some data to allow incompatible comparisons. data_dict = { "name": [ 123456, [234234, 234234], None, { 'key': 'value' }, True, 12345.12345 ], "age": [ "String", [234234, 234234], None, { 'key': 'value' }, True, 12345.12345 ], "premium_customer": [ "String", 12345567, [234234, 234234, "string"], None, { 'key': 'value' }, 123456.123456, ], "travel_history": [ "String", 12345567, None, { 'key': 'value' }, 123456.123456, ], "address": [ "String", 12345567, [234234, 134234, "string"], None, 123456.123456, ] } self.client = SDKSmartClient(RestConnection(self.master), "default", self.master) i = 0 for key in data_dict.keys(): for value in data_dict[key]: # jsonDump = json.dumps({key:value}) self.client.set("incompatible_doc_%s" % i, 0, 0, {key: value}) i += 1 self.client.close()