def test_gsi_with_crud_with_redaction_enabled(self):
# load bucket and do some ops
self.set_indexer_logLevel("trace")
json_generator = JsonGenerator()
gen_docs = json_generator.generate_all_type_documents_for_gsi(
docs_per_day=self.doc_per_day, start=0)
full_docs_list = self.generate_full_docs_list(gen_docs)
n1ql_helper = N1QLHelper(use_rest=True,
buckets=self.buckets,
full_docs_list=full_docs_list,
log=log,
input=self.input,
master=self.master)
self.load(gen_docs)
n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
query_definition_generator = SQLDefinitionGenerator()
query_definitions = query_definition_generator.generate_airlines_data_query_definitions(
)
query_definitions = query_definition_generator.filter_by_group(
"all", query_definitions)
# set log redaction level, collect logs, verify log files exist and verify them for redaction
self.set_redaction_level()
self.start_logs_collection()
# Create partial Index
for query_definition in query_definitions:
for bucket in self.buckets:
create_query = query_definition.generate_index_create_query(
bucket.name)
n1ql_helper.run_cbq_query(query=create_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
scan_query = query_definition.generate_query(
bucket=bucket.name)
n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
drop_query = query_definition.generate_index_drop_query(
bucket=bucket.name)
n1ql_helper.run_cbq_query(query=drop_query, server=n1ql_node)
result = self.monitor_logs_collection()
log.info(result)
try:
logs_path = result["perNode"]["ns_1@" +
str(self.master.ip)]["path"]
except KeyError:
logs_path = result["perNode"]["[email protected]"]["path"]
redactFileName = logs_path.split('/')[-1]
nonredactFileName = logs_path.split('/')[-1].replace('-redacted', '')
remotepath = logs_path[0:logs_path.rfind('/') + 1]
log_file = self.input.param("log_file_name", "indexer.log")
self.verify_log_files_exist(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName)
self.verify_log_redaction(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName,
logFileName="ns_server.{0}".format(log_file))
def test_gsi_with_index_rebalance_redaction_enabled(self):
# load bucket and do some ops
self.set_indexer_logLevel("trace")
json_generator = JsonGenerator()
gen_docs = json_generator.generate_all_type_documents_for_gsi(docs_per_day=self.doc_per_day, start=0)
full_docs_list = self.generate_full_docs_list(gen_docs)
n1ql_helper = N1QLHelper(use_rest=True, buckets=self.buckets, full_docs_list=full_docs_list,
log=log, input=self.input, master=self.master)
self.load(gen_docs)
n1ql_node = self.get_nodes_from_services_map(service_type="n1ql")
query_definition_generator = SQLDefinitionGenerator()
query_definitions = query_definition_generator.generate_airlines_data_query_definitions()
query_definitions = query_definition_generator.filter_by_group("all", query_definitions)
# set log redaction level, collect logs, verify log files exist and verify them for redaction
self.set_redaction_level()
self.start_logs_collection()
# Create partial Index
for query_definition in query_definitions:
for bucket in self.buckets:
create_query = query_definition.generate_index_create_query(bucket.name)
n1ql_helper.run_cbq_query(query=create_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
scan_query = query_definition.generate_query(bucket=bucket.name)
n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node)
rebalance = self.cluster.async_rebalance(self.servers[:self.nodes_init], self.nodes_in_list,
[], services=self.services_in)
rebalance.result()
self.sleep(30)
for query_definition in query_definitions:
for bucket in self.buckets:
scan_query = query_definition.generate_query(bucket=bucket.name)
n1ql_helper.run_cbq_query(query=scan_query, server=n1ql_node)
for query_definition in query_definitions:
for bucket in self.buckets:
drop_query = query_definition.generate_index_drop_query(bucket=bucket.name)
n1ql_helper.run_cbq_query(query=drop_query, server=n1ql_node)
result = self.monitor_logs_collection()
log.info(result)
try:
logs_path = result["perNode"]["ns_1@" + str(self.master.ip)]["path"]
except KeyError:
logs_path = result["perNode"]["[email protected]"]["path"]
redactFileName = logs_path.split('/')[-1]
nonredactFileName = logs_path.split('/')[-1].replace('-redacted', '')
remotepath = logs_path[0:logs_path.rfind('/') + 1]
log_file = self.input.param("log_file_name", "indexer.log")
self.verify_log_files_exist(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName)
self.verify_log_redaction(remotepath=remotepath,
redactFileName=redactFileName,
nonredactFileName=nonredactFileName,
logFileName="ns_server.{0}".format(log_file))
def generate_docs_array(self, num_items=10, start=0):
json_generator = JsonGenerator()
return json_generator.generate_all_type_documents_for_gsi(
start=start, docs_per_day=num_items)
def setupForTest(self):
# self.create_default_bucket()
self.cbas_util.createConn("default")
json_generator = JsonGenerator()
generators = json_generator.generate_all_type_documents_for_gsi(
docs_per_day=10, start=0)
tasks = self._async_load_all_buckets(self.master, generators, "create",
0)
for task in tasks:
task.get_result()
# # Create bucket on CBAS
self.cbas_util.create_bucket_on_cbas(
cbas_bucket_name=self.cbas_bucket_name,
cb_bucket_name="default",
cb_server_ip=self.cb_server_ip)
# Create dataset on the CBAS bucket
self.cbas_util.create_dataset_on_bucket(
cbas_bucket_name=self.cb_bucket_name,
cbas_dataset_name=self.cbas_dataset_name)
# Connect to Bucket
self.cbas_util.connect_to_bucket(
cbas_bucket_name=self.cbas_bucket_name,
cb_bucket_password=self.cb_bucket_password)
# Allow ingestion to complete
self.cbas_util.wait_for_ingestion_complete([self.cbas_dataset_name],
10, 300)
#load some data to allow incompatible comparisons.
data_dict = {
"name": [
123456, [234234, 234234], None, {
'key': 'value'
}, True, 12345.12345
],
"age": [
"String", [234234, 234234], None, {
'key': 'value'
}, True, 12345.12345
],
"premium_customer": [
"String",
12345567,
[234234, 234234, "string"],
None,
{
'key': 'value'
},
123456.123456,
],
"travel_history": [
"String",
12345567,
None,
{
'key': 'value'
},
123456.123456,
],
"address": [
"String",
12345567,
[234234, 134234, "string"],
None,
123456.123456,
]
}
self.client = SDKSmartClient(RestConnection(self.master), "default",
self.master)
i = 0
for key in data_dict.keys():
for value in data_dict[key]:
# jsonDump = json.dumps({key:value})
self.client.set("incompatible_doc_%s" % i, 0, 0, {key: value})
i += 1
self.client.close()
