def create_fts_index_n1ql(self):
logger.info("Creating FTS index")
definition = read_json(
self.test_config.index_settings.couchbase_fts_index_configfile)
bucket_name = self.test_config.buckets[0]
definition.update(
{'name': self.test_config.index_settings.couchbase_fts_index_name})
if self.test_config.collection.collection_map:
collection_map = self.test_config.collection.collection_map
definition["params"]["doc_config"][
"mode"] = "scope.collection.type_field"
scope_name = list(collection_map[bucket_name].keys())[1:][0]
collection_name = list(
collection_map[bucket_name][scope_name].keys())[0]
ind_type_mapping = \
copy.deepcopy(definition["params"]["mapping"]["default_mapping"])
definition["params"]["mapping"]["default_mapping"][
"enabled"] = False
new_type_mapping_name = "{}.{}".format(scope_name, collection_name)
definition["params"]["mapping"]["types"] = {
new_type_mapping_name: ind_type_mapping
}
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(
self.fts_nodes[0],
self.test_config.index_settings.couchbase_fts_index_name,
definition)
self.monitor.monitor_fts_indexing_queue(
self.fts_nodes[0],
self.test_config.index_settings.couchbase_fts_index_name,
int(self.test_config.access_settings.items))
def create_fts_indexes(self):
less_words = True
for bkt in self.test_config.buckets:
definition = read_json(self.access.couchbase_index_configfile)
if less_words:
name = "fts_less_words"
less_words = False
else:
name = "fts_more_words"
less_words = True
index_name = bkt + "-" + name
definition.update({
'name': index_name,
'sourceName': bkt,
})
mapping = definition["params"]["mapping"]["default_mapping"]
prop = definition["params"]["mapping"]["default_mapping"][
"properties"]
index = prop["fts_less_words"]
new_prop = {name: index}
mapping.update({'properties': new_prop})
index = \
definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0]
index.update({'name': name})
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node, index_name,
definition)
self.monitor.monitor_fts_indexing_queue(
self.fts_nodes[0], index_name,
int(self.test_config.access_settings.items * 0.95))
self.monitor.monitor_fts_index_persistence(self.fts_nodes,
index_name, bkt)
def create_index(self):
definition = read_json(self.test_config.fts_settings.index_configfile)
definition.update({
'name': self.index_name,
'sourceName': self.test_config.buckets[0],
})
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_host, self.index_name,
definition)
def create_index(self):
definition = read_json(self.access.couchbase_index_configfile)
definition.update({
'name': self.access.couchbase_index_name,
'sourceName': self.test_config.buckets[0],
})
if self.access.couchbase_index_type:
definition["params"]["store"]["indexType"] = self.access.couchbase_index_type
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node,
self.access.couchbase_index_name, definition)
def create_index(self):
definition = read_json(self.access.couchbase_index_configfile)
definition.update({
'name': self.access.couchbase_index_name,
'sourceName': self.test_config.buckets[0],
})
if self.access.couchbase_index_type:
definition["params"]["store"][
"indexType"] = self.access.couchbase_index_type
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node,
self.access.couchbase_index_name,
definition)
def create_fts_index_n1ql(self):
definition = read_json(self.test_config.index_settings.couchbase_fts_index_configfile)
definition.update({
'name': self.test_config.index_settings.couchbase_fts_index_name
})
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(
self.fts_nodes[0],
self.test_config.index_settings.couchbase_fts_index_name, definition)
self.monitor.monitor_fts_indexing_queue(
self.fts_nodes[0],
self.test_config.index_settings.couchbase_fts_index_name,
int(self.test_config.access_settings.items))
def create_fts_indexes(self):
less_words = True
for bkt in self.test_config.buckets:
definition = read_json(self.access.couchbase_index_configfile)
if less_words:
name = "fts_less_words"
less_words = False
else:
name = "fts_more_words"
less_words = True
index_name = bkt + "-" + name
definition.update({
'name': index_name,
'sourceName': bkt,
})
mapping = definition["params"]["mapping"]["default_mapping"]
prop = definition["params"]["mapping"]["default_mapping"]["properties"]
index = prop["fts_less_words"]
new_prop = {name: index}
mapping.update({
'properties': new_prop
})
index = \
definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0]
index.update({
'name': name
})
logger.info('Index definition: {}'.format(pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node,
index_name, definition)
self.monitor.monitor_fts_indexing_queue(self.fts_nodes[0],
index_name,
int(self.test_config.access_settings.items *
0.95))
self.monitor.monitor_fts_index_persistence(self.fts_nodes,
index_name, bkt)
def create_fts_index_definitions(self):
logger.info("In the Indexing function:")
# On a singlee fts index definition
index_def = read_json(self.access.couchbase_index_configfile)
collection_map = self.test_config.collection.collection_map
# this variable will hold the key valuesfor the custom type mapping
key_values = []
# index_type_mapping holds the type mapping part of the index
index_type_mapping = {}
index_id = 0
# on a single bucket
bucket_name = self.test_config.buckets[0]
index_def.update({
'sourceName': bucket_name,
})
if self.access.couchbase_index_type:
index_def["params"]["store"]["indexType"] = self.access.couchbase_index_type
# setting the type mapping for collection indexes
if collection_map:
index_def["params"]["doc_config"]["mode"] = "scope.collection.type_field"
# Geo queries have a slightly different index type with custom type mapping
if "types" in list(index_def["params"]["mapping"].keys()):
# if any custom type mapping is present
key_values = list(index_def["params"]["mapping"]["types"].keys())
default_index_type_mapping = {}
# for default collection settings
if collection_map and len(collection_map[bucket_name].keys()) == 1:
key_name = "{}.{}".format("_default", "_default")
if len(key_values) > 0:
# there is custom mapping
for type_name in key_values:
type_mapping_key_name = key_name + ".{}".format(type_name)
default_index_type_mapping[type_mapping_key_name] = \
index_def["params"]["mapping"]["types"][type_name]
else:
default_index_type_mapping[key_name] = index_type_mapping
index_def["params"]["mapping"]["default_mapping"]["enabled"] = False
# if collection map exists , the index is on collections
if collection_map and len(collection_map[bucket_name].keys()) > 1:
scope_names = list(collection_map[bucket_name].keys())[1:]
if "types" in list(index_def["params"]["mapping"].keys()):
# to get the custom mapping
temp = index_def["params"]["mapping"]["types"]
index_type_mapping = [temp[type_key] for type_key in key_values]
else:
index_type_mapping = \
copy.deepcopy(index_def["params"]["mapping"]["default_mapping"])
index_def["params"]["mapping"]["default_mapping"]["enabled"] = False
# on a single scope
collection_name_list = list(collection_map[bucket_name][scope_names[0]].keys())
collection_list = self.collection_split(collection_map, bucket_name, scope_names[0])
index_type_mapping_per_group = self.get_collection_index_def(collection_list,
scope_names[0],
index_type_mapping,
key_values
)
# calculating the doc per collection for persistence
num_docs_per_collection = \
self.test_config.load_settings.items // len(collection_name_list)
items_per_index = num_docs_per_collection * len(collection_list[0])
# as the test is on a single scope:
for coll_group_id, collection_type_mapping in enumerate(index_type_mapping_per_group):
for index_count in range(0, self.access.indexes_per_group):
index_name = "{}-{}".format(self.access.couchbase_index_name, index_id)
index_def.update({
'name': index_name,
})
index_def["params"]["mapping"]["types"] = collection_type_mapping
self.fts_index_map[index_name] = \
{
"bucket": bucket_name,
"scope": scope_names[0],
"collections": collection_list[coll_group_id],
"total_docs": items_per_index
}
self.fts_index_defs[index_name] = {
"index_def": index_def
}
index_id += 1
else:
# default , multiple indexes with the same index def
for num_indexes in range(0, self.access.indexes_per_group):
index_name = "{}-{}".format(self.access.couchbase_index_name, num_indexes)
index_def.update({
'name': index_name,
})
# for indexes with default collection and scope in the index
if collection_map and len(collection_map[bucket_name].keys()) == 1:
index_def["params"]["mapping"]["types"] = default_index_type_mapping
self.fts_index_map[index_name] = {
"bucket": bucket_name,
"scope": "_default",
"collections": ["_default"],
"total_docs": int(self.access.test_total_docs)
}
self.fts_index_defs[index_name] = {
"index_def": index_def
}
self.access.fts_index_map = self.fts_index_map
def create_indexes(self):
index_defs = self.access.couchbase_index_configfile.split(",")
index_defs_per_group = \
list(islice(cycle(index_defs), self.test_config.jts_access_settings.indexes_per_group))
if not self.test_config.collection.collection_map:
index_id = 0
for bucket in self.test_config.buckets:
for index_def in index_defs_per_group:
index_name = "{}-{}".format(
self.access.couchbase_index_name, index_id)
definition = read_json(index_def)
definition.update({
'name': index_name,
'sourceName': bucket,
})
if self.access.couchbase_index_type:
definition["params"]["store"]["indexType"] = \
self.access.couchbase_index_type
logger.info('Index definition: {}'.format(
pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node,
index_name, definition)
self.fts_index_map[index_name] = {
"bucket": bucket,
"scope": "_default",
"collections": ["_default"]
}
index_id += 1
else:
collection_map = self.test_config.collection.collection_map
index_id = 0
for bucket in self.test_config.buckets:
load_targets = []
for scope in collection_map[bucket].keys():
for collection in collection_map[bucket][scope].keys():
if collection_map[bucket][scope][collection][
'load'] == 1:
load_targets += [scope + "." + collection]
partitions = self.test_config.jts_access_settings.index_groups
if len(load_targets) >= partitions:
k, m = divmod(len(load_targets), partitions)
index_groups = (load_targets[i * k +
min(i, m):(i + 1) * k +
min(i + 1, m)]
for i in range(partitions))
else:
raise Exception(
"index group partitions must be <= number of collections"
)
for index_group in index_groups:
for index_def in index_defs_per_group:
index_name = "{}-{}".format(
self.access.couchbase_index_name, index_id)
definition = read_json(index_def)
definition.update({
'name': index_name,
'sourceName': bucket,
})
indexed_fld = copy.deepcopy(
definition["params"]["mapping"]["default_mapping"])
types_col = {}
definition["params"]["mapping"]["default_mapping"][
"enabled"] = False
definition["params"]["doc_config"][
"mode"] = "scope.collection.type_field"
for collection in index_group:
types_col[collection] = indexed_fld
definition["params"]["mapping"]["types"] = types_col
if self.access.couchbase_index_type:
definition["params"]["store"]["indexType"] = \
self.access.couchbase_index_type
logger.info('Index definition: {}'.format(
pretty_dict(definition)))
self.rest.create_fts_index(self.fts_master_node,
index_name, definition)
self.fts_index_map[index_name] = \
{"bucket": bucket,
"scope": index_group[0].split(".")[0],
"collections": [scope_collection.split(".")[1]
for scope_collection in index_group]}
index_id += 1
self.access.fts_index_map = self.fts_index_map
def run_and_get_stats(self, cmd: str) -> dict:
self.remote.run_magma_benchmark(cmd, self.stats_file)
data = read_json(self.stats_file)
logger.info("\nStats: {}".format(pretty_dict(data)))
return data
def create_index(self):
definition = read_json(self.test_config.fts_settings.index_configfile)
self.rest.create_elastic_index(self.master_node, self.index_name,
definition)