def create_fts_index_n1ql(self): logger.info("Creating FTS index") definition = read_json( self.test_config.index_settings.couchbase_fts_index_configfile) bucket_name = self.test_config.buckets[0] definition.update( {'name': self.test_config.index_settings.couchbase_fts_index_name}) if self.test_config.collection.collection_map: collection_map = self.test_config.collection.collection_map definition["params"]["doc_config"][ "mode"] = "scope.collection.type_field" scope_name = list(collection_map[bucket_name].keys())[1:][0] collection_name = list( collection_map[bucket_name][scope_name].keys())[0] ind_type_mapping = \ copy.deepcopy(definition["params"]["mapping"]["default_mapping"]) definition["params"]["mapping"]["default_mapping"][ "enabled"] = False new_type_mapping_name = "{}.{}".format(scope_name, collection_name) definition["params"]["mapping"]["types"] = { new_type_mapping_name: ind_type_mapping } logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, int(self.test_config.access_settings.items))
def create_fts_indexes(self): less_words = True for bkt in self.test_config.buckets: definition = read_json(self.access.couchbase_index_configfile) if less_words: name = "fts_less_words" less_words = False else: name = "fts_more_words" less_words = True index_name = bkt + "-" + name definition.update({ 'name': index_name, 'sourceName': bkt, }) mapping = definition["params"]["mapping"]["default_mapping"] prop = definition["params"]["mapping"]["default_mapping"][ "properties"] index = prop["fts_less_words"] new_prop = {name: index} mapping.update({'properties': new_prop}) index = \ definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0] index.update({'name': name}) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], index_name, int(self.test_config.access_settings.items * 0.95)) self.monitor.monitor_fts_index_persistence(self.fts_nodes, index_name, bkt)
def create_index(self): definition = read_json(self.test_config.fts_settings.index_configfile) definition.update({ 'name': self.index_name, 'sourceName': self.test_config.buckets[0], }) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_host, self.index_name, definition)
def create_index(self): definition = read_json(self.access.couchbase_index_configfile) definition.update({ 'name': self.access.couchbase_index_name, 'sourceName': self.test_config.buckets[0], }) if self.access.couchbase_index_type: definition["params"]["store"]["indexType"] = self.access.couchbase_index_type logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, self.access.couchbase_index_name, definition)
def create_index(self): definition = read_json(self.access.couchbase_index_configfile) definition.update({ 'name': self.access.couchbase_index_name, 'sourceName': self.test_config.buckets[0], }) if self.access.couchbase_index_type: definition["params"]["store"][ "indexType"] = self.access.couchbase_index_type logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, self.access.couchbase_index_name, definition)
def create_fts_index_n1ql(self): definition = read_json(self.test_config.index_settings.couchbase_fts_index_configfile) definition.update({ 'name': self.test_config.index_settings.couchbase_fts_index_name }) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, definition) self.monitor.monitor_fts_indexing_queue( self.fts_nodes[0], self.test_config.index_settings.couchbase_fts_index_name, int(self.test_config.access_settings.items))
def create_fts_indexes(self): less_words = True for bkt in self.test_config.buckets: definition = read_json(self.access.couchbase_index_configfile) if less_words: name = "fts_less_words" less_words = False else: name = "fts_more_words" less_words = True index_name = bkt + "-" + name definition.update({ 'name': index_name, 'sourceName': bkt, }) mapping = definition["params"]["mapping"]["default_mapping"] prop = definition["params"]["mapping"]["default_mapping"]["properties"] index = prop["fts_less_words"] new_prop = {name: index} mapping.update({ 'properties': new_prop }) index = \ definition["params"]["mapping"]["default_mapping"]["properties"][name]["fields"][0] index.update({ 'name': name }) logger.info('Index definition: {}'.format(pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.monitor.monitor_fts_indexing_queue(self.fts_nodes[0], index_name, int(self.test_config.access_settings.items * 0.95)) self.monitor.monitor_fts_index_persistence(self.fts_nodes, index_name, bkt)
def create_fts_index_definitions(self): logger.info("In the Indexing function:") # On a singlee fts index definition index_def = read_json(self.access.couchbase_index_configfile) collection_map = self.test_config.collection.collection_map # this variable will hold the key valuesfor the custom type mapping key_values = [] # index_type_mapping holds the type mapping part of the index index_type_mapping = {} index_id = 0 # on a single bucket bucket_name = self.test_config.buckets[0] index_def.update({ 'sourceName': bucket_name, }) if self.access.couchbase_index_type: index_def["params"]["store"]["indexType"] = self.access.couchbase_index_type # setting the type mapping for collection indexes if collection_map: index_def["params"]["doc_config"]["mode"] = "scope.collection.type_field" # Geo queries have a slightly different index type with custom type mapping if "types" in list(index_def["params"]["mapping"].keys()): # if any custom type mapping is present key_values = list(index_def["params"]["mapping"]["types"].keys()) default_index_type_mapping = {} # for default collection settings if collection_map and len(collection_map[bucket_name].keys()) == 1: key_name = "{}.{}".format("_default", "_default") if len(key_values) > 0: # there is custom mapping for type_name in key_values: type_mapping_key_name = key_name + ".{}".format(type_name) default_index_type_mapping[type_mapping_key_name] = \ index_def["params"]["mapping"]["types"][type_name] else: default_index_type_mapping[key_name] = index_type_mapping index_def["params"]["mapping"]["default_mapping"]["enabled"] = False # if collection map exists , the index is on collections if collection_map and len(collection_map[bucket_name].keys()) > 1: scope_names = list(collection_map[bucket_name].keys())[1:] if "types" in list(index_def["params"]["mapping"].keys()): # to get the custom mapping temp = index_def["params"]["mapping"]["types"] index_type_mapping = [temp[type_key] for type_key in key_values] else: index_type_mapping = \ copy.deepcopy(index_def["params"]["mapping"]["default_mapping"]) index_def["params"]["mapping"]["default_mapping"]["enabled"] = False # on a single scope collection_name_list = list(collection_map[bucket_name][scope_names[0]].keys()) collection_list = self.collection_split(collection_map, bucket_name, scope_names[0]) index_type_mapping_per_group = self.get_collection_index_def(collection_list, scope_names[0], index_type_mapping, key_values ) # calculating the doc per collection for persistence num_docs_per_collection = \ self.test_config.load_settings.items // len(collection_name_list) items_per_index = num_docs_per_collection * len(collection_list[0]) # as the test is on a single scope: for coll_group_id, collection_type_mapping in enumerate(index_type_mapping_per_group): for index_count in range(0, self.access.indexes_per_group): index_name = "{}-{}".format(self.access.couchbase_index_name, index_id) index_def.update({ 'name': index_name, }) index_def["params"]["mapping"]["types"] = collection_type_mapping self.fts_index_map[index_name] = \ { "bucket": bucket_name, "scope": scope_names[0], "collections": collection_list[coll_group_id], "total_docs": items_per_index } self.fts_index_defs[index_name] = { "index_def": index_def } index_id += 1 else: # default , multiple indexes with the same index def for num_indexes in range(0, self.access.indexes_per_group): index_name = "{}-{}".format(self.access.couchbase_index_name, num_indexes) index_def.update({ 'name': index_name, }) # for indexes with default collection and scope in the index if collection_map and len(collection_map[bucket_name].keys()) == 1: index_def["params"]["mapping"]["types"] = default_index_type_mapping self.fts_index_map[index_name] = { "bucket": bucket_name, "scope": "_default", "collections": ["_default"], "total_docs": int(self.access.test_total_docs) } self.fts_index_defs[index_name] = { "index_def": index_def } self.access.fts_index_map = self.fts_index_map
def create_indexes(self): index_defs = self.access.couchbase_index_configfile.split(",") index_defs_per_group = \ list(islice(cycle(index_defs), self.test_config.jts_access_settings.indexes_per_group)) if not self.test_config.collection.collection_map: index_id = 0 for bucket in self.test_config.buckets: for index_def in index_defs_per_group: index_name = "{}-{}".format( self.access.couchbase_index_name, index_id) definition = read_json(index_def) definition.update({ 'name': index_name, 'sourceName': bucket, }) if self.access.couchbase_index_type: definition["params"]["store"]["indexType"] = \ self.access.couchbase_index_type logger.info('Index definition: {}'.format( pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.fts_index_map[index_name] = { "bucket": bucket, "scope": "_default", "collections": ["_default"] } index_id += 1 else: collection_map = self.test_config.collection.collection_map index_id = 0 for bucket in self.test_config.buckets: load_targets = [] for scope in collection_map[bucket].keys(): for collection in collection_map[bucket][scope].keys(): if collection_map[bucket][scope][collection][ 'load'] == 1: load_targets += [scope + "." + collection] partitions = self.test_config.jts_access_settings.index_groups if len(load_targets) >= partitions: k, m = divmod(len(load_targets), partitions) index_groups = (load_targets[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(partitions)) else: raise Exception( "index group partitions must be <= number of collections" ) for index_group in index_groups: for index_def in index_defs_per_group: index_name = "{}-{}".format( self.access.couchbase_index_name, index_id) definition = read_json(index_def) definition.update({ 'name': index_name, 'sourceName': bucket, }) indexed_fld = copy.deepcopy( definition["params"]["mapping"]["default_mapping"]) types_col = {} definition["params"]["mapping"]["default_mapping"][ "enabled"] = False definition["params"]["doc_config"][ "mode"] = "scope.collection.type_field" for collection in index_group: types_col[collection] = indexed_fld definition["params"]["mapping"]["types"] = types_col if self.access.couchbase_index_type: definition["params"]["store"]["indexType"] = \ self.access.couchbase_index_type logger.info('Index definition: {}'.format( pretty_dict(definition))) self.rest.create_fts_index(self.fts_master_node, index_name, definition) self.fts_index_map[index_name] = \ {"bucket": bucket, "scope": index_group[0].split(".")[0], "collections": [scope_collection.split(".")[1] for scope_collection in index_group]} index_id += 1 self.access.fts_index_map = self.fts_index_map
def run_and_get_stats(self, cmd: str) -> dict: self.remote.run_magma_benchmark(cmd, self.stats_file) data = read_json(self.stats_file) logger.info("\nStats: {}".format(pretty_dict(data))) return data
def create_index(self): definition = read_json(self.test_config.fts_settings.index_configfile) self.rest.create_elastic_index(self.master_node, self.index_name, definition)