def benchmark_mongo4(self): rc = read_concern.ReadConcern("majority") wc = write_concern.WriteConcern("majority") batch_size = 1000 print(f"Batch size: {batch_size}") with self.collection.database.client.start_session() as session: for i in range(int(self.operations / batch_size)): ops = random.choices([READ, READMOD], [50, 50], k=batch_size) with session.start_transaction(read_concern=rc, write_concern=wc): for op in ops: if op == READ: self.num_read += 1 self.collection.find_one( {"item": self.req_set.pop()}, session=session) elif op == READMOD: self.num_readmod += 1 self.collection.find_one_and_update( {"item": self.req_set.pop()}, { "$set": { "title": f"Updated at operation {i}" } }, session=session, ) return (f"π Number of reads: {self.num_read}\n" + f"βοΈ Number of read_modify_writes: {self.num_readmod}\n" + f"π {(self.num_read / self.operations) * 100}% reads")
class Meta: connection_alias = 'Application' collection_name = 'registeredApps' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('appId', name='appIdIndex', unique=True, sparse=True), IndexModel('appHash', name='appHashIndex', unique=True, sparse=True), IndexModel('appCrypt', name='appCryptIndex', unique=True, sparse=True), IndexModel('createdDate', name='createdDateIndex', sparse=True), IndexModel('validUntil', name='validUntilIndex', sparse=True), IndexModel('appManager.email', name='managerEmailIndex', sparse=True, unique=True) ] ignore_unknown_fields = True
def benchmark_mongo4(self): rc = read_concern.ReadConcern("majority") wc = write_concern.WriteConcern("majority") batch_size = 1000 print(f"Batch size: {batch_size}") with self.collection.database.client.start_session() as session: for i in range(int(self.operations / batch_size)): ops = random.choices([READ, INSERT], [95, 5], k=batch_size) with session.start_transaction(read_concern=rc, write_concern=wc): for op in ops: if op == READ: self.num_read += 1 self.collection.find_one( {"_id": self.read_id}, session=session ) elif op == INSERT: self.num_insert += 1 self.read_id = self.collection.insert_one( { "item": self.records + i, "qty": 100 + i, "tags": ["tag"], "title": "title", } ).inserted_id return ( f"π Number of reads: {self.num_read}\n" + f"βοΈ Number of inserts: {self.num_insert}\n" + f"π {(self.num_read / self.operations) * 100}% reads" )
class Meta: connection_alias = 'Posts' collection_name = 'global_analytics' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') ignore_unknown_fields = True
class Meta: connection_alias = 'Application' collection_name = 'appsMeta' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') ignore_unknown_fields = True
def get_database(runner): client = get_client(runner) if runner in [TRANSACTIONAL_MONGO]: wc = write_concern.WriteConcern("majority") rc = read_concern.ReadConcern("majority") return client.get_database("fdb-benchmark", write_concern=wc, read_concern=rc) else: return client.get_database("fdb-benchmark")
class Meta: connection_alias = 'Comments' collection_name = 'comment_ranks' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('userId', name='rankUserIdIndex', unique=True, sparse=True), IndexModel('commentId', name='rankCommentIdIndex', unique=True, sparse=True), IndexModel('rankType', name='rankTypeIndex', unique=True, sparse=True) ] ignore_unknown_fields = True
class Meta: connection_alias = 'Channels' collection_name = 'channels' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('channelUsername', name='channelUsernameIndex', unique=True, sparse=True), IndexModel('channelId', name='channelIdIndex', unique=True, sparse=True), IndexModel('channelCreator', name='channelCreatorIndex', sparse=True), IndexModel('channelAdmins', name='channelAdminsIndex', sparse=True), IndexModel('channelBot', name='channelBotIndex', sparse=True) ] ignore_unknown_fields = True
def benchmark_mongo4(self): rc = read_concern.ReadConcern("majority") wc = write_concern.WriteConcern("majority") batch_size = 1000 print(f"Batch size: {batch_size}") with self.collection.database.client.start_session() as session: for i in range(int(self.operations / batch_size)): ops = random.choices([READ, INSERT], [95, 5], k=batch_size) with session.start_transaction(read_concern=rc, write_concern=wc): inserts = [] for op in ops: if op == READ: scan_length = random.randint(1, 10) start_id = self.req_set.pop() if start_id + scan_length > self.records: start_id = start_id - ( (start_id + scan_length) - self.records ) self.num_read += 1 self.run_scan_length += scan_length list( self.collection.find( { "item": {"$gte": start_id} }, session=session, ).limit(scan_length) ) elif op == INSERT: self.num_insert += 1 inserts.append( { "item": self.records + i, "qty": 100 + i, "tags": ["tag"], "title": "title", } ) if inserts: self.collection.insert_many(inserts, session=session) return ( f"π Number of reads: {self.num_read}\n" + f"βοΈ Number of inserts: {self.num_insert}\n" + f"π {(self.num_read / self.operations) * 100}% reads\n" + f"Average scan length: {self.run_scan_length / self.num_read}" )
def benchmark_mongo4(self): rc = read_concern.ReadConcern("majority") wc = write_concern.WriteConcern("majority") batch_size = 1000 print(f"Batch size: {batch_size}") with self.collection.database.client.start_session() as session: for i in range(int(self.operations / batch_size)): with session.start_transaction(read_concern=rc, write_concern=wc): for j in range(batch_size): self.num_read += 1 self.collection.find_one({"item": self.req_set.pop()}, session=session) return (f"Γ°ΕΈββ Number of reads: {self.num_read}\n" + f"Γ°ΕΈβΕ½ {(self.num_read / self.operations) * 100}% reads")
class Meta: connection_alias = 'Posts' collection_name = 'reactions' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('userId', name='userReactionUserIdIndex', sparse=True), IndexModel('reactionIndex', name='userReactionReactionIndexIndex', sparse=True), IndexModel('reactionDate', name='userReactionDateIndex', sparse=True), IndexModel('postId', name='postIdIndex', sparse=True) ] ignore_unknown_fields = True
class Meta: connection_alias = 'Users' collection_name = 'bots' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('botUsername', name='botUsernameIndex', unique=True, sparse=True), IndexModel('userId', name='botIdIndex', unique=True, sparse=True), IndexModel('botToken', name='botTokenIndex', unique=True, sparse=True) ] ignore_unknown_fields = True
class Meta: connection_alias = 'Posts' collection_name = 'posts' cascade = True write_concern = wc.WriteConcern(j=True) read_preference = ReadPreference.NEAREST read_concern = rc.ReadConcern(level='majority') indexes = [ IndexModel('creator', name='postCreatorIndex', sparse=True), IndexModel('postId', name='postIdIndex', unique=True, sparse=True), IndexModel('groupHash', name='postGroupHashIndex', unique=True, sparse=True), IndexModel('messageId', name='postMessageIdIndex', sparse=True), IndexModel('channelId', name='postChannelIdIndex', sparse=True), IndexModel('createdDate', name='postCreatedDateIndex', sparse=True) ] ignore_unknown_fields = True
def __init__(self, params): """init configuration acording params""" self.id = params.get('id', None) or str(uuid4()) self.admin_added = False self.login = params.get('login', '') self.password = params.get('password', '') self.auth_key = params.get('auth_key', None) self.auth_source = params.get('authSource', 'admin') self._version = params.get('version') self._configsvrs = [] self._routers = [] self._shards = {} self.tags = {} self.sslParams = params.get('sslParams', {}) self.kwargs = {} self.restart_required = self.login or self.auth_key self.x509_extra_user = False if self.sslParams: self.kwargs.update(DEFAULT_SSL_OPTIONS) self.enable_ipv6 = common.ipv6_enabled_sharded(params) # Determine what to do with config servers via mongos version. mongos_name = os.path.join(Servers().bin_path(self._version), 'mongos') mongos = Server(name=mongos_name, procParams={}) mongos_version = mongos.version mongos.cleanup() configsvr_configs = params.get('configsvrs', [{}]) self.uses_rs_configdb = (mongos_version >= (3, 1, 2) and len(configsvr_configs) == 1) self.configdb_singleton = (ReplicaSets() if self.uses_rs_configdb else Servers()) if self.uses_rs_configdb: self.__init_configrs(configsvr_configs[0]) elif mongos_version >= (3, 3, 2): raise ShardedClusterError( 'mongos >= 3.3.2 requires the config database to be backed by ' 'a replica set.') elif mongos_version >= (3, 1, 2) and len(configsvr_configs) != 3: raise ShardedClusterError( "mongos >= 3.1.2 needs a config replica set or 3 old-style " "config servers.") else: self.__init_configsvrs(configsvr_configs) for r in params.get('routers', [{}]): self.router_add(r) for cfg in params.get('shards', []): shard_params = cfg.get('shardParams', {}) shard_tags = shard_params.pop('tags', None) info = self.member_add(cfg.get('id', None), shard_params) if shard_tags: self.tags[info['id']] = shard_tags # SERVER-37631 changed 3.6 sharded cluster setup so that it's required # to run refreshLogicalSessionCacheNow on the config server followed by # each mongos. Only then will each 3.6 mongos correctly report # logicalSessionTimeoutMinutes in its isMaster responses. if mongos_version[:2] == (3, 6): router_clients = self.router_connections() is_master = router_clients[0].admin.command('isMaster') if 'logicalSessionTimeoutMinutes' not in is_master: self.config_connection().admin.command( 'refreshLogicalSessionCacheNow') for client in router_clients: client.admin.command('refreshLogicalSessionCacheNow') if self.tags: for sh_id in self.tags: logger.debug('Add tags %r to %s' % (self.tags[sh_id], sh_id)) db = self.connection().get_database( 'config', write_concern=write_concern.WriteConcern(fsync=True)) db.shards.update( {'_id': sh_id}, {'$addToSet': { 'tags': { '$each': self.tags[sh_id] } }}) shard_configs = [ s.get('shardParams', {}).get('procParams', {}) for s in params.get('shards', []) ] if self.login: # Do we need to add an extra x509 user? def only_x509(config): set_params = config.get('setParameter', {}) auth_mechs = set_params.get('authenticationMechanisms', '') auth_mechs = auth_mechs.split(',') if len(auth_mechs) == 1 and auth_mechs[0] == 'MONGODB-X509': return True return False any_only_x509 = lambda l: any(map(only_x509, l)) rs_shard_configs = [ m.get('procParams', {}) for s in params.get('shards', []) for m in s.get('shardParams', {}).get('members', []) ] router_configs = params.get('routers', []) self.x509_extra_user = (any_only_x509(configsvr_configs) or any_only_x509(shard_configs) or any_only_x509(rs_shard_configs) or any_only_x509(router_configs)) self._add_users( self.connection().get_database( self.auth_source, write_concern=write_concern.WriteConcern(fsync=True)), mongos_version) # Secondary user given from request. secondary_login = { 'name': self.login, 'roles': self._user_roles(self.connection()) } if self.password: secondary_login['password'] = self.password if mongos_version >= (3, 7, 2): secondary_login['mechanisms'] = ['SCRAM-SHA-1'] # Do the same for the shards. for shard_id, config in zip(self._shards, shard_configs): shard = self._shards[shard_id] instance_id = shard['_id'] if shard.get('isServer'): client = Servers()._storage[instance_id].connection elif shard.get('isReplicaSet'): client = ReplicaSets()._storage[instance_id].connection() db = client[self.auth_source] if self.x509_extra_user: db.add_user(DEFAULT_SUBJECT, roles=self._user_roles(client)) if self.login: db.add_user(**secondary_login) if self.restart_required: # Do we need to add clusterAuthMode back? cluster_auth_mode = None for cfg in shard_configs: cam = cfg.get('clusterAuthMode') if cam: cluster_auth_mode = cam break def restart_with_auth(server_or_rs): server_or_rs.x509_extra_user = self.x509_extra_user server_or_rs.auth_source = self.auth_source server_or_rs.ssl_params = self.sslParams server_or_rs.login = self.login server_or_rs.password = self.password server_or_rs.auth_key = self.auth_key def add_auth(cfg): if self.auth_key: cfg['keyFile'] = self.key_file # Add clusterAuthMode back in. if cluster_auth_mode: cfg['clusterAuthMode'] = cam return cfg server_or_rs.restart(config_callback=add_auth) for config_id in self._configsvrs: restart_with_auth(self.configdb_singleton._storage[config_id]) for server_id in self._routers: server = Servers()._storage[server_id] restart_with_auth(server) for shard_id in self._shards: shard = self._shards[shard_id] instance_id = shard['_id'] klass = ReplicaSets if shard.get('isReplicaSet') else Servers instance = klass()._storage[instance_id] restart_with_auth(instance) self.restart_required = False
def getWrite(self,db,collection): db_ = self.client.get_database(db,write_concern=write_concern.WriteConcern()); collection_ = db_.get_collection(collection); return collection_;
def start(self): self.session.start_transaction(read_concern.ReadConcern('snapshot'), write_concern.WriteConcern('majority'))