def benchmark_mongo4(self):
rc = read_concern.ReadConcern("majority")
wc = write_concern.WriteConcern("majority")
batch_size = 1000
print(f"Batch size: {batch_size}")
with self.collection.database.client.start_session() as session:
for i in range(int(self.operations / batch_size)):
ops = random.choices([READ, READMOD], [50, 50], k=batch_size)
with session.start_transaction(read_concern=rc,
write_concern=wc):
for op in ops:
if op == READ:
self.num_read += 1
self.collection.find_one(
{"item": self.req_set.pop()}, session=session)
elif op == READMOD:
self.num_readmod += 1
self.collection.find_one_and_update(
{"item": self.req_set.pop()},
{
"$set": {
"title": f"Updated at operation {i}"
}
},
session=session,
)
return (f"📖 Number of reads: {self.num_read}\n" +
f"✍️ Number of read_modify_writes: {self.num_readmod}\n" +
f"🔎 {(self.num_read / self.operations) * 100}% reads")
class Meta:
connection_alias = 'Application'
collection_name = 'registeredApps'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('appId', name='appIdIndex', unique=True, sparse=True),
IndexModel('appHash',
name='appHashIndex',
unique=True,
sparse=True),
IndexModel('appCrypt',
name='appCryptIndex',
unique=True,
sparse=True),
IndexModel('createdDate', name='createdDateIndex', sparse=True),
IndexModel('validUntil', name='validUntilIndex', sparse=True),
IndexModel('appManager.email',
name='managerEmailIndex',
sparse=True,
unique=True)
]
ignore_unknown_fields = True
def benchmark_mongo4(self):
rc = read_concern.ReadConcern("majority")
wc = write_concern.WriteConcern("majority")
batch_size = 1000
print(f"Batch size: {batch_size}")
with self.collection.database.client.start_session() as session:
for i in range(int(self.operations / batch_size)):
ops = random.choices([READ, INSERT], [95, 5], k=batch_size)
with session.start_transaction(read_concern=rc, write_concern=wc):
for op in ops:
if op == READ:
self.num_read += 1
self.collection.find_one(
{"_id": self.read_id}, session=session
)
elif op == INSERT:
self.num_insert += 1
self.read_id = self.collection.insert_one(
{
"item": self.records + i,
"qty": 100 + i,
"tags": ["tag"],
"title": "title",
}
).inserted_id
return (
f"📖 Number of reads: {self.num_read}\n"
+ f"✍️ Number of inserts: {self.num_insert}\n"
+ f"🔎 {(self.num_read / self.operations) * 100}% reads"
)
class Meta:
connection_alias = 'Posts'
collection_name = 'global_analytics'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
ignore_unknown_fields = True
class Meta:
connection_alias = 'Application'
collection_name = 'appsMeta'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
ignore_unknown_fields = True
def get_database(runner):
client = get_client(runner)
if runner in [TRANSACTIONAL_MONGO]:
wc = write_concern.WriteConcern("majority")
rc = read_concern.ReadConcern("majority")
return client.get_database("fdb-benchmark", write_concern=wc, read_concern=rc)
else:
return client.get_database("fdb-benchmark")
class Meta:
connection_alias = 'Comments'
collection_name = 'comment_ranks'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('userId', name='rankUserIdIndex', unique=True, sparse=True),
IndexModel('commentId', name='rankCommentIdIndex', unique=True, sparse=True),
IndexModel('rankType', name='rankTypeIndex', unique=True, sparse=True)
]
ignore_unknown_fields = True
class Meta:
connection_alias = 'Channels'
collection_name = 'channels'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('channelUsername', name='channelUsernameIndex', unique=True, sparse=True),
IndexModel('channelId', name='channelIdIndex', unique=True, sparse=True),
IndexModel('channelCreator', name='channelCreatorIndex', sparse=True),
IndexModel('channelAdmins', name='channelAdminsIndex', sparse=True),
IndexModel('channelBot', name='channelBotIndex', sparse=True)
]
ignore_unknown_fields = True
def benchmark_mongo4(self):
rc = read_concern.ReadConcern("majority")
wc = write_concern.WriteConcern("majority")
batch_size = 1000
print(f"Batch size: {batch_size}")
with self.collection.database.client.start_session() as session:
for i in range(int(self.operations / batch_size)):
ops = random.choices([READ, INSERT], [95, 5], k=batch_size)
with session.start_transaction(read_concern=rc, write_concern=wc):
inserts = []
for op in ops:
if op == READ:
scan_length = random.randint(1, 10)
start_id = self.req_set.pop()
if start_id + scan_length > self.records:
start_id = start_id - (
(start_id + scan_length) - self.records
)
self.num_read += 1
self.run_scan_length += scan_length
list(
self.collection.find(
{
"item": {"$gte": start_id}
},
session=session,
).limit(scan_length)
)
elif op == INSERT:
self.num_insert += 1
inserts.append(
{
"item": self.records + i,
"qty": 100 + i,
"tags": ["tag"],
"title": "title",
}
)
if inserts:
self.collection.insert_many(inserts, session=session)
return (
f"📖 Number of reads: {self.num_read}\n"
+ f"✍️ Number of inserts: {self.num_insert}\n"
+ f"🔎 {(self.num_read / self.operations) * 100}% reads\n"
+ f"Average scan length: {self.run_scan_length / self.num_read}"
)
def benchmark_mongo4(self):
rc = read_concern.ReadConcern("majority")
wc = write_concern.WriteConcern("majority")
batch_size = 1000
print(f"Batch size: {batch_size}")
with self.collection.database.client.start_session() as session:
for i in range(int(self.operations / batch_size)):
with session.start_transaction(read_concern=rc,
write_concern=wc):
for j in range(batch_size):
self.num_read += 1
self.collection.find_one({"item": self.req_set.pop()},
session=session)
return (f"📖 Number of reads: {self.num_read}\n" +
f"🔎 {(self.num_read / self.operations) * 100}% reads")
class Meta:
connection_alias = 'Posts'
collection_name = 'reactions'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('userId', name='userReactionUserIdIndex', sparse=True),
IndexModel('reactionIndex',
name='userReactionReactionIndexIndex',
sparse=True),
IndexModel('reactionDate',
name='userReactionDateIndex',
sparse=True),
IndexModel('postId', name='postIdIndex', sparse=True)
]
ignore_unknown_fields = True
class Meta:
connection_alias = 'Users'
collection_name = 'bots'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('botUsername',
name='botUsernameIndex',
unique=True,
sparse=True),
IndexModel('userId', name='botIdIndex', unique=True, sparse=True),
IndexModel('botToken',
name='botTokenIndex',
unique=True,
sparse=True)
]
ignore_unknown_fields = True
class Meta:
connection_alias = 'Posts'
collection_name = 'posts'
cascade = True
write_concern = wc.WriteConcern(j=True)
read_preference = ReadPreference.NEAREST
read_concern = rc.ReadConcern(level='majority')
indexes = [
IndexModel('creator', name='postCreatorIndex', sparse=True),
IndexModel('postId', name='postIdIndex', unique=True, sparse=True),
IndexModel('groupHash',
name='postGroupHashIndex',
unique=True,
sparse=True),
IndexModel('messageId', name='postMessageIdIndex', sparse=True),
IndexModel('channelId', name='postChannelIdIndex', sparse=True),
IndexModel('createdDate', name='postCreatedDateIndex', sparse=True)
]
ignore_unknown_fields = True
def __init__(self, params):
"""init configuration acording params"""
self.id = params.get('id', None) or str(uuid4())
self.admin_added = False
self.login = params.get('login', '')
self.password = params.get('password', '')
self.auth_key = params.get('auth_key', None)
self.auth_source = params.get('authSource', 'admin')
self._version = params.get('version')
self._configsvrs = []
self._routers = []
self._shards = {}
self.tags = {}
self.sslParams = params.get('sslParams', {})
self.kwargs = {}
self.restart_required = self.login or self.auth_key
self.x509_extra_user = False
if self.sslParams:
self.kwargs.update(DEFAULT_SSL_OPTIONS)
self.enable_ipv6 = common.ipv6_enabled_sharded(params)
# Determine what to do with config servers via mongos version.
mongos_name = os.path.join(Servers().bin_path(self._version), 'mongos')
mongos = Server(name=mongos_name, procParams={})
mongos_version = mongos.version
mongos.cleanup()
configsvr_configs = params.get('configsvrs', [{}])
self.uses_rs_configdb = (mongos_version >= (3, 1, 2)
and len(configsvr_configs) == 1)
self.configdb_singleton = (ReplicaSets()
if self.uses_rs_configdb else Servers())
if self.uses_rs_configdb:
self.__init_configrs(configsvr_configs[0])
elif mongos_version >= (3, 3, 2):
raise ShardedClusterError(
'mongos >= 3.3.2 requires the config database to be backed by '
'a replica set.')
elif mongos_version >= (3, 1, 2) and len(configsvr_configs) != 3:
raise ShardedClusterError(
"mongos >= 3.1.2 needs a config replica set or 3 old-style "
"config servers.")
else:
self.__init_configsvrs(configsvr_configs)
for r in params.get('routers', [{}]):
self.router_add(r)
for cfg in params.get('shards', []):
shard_params = cfg.get('shardParams', {})
shard_tags = shard_params.pop('tags', None)
info = self.member_add(cfg.get('id', None), shard_params)
if shard_tags:
self.tags[info['id']] = shard_tags
# SERVER-37631 changed 3.6 sharded cluster setup so that it's required
# to run refreshLogicalSessionCacheNow on the config server followed by
# each mongos. Only then will each 3.6 mongos correctly report
# logicalSessionTimeoutMinutes in its isMaster responses.
if mongos_version[:2] == (3, 6):
router_clients = self.router_connections()
is_master = router_clients[0].admin.command('isMaster')
if 'logicalSessionTimeoutMinutes' not in is_master:
self.config_connection().admin.command(
'refreshLogicalSessionCacheNow')
for client in router_clients:
client.admin.command('refreshLogicalSessionCacheNow')
if self.tags:
for sh_id in self.tags:
logger.debug('Add tags %r to %s' % (self.tags[sh_id], sh_id))
db = self.connection().get_database(
'config',
write_concern=write_concern.WriteConcern(fsync=True))
db.shards.update(
{'_id': sh_id},
{'$addToSet': {
'tags': {
'$each': self.tags[sh_id]
}
}})
shard_configs = [
s.get('shardParams', {}).get('procParams', {})
for s in params.get('shards', [])
]
if self.login:
# Do we need to add an extra x509 user?
def only_x509(config):
set_params = config.get('setParameter', {})
auth_mechs = set_params.get('authenticationMechanisms', '')
auth_mechs = auth_mechs.split(',')
if len(auth_mechs) == 1 and auth_mechs[0] == 'MONGODB-X509':
return True
return False
any_only_x509 = lambda l: any(map(only_x509, l))
rs_shard_configs = [
m.get('procParams', {}) for s in params.get('shards', [])
for m in s.get('shardParams', {}).get('members', [])
]
router_configs = params.get('routers', [])
self.x509_extra_user = (any_only_x509(configsvr_configs)
or any_only_x509(shard_configs)
or any_only_x509(rs_shard_configs)
or any_only_x509(router_configs))
self._add_users(
self.connection().get_database(
self.auth_source,
write_concern=write_concern.WriteConcern(fsync=True)),
mongos_version)
# Secondary user given from request.
secondary_login = {
'name': self.login,
'roles': self._user_roles(self.connection())
}
if self.password:
secondary_login['password'] = self.password
if mongos_version >= (3, 7, 2):
secondary_login['mechanisms'] = ['SCRAM-SHA-1']
# Do the same for the shards.
for shard_id, config in zip(self._shards, shard_configs):
shard = self._shards[shard_id]
instance_id = shard['_id']
if shard.get('isServer'):
client = Servers()._storage[instance_id].connection
elif shard.get('isReplicaSet'):
client = ReplicaSets()._storage[instance_id].connection()
db = client[self.auth_source]
if self.x509_extra_user:
db.add_user(DEFAULT_SUBJECT,
roles=self._user_roles(client))
if self.login:
db.add_user(**secondary_login)
if self.restart_required:
# Do we need to add clusterAuthMode back?
cluster_auth_mode = None
for cfg in shard_configs:
cam = cfg.get('clusterAuthMode')
if cam:
cluster_auth_mode = cam
break
def restart_with_auth(server_or_rs):
server_or_rs.x509_extra_user = self.x509_extra_user
server_or_rs.auth_source = self.auth_source
server_or_rs.ssl_params = self.sslParams
server_or_rs.login = self.login
server_or_rs.password = self.password
server_or_rs.auth_key = self.auth_key
def add_auth(cfg):
if self.auth_key:
cfg['keyFile'] = self.key_file
# Add clusterAuthMode back in.
if cluster_auth_mode:
cfg['clusterAuthMode'] = cam
return cfg
server_or_rs.restart(config_callback=add_auth)
for config_id in self._configsvrs:
restart_with_auth(self.configdb_singleton._storage[config_id])
for server_id in self._routers:
server = Servers()._storage[server_id]
restart_with_auth(server)
for shard_id in self._shards:
shard = self._shards[shard_id]
instance_id = shard['_id']
klass = ReplicaSets if shard.get('isReplicaSet') else Servers
instance = klass()._storage[instance_id]
restart_with_auth(instance)
self.restart_required = False
def getWrite(self,db,collection):
db_ = self.client.get_database(db,write_concern=write_concern.WriteConcern());
collection_ = db_.get_collection(collection);
return collection_;
def start(self):
self.session.start_transaction(read_concern.ReadConcern('snapshot'),
write_concern.WriteConcern('majority'))
