def test_maybe_add_read_preference(self):
# Primary doesn't add $readPreference
out = _maybe_add_read_preference({}, Primary())
self.assertEqual(out, {})
pref = PrimaryPreferred()
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = PrimaryPreferred(tag_sets=[{'dc': 'nyc'}])
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = Secondary()
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = Secondary(tag_sets=[{'dc': 'nyc'}])
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
# SecondaryPreferred without tag_sets or max_staleness doesn't add
# $readPreference
pref = SecondaryPreferred()
out = _maybe_add_read_preference({}, pref)
self.assertEqual(out, {})
pref = SecondaryPreferred(tag_sets=[{'dc': 'nyc'}])
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = SecondaryPreferred(max_staleness=120)
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = Nearest()
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
pref = Nearest(tag_sets=[{'dc': 'nyc'}])
out = _maybe_add_read_preference({}, pref)
self.assertEqual(
out, SON([("$query", {}), ("$readPreference", pref.document)]))
criteria = SON([("$query", {}), ("$orderby", SON([("_id", 1)]))])
pref = Nearest()
out = _maybe_add_read_preference(criteria, pref)
self.assertEqual(
out,
SON([("$query", {}), ("$orderby", SON([("_id", 1)])),
("$readPreference", pref.document)]))
pref = Nearest(tag_sets=[{'dc': 'nyc'}])
out = _maybe_add_read_preference(criteria, pref)
self.assertEqual(
out,
SON([("$query", {}), ("$orderby", SON([("_id", 1)])),
("$readPreference", pref.document)]))
def test_read_preference(self):
# Check the default
cx = motor_asyncio.AsyncIOMotorClient(test.env.uri, io_loop=self.loop)
self.assertEqual(ReadPreference.PRIMARY, cx.read_preference)
# We can set mode, tags, and latency.
cx = self.asyncio_client(
read_preference=Secondary(tag_sets=[{
'foo': 'bar'
}]),
localThresholdMS=42)
self.assertEqual(ReadPreference.SECONDARY.mode,
cx.read_preference.mode)
self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets)
self.assertEqual(42, cx.local_threshold_ms)
# Make a MotorCursor and get its PyMongo Cursor
collection = cx.motor_test.test_collection.with_options(
read_preference=Nearest(tag_sets=[{
'yay': 'jesse'
}]))
motor_cursor = collection.find()
cursor = motor_cursor.delegate
self.assertEqual(Nearest(tag_sets=[{
'yay': 'jesse'
}]), cursor._Cursor__read_preference)
cx.close()
def create(self, name: str) -> Client:
client = self.clients.get(name)
if client != None:
return client
else:
opts = self.load_options(name)
if opts == None:
return
mode = Primary()
if opts.read_preference == READ_PREFERENCE_PRIMARY_PREFERRED:
mode = PrimaryPreferred()
elif opts.read_preference == READ_PREFERENCE_SECONDARY:
mode = Secondary()
elif opts.read_preference == READ_PREFERENCE_SECONDARY_PREFERRED:
mode = SecondaryPreferred()
elif opts.read_preference == READ_PREFERENCE_NEAREST:
mode = Nearest()
kwargs = {
"read_preference" : mode,
"maxPoolSize" : opts.max_pool_size,
"minPoolSize" : opts.min_pool_size,
"socketTimeoutMS" : opts.socket_time_out,
"connectTimeoutMS" : opts.connect_time_out
}
_client = MongoClient(host=opts.uri, **kwargs)
client = Client(db=name, c=_client, opts=opts)
self.clients.set(name, client)
return client
def test_query_and_read_mode_sharded_op_msg(self):
"""Test OP_MSG sends non-primary $readPreference and never $query."""
server = MockupDB()
server.autoresponds('ismaster', ismaster=True, msg='isdbgrid',
minWireVersion=2, maxWireVersion=6)
server.run()
self.addCleanup(server.stop)
client = MongoClient(server.uri)
self.addCleanup(client.close)
read_prefs = (
Primary(),
SecondaryPreferred(),
PrimaryPreferred(),
Secondary(),
Nearest(),
SecondaryPreferred([{'tag': 'value'}]),)
for query in ({'a': 1}, {'$query': {'a': 1}},):
for mode in read_prefs:
collection = client.db.get_collection('test',
read_preference=mode)
cursor = collection.find(query.copy())
with going(next, cursor):
request = server.receives()
# Command is not nested in $query.
request.assert_matches(OpMsg(
SON([('find', 'test'),
('filter', {'a': 1}),
('$readPreference', mode.document)])))
request.replies({'cursor': {'id': 0, 'firstBatch': [{}]}})
def test_mongos(self):
shard = client_context.client.config.shards.find_one()['host']
num_members = shard.count(',') + 1
if num_members == 1:
raise SkipTest("Need a replica set shard to test.")
coll = client_context.client.pymongo_test.get_collection(
"test",
write_concern=WriteConcern(w=num_members))
coll.drop()
res = coll.insert_many([{} for _ in range(5)])
first_id = res.inserted_ids[0]
last_id = res.inserted_ids[-1]
# Note - this isn't a perfect test since there's no way to
# tell what shard member a query ran on.
for pref in (Primary(),
PrimaryPreferred(),
Secondary(),
SecondaryPreferred(),
Nearest()):
qcoll = coll.with_options(read_preference=pref)
results = list(qcoll.find().sort([("_id", 1)]))
self.assertEqual(first_id, results[0]["_id"])
self.assertEqual(last_id, results[-1]["_id"])
results = list(qcoll.find().sort([("_id", -1)]))
self.assertEqual(first_id, results[-1]["_id"])
self.assertEqual(last_id, results[0]["_id"])
def test_properties(self):
c = client_context.client
c.admin.command('ping')
wait_until(lambda: c.primary == self.primary, "discover primary")
wait_until(lambda: c.secondaries == self.secondaries,
"discover secondaries")
# SERVER-32845
if not (client_context.version >= (3, 7, 2)
and client_context.auth_enabled and client_context.is_rs):
wait_until(lambda: c.arbiters == self.arbiters,
"discover arbiters")
self.assertEqual(c.arbiters, self.arbiters)
self.assertEqual(c.primary, self.primary)
self.assertEqual(c.secondaries, self.secondaries)
self.assertEqual(c.max_pool_size, 100)
# Make sure MongoClient's properties are copied to Database and
# Collection.
for obj in c, c.pymongo_test, c.pymongo_test.test:
self.assertEqual(obj.codec_options, CodecOptions())
self.assertEqual(obj.read_preference, ReadPreference.PRIMARY)
self.assertEqual(obj.write_concern, WriteConcern())
cursor = c.pymongo_test.test.find()
self.assertEqual(
ReadPreference.PRIMARY, cursor._Cursor__read_preference)
tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}]
secondary = Secondary(tag_sets=tag_sets)
c = rs_client(
maxPoolSize=25,
document_class=SON,
tz_aware=True,
read_preference=secondary,
localThresholdMS=77,
j=True)
self.assertEqual(c.max_pool_size, 25)
for obj in c, c.pymongo_test, c.pymongo_test.test:
self.assertEqual(obj.codec_options, CodecOptions(SON, True))
self.assertEqual(obj.read_preference, secondary)
self.assertEqual(obj.write_concern, WriteConcern(j=True))
cursor = c.pymongo_test.test.find()
self.assertEqual(
secondary, cursor._Cursor__read_preference)
nearest = Nearest(tag_sets=[{'dc': 'ny'}, {}])
cursor = c.pymongo_test.get_collection(
"test", read_preference=nearest).find()
self.assertEqual(nearest, cursor._Cursor__read_preference)
self.assertEqual(c.max_bson_size, 16777216)
c.close()
class TestReadPreferenceObjects(unittest.TestCase):
prefs = [Primary(), Secondary(), Nearest(tag_sets=[{'a': 1}, {'b': 2}])]
def test_pickle(self):
for pref in self.prefs:
self.assertEqual(pref, pickle.loads(pickle.dumps(pref)))
def test_copy(self):
for pref in self.prefs:
self.assertEqual(pref, copy.copy(pref))
def test_query_and_read_mode_sharded_op_query(self):
server = MockupDB()
server.autoresponds('ismaster', ismaster=True, msg='isdbgrid',
minWireVersion=2, maxWireVersion=5)
server.run()
self.addCleanup(server.stop)
client = MongoClient(server.uri)
self.addCleanup(client.close)
modes_without_query = (
Primary(),
SecondaryPreferred(),)
modes_with_query = (
PrimaryPreferred(),
Secondary(),
Nearest(),
SecondaryPreferred([{'tag': 'value'}]),)
find_command = SON([('find', 'test'), ('filter', {'a': 1})])
for query in ({'a': 1}, {'$query': {'a': 1}},):
for mode in modes_with_query + modes_without_query:
collection = client.db.get_collection('test',
read_preference=mode)
cursor = collection.find(query.copy())
with going(next, cursor):
request = server.receives()
if mode in modes_without_query:
# Filter is hoisted out of $query.
request.assert_matches(Command(find_command))
self.assertFalse('$readPreference' in request)
else:
# Command is nested in $query.
request.assert_matches(Command(
SON([('$query', find_command),
('$readPreference', mode.document)])))
request.replies({'cursor': {'id': 0, 'firstBatch': [{}]}})
def test_query_and_read_mode_sharded(self):
server = MockupDB()
server.autoresponds('ismaster', ismaster=True, msg='isdbgrid')
server.run()
self.addCleanup(server.stop)
client = MongoClient(server.uri)
self.addCleanup(client.close)
modes_without_query = (
Primary(),
SecondaryPreferred(),)
modes_with_query = (
PrimaryPreferred(),
Secondary(),
Nearest(),
SecondaryPreferred([{'tag': 'value'}]),)
for query in ({'a': 1}, {'$query': {'a': 1}},):
for mode in modes_with_query + modes_without_query:
collection = client.db.get_collection('test',
read_preference=mode)
cursor = collection.find(query.copy())
with going(next, cursor):
request = server.receives(OpQuery)
if mode in modes_without_query:
# Query is not edited: {'a': 1} is not nested in $query,
# {'$query': {'a': 1}} is not hoisted.
request.assert_matches(query)
self.assertFalse('$readPreference' in request)
else:
# {'a': 1} is *always* nested in $query.
request.assert_matches({
'$query': {'a': 1},
'$readPreference': mode.document
})
request.replies({})
def test_read_preference_document(self):
pref = Primary()
self.assertEqual(pref.document, {'mode': 'primary'})
pref = PrimaryPreferred()
self.assertEqual(pref.document, {'mode': 'primaryPreferred'})
pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'primaryPreferred',
'tags': [{
'dc': 'sf'
}]
})
pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(
pref.document, {
'mode': 'primaryPreferred',
'tags': [{
'dc': 'sf'
}],
'maxStalenessSeconds': 30
})
pref = Secondary()
self.assertEqual(pref.document, {'mode': 'secondary'})
pref = Secondary(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'secondary',
'tags': [{
'dc': 'sf'
}]
})
pref = Secondary(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(pref.document, {
'mode': 'secondary',
'tags': [{
'dc': 'sf'
}],
'maxStalenessSeconds': 30
})
pref = SecondaryPreferred()
self.assertEqual(pref.document, {'mode': 'secondaryPreferred'})
pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'secondaryPreferred',
'tags': [{
'dc': 'sf'
}]
})
pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(
pref.document, {
'mode': 'secondaryPreferred',
'tags': [{
'dc': 'sf'
}],
'maxStalenessSeconds': 30
})
pref = Nearest()
self.assertEqual(pref.document, {'mode': 'nearest'})
pref = Nearest(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'nearest',
'tags': [{
'dc': 'sf'
}]
})
pref = Nearest(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(pref.document, {
'mode': 'nearest',
'tags': [{
'dc': 'sf'
}],
'maxStalenessSeconds': 30
})
with self.assertRaises(TypeError):
Nearest(max_staleness=1.5) # Float is prohibited.
with self.assertRaises(ValueError):
Nearest(max_staleness=0)
with self.assertRaises(ValueError):
Nearest(max_staleness=-2)
def test_read_preference_document(self):
pref = Primary()
self.assertEqual(pref.document, {'mode': 'primary'})
pref = PrimaryPreferred()
self.assertEqual(pref.document, {'mode': 'primaryPreferred'})
pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'primaryPreferred',
'tags': [{
'dc': 'sf'
}]
})
pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(
pref.document, {
'mode': 'primaryPreferred',
'tags': [{
'dc': 'sf'
}],
'maxStalenessMS': 30000
})
pref = Secondary()
self.assertEqual(pref.document, {'mode': 'secondary'})
pref = Secondary(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'secondary',
'tags': [{
'dc': 'sf'
}]
})
pref = Secondary(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(pref.document, {
'mode': 'secondary',
'tags': [{
'dc': 'sf'
}],
'maxStalenessMS': 30000
})
pref = SecondaryPreferred()
self.assertEqual(pref.document, {'mode': 'secondaryPreferred'})
pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'secondaryPreferred',
'tags': [{
'dc': 'sf'
}]
})
pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(
pref.document, {
'mode': 'secondaryPreferred',
'tags': [{
'dc': 'sf'
}],
'maxStalenessMS': 30000
})
pref = Nearest()
self.assertEqual(pref.document, {'mode': 'nearest'})
pref = Nearest(tag_sets=[{'dc': 'sf'}])
self.assertEqual(pref.document, {
'mode': 'nearest',
'tags': [{
'dc': 'sf'
}]
})
pref = Nearest(tag_sets=[{'dc': 'sf'}], max_staleness=30)
self.assertEqual(pref.document, {
'mode': 'nearest',
'tags': [{
'dc': 'sf'
}],
'maxStalenessMS': 30000
})
def create_app(script_info=None):
app = Flask(__name__)
# set config
app.config.from_object(APP_SETTINGS)
app.config["MONGODB_SETTINGS"] = {
"db": conf.DATABASE_NAME,
"host": conf.database_uri() if not conf.DATABASE_URI else conf.DATABASE_URI,
"connect": False, # prevents prefork connection
"replicaset": conf.REPLICA_SET,
"read_preference": Nearest()
}
# set up extensions
db.init_app(app)
toolbar.init_app(app)
celery.config_from_object(app.config)
configure_blueprints(app)
# shell context for flask cli
@app.shell_context_processor
def ctx():
return {"app": app, "db": db, "celery": celery}
@app.before_request
def before_request():
g.start = time.time()
request.id = shortuuid.uuid()
request.should_log = random.random() < conf.WEB_LOG_SAMPLE_FRAC # pairs request/response logs # noqa
request.arg_counts = {
k: len(ensure_list(v)) for k, v in (request.args or {}).items()
}
request.arg_count_str = " ".join(
[f" {k}s={v}" for k, v in request.arg_counts.items()]
)
if conf.WEB_LOG_REQUESTS:
attrs = {
"request": {
"request_at": utcnow().strftime("%d/%b/%Y:%H:%M:%S.%f")[:-3],
"remote_addr": request.remote_addr,
"method": request.method,
"path": request.path,
"query_string": request.query_string,
"scheme": request.scheme,
"referrer": request.referrer,
"user_agent": request.user_agent,
"headers": request.headers,
"args": request.args,
},
}
if request.should_log:
logger.info(
f"[{request.id}] {request.method} - {request.scheme}:{request.path}{request.arg_count_str}", # noqa
extra=attrs,
)
@app.after_request
def after_request(response):
""" Logging after every request. """
now = time.time()
duration = round(now - g.start, 2) # seconds
is_slow_response = duration >= conf.WEB_LOG_SLOW_RESPONSE_THRESHOLD
if conf.WEB_LOG_RESPONSES:
attrs = {
"request": {
"request_id": request.id,
"remote_addr": request.remote_addr,
"method": request.method,
"path": request.path,
"query_string": request.query_string,
"scheme": request.scheme,
"referrer": request.referrer,
"user_agent": request.user_agent,
"headers": request.headers,
"args": request.args,
"arg_counts": request.arg_counts,
"duration": duration,
"should_log": request.should_log,
},
"response": {
"status": response.status,
"status_code": response.status_code,
"content_length": response.content_length,
"is_slow": is_slow_response
},
}
if request.should_log or is_slow_response: # always log slow responses
logger.info(
f"[{request.id}] RESPONSE - {request.scheme}:{request.path}{request.arg_count_str} -> {response.status} ({duration}s)", # noqa
extra=attrs,
)
return response
return app