def test_maybe_add_read_preference(self): # Primary doesn't add $readPreference out = _maybe_add_read_preference({}, Primary()) self.assertEqual(out, {}) pref = PrimaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = PrimaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Secondary(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) # SecondaryPreferred without tag_sets or max_staleness doesn't add # $readPreference pref = SecondaryPreferred() out = _maybe_add_read_preference({}, pref) self.assertEqual(out, {}) pref = SecondaryPreferred(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = SecondaryPreferred(max_staleness=120) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest() out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference({}, pref) self.assertEqual( out, SON([("$query", {}), ("$readPreference", pref.document)])) criteria = SON([("$query", {}), ("$orderby", SON([("_id", 1)]))]) pref = Nearest() out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)])) pref = Nearest(tag_sets=[{'dc': 'nyc'}]) out = _maybe_add_read_preference(criteria, pref) self.assertEqual( out, SON([("$query", {}), ("$orderby", SON([("_id", 1)])), ("$readPreference", pref.document)]))
def test_read_preference(self): # Check the default cx = motor_asyncio.AsyncIOMotorClient(test.env.uri, io_loop=self.loop) self.assertEqual(ReadPreference.PRIMARY, cx.read_preference) # We can set mode, tags, and latency. cx = self.asyncio_client( read_preference=Secondary(tag_sets=[{ 'foo': 'bar' }]), localThresholdMS=42) self.assertEqual(ReadPreference.SECONDARY.mode, cx.read_preference.mode) self.assertEqual([{'foo': 'bar'}], cx.read_preference.tag_sets) self.assertEqual(42, cx.local_threshold_ms) # Make a MotorCursor and get its PyMongo Cursor collection = cx.motor_test.test_collection.with_options( read_preference=Nearest(tag_sets=[{ 'yay': 'jesse' }])) motor_cursor = collection.find() cursor = motor_cursor.delegate self.assertEqual(Nearest(tag_sets=[{ 'yay': 'jesse' }]), cursor._Cursor__read_preference) cx.close()
def create(self, name: str) -> Client: client = self.clients.get(name) if client != None: return client else: opts = self.load_options(name) if opts == None: return mode = Primary() if opts.read_preference == READ_PREFERENCE_PRIMARY_PREFERRED: mode = PrimaryPreferred() elif opts.read_preference == READ_PREFERENCE_SECONDARY: mode = Secondary() elif opts.read_preference == READ_PREFERENCE_SECONDARY_PREFERRED: mode = SecondaryPreferred() elif opts.read_preference == READ_PREFERENCE_NEAREST: mode = Nearest() kwargs = { "read_preference" : mode, "maxPoolSize" : opts.max_pool_size, "minPoolSize" : opts.min_pool_size, "socketTimeoutMS" : opts.socket_time_out, "connectTimeoutMS" : opts.connect_time_out } _client = MongoClient(host=opts.uri, **kwargs) client = Client(db=name, c=_client, opts=opts) self.clients.set(name, client) return client
def test_query_and_read_mode_sharded_op_msg(self): """Test OP_MSG sends non-primary $readPreference and never $query.""" server = MockupDB() server.autoresponds('ismaster', ismaster=True, msg='isdbgrid', minWireVersion=2, maxWireVersion=6) server.run() self.addCleanup(server.stop) client = MongoClient(server.uri) self.addCleanup(client.close) read_prefs = ( Primary(), SecondaryPreferred(), PrimaryPreferred(), Secondary(), Nearest(), SecondaryPreferred([{'tag': 'value'}]),) for query in ({'a': 1}, {'$query': {'a': 1}},): for mode in read_prefs: collection = client.db.get_collection('test', read_preference=mode) cursor = collection.find(query.copy()) with going(next, cursor): request = server.receives() # Command is not nested in $query. request.assert_matches(OpMsg( SON([('find', 'test'), ('filter', {'a': 1}), ('$readPreference', mode.document)]))) request.replies({'cursor': {'id': 0, 'firstBatch': [{}]}})
def test_mongos(self): shard = client_context.client.config.shards.find_one()['host'] num_members = shard.count(',') + 1 if num_members == 1: raise SkipTest("Need a replica set shard to test.") coll = client_context.client.pymongo_test.get_collection( "test", write_concern=WriteConcern(w=num_members)) coll.drop() res = coll.insert_many([{} for _ in range(5)]) first_id = res.inserted_ids[0] last_id = res.inserted_ids[-1] # Note - this isn't a perfect test since there's no way to # tell what shard member a query ran on. for pref in (Primary(), PrimaryPreferred(), Secondary(), SecondaryPreferred(), Nearest()): qcoll = coll.with_options(read_preference=pref) results = list(qcoll.find().sort([("_id", 1)])) self.assertEqual(first_id, results[0]["_id"]) self.assertEqual(last_id, results[-1]["_id"]) results = list(qcoll.find().sort([("_id", -1)])) self.assertEqual(first_id, results[-1]["_id"]) self.assertEqual(last_id, results[0]["_id"])
def test_properties(self): c = client_context.client c.admin.command('ping') wait_until(lambda: c.primary == self.primary, "discover primary") wait_until(lambda: c.secondaries == self.secondaries, "discover secondaries") # SERVER-32845 if not (client_context.version >= (3, 7, 2) and client_context.auth_enabled and client_context.is_rs): wait_until(lambda: c.arbiters == self.arbiters, "discover arbiters") self.assertEqual(c.arbiters, self.arbiters) self.assertEqual(c.primary, self.primary) self.assertEqual(c.secondaries, self.secondaries) self.assertEqual(c.max_pool_size, 100) # Make sure MongoClient's properties are copied to Database and # Collection. for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions()) self.assertEqual(obj.read_preference, ReadPreference.PRIMARY) self.assertEqual(obj.write_concern, WriteConcern()) cursor = c.pymongo_test.test.find() self.assertEqual( ReadPreference.PRIMARY, cursor._Cursor__read_preference) tag_sets = [{'dc': 'la', 'rack': '2'}, {'foo': 'bar'}] secondary = Secondary(tag_sets=tag_sets) c = rs_client( maxPoolSize=25, document_class=SON, tz_aware=True, read_preference=secondary, localThresholdMS=77, j=True) self.assertEqual(c.max_pool_size, 25) for obj in c, c.pymongo_test, c.pymongo_test.test: self.assertEqual(obj.codec_options, CodecOptions(SON, True)) self.assertEqual(obj.read_preference, secondary) self.assertEqual(obj.write_concern, WriteConcern(j=True)) cursor = c.pymongo_test.test.find() self.assertEqual( secondary, cursor._Cursor__read_preference) nearest = Nearest(tag_sets=[{'dc': 'ny'}, {}]) cursor = c.pymongo_test.get_collection( "test", read_preference=nearest).find() self.assertEqual(nearest, cursor._Cursor__read_preference) self.assertEqual(c.max_bson_size, 16777216) c.close()
class TestReadPreferenceObjects(unittest.TestCase): prefs = [Primary(), Secondary(), Nearest(tag_sets=[{'a': 1}, {'b': 2}])] def test_pickle(self): for pref in self.prefs: self.assertEqual(pref, pickle.loads(pickle.dumps(pref))) def test_copy(self): for pref in self.prefs: self.assertEqual(pref, copy.copy(pref))
def test_query_and_read_mode_sharded_op_query(self): server = MockupDB() server.autoresponds('ismaster', ismaster=True, msg='isdbgrid', minWireVersion=2, maxWireVersion=5) server.run() self.addCleanup(server.stop) client = MongoClient(server.uri) self.addCleanup(client.close) modes_without_query = ( Primary(), SecondaryPreferred(),) modes_with_query = ( PrimaryPreferred(), Secondary(), Nearest(), SecondaryPreferred([{'tag': 'value'}]),) find_command = SON([('find', 'test'), ('filter', {'a': 1})]) for query in ({'a': 1}, {'$query': {'a': 1}},): for mode in modes_with_query + modes_without_query: collection = client.db.get_collection('test', read_preference=mode) cursor = collection.find(query.copy()) with going(next, cursor): request = server.receives() if mode in modes_without_query: # Filter is hoisted out of $query. request.assert_matches(Command(find_command)) self.assertFalse('$readPreference' in request) else: # Command is nested in $query. request.assert_matches(Command( SON([('$query', find_command), ('$readPreference', mode.document)]))) request.replies({'cursor': {'id': 0, 'firstBatch': [{}]}})
def test_query_and_read_mode_sharded(self): server = MockupDB() server.autoresponds('ismaster', ismaster=True, msg='isdbgrid') server.run() self.addCleanup(server.stop) client = MongoClient(server.uri) self.addCleanup(client.close) modes_without_query = ( Primary(), SecondaryPreferred(),) modes_with_query = ( PrimaryPreferred(), Secondary(), Nearest(), SecondaryPreferred([{'tag': 'value'}]),) for query in ({'a': 1}, {'$query': {'a': 1}},): for mode in modes_with_query + modes_without_query: collection = client.db.get_collection('test', read_preference=mode) cursor = collection.find(query.copy()) with going(next, cursor): request = server.receives(OpQuery) if mode in modes_without_query: # Query is not edited: {'a': 1} is not nested in $query, # {'$query': {'a': 1}} is not hoisted. request.assert_matches(query) self.assertFalse('$readPreference' in request) else: # {'a': 1} is *always* nested in $query. request.assert_matches({ '$query': {'a': 1}, '$readPreference': mode.document }) request.replies({})
def test_read_preference_document(self): pref = Primary() self.assertEqual(pref.document, {'mode': 'primary'}) pref = PrimaryPreferred() self.assertEqual(pref.document, {'mode': 'primaryPreferred'}) pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'primaryPreferred', 'tags': [{ 'dc': 'sf' }] }) pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, { 'mode': 'primaryPreferred', 'tags': [{ 'dc': 'sf' }], 'maxStalenessSeconds': 30 }) pref = Secondary() self.assertEqual(pref.document, {'mode': 'secondary'}) pref = Secondary(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'secondary', 'tags': [{ 'dc': 'sf' }] }) pref = Secondary(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual(pref.document, { 'mode': 'secondary', 'tags': [{ 'dc': 'sf' }], 'maxStalenessSeconds': 30 }) pref = SecondaryPreferred() self.assertEqual(pref.document, {'mode': 'secondaryPreferred'}) pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'secondaryPreferred', 'tags': [{ 'dc': 'sf' }] }) pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, { 'mode': 'secondaryPreferred', 'tags': [{ 'dc': 'sf' }], 'maxStalenessSeconds': 30 }) pref = Nearest() self.assertEqual(pref.document, {'mode': 'nearest'}) pref = Nearest(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'nearest', 'tags': [{ 'dc': 'sf' }] }) pref = Nearest(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual(pref.document, { 'mode': 'nearest', 'tags': [{ 'dc': 'sf' }], 'maxStalenessSeconds': 30 }) with self.assertRaises(TypeError): Nearest(max_staleness=1.5) # Float is prohibited. with self.assertRaises(ValueError): Nearest(max_staleness=0) with self.assertRaises(ValueError): Nearest(max_staleness=-2)
def test_read_preference_document(self): pref = Primary() self.assertEqual(pref.document, {'mode': 'primary'}) pref = PrimaryPreferred() self.assertEqual(pref.document, {'mode': 'primaryPreferred'}) pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'primaryPreferred', 'tags': [{ 'dc': 'sf' }] }) pref = PrimaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, { 'mode': 'primaryPreferred', 'tags': [{ 'dc': 'sf' }], 'maxStalenessMS': 30000 }) pref = Secondary() self.assertEqual(pref.document, {'mode': 'secondary'}) pref = Secondary(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'secondary', 'tags': [{ 'dc': 'sf' }] }) pref = Secondary(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual(pref.document, { 'mode': 'secondary', 'tags': [{ 'dc': 'sf' }], 'maxStalenessMS': 30000 }) pref = SecondaryPreferred() self.assertEqual(pref.document, {'mode': 'secondaryPreferred'}) pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'secondaryPreferred', 'tags': [{ 'dc': 'sf' }] }) pref = SecondaryPreferred(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual( pref.document, { 'mode': 'secondaryPreferred', 'tags': [{ 'dc': 'sf' }], 'maxStalenessMS': 30000 }) pref = Nearest() self.assertEqual(pref.document, {'mode': 'nearest'}) pref = Nearest(tag_sets=[{'dc': 'sf'}]) self.assertEqual(pref.document, { 'mode': 'nearest', 'tags': [{ 'dc': 'sf' }] }) pref = Nearest(tag_sets=[{'dc': 'sf'}], max_staleness=30) self.assertEqual(pref.document, { 'mode': 'nearest', 'tags': [{ 'dc': 'sf' }], 'maxStalenessMS': 30000 })
def create_app(script_info=None): app = Flask(__name__) # set config app.config.from_object(APP_SETTINGS) app.config["MONGODB_SETTINGS"] = { "db": conf.DATABASE_NAME, "host": conf.database_uri() if not conf.DATABASE_URI else conf.DATABASE_URI, "connect": False, # prevents prefork connection "replicaset": conf.REPLICA_SET, "read_preference": Nearest() } # set up extensions db.init_app(app) toolbar.init_app(app) celery.config_from_object(app.config) configure_blueprints(app) # shell context for flask cli @app.shell_context_processor def ctx(): return {"app": app, "db": db, "celery": celery} @app.before_request def before_request(): g.start = time.time() request.id = shortuuid.uuid() request.should_log = random.random() < conf.WEB_LOG_SAMPLE_FRAC # pairs request/response logs # noqa request.arg_counts = { k: len(ensure_list(v)) for k, v in (request.args or {}).items() } request.arg_count_str = " ".join( [f" {k}s={v}" for k, v in request.arg_counts.items()] ) if conf.WEB_LOG_REQUESTS: attrs = { "request": { "request_at": utcnow().strftime("%d/%b/%Y:%H:%M:%S.%f")[:-3], "remote_addr": request.remote_addr, "method": request.method, "path": request.path, "query_string": request.query_string, "scheme": request.scheme, "referrer": request.referrer, "user_agent": request.user_agent, "headers": request.headers, "args": request.args, }, } if request.should_log: logger.info( f"[{request.id}] {request.method} - {request.scheme}:{request.path}{request.arg_count_str}", # noqa extra=attrs, ) @app.after_request def after_request(response): """ Logging after every request. """ now = time.time() duration = round(now - g.start, 2) # seconds is_slow_response = duration >= conf.WEB_LOG_SLOW_RESPONSE_THRESHOLD if conf.WEB_LOG_RESPONSES: attrs = { "request": { "request_id": request.id, "remote_addr": request.remote_addr, "method": request.method, "path": request.path, "query_string": request.query_string, "scheme": request.scheme, "referrer": request.referrer, "user_agent": request.user_agent, "headers": request.headers, "args": request.args, "arg_counts": request.arg_counts, "duration": duration, "should_log": request.should_log, }, "response": { "status": response.status, "status_code": response.status_code, "content_length": response.content_length, "is_slow": is_slow_response }, } if request.should_log or is_slow_response: # always log slow responses logger.info( f"[{request.id}] RESPONSE - {request.scheme}:{request.path}{request.arg_count_str} -> {response.status} ({duration}s)", # noqa extra=attrs, ) return response return app