def test_Priority(self):
"""
Check that connection-level, database-level, collection-level
and query-level write concerns are respected with correct priority
"""
conn = MongoConnection(mongo_host, mongo_port, w=1, wtimeout=500)
try:
with self.mock_gle() as mock:
yield conn.mydb.mycol.insert({'x': 42})
mock.assert_called_once_with("mydb", w=1, wtimeout=500)
db_w0 = Database(conn, "mydb", write_concern=WriteConcern(w=0))
with self.mock_gle() as mock:
yield db_w0.mycol.insert({'x': 42})
self.assertFalse(mock.called)
coll = Collection(db_w0, "mycol", write_concern=WriteConcern(w=2))
with self.mock_gle() as mock:
yield coll.insert({'x': 42})
mock.assert_called_once_with("mydb", w=2)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, j=True)
mock.assert_called_once_with("mydb", j=True)
finally:
yield conn.mydb.mycol.drop()
yield conn.disconnect()
def test_Priority(self):
"""
Check that connection-level, database-level, collection-level
and query-level write concerns are respected with correct priority
"""
conn = MongoConnection(mongo_host, mongo_port, w=1, wtimeout=500)
try:
with self.mock_gle() as mock:
yield conn.mydb.mycol.insert({'x': 42})
mock.assert_called_once_with("mydb", w=1, wtimeout=500)
db_w0 = Database(conn, "mydb", write_concern=WriteConcern(w=0))
with self.mock_gle() as mock:
yield db_w0.mycol.insert({'x': 42})
self.assertFalse(mock.called)
coll = Collection(db_w0, "mycol", write_concern=WriteConcern(w=2))
with self.mock_gle() as mock:
yield coll.insert({'x': 42})
mock.assert_called_once_with("mydb", w=2)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, j=True)
mock.assert_called_once_with("mydb", j=True)
finally:
yield conn.mydb.mycol.drop()
yield conn.disconnect()
def test_WriteToMaster(self):
conn = MongoConnection("localhost", self.ports[0])
try:
coll = conn.db.coll
yield coll.insert({'x': 42}, safe=True)
result = yield coll.find_one()
self.assertEqual(result['x'], 42)
finally:
yield conn.disconnect()
def test_SwitchToMasterOnConnect(self):
# Reverse hosts order
try:
conn = MongoConnection("localhost", self.ports[1])
result = yield conn.db.coll.find({'x': 42})
self.assertEqual(result, [])
finally:
yield conn.disconnect()
# txmongo will do log.err() for AutoReconnects
self.flushLoggedErrors(AutoReconnect)
def test_AllOperations(self):
conn = MongoConnection(mongo_host, mongo_port, w=0)
coll = conn.mydb.mycol
try:
yield self.__test_operation(coll, "insert", {'x': 42})
yield self.__test_operation(coll, "update", {}, {'x': 42})
yield self.__test_operation(coll, "save", {'x': 42})
yield self.__test_operation(coll, "remove", {})
finally:
yield coll.drop()
yield conn.disconnect()
def test_AllOperations(self):
conn = MongoConnection(mongo_host, mongo_port, w=0)
coll = conn.mydb.mycol
try:
yield self.__test_operation(coll, "insert", {'x': 42})
yield self.__test_operation(coll, "update", {}, {'x': 42})
yield self.__test_operation(coll, "save", {'x': 42})
yield self.__test_operation(coll, "remove", {})
finally:
yield coll.drop()
yield conn.disconnect()
def test_StaleConnection(self):
conn = MongoConnection("localhost", self.ports[0])
try:
yield conn.db.coll.count()
self.__mongod[0].kill(signal.SIGSTOP)
yield self.__sleep(0.2)
while True:
try:
yield conn.db.coll.count()
break
except AutoReconnect:
pass
finally:
self.__mongod[0].kill(signal.SIGCONT)
yield conn.disconnect()
def test_Levels(self):
as_son = CodecOptions(document_class=SON)
doc = yield self.coll.find_one()
self.assertIsInstance(doc, dict)
try:
conn = MongoConnection(mongo_host, mongo_port, codec_options=as_son)
doc = yield conn.db.coll.find_one()
self.assertIsInstance(doc, SON)
finally:
yield conn.disconnect()
doc = yield Database(self.conn, "db", codec_options=as_son).coll.find_one()
self.assertIsInstance(doc, SON)
doc = yield self.coll.with_options(codec_options=as_son).find_one()
self.assertIsInstance(doc, SON)
def test_StaleConnection(self):
conn = MongoConnection("localhost", self.ports[0], ping_interval = 5, ping_timeout = 5)
try:
yield conn.db.coll.count()
# check that 5s pingers won't break connection if it is healthy
yield self.__sleep(6)
yield conn.db.coll.count()
self.__mongod[0].kill(signal.SIGSTOP)
yield self.__sleep(0.2)
while True:
try:
yield conn.db.coll.count()
break
except AutoReconnect:
pass
finally:
self.__mongod[0].kill(signal.SIGCONT)
yield conn.disconnect()
def test_StaleConnection(self):
conn = MongoConnection("localhost",
self.ports[0],
watchdog_interval=10,
watchdog_timeout=5)
try:
yield conn.db.coll.count()
self.__mongod[0].kill(signal.SIGSTOP)
yield self.__sleep(0.2)
while True:
try:
yield conn.db.coll.count()
break
except AutoReconnect:
pass
finally:
self.__mongod[0].kill(signal.SIGCONT)
yield conn.disconnect()
def test_SafeWithDefaultW0(self):
conn = MongoConnection(mongo_host, mongo_port, w=0)
coll = conn.mydb.mycol
try:
with self.mock_gle() as mock:
yield coll.insert({'x': 42})
self.assertFalse(mock.called)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, safe=False)
self.assertFalse(mock.called)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, safe=True)
mock.assert_called_once_with("mydb")
finally:
yield coll.drop()
yield conn.disconnect()
def test_SafeWithDefaultW0(self):
conn = MongoConnection(mongo_host, mongo_port, w=0)
coll = conn.mydb.mycol
try:
with self.mock_gle() as mock:
yield coll.insert({'x': 42})
self.assertFalse(mock.called)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, safe=False)
self.assertFalse(mock.called)
with self.mock_gle() as mock:
yield coll.insert({'x': 42}, safe=True)
mock.assert_called_once_with("mydb")
finally:
yield coll.drop()
yield conn.disconnect()
class DBPipeline(object):
def __init__(self, mongo_uri, mongo_db, collection_name):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
self.collection_name = collection_name
@classmethod
def from_crawler(cls, crawler):
return cls(
mongo_uri=crawler.settings.get('MONGO_URI'),
mongo_db=crawler.settings.get('MONGO_DATABASE', 'items'),
collection_name=crawler.settings.get('MONGO_COLLECTION_NAME'))
def open_spider(self, spider):
self.client = MongoConnection(self.mongo_uri)
self.db = self.client[self.mongo_db]
def close_spider(self, spider):
self.client.close()
def process_item(self, item, spider):
if self.get_item({
"company": item["company"],
"title": item["title"]
}) is None:
self.insert_item(item)
else:
raise DropItem(
u'Existing item with same company and title found, skipping.')
def get_item(self, item):
return self.db[self.collection_name].find_one(item)
def insert_item(self, item):
try:
self.db[self.collection_name].insert_one(dict(item))
return item
except Exception as e:
raise DropItem(
u'Inserting of item into database failed with error: %s' %
str(e))
def test_Levels(self):
as_son = CodecOptions(document_class=SON)
doc = yield self.coll.find_one()
self.assertIsInstance(doc, dict)
try:
conn = MongoConnection(mongo_host,
mongo_port,
codec_options=as_son)
doc = yield conn.db.coll.find_one()
self.assertIsInstance(doc, SON)
finally:
yield conn.disconnect()
doc = yield Database(self.conn, "db",
codec_options=as_son).coll.find_one()
self.assertIsInstance(doc, SON)
doc = yield self.coll.with_options(codec_options=as_son).find_one()
self.assertIsInstance(doc, SON)
class TestCodecOptions(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.conn = MongoConnection(mongo_host, mongo_port)
self.db = self.conn.db
self.coll = self.db.coll
yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
@defer.inlineCallbacks
def tearDown(self):
yield self.coll.drop()
yield self.conn.disconnect()
@defer.inlineCallbacks
def test_Levels(self):
as_son = CodecOptions(document_class=SON)
doc = yield self.coll.find_one()
self.assertIsInstance(doc, dict)
try:
conn = MongoConnection(mongo_host,
mongo_port,
codec_options=as_son)
doc = yield conn.db.coll.find_one()
self.assertIsInstance(doc, SON)
finally:
yield conn.disconnect()
doc = yield Database(self.conn, "db",
codec_options=as_son).coll.find_one()
self.assertIsInstance(doc, SON)
doc = yield self.coll.with_options(codec_options=as_son).find_one()
self.assertIsInstance(doc, SON)
@defer.inlineCallbacks
def test_TzAware(self):
doc = yield self.coll.find_one()
self.assertIsNone(doc['y'].tzinfo, None)
tz_aware = CodecOptions(tz_aware=True)
doc = yield self.coll.with_options(codec_options=tz_aware).find_one()
self.assertIsNotNone(doc['y'].tzinfo, None)
class TestCodecOptions(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.conn = MongoConnection(mongo_host, mongo_port)
self.db = self.conn.db
self.coll = self.db.coll
yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
@defer.inlineCallbacks
def tearDown(self):
yield self.coll.drop()
yield self.conn.disconnect()
@defer.inlineCallbacks
def test_Levels(self):
as_son = CodecOptions(document_class=SON)
doc = yield self.coll.find_one()
self.assertIsInstance(doc, dict)
try:
conn = MongoConnection(mongo_host, mongo_port, codec_options=as_son)
doc = yield conn.db.coll.find_one()
self.assertIsInstance(doc, SON)
finally:
yield conn.disconnect()
doc = yield Database(self.conn, "db", codec_options=as_son).coll.find_one()
self.assertIsInstance(doc, SON)
doc = yield self.coll.with_options(codec_options=as_son).find_one()
self.assertIsInstance(doc, SON)
@defer.inlineCallbacks
def test_TzAware(self):
doc = yield self.coll.find_one()
self.assertIsNone(doc['y'].tzinfo, None)
tz_aware = CodecOptions(tz_aware=True)
doc = yield self.coll.with_options(codec_options=tz_aware).find_one()
self.assertIsNotNone(doc['y'].tzinfo, None)
def setUp(self):
self.conn = MongoConnection(mongo_host, mongo_port)
self.db = self.conn.db
self.coll = self.db.coll
yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
def setUp(self):
self.conn = MongoConnection(mongo_host, mongo_port)
self.db = self.conn.db
self.coll = self.db.coll
yield self.coll.insert_one({'x': 42, 'y': datetime.datetime.now()})
def open_spider(self, spider):
self.client = MongoConnection(self.mongo_uri)
self.db = self.client[self.mongo_db]
def __init__(self):
self.connection_host = os.getenv("DB_SERVICE_HOST", "172.22.22.20")
self.connection_port = os.getenv("DB_SERVICE_PORT", 27017)
self.connection = MongoConnection(self.connection_host,
self.connection_port)
self.db = self.connection.demo_chat
