def test_create_collection(self):
self.initOplogThread()
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
assert_soon(lambda: self.docman.commands)
command = self.docman.commands[0]
self.assertEqual(command['create'], 'test')
def check_renamed_insert(self, target_coll):
target_db, target_coll = target_coll.split('.', 1)
mongo_target = self.mongo_conn[target_db][target_coll]
assert_soon(lambda: len(list(mongo_target.find({}))))
target_docs = list(mongo_target.find({}))
self.assertEqual(len(target_docs), 1)
self.assertEqual(target_docs[0]["renamed"], 1)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert_one(
{"name": "kermit", "color": "green"}
)
self.primary_conn["test"]["test"].insert_one(
{"name": "elmo", "color": "firetruck red"}
)
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets",
)
self.primary_conn["test"]["test"].delete_one({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets",
)
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_skipped_oplog_entry_updates_checkpoint(self):
repl_set = ReplicaSetSingle().start()
conn = repl_set.client()
opman = OplogThread(
primary_client=conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
namespace_config=NamespaceConfig(namespace_set=["test.test"]),
)
opman.start()
# Insert a document into an included collection
conn["test"]["test"].insert_one({"test": 1})
last_ts = opman.get_last_oplog_timestamp()
assert_soon(
lambda: last_ts == opman.checkpoint,
"OplogThread never updated checkpoint to non-skipped " "entry.",
)
self.assertEqual(len(opman.doc_managers[0]._search()), 1)
# Make sure that the oplog thread updates its checkpoint on every
# oplog entry.
conn["test"]["ignored"].insert_one({"test": 1})
last_ts = opman.get_last_oplog_timestamp()
assert_soon(
lambda: last_ts == opman.checkpoint,
"OplogThread never updated checkpoint to skipped entry.",
)
opman.join()
conn.close()
repl_set.stop()
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.primary_client
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {
"a": 1, "b": 2, "c": 3,
"d": 4, "e": 5, "f": 6,
"_id": 1
}
db = conn['test']['test']
db.insert_one(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_remove(self):
"""Tests remove operations."""
self.conn["test"]["test"].insert_one({"name": "paulie"})
assert_soon(lambda: self._count() == 1)
self.conn["test"]["test"].delete_one({"name": "paulie"})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'drop': 'test'})
def test_drop_database(self):
self.initOplogThread()
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
self.primary_conn.drop_database('test')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'dropDatabase': 1})
def test_connector_minimum_privileges(self):
"""Test the Connector works with a user with minimum privileges."""
if not (db_user and db_password):
raise SkipTest("Need to set a user/password to test this.")
client = self.repl_set.client()
minimum_user = "******"
minimum_pwd = "password"
client.admin.add_user(
minimum_user,
minimum_pwd,
roles=[
{"role": "read", "db": "test"},
{"role": "read", "db": "wildcard"},
{"role": "read", "db": "local"},
],
)
client.test.test.insert_one({"replicated": 1})
client.test.ignored.insert_one({"replicated": 0})
client.ignored.ignored.insert_one({"replicated": 0})
client.wildcard.test.insert_one({"replicated": 1})
conn = Connector(
mongo_address=self.repl_set.primary.uri,
auth_username=minimum_user,
auth_key=minimum_pwd,
namespace_options={"test.test": True, "wildcard.*": True},
)
conn.start()
try:
assert_soon(conn.doc_managers[0]._search)
finally:
conn.join()
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn["test"], "test", create=True
)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {"drop": "test"})
def test_drop_database_renamed(self):
"""Test the dropDatabase command on a renamed database."""
self.create_renamed_collection("rename.me", "new.target")
self.create_renamed_collection("rename.me2", "new2.target2")
# test that drop database removes target databases
self.conn.drop_database("rename")
assert_soon(lambda: "new" not in self.mongo_conn.database_names())
assert_soon(lambda: "new2" not in self.mongo_conn.database_names())
def test_remove_file(self):
"""Tests removing a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn["test"], "test", create=True
)
coll.rename("test2")
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1].get("renameCollection"), "test.test")
self.assertEqual(self.docman.commands[1].get("to"), "test.test2")
def test_rename_collection_renamed(self):
"""Test the renameCollection command on a renamed collection to a
renamed collection.
"""
self.create_renamed_collection("rename.me", "new.target")
self.conn.admin.command("renameCollection", "rename.me", to="rename.me2")
# In the target, 'new.target' should be renamed to 'new2.target2'
assert_soon(lambda: "target" not in self.mongo_conn.new.collection_names())
self.check_renamed_insert("new2.target2")
def test_bad_int_value(self):
self.conn.test.test.insert_one(
{"inf": float("inf"), "nan": float("nan"), "still_exists": True}
)
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn("inf", doc)
self.assertNotIn("nan", doc)
self.assertTrue(doc["still_exists"])
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'),
('query', {"a": 0}),
('update', update_spec),
('new', True)]))['value']
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
assert_soon(lambda: next(self._search()) == updated)
def test_drop_collection_renamed(self):
"""Test the drop collection command on a renamed collection."""
self.create_renamed_collection("rename.me", "new.target")
self.create_renamed_collection("rename.me2", "new2.target2")
# test that drop collection removes target collection
self.conn.rename.drop_collection("me")
assert_soon(lambda: "target" not in self.mongo_conn.new.collection_names())
self.conn.rename.drop_collection("me2")
assert_soon(lambda: "target2" not in self.mongo_conn.new2.collection_names())
def test_remove(self):
"""Tests remove
"""
self.conn["test"]["test"].insert_one({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.conn["test"]["test"].delete_one({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self._search()) != 1)
self.assertEqual(sum(1 for _ in self._search()), 0)
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) != 1)
self.assertEqual(sum(1 for _ in self._search()), 0)
def test_bad_int_value(self):
self.conn.test.test.insert_one({
'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_insert(self):
"""Test insert operations."""
self.conn["test"]["test"].insert_one({"name": "paulie"})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn["test"]["test"].find_one()
for item in result_set_1:
self.assertEqual(item["_id"], str(result_set_2["_id"]))
self.assertEqual(item["name"], result_set_2["name"])
def test_create_collection_skipped(self):
self.initOplogThread(["test.test"])
pymongo.collection.Collection(self.primary_conn["test2"], "test2", create=True)
pymongo.collection.Collection(self.primary_conn["test"], "test", create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
command = self.docman.commands[0]
self.assertEqual(command["create"], "test")
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON(
[
("findAndModify", "test"),
("query", {"a": 0}),
("update", update_spec),
("new", True),
]
)
)["value"]
def update_worked():
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
return replicated == updated
# Allow some time for update to propagate
assert_soon(update_worked)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by un-setting an array element.
check_update({"$unset": {"b.10": True}})
# Update by un-setting a non-existent attribute.
check_update({"$unset": {"not-present": True}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_bad_int_value(self):
self.conn.test.test.insert_one({
"inf": float("inf"),
"nan": float("nan"),
"still_exists": True
})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn("inf", doc)
self.assertNotIn("nan", doc)
self.assertTrue(doc["still_exists"])
def test_create_collection_skipped(self):
self.initOplogThread(['test.test'])
pymongo.collection.Collection(
self.primary_conn['test2'], 'test2', create=True)
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_bad_int_value(self):
self.conn.test.test.insert_one({
'inf': float('inf'),
'nan': float('nan'),
'still_exists': True
})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_drop_collection_renamed(self):
"""Test the drop collection command on a renamed collection."""
self.create_renamed_collection("rename.me", "new.target")
self.create_renamed_collection("rename.me2", "new2.target2")
# test that drop collection removes target collection
self.conn.rename.drop_collection("me")
assert_soon(
lambda: "target" not in self.mongo_conn.new.collection_names())
self.conn.rename.drop_collection("me2")
assert_soon(
lambda: "target2" not in self.mongo_conn.new2.collection_names())
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
result_set_1 = self._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_with_chunk_migration(self):
"""Test that DocManagers have proper state after both a successful
and an unsuccessful chunk migration
"""
# Start replicating to dummy doc managers
self.opman1.start()
self.opman2.start()
collection = self.mongos_conn["test"]["mcsharded"]
for i in range(1000):
collection.insert_one({"i": i + 500})
# Assert current state of the mongoverse
self.assertEqual(self.shard1_conn["test"]["mcsharded"].find().count(),
500)
self.assertEqual(self.shard2_conn["test"]["mcsharded"].find().count(),
500)
assert_soon(lambda: len(self.opman1.doc_managers[0]._search()) == 1000)
# Test successful chunk move from shard 1 to shard 2
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-1"
)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
# Mark the collection as "dropped". This will cause migration to fail.
self.mongos_conn["config"]["collections"].update_one(
{"_id": "test.mcsharded"},
{"$set": {"dropped": True}}
)
# Test unsuccessful chunk move from shard 2 to shard 1
def fail_to_move_chunk():
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-0"
)
self.assertRaises(pymongo.errors.OperationFailure, fail_to_move_chunk)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
result_set_1 = self._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.rename('test2')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(
self.docman.commands[1].get('renameCollection'),
'test.test')
self.assertEqual(
self.docman.commands[1].get('to'),
'test.test2')
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'), ('query', {
"a": 0
}), ('update', update_spec), ('new', True)]))['value']
def update_worked():
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
return replicated == updated
# Allow some time for update to propagate
assert_soon(update_worked)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([
("findAndModify", "test"),
("query", {
"a": 0
}),
("update", update_spec),
("new", True),
]))["value"]
def update_worked():
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
return replicated == updated
# Allow some time for update to propagate
assert_soon(update_worked)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by un-setting an array element.
check_update({"$unset": {"b.10": True}})
# Update by un-setting a non-existent attribute.
check_update({"$unset": {"not-present": True}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rename_collection_renamed(self):
"""Test the renameCollection command on a renamed collection to a
renamed collection.
"""
self.create_renamed_collection("rename.me", "new.target")
self.conn.admin.command("renameCollection",
"rename.me",
to="rename.me2")
# In the target, 'new.target' should be renamed to 'new2.target2'
assert_soon(
lambda: "target" not in self.mongo_conn.new.collection_names())
self.check_renamed_insert("new2.target2")
def test_with_chunk_migration(self):
"""Test that DocManagers have proper state after both a successful
and an unsuccessful chunk migration
"""
# Start replicating to dummy doc managers
self.opman1.start()
self.opman2.start()
collection = self.mongos_conn["test"]["mcsharded"]
for i in range(1000):
collection.insert_one({"i": i + 500})
# Assert current state of the mongoverse
self.assertEqual(self.shard1_conn["test"]["mcsharded"].find().count(),
500)
self.assertEqual(self.shard2_conn["test"]["mcsharded"].find().count(),
500)
assert_soon(lambda: len(self.opman1.doc_managers[0]._search()) == 1000)
# Test successful chunk move from shard 1 to shard 2
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-1"
)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
# Mark the collection as "dropped". This will cause migration to fail.
self.mongos_conn["config"]["collections"].update_one(
{"_id": "test.mcsharded"},
{"$set": {"dropped": True}}
)
# Test unsuccessful chunk move from shard 2 to shard 1
def fail_to_move_chunk():
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-0"
)
self.assertRaises(pymongo.errors.OperationFailure, fail_to_move_chunk)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('name:paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_drop_database_renamed(self):
"""Test the dropDatabase command on a renamed database."""
if not self.desination_version.at_least(3, 0, 7) or (
self.desination_version.at_least(3, 1)
and not self.desination_version.at_least(3, 1, 9)):
raise SkipTest("This test fails often because of SERVER-13212")
self.create_renamed_collection("rename.me", "new.target")
self.create_renamed_collection("rename.me2", "new2.target2")
# test that drop database removes target databases
self.conn.drop_database("rename")
assert_soon(lambda: "new" not in self.mongo_conn.database_names())
assert_soon(lambda: "new2" not in self.mongo_conn.database_names())
def test_insert(self):
"""Tests insert
"""
self.conn["test"]["test"].insert_one({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
result_set_1 = self._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn["test"]["test"].find_one()
for item in result_set_1:
self.assertEqual(item["_id"], result_set_2["_id"])
self.assertEqual(item["name"], result_set_2["name"])
def test_create_collection_skipped(self):
self.initOplogThread(['test.test'])
pymongo.collection.Collection(self.primary_conn['test2'],
'test2',
create=True)
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def check_update(update_spec):
updated = self.conn.test.command(
SON([
("findAndModify", "test"),
("query", {
"a": 0
}),
("update", update_spec),
("new", True),
]))["value"]
# Stringify _id to match what will be retrieved from ES
updated["_id"] = str(updated["_id"])
assert_soon(lambda: next(self._search()) == updated)
def setUpClass(cls):
if db_user and db_password:
auth_args = dict(auth_username=db_user, auth_key=db_password)
else:
auth_args = {}
cls.cluster = ShardedClusterSingle().start()
cls.main_uri = cls.cluster.uri + '/?readPreference=primaryPreferred'
cls.dm = DocManager()
cls.connector = Connector(mongo_address=cls.main_uri,
doc_managers=[cls.dm],
**auth_args)
cls.connector.start()
assert_soon(lambda: len(cls.connector.shard_set) == 2,
message='connector failed to find both shards!')
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn["test"], "test")
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding="utf8")
assert_soon(lambda: sum(1 for _ in self._search()) > 0)
res = list(self._search())
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc["filename"], "test.txt")
self.assertEqual(doc["_id"], id)
self.assertEqual(doc["content"], test_data)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) > 0)
res = list(self._search())
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], id)
self.assertEqual(doc['content'], test_data)
def test_create_collection_skipped(self):
self.initOplogThread(["test.test"])
pymongo.collection.Collection(self.primary_conn["test2"],
"test2",
create=True)
pymongo.collection.Collection(self.primary_conn["test"],
"test",
create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
command = self.docman.commands[0]
self.assertEqual(command["create"], "test")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="billing.address.street" type="string" ... />
<field name="billing.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
# Connector is already running
self.conn["test"]["test"].insert_one({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert_one({
"numbers": ["one", "two", "three"],
"characters": [{
"name": "Big Bird",
"color": "yellow"
}, {
"name": "Elmo",
"color": "red"
}, "Cookie Monster"]
})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(
next(iter(results))["billing.address.state"], "California")
# Search for second document
results = self.solr_conn.search("characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn["test"], "test")
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding="utf8")
assert_soon(lambda: self._count() > 0)
query = {"match": {"attachment.content": "test_insert_file"}}
res = list(self._search(query))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc["filename"], "test.txt")
self.assertEqual(doc["_id"], str(id))
self.assertEqual(base64.b64decode(doc["content"]), test_data)
def test_start_with_auth(self):
dm = DocManager()
connector = Connector(mongo_address=self.cluster.uri,
doc_managers=[dm],
auth_username=db_user,
auth_key=db_password)
connector.start()
# Insert some documents into the sharded cluster. These
# should go to the DocManager, and the connector should not
# have an auth failure.
self.cluster.client().test.test.insert_one({'auth_failure': False})
assert_soon(lambda: len(dm._search()) > 0)
connector.join()
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: self._count() > 0)
query = {"match": {"_all": "test_insert_file"}}
res = list(self._search(query))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], str(id))
self.assertEqual(base64.b64decode(doc['content']), test_data)
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
# This doc can be picked up in the collection dump
self.conn['test']['test'].insert_one({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
# This doc is definitely not picked up by collection dump
self.conn['test']['test'].insert_one({'name': 'pauly'})
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert_one,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 3)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 3)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 2)
if result_set_1[0]['name'] == 'paul':
self.assertEqual(result_set_1[1]['name'], 'pauly')
elif result_set_1[0]['name'] == 'pauly':
self.assertEqual(result_set_1[1]['name'], 'paul')
else:
self.assertTrue(0, 'Unknown document retrieved')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 2)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert_one({
'name':
'test_valid'
}).inserted_id
self.conn['test']['test'].update_one({'_id': inserted_obj},
{'$set': {
'popularity': 1
}})
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(sum(1 for _ in self._search("name:test_valid")), 1)
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(mongo_address=self.repl_set.uri, **connector_opts)
conn.start()
assert_soon(lambda: bool(conn.shard_set))
# Make sure get_mininum_mongodb_version returns the current version.
self.assertEqual(Version.from_client(self.repl_set.client()),
get_mininum_mongodb_version())
conn.join()
# Make sure the connector is shutdown correctly
self.assertFalse(conn.can_run)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def setUp(self):
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self._remove()
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(self.mongo_doc, ),
gridfs_set=['test.test'],
**connector_opts)
self.conn.drop_database('test')
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def setUpClass(cls):
""" Initializes the cluster
"""
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
cls.connector = Connector(mongo_address=cls.repl_set.uri,
ns_set=['test.test'],
**connector_opts)
cls.synchronizer = cls.connector.doc_managers[0]
cls.connector.start()
assert_soon(lambda: len(cls.connector.shard_set) != 0)
def check_update(update_spec):
updated = self.conn.test.command(
SON([
("findAndModify", "test"),
("query", {
"a": 0
}),
("update", update_spec),
("new", True),
]))["value"]
def update_worked():
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
return replicated == updated
# Allow some time for update to propagate
assert_soon(update_worked)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
self.conn["test"]["test"].insert_one({"name": "paul"})
condition = (lambda: self.conn["test"]["test"].find_one(
{"name": "paul"}) is not None)
assert_soon(condition)
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn["admin"]
def condition():
return admin.command("isMaster")["ismaster"]
assert_soon(lambda: retry_until_ok(condition))
retry_until_ok(self.conn.test.test.insert_one, {"name": "pauline"})
assert_soon(lambda: sum(1 for _ in self._search()) == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn["test"]["test"].find_one({"name": "pauline"})
self.assertEqual(len(result_set_1), 2)
# make sure pauline is there
for item in result_set_1:
if item["name"] == "pauline":
self.assertEqual(item["_id"], result_set_2["_id"])
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
assert_soon(
lambda: primary_conn["admin"].command("isMaster")["ismaster"])
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item["name"], "paul")
find_cursor = retry_until_ok(self.conn["test"]["test"].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = "test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
res = list(self.solr_conn.search('content:*test_insert_file*'))
if not res:
res = list(self.solr_conn.search('_text_:*test_insert_file*'))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], "test.txt")
self.assertEqual(doc['_id'], str(id))
content = doc.get('content', doc.get('_text_', None))
self.assertTrue(content)
self.assertIn(test_data.strip(), content[0].strip())
def setUp(self):
self._remove()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
docman = DocManager(solr_url, auto_commit_interval=0)
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman, ),
gridfs_set=['test.test'])
retry_until_ok(self.conn.test.test.drop)
retry_until_ok(self.conn.test.test.files.drop)
retry_until_ok(self.conn.test.test.chunks.drop)
self._remove()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
