def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Allow some time for update to propagate
time.sleep(2)
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
self.assertEqual(replicated, updated)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_drop_database(self):
self.initOplogThread()
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
self.primary_conn.drop_database('test')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'dropDatabase': 1})
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'drop': 'test'})
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_with_orphan_documents(self):
"""Test that DocManagers have proper state after a chunk migration
that resuts in orphaned documents.
"""
# Start replicating to dummy doc managers
self.opman1.start()
self.opman2.start()
collection = self.mongos_conn["test"]["mcsharded"]
collection.insert({"i": i + 500} for i in range(1000))
# Assert current state of the mongoverse
self.assertEqual(self.shard1_conn["test"]["mcsharded"].find().count(),
500)
self.assertEqual(self.shard2_conn["test"]["mcsharded"].find().count(),
500)
assert_soon(lambda: len(self.opman1.doc_managers[0]._search()) == 1000)
# Stop replication using the 'rsSyncApplyStop' failpoint
self.shard1_conn.admin.command(
"configureFailPoint", "rsSyncApplyStop",
mode="alwaysOn"
)
# Move a chunk from shard2 to shard1
def move_chunk():
try:
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1000},
to="demo-set-0"
)
except pymongo.errors.OperationFailure:
pass
# moveChunk will never complete, so use another thread to continue
mover = threading.Thread(target=move_chunk)
mover.start()
# wait for documents to start moving to shard 1
assert_soon(lambda: self.shard1_conn.test.mcsharded.count() > 500)
# Get opid for moveChunk command
operations = self.mongos_conn.test.current_op()
opid = None
for op in operations["inprog"]:
if op.get("query", {}).get("moveChunk"):
opid = op["opid"]
self.assertNotEqual(opid, None, "could not find moveChunk operation")
# Kill moveChunk with the opid
self.mongos_conn["test"]["$cmd.sys.killop"].find_one({"op": opid})
# Mongo Connector should not become confused by unsuccessful chunk move
docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(docs), 1000)
self.assertEqual(sorted(d["i"] for d in docs),
list(range(500, 1500)))
# cleanup
mover.join()
def test_subdocument_with_id(self):
"""
Test inserting a document with a subdocument containing an _id property.
In the current version of translating from document data model to property graph, the root level _id
field is included in all subdocument nodes in the graph. This test verifies there is no error when
inserting a document containing a subdocument with an _id property.
See https://github.com/neo4j-contrib/neo4j_doc_manager/issues/56
"""
doc = {
'_id': 'root_level_object_id',
'name': 'Bob',
'location': {
'_id': 'sub_document_object_id',
'city': 'Missoula',
'state': 'Montana'
}
}
self.connector.doc_managers[0].upsert(doc, "test.test_id", 1)
assert_soon(lambda: self._count() > 0)
result_set = self.neo4j_conn.find_one("location")
self.assertNotEqual(result_set, None)
self.assertEqual(
result_set['_id'],
'root_level_object_id') # expect subdocument _id to be ignored
def start_replica_set(set_name, num_members=3, num_arbiters=1):
"""Start a replica set with a given name."""
# Start members
repl_ports = [
start_mongo_proc(
options=["--replSet", set_name, "--noprealloc", "--nojournal"])
for i in range(num_members)
]
# Initialize set
client = MongoClient(mongo_host, repl_ports[-1])
client.admin.command(
'replSetInitiate', {
'_id':
set_name,
'members': [{
'_id': i,
'host': '%s:%d' % (mongo_host, p),
'arbiterOnly': (i < num_arbiters)
} for i, p in enumerate(repl_ports)]
})
# Wait for primary
assert_soon(lambda: client.admin.command("isMaster")['ismaster'])
# Wait for secondaries
for port in repl_ports[num_arbiters:-1]:
secondary_client = MongoClient(mongo_host, port)
assert_soon(lambda: secondary_client.admin.command('replSetGetStatus')[
'myState'] == 2)
# Tag primary
nodes[client.port]['master'] = True
# Tag arbiters
for port in repl_ports[:num_arbiters]:
nodes[port]['arbiter'] = True
return repl_ports
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def setUp(self):
""" Starts a new connector for every test
"""
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0
)
# Clean out test databases
try:
self.elastic_doc._remove()
except OperationFailed:
try:
# Create test.test index if necessary
client = Elasticsearch(hosts=[elastic_pair])
idx_client = IndicesClient(client)
idx_client.create(index='test.test')
except es_exceptions.TransportError:
pass
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def setUp(self):
""" Starts a new connector for every test
"""
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0)
# Clean out test databases
try:
self.elastic_doc._remove()
except OperationFailed:
try:
# Create test.test index if necessary
client = Elasticsearch(hosts=[elastic_pair])
idx_client = IndicesClient(client)
idx_client.create(index='test.test')
except es_exceptions.TransportError:
pass
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.main_connection
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {
"a": 1, "b": 2, "c": 3,
"d": 4, "e": 5, "f": 6,
"_id": 1
}
db = conn['test']['test']
db.insert(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert({
"name": "kermit",
"color": "green"
})
self.primary_conn["test"]["test"].insert({
"name": "elmo",
"color": "firetruck red"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets")
self.primary_conn["test"]["test"].remove({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets")
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_create_collection(self):
self.initOplogThread()
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert({"name": "kermit", "color": "green"})
self.primary_conn["test"]["test"].insert({"name": "elmo", "color": "firetruck red"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets",
)
self.primary_conn["test"]["test"].remove({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets",
)
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_subdocument_with_id(self):
"""
Test inserting a document with a subdocument containing an _id property.
In the current version of translating from document data model to property graph, the root level _id
field is included in all subdocument nodes in the graph. This test verifies there is no error when
inserting a document containing a subdocument with an _id property.
See https://github.com/neo4j-contrib/neo4j_doc_manager/issues/56
"""
doc = {
'_id': 'root_level_object_id',
'name': 'Bob',
'location': {
'_id': 'sub_document_object_id',
'city': 'Missoula',
'state': 'Montana'
}
}
self.connector.doc_managers[0].upsert(doc, "test.test_id", 1)
assert_soon(lambda: self._count() > 0)
result_set = self.neo4j_conn.find_one("location")
self.assertNotEqual(result_set, None)
self.assertEqual(result_set['_id'], 'root_level_object_id') # expect subdocument _id to be ignored
def test_remove(self):
"""Tests remove
"""
self.conn["test"]["test"].insert({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn["test"]["test"].remove({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify({"a": 0},
update_spec,
new=True)
# Allow some time for update to propagate
time.sleep(2)
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
self.assertEqual(replicated, updated)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.main_connection
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {
"a": 1, "b": 2, "c": 3,
"d": 4, "e": 5, "f": 6,
"_id": 1
}
db = conn['test']['test']
db.insert(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_update(self):
"""Test update operations on Solr.
Need to have the following defined in schema.xml:
<field name="a" type="int" indexed="true" stored="true" />
<field name="b.0.c" type="int" indexed="true" stored="true" />
<field name="b.10.c" type="int" indexed="true" stored="true" />
<field name="b.0.e" type="int" indexed="true" stored="true" />
<field name="b.1.d" type="int" indexed="true" stored="true" />
<field name="b.1.f" type="int" indexed="true" stored="true" />
<field name="b.2.e" type="int" indexed="true" stored="true" />
"""
docman = self.connector.doc_managers[0]
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify({"a": 0}, update_spec, new=True)
# Stringify _id to match what will be retrieved from Solr
updated["_id"] = str(updated["_id"])
# Flatten the MongoDB document to match Solr
updated = docman._clean_doc(updated)
# Allow some time for update to propagate
time.sleep(1)
replicated = list(docman._search("a:0"))[0]
# Remove add'l fields until these are stored in a separate Solr core
replicated.pop("_ts")
replicated.pop("ns")
# Remove field added by Solr
replicated.pop("_version_")
self.assertEqual(replicated, docman._clean_doc(updated))
# Update by adding a field.
# Note that Solr can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by replacing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_remove_file(self):
"""Tests removing a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_drop_database(self):
self.initOplogThread()
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
self.primary_conn.drop_database('test')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'dropDatabase': 1})
def test_remove_file(self):
"""Tests removing a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'drop': 'test'})
def test_update(self):
"""Test update operations on Solr.
Need to have the following defined in schema.xml:
<field name="a" type="int" indexed="true" stored="true" />
<field name="b.0.c" type="int" indexed="true" stored="true" />
<field name="b.0.e" type="int" indexed="true" stored="true" />
<field name="b.1.d" type="int" indexed="true" stored="true" />
<field name="b.1.f" type="int" indexed="true" stored="true" />
<field name="b.2.e" type="int" indexed="true" stored="true" />
"""
docman = self.connector.doc_managers[0]
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify({"a": 0},
update_spec,
new=True)
# Stringify _id to match what will be retrieved from Solr
updated['_id'] = str(updated['_id'])
# Flatten the MongoDB document to match Solr
updated = docman._clean_doc(updated)
# Allow some time for update to propagate
time.sleep(1)
replicated = list(docman._search("a:0"))[0]
# Remove add'l fields until these are stored in a separate Solr core
replicated.pop("_ts")
replicated.pop("ns")
# Remove field added by Solr
replicated.pop("_version_")
self.assertEqual(replicated, docman._clean_doc(updated))
# Update by adding a field.
# Note that Solr can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by replacing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'),
('query', {"a": 0}),
('update', update_spec),
('new', True)]))['value']
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
assert_soon(lambda: next(self._search()) == updated)
def test_bad_int_value(self):
self.conn.test.test.insert_one({
'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.rename('test2')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(
self.docman.commands[1],
{'renameCollection': 'test.test', 'to': 'test.test2'})
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_single_target(self):
"""Test with a single replication target"""
self.opman.start()
# Insert first document with primary up
self.main_conn["test"]["mc"].insert({"i": 0})
self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 1)
# Make sure the insert is replicated
secondary = self.secondary_conn
assert_soon(lambda: secondary["test"]["mc"].count() == 1,
"first write didn't replicate to secondary")
# Kill the primary
kill_mongo_proc(self.primary_p, destroy=False)
# Wait for the secondary to be promoted
while not secondary["admin"].command("isMaster")["ismaster"]:
time.sleep(1)
# Insert another document. This will be rolled back later
retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": 1})
self.assertEqual(secondary["test"]["mc"].count(), 2)
# Wait for replication to doc manager
assert_soon(lambda: len(self.opman.doc_managers[0]._search()) == 2,
"not all writes were replicated to doc manager")
# Kill the new primary
kill_mongo_proc(self.secondary_p, destroy=False)
# Start both servers back up
restart_mongo_proc(self.primary_p)
primary_admin = self.primary_conn["admin"]
assert_soon(lambda: primary_admin.command("isMaster")["ismaster"],
"restarted primary never resumed primary status")
restart_mongo_proc(self.secondary_p)
assert_soon(lambda: retry_until_ok(secondary.admin.command,
'replSetGetStatus')['myState'] == 2,
"restarted secondary never resumed secondary status")
assert_soon(lambda:
retry_until_ok(self.main_conn.test.mc.find().count) > 0,
"documents not found after primary/secondary restarted")
# Only first document should exist in MongoDB
self.assertEqual(self.main_conn["test"]["mc"].count(), 1)
self.assertEqual(self.main_conn["test"]["mc"].find_one()["i"], 0)
# Same case should hold for the doc manager
doc_manager = self.opman.doc_managers[0]
self.assertEqual(len(doc_manager._search()), 1)
self.assertEqual(doc_manager._search()[0]["i"], 0)
# cleanup
self.opman.join()
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_stress(self):
"""Test stress by inserting and removing a large number of documents"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
self.assertEqual(set('Paul ' + str(i) for i in range(STRESS_COUNT)),
set(item['name'] for item in self._search()))
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self.
conn['test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn['admin']
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search())
== self.conn['test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn['admin']
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
def test_create_collection_skipped(self):
self.initOplogThread(['test.test'])
pymongo.collection.Collection(
self.primary_conn['test2'], 'test2', create=True)
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn["test"]["test"].insert({"name": "test_valid"})
self.conn["test"]["test"].update({"_id": inserted_obj}, {"$set": {"popularity": 1}})
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn("popularity", result)
self.assertEqual(sum(1 for _ in docman._search("name=test_valid")), 1)
def test_stress(self):
"""Stress test for inserting and removing many documents."""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
self.assertEqual(
set('Paul ' + str(i) for i in range(STRESS_COUNT)),
set(item['name'] for item in self._search())
)
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set = self.conn['test']['test'].find_one()
# self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.connector.doc_managers[0].remove(str(result_set['_id']), 'test.test', 1)
self.assertEqual(self._count(), 0)
self.connector.doc_managers[0].graph.delete_all()
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set_2 = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set_2['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() > 0)
result_set_1 = self.neo4j_conn.find_one("test")
self.assertNotEqual(result_set_1, None)
self.assertEqual(result_set_1['_id'], str(result_set_2['_id']))
self.assertEqual(result_set_1['name'], result_set_2['name'])
self.connector.doc_managers[0].graph.delete_all()
def test_bad_int_value(self):
self.conn['test']['test'].insert({'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
result_set = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']), 'inf': float('inf'), 'nan': float('nan'),
'still_exists': True }, "test.test", 1)
assert_soon(lambda: self._count() > 0)
doc = self.neo4j_conn.find_one("test")
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_bad_int_value(self):
self.conn.test.test.insert({
'inf': float('inf'),
'nan': float('nan'),
'still_exists': True
})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
coll.rename('test2')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {
'renameCollection': 'test.test',
'to': 'test.test2'
})
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_insert(self):
"""Tests insert
"""
self.conn["test"]["test"].insert({"name": "paulie"})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0)
result_set_1 = list(self.solr_conn.search("paulie"))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn["test"]["test"].find_one()
for item in result_set_1:
self.assertEqual(item["_id"], str(result_set_2["_id"]))
self.assertEqual(item["name"], result_set_2["name"])
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set = self.conn['test']['test'].find_one()
# self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.connector.doc_managers[0].remove(str(result_set['_id']),
'test.test', 1)
self.assertEqual(self._count(), 0)
self.connector.doc_managers[0].graph.delete_all()
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_with_chunk_migration(self):
"""Test that DocManagers have proper state after both a successful
and an unsuccessful chunk migration
"""
# Start replicating to dummy doc managers
self.opman1.start()
self.opman2.start()
collection = self.mongos_conn["test"]["mcsharded"]
for i in range(1000):
collection.insert({"i": i + 500})
# Assert current state of the mongoverse
self.assertEqual(self.shard1_conn["test"]["mcsharded"].find().count(),
500)
self.assertEqual(self.shard2_conn["test"]["mcsharded"].find().count(),
500)
assert_soon(lambda: len(self.opman1.doc_managers[0]._search()) == 1000)
# Test successful chunk move from shard 1 to shard 2
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-1"
)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
# Mark the collection as "dropped". This will cause migration to fail.
self.mongos_conn["config"]["collections"].update(
{"_id": "test.mcsharded"},
{"$set": {"dropped": True}}
)
# Test unsuccessful chunk move from shard 2 to shard 1
def fail_to_move_chunk():
self.mongos_conn["admin"].command(
"moveChunk",
"test.mcsharded",
find={"i": 1},
to="demo-set-0"
)
self.assertRaises(pymongo.errors.OperationFailure, fail_to_move_chunk)
# doc manager should still have all docs
all_docs = self.opman1.doc_managers[0]._search()
self.assertEqual(len(all_docs), 1000)
for i, doc in enumerate(sorted(all_docs, key=lambda x: x["i"])):
self.assertEqual(doc["i"], i + 500)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = "test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
res = list(self.solr_conn.search('content:*test_insert_file*'))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], "test.txt")
self.assertEqual(doc['_id'], str(id))
self.assertIn(test_data.strip(), doc['content'][0].strip())
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search(
'*:*', rows=STRESS_COUNT)) == STRESS_COUNT)
for i in range(0, STRESS_COUNT):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def setUp(self):
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint="config.txt",
target_url='%s:%d' % (mongo_host, self.standalone_port),
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py')
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
self.conn['test']['test'].remove()
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) > 0)
res = list(self._search())
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], id)
self.assertEqual(doc['content'], test_data)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert({'name': 'test_valid'})
self.conn['test']['test'].update({'_id': inserted_obj},
{'$set': {
'popularity': 1
}})
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(sum(1 for _ in self._search("name=test_valid")), 1)