def test_connector_minimum_privileges(self):
"""Test the Connector works with a user with minimum privileges."""
if not (db_user and db_password):
raise SkipTest("Need to set a user/password to test this.")
client = self.repl_set.client()
minimum_user = "******"
minimum_pwd = "password"
client.admin.add_user(
minimum_user,
minimum_pwd,
roles=[
{"role": "read", "db": "test"},
{"role": "read", "db": "wildcard"},
{"role": "read", "db": "local"},
],
)
client.test.test.insert_one({"replicated": 1})
client.test.ignored.insert_one({"replicated": 0})
client.ignored.ignored.insert_one({"replicated": 0})
client.wildcard.test.insert_one({"replicated": 1})
conn = Connector(
mongo_address=self.repl_set.primary.uri,
auth_username=minimum_user,
auth_key=minimum_pwd,
namespace_options={"test.test": True, "wildcard.*": True},
)
conn.start()
try:
assert_soon(conn.doc_managers[0]._search)
finally:
conn.join()
def test_connector_minimum_privileges(self):
"""Test the Connector works with a user with minimum privileges."""
if not (db_user and db_password):
raise SkipTest('Need to set a user/password to test this.')
client = self.repl_set.client()
minimum_user = '******'
minimum_pwd = 'password'
client.admin.add_user(minimum_user, minimum_pwd,
roles=[{'role': 'read', 'db': 'test'},
{'role': 'read', 'db': 'wildcard'},
{'role': 'read', 'db': 'local'}])
client.test.test.insert_one({"replicated": 1})
client.test.ignored.insert_one({"replicated": 0})
client.ignored.ignored.insert_one({"replicated": 0})
client.wildcard.test.insert_one({"replicated": 1})
conn = Connector(
mongo_address=self.repl_set.primary.uri,
auth_username=minimum_user,
auth_key=minimum_pwd,
namespace_options={'test.test': True, 'wildcard.*': True}
)
conn.start()
try:
assert_soon(conn.doc_managers[0]._search)
finally:
conn.join()
def test_start_with_auth(self):
dm = DocManager()
connector = Connector(mongo_address=self.cluster.uri,
doc_managers=[dm],
auth_username=db_user,
auth_key=db_password)
connector.start()
# Insert some documents into the sharded cluster. These
# should go to the DocManager, and the connector should not
# have an auth failure.
self.cluster.client().test.test.insert_one({'auth_failure': False})
assert_soon(lambda: len(dm._search()) > 0)
connector.join()
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
**connector_opts)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_start_with_auth(self):
dm = DocManager()
connector = Connector(
mongo_address=self.cluster.uri,
doc_managers=[dm],
auth_username=db_user,
auth_key=db_password
)
connector.start()
# Insert some documents into the sharded cluster. These
# should go to the DocManager, and the connector should not
# have an auth failure.
self.cluster.client().test.test.insert({'auth_failure': False})
assert_soon(lambda: len(dm._search()) > 0)
connector.join()
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(mongo_address=self.repl_set.uri, **connector_opts)
conn.start()
assert_soon(lambda: bool(conn.shard_set))
# Make sure get_mininum_mongodb_version returns the current version.
self.assertEqual(Version.from_client(self.repl_set.client()),
get_mininum_mongodb_version())
conn.join()
# Make sure the connector is shutdown correctly
self.assertFalse(conn.can_run)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(
mongo_address=self.repl_set.uri,
**connector_opts
)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(mongo_address=self.repl_set.uri, **connector_opts)
conn.start()
assert_soon(lambda: bool(conn.shard_set))
# Make sure get_mininum_mongodb_version returns the current version.
self.assertEqual(
Version.from_client(self.repl_set.client()), get_mininum_mongodb_version()
)
conn.join()
# Make sure the connector is shutdown correctly
self.assertFalse(conn.can_run)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_connector(self):
"""Test whether the connector initiates properly
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
conn = Connector(MAIN_ADDRESS, CONFIG, None, ['test.test'],
'_id', None, None)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_connector_minimum_privileges(self):
"""Test the Connector works with a user with minimum privileges."""
if not (db_user and db_password):
raise SkipTest("Need to set a user/password to test this.")
client = self.repl_set.client()
minimum_user = "******"
minimum_pwd = "password"
client.admin.add_user(
minimum_user,
minimum_pwd,
roles=[
{
"role": "read",
"db": "test"
},
{
"role": "read",
"db": "wildcard"
},
{
"role": "read",
"db": "local"
},
],
)
client.test.test.insert_one({"replicated": 1})
client.test.ignored.insert_one({"replicated": 0})
client.ignored.ignored.insert_one({"replicated": 0})
client.wildcard.test.insert_one({"replicated": 1})
conn = Connector(
mongo_address=self.repl_set.primary.uri,
auth_username=minimum_user,
auth_key=minimum_pwd,
namespace_options={
"test.test": True,
"wildcard.*": True
},
)
conn.start()
try:
assert_soon(conn.doc_managers[0]._search)
finally:
conn.join()
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(address='%s:%d' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=None,
ns_set=['test.test'],
u_key='_id',
auth_key=None)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
class TestConnectorSharded(unittest.TestCase):
def setUp(self):
if db_user and db_password:
auth_args = dict(auth_username=db_user, auth_key=db_password)
else:
auth_args = {}
self.cluster = ShardedClusterSingle().start()
self.dm = DocManager()
self.connector = Connector(mongo_address=self.cluster.uri,
doc_managers=[self.dm],
**auth_args)
self.connector.start()
def tearDown(self):
self.connector.join()
try:
os.unlink('oplog.timestamp')
except OSError:
pass
self.cluster.stop()
def test_connector(self):
"""Test whether the connector initiates properly
"""
conn = Connector(
address='%s:%d' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=None,
ns_set=['test.test'],
u_key='_id',
auth_key=None
)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
def test_connector(self):
"""Test whether the connector initiates properly
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
conn = Connector(address=MAIN_ADDRESS,
oplog_checkpoint=CONFIG,
target_url=None,
ns_set=['test.test'],
u_key='_id',
auth_key=None)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
class MongoReplicaSetTestCase(MongoTestCase):
def setUp(self):
self.repl_set = self.replica_set_class().start()
self.conn = self.repl_set.client()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self._remove()
self.connector = Connector(
mongo_address=self.repl_set.uri,
doc_managers=(self.mongo_doc,),
namespace_options={
'test.test': {'gridfs': True},
'rename.me': 'new.target',
'rename.me2': 'new2.target2'
},
**connector_opts
)
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def drop_all_databases(self):
for name in self.mongo_conn.database_names():
if name not in ["local", "admin"]:
self.mongo_conn.drop_database(name)
for name in self.conn.database_names():
if name not in ["local", "admin"]:
self.conn.drop_database(name)
def tearDown(self):
self.connector.join()
self.drop_all_databases()
self.repl_set.stop()
class TestConnectorSharded(unittest.TestCase):
def setUp(self):
if db_user and db_password:
auth_args = dict(auth_username=db_user, auth_key=db_password)
else:
auth_args = {}
self.cluster = ShardedClusterSingle().start()
self.dm = DocManager()
self.connector = Connector(
mongo_address=self.cluster.uri,
doc_managers=[self.dm],
**auth_args
)
self.connector.start()
def tearDown(self):
self.connector.join()
try:
os.unlink('oplog.timestamp')
except OSError:
pass
self.cluster.stop()
class MongoReplicaSetTestCase(MongoTestCase):
def setUp(self):
self.repl_set = self.replica_set_class().start()
self.conn = self.repl_set.client()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self._remove()
self.connector = Connector(mongo_address=self.repl_set.uri,
doc_managers=(self.mongo_doc, ),
namespace_options={
"test.test": {
"gridfs": True
},
"rename.me": "new.target",
"rename.me2": "new2.target2",
},
**connector_opts)
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def drop_all_databases(self):
for name in self.mongo_conn.database_names():
if name not in ["local", "admin"]:
self.mongo_conn.drop_database(name)
for name in self.conn.database_names():
if name not in ["local", "admin"]:
self.conn.drop_database(name)
def tearDown(self):
self.connector.join()
self.drop_all_databases()
self.repl_set.stop()
def test_connector(self):
"""Test whether the connector initiates properly
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
conn = Connector(
address=MAIN_ADDRESS,
oplog_checkpoint=CONFIG,
target_url=None,
ns_set=['test.test'],
u_key='_id',
auth_key=None
)
conn.start()
while len(conn.shard_set) != 1:
time.sleep(2)
conn.join()
self.assertFalse(conn.can_run)
time.sleep(5)
for thread in conn.shard_set.values():
self.assertFalse(thread.running)
class TestSolr(SolrTestCase):
""" Tests Solr
"""
@classmethod
def setUpClass(cls):
SolrTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
cls.repl_set.stop()
def setUp(self):
self._remove()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
docman = DocManager('http://%s/solr' % solr_pair,
auto_commit_interval=0)
self.connector = Connector(
mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman,),
gridfs_set=['test.test']
)
retry_until_ok(self.conn.test.test.drop)
retry_until_ok(self.conn.test.test.files.drop)
retry_until_ok(self.conn.test.test.chunks.drop)
self._remove()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
def tearDown(self):
self.connector.join()
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = "test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
res = list(self.solr_conn.search('test_insert_file'))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], "test.txt")
self.assertEqual(doc['_id'], str(id))
self.assertEqual(doc['content'][0].strip(), test_data.strip())
def test_remove_file(self):
"""Tests removing a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_update(self):
"""Test update operations on Solr.
Need to have the following defined in schema.xml:
<field name="a" type="int" indexed="true" stored="true" />
<field name="b.0.c" type="int" indexed="true" stored="true" />
<field name="b.10.c" type="int" indexed="true" stored="true" />
<field name="b.0.e" type="int" indexed="true" stored="true" />
<field name="b.1.d" type="int" indexed="true" stored="true" />
<field name="b.1.f" type="int" indexed="true" stored="true" />
<field name="b.2.e" type="int" indexed="true" stored="true" />
"""
docman = self.connector.doc_managers[0]
# Use diabolical value for _id to test string escaping as well.
self.conn.test.test.insert({"_id": u'+-&え|!(){}[]^"~*?:\\/', "a": 0})
assert_soon(lambda: sum(1 for _ in self._search("*:*")) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Stringify _id to match what will be retrieved from Solr
updated['_id'] = u(updated['_id'])
# Flatten the MongoDB document to match Solr
updated = docman._clean_doc(updated, 'dummy.namespace', 0)
# Allow some time for update to propagate
time.sleep(1)
replicated = list(self._search("a:0"))[0]
# Remove add'l fields until these are stored in a separate Solr core
updated.pop('_ts')
replicated.pop('_ts')
updated.pop('ns')
replicated.pop('ns')
# Remove field added by Solr
replicated.pop("_version_")
self.assertEqual(replicated, updated)
# Update by adding a field.
# Note that Solr can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by replacing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['test'].insert({'name': 'paul'})
assert_soon(
lambda: self.conn.test.test.find({'name': 'paul'}).count() == 1)
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2)
result_set_1 = list(self.solr_conn.search('pauline'))
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(sum(1 for _ in result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(sum(1 for _ in result_set_2), 1)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_valid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'popularity': 1}}
)
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(sum(1 for _ in self._search(
"name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_invalid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'break_this_test': 1}}
)
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(sum(1 for _ in self._search(
"name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert(match_first)
self.conn["test"]["test"].insert(match_second)
self.conn["test"]["test"].insert(match_none)
# Should have documents in Solr now
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="person.address.street" type="string" ... />
<field name="person.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
# Connector is already running
self.conn["test"]["test"].insert({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert({
"numbers": ["one", "two", "three"],
"characters": [
{"name": "Big Bird",
"color": "yellow"},
{"name": "Elmo",
"color": "red"},
"Cookie Monster"
]
})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["billing.address.state"],
"California")
# Search for second document
results = self.solr_conn.search(
"characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase):
"""Integration tests for mongo-connector + Elasticsearch."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
super(TestElastic, cls).setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
close_client(cls.conn)
cls.repl_set.stop()
def tearDown(self):
"""Stop the Connector thread."""
super(TestElastic, self).tearDown()
self.connector.join()
def setUp(self):
"""Start a new Connector for each test."""
super(TestElastic, self).setUp()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self.connector = Connector(
mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(self.elastic_doc,),
gridfs_set=['test.test']
)
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: self._count() > 0)
query = {"match": {"_all": "test_insert_file"}}
res = list(self._search(query))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], str(id))
self.assertEqual(base64.b64decode(doc['content']), test_data)
def test_remove_file(self):
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: self._count() == 1)
fs.delete(id)
assert_soon(lambda: self._count() == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'),
('query', {"a": 0}),
('update', update_spec),
('new', True)]))['value']
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
assert_soon(lambda: next(self._search()) == updated)
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
# This doc can be picked up in the collection dump
self.conn['test']['test'].insert_one({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
# This doc is definitely not picked up by collection dump
self.conn['test']['test'].insert_one({'name': 'pauly'})
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert_one,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 3)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 3)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 2)
if result_set_1[0]['name'] == 'paul':
self.assertEqual(result_set_1[1]['name'], 'pauly')
elif result_set_1[0]['name'] == 'pauly':
self.assertEqual(result_set_1[1]['name'], 'paul')
else:
self.assertTrue(0, 'Unknown document retrieved')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 2)
def test_bad_int_value(self):
self.conn.test.test.insert_one({
'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
class TestElastic(unittest.TestCase):
""" Tests the Elastic instance
"""
def runTest(self):
""" Runs the tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
""" Starts the cluster
"""
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
cls.elastic_doc = DocManager('localhost:9200')
cls.elastic_doc._remove()
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']),
replicaSet="demo-repl")
import logging
logger = logging.getLogger()
loglevel = logging.INFO
logger.setLevel(loglevel)
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_all()
def tearDown(self):
""" Ends the connector
"""
self.connector.join()
def setUp(self):
""" Starts a new connector for every test
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector(
address='%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']),
oplog_checkpoint=CONFIG,
target_url='localhost:9200',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
self.conn['test']['test'].remove(safe=True)
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) > 0)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
condition1 = lambda: self.conn['test']['test'].find({
'name': 'paul'
}).count() == 1
condition2 = lambda: sum(1 for _ in self.elastic_doc._search()) == 1
wait_for(condition1)
wait_for(condition2)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = 0
while True:
try:
self.conn['test']['test'].insert({'name': 'pauline'},
safe=True)
break
except OperationFailure:
time.sleep(1)
count += 1
if count >= 60:
sys.exit(1)
continue
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 2)
result_set_1 = list(self.elastic_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if (item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)},
safe=True)
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)}, safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == self.
conn['test']['test'].find().count())
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
db_admin = primary_conn["admin"]
wait_for(lambda: db_admin.command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
def test_non_standard_fields(self):
""" Tests ObjectIds, DBrefs, etc
"""
# This test can break if it attempts to insert before the dump takes
# place- this prevents it (other tests affected too actually)
while (self.connector.shard_set['demo-repl'].checkpoint is None):
time.sleep(1)
docs = [{
'foo': [1, 2]
}, {
'bar': {
'hello': 'world'
}
}, {
'code': Code("function x() { return 1; }")
}, {
'dbref': {
'_ref': DBRef('simple', ObjectId('509b8db456c02c5ab7e63c34'))
}
}]
try:
self.conn['test']['test'].insert(docs)
except OperationFailure:
self.fail("Cannot insert documents into Elastic!")
search = self.elastic_doc._search
if not wait_for(lambda: sum(1 for _ in search()) == len(docs)):
self.fail("Did not get all expected documents")
self.assertIn("dbref", self.elastic_doc.get_last_doc())
class TestElastic(unittest.TestCase):
""" Tests the Elastic instance
"""
@classmethod
def setUpClass(cls):
""" Starts the cluster
"""
cls.elastic_doc = DocManager(elastic_pair, auto_commit=False)
_, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-elastic')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_replica_set('test-elastic')
def tearDown(self):
""" Ends the connector
"""
self.connector.join()
def setUp(self):
""" Starts a new connector for every test
"""
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0
)
# Clean out test databases
try:
self.elastic_doc._remove()
except OperationFailed:
try:
# Create test.test index if necessary
client = Elasticsearch(hosts=[elastic_pair])
idx_client = IndicesClient(client)
idx_client.create(index='test.test')
except es_exceptions.TransportError:
pass
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) > 0)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: sum(1 for _ in self.elastic_doc._search()) == 1
assert_soon(condition1)
assert_soon(condition2)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 2)
result_set_1 = list(self.elastic_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing a large number of documents"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
for i in range(0, STRESS_COUNT):
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if(item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search())
== self.conn['test']['test'].find().count())
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn["admin"]
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self.elastic_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSolr(SolrTestCase):
""" Tests Solr
"""
@classmethod
def setUpClass(cls):
SolrTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
cls.repl_set.stop()
def setUp(self):
self._remove()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
docman = DocManager(solr_url, auto_commit_interval=0)
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman, ),
gridfs_set=['test.test'])
retry_until_ok(self.conn.test.test.drop)
retry_until_ok(self.conn.test.test.files.drop)
retry_until_ok(self.conn.test.test.chunks.drop)
self._remove()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
def tearDown(self):
self.connector.join()
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('name:paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = "test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
res = list(self.solr_conn.search('content:*test_insert_file*'))
if not res:
res = list(self.solr_conn.search('_text_:*test_insert_file*'))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], "test.txt")
self.assertEqual(doc['_id'], str(id))
content = doc.get('content', doc.get('_text_', None))
self.assertTrue(content)
self.assertIn(test_data.strip(), content[0].strip())
def test_remove_file(self):
"""Tests removing a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_update(self):
"""Test update operations on Solr.
Need to have the following defined in schema.xml:
<field name="a" type="int" indexed="true" stored="true" />
<field name="b.0.c" type="int" indexed="true" stored="true" />
<field name="b.10.c" type="int" indexed="true" stored="true" />
<field name="b.0.e" type="int" indexed="true" stored="true" />
<field name="b.1.d" type="int" indexed="true" stored="true" />
<field name="b.1.f" type="int" indexed="true" stored="true" />
<field name="b.2.e" type="int" indexed="true" stored="true" />
"""
docman = self.connector.doc_managers[0]
# Use diabolical value for _id to test string escaping as well.
self.conn.test.test.insert_one({
"_id": u'+-&え|!(){}[]^"~*?:\\/',
"a": 0
})
assert_soon(lambda: sum(1 for _ in self._search("*:*")) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'), ('query', {
"a": 0
}), ('update', update_spec), ('new', True)]))['value']
# Stringify _id to match what will be retrieved from Solr
updated[u('_id')] = u(updated['_id'])
# Flatten the MongoDB document to match Solr
updated = docman._clean_doc(updated, 'dummy.namespace', 0)
# Allow some time for update to propagate
time.sleep(3)
replicated = list(self._search("a:0"))[0]
# Remove add'l fields until these are stored in a separate Solr core
updated.pop('_ts')
replicated.pop('_ts')
updated.pop('ns')
replicated.pop('ns')
# Remove field added by Solr
replicated.pop("_version_")
self.assertEqual(replicated, updated)
# Update by adding a field.
# Note that Solr can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by replacing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['test'].insert_one({'name': 'paul'})
assert_soon(lambda: self.conn.test.test.find({
'name': 'paul'
}).count() == 1)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2)
result_set_1 = list(self.solr_conn.search('name:pauline'))
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = self.solr_conn.search('name:pauline')
self.assertEqual(sum(1 for _ in result_set_1), 0)
result_set_2 = self.solr_conn.search('name:paul')
self.assertEqual(sum(1 for _ in result_set_2), 1)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert_one({
'name':
'test_valid'
}).inserted_id
self.conn['test']['test'].update_one({'_id': inserted_obj},
{'$set': {
'popularity': 1
}})
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(sum(1 for _ in self._search("name:test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert_one({
'name':
'test_invalid'
}).inserted_id
self.conn['test']['test'].update_one({'_id': inserted_obj},
{'$set': {
'break_this_test': 1
}})
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(sum(1 for _ in self._search("name:test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert_one(match_first)
self.conn["test"]["test"].insert_one(match_second)
self.conn["test"]["test"].insert_one(match_none)
# Should have documents in Solr now
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="billing.address.street" type="string" ... />
<field name="billing.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
# Connector is already running
self.conn["test"]["test"].insert_one({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert_one({
"numbers": ["one", "two", "three"],
"characters": [{
"name": "Big Bird",
"color": "yellow"
}, {
"name": "Elmo",
"color": "red"
}, "Cookie Monster"]
})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(
next(iter(results))["billing.address.state"], "California")
# Search for second document
results = self.solr_conn.search("characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase):
"""Integration tests for mongo-connector + Elasticsearch."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
super(TestElastic, cls).setUpClass()
_, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-elastic')
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
kill_replica_set('test-elastic')
def tearDown(self):
"""Stop the Connector thread."""
super(TestElastic, self).tearDown()
self.connector.join()
def setUp(self):
"""Start a new Connector for each test."""
super(TestElastic, self).setUp()
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0
)
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_shard_length(self):
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
# Allow some time for update to propagate
time.sleep(1)
replicated = next(self._search())
self.assertEqual(replicated, updated)
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Stress test for inserting and removing many documents."""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
self.assertEqual(
set('Paul ' + str(i) for i in range(STRESS_COUNT)),
set(item['name'] for item in self._search())
)
def test_stressed_rollback(self):
"""Stress test for a rollback with many documents."""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: self._count()
== self.conn['test']['test'].find().count())
result_set_1 = self._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn["admin"]
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase):
""" Tests Solr
"""
@classmethod
def setUpClass(cls):
_, cls.secondary_p, cls.primary_p = start_replica_set('test-solr')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-solr')
cls.solr_conn = Solr('http://%s/solr' % solr_pair)
cls.solr_conn.delete(q='*:*')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_replica_set('test-solr')
def setUp(self):
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url='http://localhost:8983/solr',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/solr_doc_manager.py',
auto_commit_interval=0
)
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
retry_until_ok(self.conn.test.test.remove)
assert_soon(lambda: len(self.solr_conn.search('*:*')) == 0)
def tearDown(self):
self.connector.join()
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
while (len(self.solr_conn.search('*:*')) == 0):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: len(self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: len(self.solr_conn.search("*:*")) == 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
time.sleep(1)
while len(self.solr_conn.search('*:*')) != 1:
time.sleep(1)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
while (len(self.solr_conn.search('*:*')) != 2):
time.sleep(1)
result_set_1 = self.solr_conn.search('pauline')
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(len(result_set_2), 1)
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT))
!= STRESS_COUNT):
time.sleep(5)
for i in range(0, STRESS_COUNT):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with a large number of documents"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert(
{'name': 'Paul ' + str(i)})
while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT))
!= STRESS_COUNT):
time.sleep(1)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT * 2)) !=
self.conn['test']['test'].find().count()):
time.sleep(1)
result_set_1 = self.solr_conn.search(
'Pauline',
rows=STRESS_COUNT * 2, sort='_id asc'
)
for item in result_set_1:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
while (len(self.solr_conn.search(
'Pauline',
rows=STRESS_COUNT * 2)) != 0):
time.sleep(15)
result_set_1 = self.solr_conn.search(
'Pauline',
rows=STRESS_COUNT * 2
)
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search(
'Paul',
rows=STRESS_COUNT * 2
)
self.assertEqual(len(result_set_2), STRESS_COUNT)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_valid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'popularity': 1}}
)
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(len(docman._search(
"name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_invalid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'break_this_test': 1}}
)
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(len(docman._search(
"name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert(match_first)
self.conn["test"]["test"].insert(match_second)
self.conn["test"]["test"].insert(match_none)
# Should have documents in Solr now
assert_soon(lambda: len(self.solr_conn.search("*:*")) > 0,
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="person.address.street" type="string" ... />
<field name="person.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
# Connector is already running
self.conn["test"]["test"].insert({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert({
"numbers": ["one", "two", "three"],
"characters": [
{"name": "Big Bird",
"color": "yellow"},
{"name": "Elmo",
"color": "red"},
"Cookie Monster"
]
})
assert_soon(lambda: len(self.solr_conn.search("*:*")) > 0,
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["billing.address.state"],
"California")
# Search for second document
results = self.solr_conn.search(
"characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestNeo4j(Neo4jTestCase):
"""Integration tests for mongo-connector + Neo4j."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
Neo4jTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
cls.repl_set.stop()
def tearDown(self):
"""Stop the Connector thread."""
super(TestNeo4j, self).tearDown()
self.connector.join()
time.sleep(10)
def setUp(self):
"""Start a new Connector for each test."""
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
docman = DocManager('http://localhost:7474/db/data',
auto_commit_interval=0)
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman, ),
gridfs_set=['test.test'])
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
self.neo4j_conn.delete_all()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
time.sleep(5)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set_2 = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert(
{
'_id': str(result_set_2['_id']),
'name': 'paulie'
}, "test.test", 1)
assert_soon(lambda: self._count() > 0)
result_set_1 = self.neo4j_conn.find_one("test")
self.assertNotEqual(result_set_1, None)
self.assertEqual(result_set_1['_id'], str(result_set_2['_id']))
self.assertEqual(result_set_1['name'], result_set_2['name'])
self.connector.doc_managers[0].graph.delete_all()
def test_subdocument_with_id(self):
"""
Test inserting a document with a subdocument containing an _id property.
In the current version of translating from document data model to property graph, the root level _id
field is included in all subdocument nodes in the graph. This test verifies there is no error when
inserting a document containing a subdocument with an _id property.
See https://github.com/neo4j-contrib/neo4j_doc_manager/issues/56
"""
doc = {
'_id': 'root_level_object_id',
'name': 'Bob',
'location': {
'_id': 'sub_document_object_id',
'city': 'Missoula',
'state': 'Montana'
}
}
self.connector.doc_managers[0].upsert(doc, "test.test_id", 1)
assert_soon(lambda: self._count() > 0)
result_set = self.neo4j_conn.find_one("location")
self.assertNotEqual(result_set, None)
self.assertEqual(
result_set['_id'],
'root_level_object_id') # expect subdocument _id to be ignored
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set = self.conn['test']['test'].find_one()
# self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.connector.doc_managers[0].remove(str(result_set['_id']),
'test.test', 1)
self.assertEqual(self._count(), 0)
self.connector.doc_managers[0].graph.delete_all()
def test_update(self):
"""Test update operations."""
# Insert
self.connector.doc_managers[0].graph.delete_all()
self.conn['test']['updt'].insert({"a": 0})
result_set = self.conn['test']['updt'].find_one()
self.connector.doc_managers[0].upsert(
{
'_id': str(result_set['_id']),
'a': '0'
}, "test.updt", 1)
assert_soon(lambda: self._count() == 1)
def check_update(update_spec):
updated = self.conn.test.updt.find_and_modify({"a": 0},
update_spec,
new=True)
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
# Allow some time for update to propagate
time.sleep(5)
replicated = self.neo4j_conn.find_one("updt")['_id']
self.assertEqual(replicated, updated['_id'])
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
self.connector.doc_managers[0].graph.delete_all()
def test_bad_int_value(self):
self.conn['test']['test'].insert({
'inf': float('inf'),
'nan': float('nan'),
'still_exists': True
})
result_set = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert(
{
'_id': str(result_set['_id']),
'inf': float('inf'),
'nan': float('nan'),
'still_exists': True
}, "test.test", 1)
assert_soon(lambda: self._count() > 0)
doc = self.neo4j_conn.find_one("test")
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['rollback'].insert({'name': 'paul'})
result_set = self.conn['test']['rollback'].find_one()
self.connector.doc_managers[0].upsert(
{
'_id': str(result_set['_id']),
'name': 'paul'
}, "test.rollback", 1)
condition1 = lambda: self.conn['test']['rollback'].find({
'name': 'paul'
}).count() == 1
assert_soon(condition1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.rollback.insert, {'name': 'pauline'})
assert_soon(lambda: self._count() > 0)
result_set_2 = self.conn['test']['rollback'].find_one(
{'name': 'pauline'})
self.connector.doc_managers[0].upsert(
{
'_id': str(result_set_2['_id']),
'name': 'pauline'
}, "test.rollback", 1)
result_set_1 = self.neo4j_conn.find_one("rollback", "name", "pauline")
self.assertNotEqual(result_set_1, None)
#make sure pauline is there
if result_set_1['name'] == 'pauline':
self.assertEqual(result_set_1['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
node = self.neo4j_conn.find_one("rollback")
self.assertEqual(node['name'], 'paul')
node = self.neo4j_conn.find_one("rollback", "name", "pauline")
self.assertNotEqual(node, None)
find_cursor = retry_until_ok(self.conn['test']['rollback'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestMongo(MongoTestCase):
""" Tests the mongo instance
"""
@classmethod
def setUpClass(cls):
MongoTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
MongoTestCase.tearDownClass()
cls.repl_set.stop()
def tearDown(self):
self.connector.join()
def setUp(self):
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self._remove()
self.connector = Connector(mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(self.mongo_doc, ),
gridfs_set=['test.test'],
**connector_opts)
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
result_set_1 = self._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) != 1)
self.assertEqual(sum(1 for _ in self._search()), 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) > 0)
res = list(self._search())
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], id)
self.assertEqual(doc['content'], test_data)
def test_remove_file(self):
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'), ('query', {
"a": 0
}), ('update', update_spec), ('new', True)]))['value']
# Allow some time for update to propagate
time.sleep(2)
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
self.assertEqual(replicated, updated)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['test'].insert_one({'name': 'paul'})
condition = lambda: self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
assert_soon(condition)
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
condition = lambda: admin.command("isMaster")['ismaster']
assert_soon(lambda: retry_until_ok(condition))
retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self._search()) == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
assert_soon(
lambda: primary_conn['admin'].command("isMaster")['ismaster'])
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestSynchronizer(unittest.TestCase):
""" Tests Solr
"""
@classmethod
def setUpClass(cls):
_, cls.secondary_p, cls.primary_p = start_replica_set('test-solr')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-solr')
cls.solr_conn = Solr('http://%s/solr' % solr_pair)
cls.solr_conn.delete(q='*:*')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_replica_set('test-solr')
def setUp(self):
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url='http://localhost:8983/solr',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/solr_doc_manager.py',
auto_commit_interval=0
)
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
retry_until_ok(self.conn.test.test.remove)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 0)
def tearDown(self):
self.connector.join()
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0)
result_set_1 = list(self.solr_conn.search('paulie'))
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0)
def test_update(self):
"""Test update operations on Solr.
Need to have the following defined in schema.xml:
<field name="a" type="int" indexed="true" stored="true" />
<field name="b.0.c" type="int" indexed="true" stored="true" />
<field name="b.0.e" type="int" indexed="true" stored="true" />
<field name="b.1.d" type="int" indexed="true" stored="true" />
<field name="b.1.f" type="int" indexed="true" stored="true" />
"""
docman = self.connector.doc_managers[0]
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Stringify _id to match what will be retrieved from Solr
updated['_id'] = str(updated['_id'])
# Flatten the MongoDB document to match Solr
updated = docman._clean_doc(updated)
# Allow some time for update to propagate
time.sleep(1)
replicated = list(docman._search("a:0"))[0]
# Remove add'l fields until these are stored in a separate Solr core
replicated.pop("_ts")
replicated.pop("ns")
# Remove field added by Solr
replicated.pop("_version_")
self.assertEqual(replicated, docman._clean_doc(updated))
# Update by adding a field.
# Note that Solr can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by replacing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
assert_soon(
lambda: self.conn.test.test.find({'name': 'paul'}).count() == 1)
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2)
result_set_1 = list(self.solr_conn.search('pauline'))
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(sum(1 for _ in result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(sum(1 for _ in result_set_2), 1)
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search(
'*:*', rows=STRESS_COUNT)) == STRESS_COUNT)
for i in range(0, STRESS_COUNT):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with a large number of documents"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert(
{'name': 'Paul ' + str(i)})
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search(
'*:*', rows=STRESS_COUNT)) == STRESS_COUNT)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
collection_size = self.conn['test']['test'].find().count()
assert_soon(
lambda: sum(1 for _ in self.solr_conn.search(
'*:*', rows=STRESS_COUNT * 2)) == collection_size)
result_set_1 = self.solr_conn.search(
'Pauline',
rows=STRESS_COUNT * 2, sort='_id asc'
)
for item in result_set_1:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
assert_soon(lambda: sum(1 for _ in self.solr_conn.search(
'Pauline', rows=STRESS_COUNT * 2)) == 0)
result_set_1 = list(self.solr_conn.search(
'Pauline',
rows=STRESS_COUNT * 2
))
self.assertEqual(len(result_set_1), 0)
result_set_2 = list(self.solr_conn.search(
'Paul',
rows=STRESS_COUNT * 2
))
self.assertEqual(len(result_set_2), STRESS_COUNT)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_valid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'popularity': 1}}
)
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(sum(1 for _ in docman._search(
"name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_invalid'})
self.conn['test']['test'].update(
{'_id': inserted_obj},
{'$set': {'break_this_test': 1}}
)
docman = self.connector.doc_managers[0]
assert_soon(lambda: sum(1 for _ in docman._search("*:*")) > 0)
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(sum(1 for _ in docman._search(
"name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert(match_first)
self.conn["test"]["test"].insert(match_second)
self.conn["test"]["test"].insert(match_none)
# Should have documents in Solr now
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="person.address.street" type="string" ... />
<field name="person.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
# Connector is already running
self.conn["test"]["test"].insert({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert({
"numbers": ["one", "two", "three"],
"characters": [
{"name": "Big Bird",
"color": "yellow"},
{"name": "Elmo",
"color": "red"},
"Cookie Monster"
]
})
assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0,
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["billing.address.state"],
"California")
# Search for second document
results = self.solr_conn.search(
"characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestSynchronizer(unittest.TestCase):
""" Tests the mongo instance
"""
@classmethod
def setUpClass(cls):
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
cls.standalone_port = start_mongo_proc(options=['--nojournal',
'--noprealloc'])
cls.mongo_doc = DocManager('%s:%d' % (mongo_host, cls.standalone_port))
cls.mongo_doc._remove()
_, cls.secondary_p, cls.primary_p = start_replica_set('test-mongo')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-mongo')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_mongo_proc(cls.standalone_port)
kill_replica_set('test-mongo')
def tearDown(self):
self.connector.join()
def setUp(self):
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint="config.txt",
target_url='%s:%d' % (mongo_host, self.standalone_port),
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py'
)
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
self.conn['test']['test'].remove()
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Allow some time for update to propagate
time.sleep(2)
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
self.assertEqual(replicated, updated)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition = lambda: self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
assert_soon(condition)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
condition = lambda: admin.command("isMaster")['ismaster']
assert_soon(lambda: retry_until_ok(condition))
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 2)
result_set_1 = list(self.mongo_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
assert_soon(
lambda: primary_conn['admin'].command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
for i in range(0, STRESS_COUNT):
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if(item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search())
== self.conn['test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn['admin']
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase):
""" Tests Solr
"""
def runTest(self):
""" Runs tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']),
replicaSet="demo-repl")
# Creating a Solr object with an invalid URL
# doesn't create an exception
cls.solr_conn = Solr('http://localhost:8983/solr')
try:
cls.solr_conn.commit()
except (SolrError, MissingSchema):
cls.err_msg = "Cannot connect to Solr!"
cls.flag = False
if cls.flag:
cls.solr_conn.delete(q='*:*')
else:
cls.err_msg = "Shards cannot be added to mongos"
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_all()
def setUp(self):
if not self.flag:
self.fail(self.err_msg)
self.connector = Connector(('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])),
CONFIG, 'http://localhost:8983/solr', ['test.test'], '_id',
None,
'mongo_connector/doc_managers/solr_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
time.sleep(1)
count = 0
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except (AutoReconnect, OperationFailure):
time.sleep(1)
count += 1
if count > 60:
unittest.SkipTest('Call to remove failed too '
'many times in setup')
while (len(self.solr_conn.search('*:*')) != 0):
time.sleep(1)
def tearDown(self):
self.connector.doc_manager.auto_commit = False
time.sleep(2)
self.connector.join()
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except OperationFailure:
continue
self.solr_conn.delete(q='*:*')
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(len(self.solr_conn.search('*:*')), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 0):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 1):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
time.sleep(1)
while len(self.solr_conn.search('*:*')) != 1:
time.sleep(1)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
self.conn['test']['test'].insert(
{'name': 'pauline'}, safe=True)
break
except OperationFailure:
count += 1
if count > 60:
self.fail('Call to insert failed too '
'many times in test_rollback')
time.sleep(1)
continue
while (len(self.solr_conn.search('*:*')) != 2):
time.sleep(1)
result_set_1 = self.solr_conn.search('pauline')
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(len(result_set_2), 1)
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS))
!= NUMBER_OF_DOC_DIRS):
time.sleep(5)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. The rollback is performed the same way as before
but with more docs
"""
self.conn['test']['test'].remove()
while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0:
time.sleep(1)
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert(
{'name': 'Paul ' + str(i)}, safe=True)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS))
!= NUMBER_OF_DOC_DIRS):
time.sleep(1)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)},
safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) !=
self.conn['test']['test'].find().count()):
time.sleep(1)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc')
for item in result_set_1:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
while (len(self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2)) != 0):
time.sleep(15)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('Paul',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name':'test_valid'})
self.conn['test']['test'].update({'_id' : inserted_obj},
{'$set':{'popularity' : 1 }})
for _ in range(60):
if len(self.connector.doc_manager._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = self.connector.doc_manager.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(len(self.connector.doc_manager._search(
"name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name':'test_invalid'})
self.conn['test']['test'].update({'_id' : inserted_obj},
{'$set':{'break_this_test' : 1 }})
for _ in range(60):
if len(self.connector.doc_manager._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = self.connector.doc_manager.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(len(self.connector.doc_manager._search(
"name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following field in the supplied schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
"""
inserted_obj = self.conn['test']['test'].insert({
'name':'test_dynamic',
'foo_i':1,
'i_foo':1})
self.assertEqual(self.conn['test']['test'].find().count(), 1)
for _ in range(60):
if len(self.connector.doc_manager._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = self.connector.doc_manager.get_last_doc()
self.assertIn('i_foo', result)
self.assertIn('foo_i', result)
self.assertEqual(len(self.connector.doc_manager._search(
"i_foo:1")), 1)
self.assertEqual(len(self.connector.doc_manager._search(
"foo_i:1")), 1)
class TestElastic(ElasticsearchTestCase):
"""Integration tests for mongo-connector + Elasticsearch."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
super(TestElastic, cls).setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
close_client(cls.conn)
cls.repl_set.stop()
def tearDown(self):
"""Stop the Connector thread."""
super(TestElastic, self).tearDown()
self.connector.join()
def setUp(self):
"""Start a new Connector for each test."""
super(TestElastic, self).setUp()
try:
os.unlink("oplog.timestamp")
except OSError:
pass
docman = DocManager(elastic_pair)
self.connector = Connector(
mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman,),
gridfs_set=['test.test']
)
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: self._count() > 0)
query = {"match": {"_all": "test_insert_file"}}
res = list(self._search(query))
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], str(id))
self.assertEqual(base64.b64decode(doc['content']), test_data)
def test_remove_file(self):
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: self._count() == 1)
fs.delete(id)
assert_soon(lambda: self._count() == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'),
('query', {"a": 0}),
('update', update_spec),
('new', True)]))['value']
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
assert_soon(lambda: next(self._search()) == updated)
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
# This doc can be picked up in the collection dump
self.conn['test']['test'].insert_one({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
# This doc is definitely not picked up by collection dump
self.conn['test']['test'].insert_one({'name': 'pauly'})
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert_one,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 3)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 3)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 2)
if result_set_1[0]['name'] == 'paul':
self.assertEqual(result_set_1[1]['name'], 'pauly')
elif result_set_1[0]['name'] == 'pauly':
self.assertEqual(result_set_1[1]['name'], 'paul')
else:
self.assertTrue(0, 'Unknown document retrieved')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 2)
def test_bad_int_value(self):
self.conn.test.test.insert_one({
'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
assert_soon(lambda: self._count() > 0)
for doc in self._search():
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
class TestElastic(ElasticsearchTestCase):
""" Tests the Elastic instance
"""
@classmethod
def setUpClass(cls):
""" Starts the cluster
"""
super(TestElastic, cls).setUpClass()
_, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-elastic')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_replica_set('test-elastic')
def tearDown(self):
""" Ends the connector
"""
super(TestElastic, self).tearDown()
self.connector.join()
def setUp(self):
""" Starts a new connector for every test
"""
super(TestElastic, self).setUp()
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0
)
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing a large number of documents"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
self.assertEqual(
set('Paul ' + str(i) for i in range(STRESS_COUNT)),
set(item['name'] for item in self._search())
)
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: self._count()
== self.conn['test']['test'].find().count())
result_set_1 = self._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn["admin"]
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase):
""" Tests the mongo instance
"""
def runTest(self):
""" Runs the tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
start_single_mongod_instance("30000", "/MC", "MC_log")
cls.mongo_doc = DocManager("localhost:30000")
cls.mongo_doc._remove()
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection("%s:%s" % (HOSTNAME, PORTS_ONE['MONGOS']),
replicaSet="demo-repl")
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_mongo_proc(HOSTNAME, 30000)
kill_all()
def tearDown(self):
self.connector.join()
def setUp(self):
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector("%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]),
CONFIG, '%s:30000' % (HOSTNAME),
['test.test'],
'_id', None,
'mongo_connector/doc_managers/mongo_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
self.conn['test']['test'].remove(safe=True)
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
condition = lambda : self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
wait_for(condition)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
condition = lambda : admin.command("isMaster")['ismaster']
wait_for(condition)
time.sleep(5)
count = 0
while True:
try:
result_set_1 = self.conn['test']['test'].insert(
{'name': 'pauline'}, safe=True)
break
except OperationFailure:
time.sleep(1)
count += 1
if count >= 60:
sys.exit(1)
continue
wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 2)
result_set_1 = list(self.mongo_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
wait_for(lambda : primary_conn['admin'].command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.mongo_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if(item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)},
safe=True)
search = self.mongo_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda : admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert({'name': 'Pauline ' +
str(count)}, safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
wait_for(lambda : sum(1 for _ in self.mongo_doc._search())
== self.conn['test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
db_admin = primary_conn['admin']
wait_for(lambda : db_admin.command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
search = self.mongo_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
class TestMongo(MongoTestCase):
""" Tests the mongo instance
"""
@classmethod
def setUpClass(cls):
MongoTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
MongoTestCase.tearDownClass()
cls.repl_set.stop()
def tearDown(self):
self.connector.join()
def setUp(self):
try:
os.unlink("oplog.timestamp")
except OSError:
pass
self._remove()
self.connector = Connector(
mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(self.mongo_doc,),
gridfs_set=['test.test'],
**connector_opts
)
self.conn.drop_database('test')
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
result_set_1 = self._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.conn['test']['test'].delete_one({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self._search()) != 1)
self.assertEqual(sum(1 for _ in self._search()), 0)
def test_insert_file(self):
"""Tests inserting a gridfs file
"""
fs = GridFS(self.conn['test'], 'test')
test_data = b"test_insert_file test file"
id = fs.put(test_data, filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) > 0)
res = list(self._search())
self.assertEqual(len(res), 1)
doc = res[0]
self.assertEqual(doc['filename'], 'test.txt')
self.assertEqual(doc['_id'], id)
self.assertEqual(doc['content'], test_data)
def test_remove_file(self):
fs = GridFS(self.conn['test'], 'test')
id = fs.put("test file", filename="test.txt", encoding='utf8')
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
fs.delete(id)
assert_soon(lambda: sum(1 for _ in self._search()) == 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert_one({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.command(
SON([('findAndModify', 'test'),
('query', {"a": 0}),
('update', update_spec),
('new', True)]))['value']
def update_worked():
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
return replicated == updated
# Allow some time for update to propagate
assert_soon(update_worked)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['test'].insert_one({'name': 'paul'})
condition = lambda: self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
assert_soon(condition)
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
condition = lambda: admin.command("isMaster")['ismaster']
assert_soon(lambda: retry_until_ok(condition))
retry_until_ok(self.conn.test.test.insert_one,
{'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self._search()) == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
# make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
assert_soon(
lambda: primary_conn['admin'].command("isMaster")['ismaster'])
self.repl_set.secondary.start()
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestNeo4j(Neo4jTestCase):
"""Integration tests for mongo-connector + Neo4j."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
Neo4jTestCase.setUpClass()
cls.repl_set = ReplicaSet().start()
cls.conn = cls.repl_set.client()
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
cls.repl_set.stop()
def tearDown(self):
"""Stop the Connector thread."""
super(TestNeo4j, self).tearDown()
self.connector.join()
time.sleep(10)
def setUp(self):
"""Start a new Connector for each test."""
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
docman = DocManager('http://localhost:7474/db/data',auto_commit_interval=0)
self.connector = Connector(
mongo_address=self.repl_set.uri,
ns_set=['test.test'],
doc_managers=(docman,),
gridfs_set=['test.test']
)
self.conn.test.test.drop()
self.conn.test.test.files.drop()
self.conn.test.test.chunks.drop()
self.connector.start()
self.neo4j_conn.delete_all()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
time.sleep(5)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set_2 = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set_2['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() > 0)
result_set_1 = self.neo4j_conn.find_one("test")
self.assertNotEqual(result_set_1, None)
self.assertEqual(result_set_1['_id'], str(result_set_2['_id']))
self.assertEqual(result_set_1['name'], result_set_2['name'])
self.connector.doc_managers[0].graph.delete_all()
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
result_set = self.conn['test']['test'].find_one()
# self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1)
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.connector.doc_managers[0].remove(str(result_set['_id']), 'test.test', 1)
self.assertEqual(self._count(), 0)
self.connector.doc_managers[0].graph.delete_all()
def test_update(self):
"""Test update operations."""
# Insert
self.connector.doc_managers[0].graph.delete_all()
self.conn['test']['updt'].insert({"a": 0})
result_set = self.conn['test']['updt'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'a': '0'}, "test.updt", 1)
assert_soon(lambda: self._count() == 1)
def check_update(update_spec):
updated = self.conn.test.updt.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
# Allow some time for update to propagate
time.sleep(5)
replicated = self.neo4j_conn.find_one("updt")['_id']
self.assertEqual(replicated, updated['_id'])
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
self.connector.doc_managers[0].graph.delete_all()
def test_bad_int_value(self):
self.conn['test']['test'].insert({'inf': float('inf'), 'nan': float('nan'),
'still_exists': True})
result_set = self.conn['test']['test'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']), 'inf': float('inf'), 'nan': float('nan'),
'still_exists': True }, "test.test", 1)
assert_soon(lambda: self._count() > 0)
doc = self.neo4j_conn.find_one("test")
self.assertNotIn('inf', doc)
self.assertNotIn('nan', doc)
self.assertTrue(doc['still_exists'])
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = self.repl_set.primary.client()
self.conn['test']['rollback'].insert({'name': 'paul'})
result_set = self.conn['test']['rollback'].find_one()
self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paul'}, "test.rollback", 1)
condition1 = lambda: self.conn['test']['rollback'].find(
{'name': 'paul'}).count() == 1
assert_soon(condition1)
self.repl_set.primary.stop(destroy=False)
new_primary_conn = self.repl_set.secondary.client()
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.rollback.insert, {'name': 'pauline'})
assert_soon(lambda: self._count() > 0)
result_set_2 = self.conn['test']['rollback'].find_one({'name': 'pauline'})
self.connector.doc_managers[0].upsert({'_id': str(result_set_2['_id']),'name': 'pauline'}, "test.rollback", 1)
result_set_1 = self.neo4j_conn.find_one("rollback", "name", "pauline")
self.assertNotEqual(result_set_1, None)
#make sure pauline is there
if result_set_1['name'] == 'pauline':
self.assertEqual(result_set_1['_id'], str(result_set_2['_id']))
self.repl_set.secondary.stop(destroy=False)
self.repl_set.primary.start()
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
self.repl_set.secondary.start()
time.sleep(2)
node = self.neo4j_conn.find_one("rollback")
self.assertEqual(node['name'], 'paul')
node = self.neo4j_conn.find_one("rollback", "name", "pauline")
self.assertNotEqual(node, None)
find_cursor = retry_until_ok(self.conn['test']['rollback'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestSynchronizer(unittest.TestCase):
""" Tests the mongo instance
"""
@classmethod
def setUpClass(cls):
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
cls.standalone_port = start_mongo_proc(
options=['--nojournal', '--noprealloc'])
cls.mongo_doc = DocManager('%s:%d' % (mongo_host, cls.standalone_port))
cls.mongo_doc._remove()
_, cls.secondary_p, cls.primary_p = start_replica_set('test-mongo')
cls.conn = MongoClient(mongo_host,
cls.primary_p,
replicaSet='test-mongo')
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_mongo_proc(cls.standalone_port)
kill_replica_set('test-mongo')
def tearDown(self):
self.connector.join()
def setUp(self):
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint="config.txt",
target_url='%s:%d' % (mongo_host, self.standalone_port),
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py')
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
self.conn['test']['test'].remove()
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify({"a": 0},
update_spec,
new=True)
# Allow some time for update to propagate
time.sleep(2)
replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0})
self.assertEqual(replicated, updated)
# Update by adding a field
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition = lambda: self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
assert_soon(condition)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
condition = lambda: admin.command("isMaster")['ismaster']
assert_soon(lambda: retry_until_ok(condition))
retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'})
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 2)
result_set_1 = list(self.mongo_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
assert_soon(
lambda: primary_conn['admin'].command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
for i in range(0, STRESS_COUNT):
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if (item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self.
conn['test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn['admin']
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase):
""" Tests Solr
"""
def runTest(self):
""" Runs tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']),
replicaSet="demo-repl")
# Creating a Solr object with an invalid URL
# doesn't create an exception
cls.solr_conn = Solr('http://localhost:8983/solr')
try:
cls.solr_conn.commit()
except (SolrError, MissingSchema):
cls.err_msg = "Cannot connect to Solr!"
cls.flag = False
if cls.flag:
cls.solr_conn.delete(q='*:*')
else:
cls.err_msg = "Shards cannot be added to mongos"
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_all()
def setUp(self):
if not self.flag:
self.fail(self.err_msg)
self.connector = Connector(
address=('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])),
oplog_checkpoint=CONFIG,
target_url='http://localhost:8983/solr',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/solr_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
time.sleep(1)
count = 0
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except (AutoReconnect, OperationFailure):
time.sleep(1)
count += 1
if count > 60:
unittest.SkipTest('Call to remove failed too '
'many times in setup')
while (len(self.solr_conn.search('*:*')) != 0):
time.sleep(1)
def tearDown(self):
self.connector.join()
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except OperationFailure:
continue
self.solr_conn.delete(q='*:*')
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(len(self.solr_conn.search('*:*')), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 0):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 1):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
time.sleep(1)
while len(self.solr_conn.search('*:*')) != 1:
time.sleep(1)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
self.conn['test']['test'].insert({'name': 'pauline'},
safe=True)
break
except OperationFailure:
count += 1
if count > 60:
self.fail('Call to insert failed too '
'many times in test_rollback')
time.sleep(1)
continue
while (len(self.solr_conn.search('*:*')) != 2):
time.sleep(1)
result_set_1 = self.solr_conn.search('pauline')
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(len(result_set_2), 1)
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) !=
NUMBER_OF_DOC_DIRS):
time.sleep(5)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. The rollback is performed the same way as before
but with more docs
"""
self.conn['test']['test'].remove()
while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0:
time.sleep(1)
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)},
safe=True)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) !=
NUMBER_OF_DOC_DIRS):
time.sleep(1)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)}, safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2))
!= self.conn['test']['test'].find().count()):
time.sleep(1)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2,
sort='_id asc')
for item in result_set_1:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
while (len(
self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2))
!= 0):
time.sleep(15)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('Paul',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert({'name': 'test_valid'})
self.conn['test']['test'].update({'_id': inserted_obj},
{'$set': {
'popularity': 1
}})
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(len(docman._search("name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name': 'test_invalid'})
self.conn['test']['test'].update({'_id': inserted_obj},
{'$set': {
'break_this_test': 1
}})
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(len(docman._search("name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert(match_first)
self.conn["test"]["test"].insert(match_second)
self.conn["test"]["test"].insert(match_none)
# Should have documents in Solr now
self.assertTrue(
wait_for(lambda: len(self.solr_conn.search("*:*")) > 0),
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="person.address.street" type="string" ... />
<field name="person.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
self.solr_conn.delete(q='*:*')
# Connector is already running
self.conn["test"]["test"].insert({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert({
"numbers": ["one", "two", "three"],
"characters": [{
"name": "Big Bird",
"color": "yellow"
}, {
"name": "Elmo",
"color": "red"
}, "Cookie Monster"]
})
self.assertTrue(
wait_for(lambda: len(self.solr_conn.search("*:*")) > 0),
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(
next(iter(results))["billing.address.state"], "California")
# Search for second document
results = self.solr_conn.search("characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase):
"""Integration tests for mongo-connector + Elasticsearch."""
@classmethod
def setUpClass(cls):
"""Start the cluster."""
super(TestElastic, cls).setUpClass()
_, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic')
cls.conn = MongoClient(mongo_host, cls.primary_p,
replicaSet='test-elastic')
@classmethod
def tearDownClass(cls):
"""Kill the cluster."""
kill_replica_set('test-elastic')
def tearDown(self):
"""Stop the Connector thread."""
super(TestElastic, self).tearDown()
self.connector.join()
def setUp(self):
"""Start a new Connector for each test."""
super(TestElastic, self).setUp()
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0
)
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: self._count() == 0)
def test_shard_length(self):
self.assertEqual(len(self.connector.shard_set), 1)
def test_insert(self):
"""Test insert operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() > 0)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove operations."""
self.conn['test']['test'].insert({'name': 'paulie'})
assert_soon(lambda: self._count() == 1)
self.conn['test']['test'].remove({'name': 'paulie'})
assert_soon(lambda: self._count() != 1)
self.assertEqual(self._count(), 0)
def test_update(self):
"""Test update operations."""
# Insert
self.conn.test.test.insert({"a": 0})
assert_soon(lambda: sum(1 for _ in self._search()) == 1)
def check_update(update_spec):
updated = self.conn.test.test.find_and_modify(
{"a": 0},
update_spec,
new=True
)
# Stringify _id to match what will be retrieved from ES
updated['_id'] = str(updated['_id'])
# Allow some time for update to propagate
time.sleep(1)
replicated = next(self._search())
self.assertEqual(replicated, updated)
# Update by adding a field. Note that ES can't mix types within an array
check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}})
# Update by setting an attribute of a sub-document beyond end of array.
check_update({"$set": {"b.10.c": 42}})
# Update by changing a value within a sub-document (contains array)
check_update({"$inc": {"b.0.c": 1}})
# Update by changing the value within an array
check_update({"$inc": {"b.1.f": 12}})
# Update by adding new bucket to list
check_update({"$push": {"b": {"e": 12}}})
# Update by changing an entire sub-document
check_update({"$set": {"b.0": {"e": 4}}})
# Update by adding a sub-document
check_update({"$set": {"b": {"0": {"c": 100}}}})
# Update whole document
check_update({"a": 0, "b": {"1": {"d": 10000}}})
def test_rollback(self):
"""Test behavior during a MongoDB rollback.
We force a rollback by adding a doc, killing the primary,
adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = MongoClient(mongo_host, self.primary_p)
self.conn['test']['test'].insert({'name': 'paul'})
condition1 = lambda: self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda: self._count() == 1
assert_soon(condition1)
assert_soon(condition2)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
retry_until_ok(self.conn.test.test.insert,
{'name': 'pauline'})
assert_soon(lambda: self._count() == 2)
result_set_1 = list(self._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
restart_mongo_proc(self.secondary_p)
time.sleep(2)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Stress test for inserting and removing many documents."""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
self.assertEqual(
set('Paul ' + str(i) for i in range(STRESS_COUNT)),
set(item['name'] for item in self._search())
)
def test_stressed_rollback(self):
"""Stress test for a rollback with many documents."""
for i in range(0, STRESS_COUNT):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
condition = lambda: self._count() == STRESS_COUNT
assert_soon(condition)
primary_conn = MongoClient(mongo_host, self.primary_p)
kill_mongo_proc(self.primary_p, destroy=False)
new_primary_conn = MongoClient(mongo_host, self.secondary_p)
admin = new_primary_conn['admin']
assert_soon(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < STRESS_COUNT:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)})
except (OperationFailure, AutoReconnect):
time.sleep(1)
assert_soon(lambda: self._count()
== self.conn['test']['test'].find().count())
result_set_1 = self._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(self.secondary_p, destroy=False)
restart_mongo_proc(self.primary_p)
db_admin = primary_conn["admin"]
assert_soon(lambda: db_admin.command("isMaster")['ismaster'])
restart_mongo_proc(self.secondary_p)
search = self._search
condition = lambda: sum(1 for _ in search()) == STRESS_COUNT
assert_soon(condition)
result_set_1 = list(self._search())
self.assertEqual(len(result_set_1), STRESS_COUNT)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase):
""" Tests Solr
"""
def runTest(self):
""" Runs tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']),
replicaSet="demo-repl")
# Creating a Solr object with an invalid URL
# doesn't create an exception
cls.solr_conn = Solr('http://localhost:8983/solr')
try:
cls.solr_conn.commit()
except (SolrError, MissingSchema):
cls.err_msg = "Cannot connect to Solr!"
cls.flag = False
if cls.flag:
cls.solr_conn.delete(q='*:*')
else:
cls.err_msg = "Shards cannot be added to mongos"
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_all()
def setUp(self):
if not self.flag:
self.fail(self.err_msg)
self.connector = Connector(
address=('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])),
oplog_checkpoint=CONFIG,
target_url='http://localhost:8983/solr',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/solr_doc_manager.py'
)
self.connector.start()
while len(self.connector.shard_set) == 0:
time.sleep(1)
count = 0
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except (AutoReconnect, OperationFailure):
time.sleep(1)
count += 1
if count > 60:
unittest.SkipTest('Call to remove failed too '
'many times in setup')
while (len(self.solr_conn.search('*:*')) != 0):
time.sleep(1)
def tearDown(self):
self.connector.join()
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
while (True):
try:
self.conn['test']['test'].remove(safe=True)
break
except OperationFailure:
continue
self.solr_conn.delete(q='*:*')
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(len(self.solr_conn.search('*:*')), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 0):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
while (len(self.solr_conn.search('*:*')) == 1):
time.sleep(1)
result_set_1 = self.solr_conn.search('paulie')
self.assertEqual(len(result_set_1), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by inserting one doc, killing
primary, adding another doc, killing the new primary, and
restarting both the servers.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
time.sleep(1)
while len(self.solr_conn.search('*:*')) != 1:
time.sleep(1)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
self.conn['test']['test'].insert(
{'name': 'pauline'}, safe=True)
break
except OperationFailure:
count += 1
if count > 60:
self.fail('Call to insert failed too '
'many times in test_rollback')
time.sleep(1)
continue
while (len(self.solr_conn.search('*:*')) != 2):
time.sleep(1)
result_set_1 = self.solr_conn.search('pauline')
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = self.solr_conn.search('pauline')
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('paul')
self.assertEqual(len(result_set_2), 1)
def test_stress(self):
"""Test stress by inserting and removing a large amount of docs.
"""
#stress test
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS))
!= NUMBER_OF_DOC_DIRS):
time.sleep(5)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.solr_conn.search('Paul ' + str(i))
for item in result_set_1:
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. The rollback is performed the same way as before
but with more docs
"""
self.conn['test']['test'].remove()
while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0:
time.sleep(1)
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert(
{'name': 'Paul ' + str(i)}, safe=True)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS))
!= NUMBER_OF_DOC_DIRS):
time.sleep(1)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)},
safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) !=
self.conn['test']['test'].find().count()):
time.sleep(1)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc')
for item in result_set_1:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
while (len(self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2)) != 0):
time.sleep(15)
result_set_1 = self.solr_conn.search('Pauline',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_1), 0)
result_set_2 = self.solr_conn.search('Paul',
rows=NUMBER_OF_DOC_DIRS * 2)
self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)
def test_valid_fields(self):
""" Tests documents with field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name':'test_valid'})
self.conn['test']['test'].update({'_id' : inserted_obj},
{'$set':{'popularity' : 1 }})
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertIn('popularity', result)
self.assertEqual(len(docman._search(
"name=test_valid")), 1)
def test_invalid_fields(self):
""" Tests documents without field definitions
"""
inserted_obj = self.conn['test']['test'].insert(
{'name':'test_invalid'})
self.conn['test']['test'].update({'_id' : inserted_obj},
{'$set':{'break_this_test' : 1 }})
docman = self.connector.doc_managers[0]
for _ in range(60):
if len(docman._search("*:*")) != 0:
break
time.sleep(1)
else:
self.fail("Timeout when removing docs from Solr")
result = docman.get_last_doc()
self.assertNotIn('break_this_test', result)
self.assertEqual(len(docman._search(
"name=test_invalid")), 1)
def test_dynamic_fields(self):
""" Tests dynamic field definitions
The following fields are supplied in the provided schema.xml:
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="i_*" type="int" indexed="true" stored="true"/>
Cases:
1. Match on first definition
2. Match on second definition
3. No match
"""
self.solr_conn.delete(q='*:*')
match_first = {"_id": 0, "foo_i": 100}
match_second = {"_id": 1, "i_foo": 200}
match_none = {"_id": 2, "foo": 300}
# Connector is already running
self.conn["test"]["test"].insert(match_first)
self.conn["test"]["test"].insert(match_second)
self.conn["test"]["test"].insert(match_none)
# Should have documents in Solr now
self.assertTrue(wait_for(lambda: len(self.solr_conn.search("*:*")) > 0),
"Solr doc manager should allow dynamic fields")
# foo_i and i_foo should be indexed, foo field should not exist
self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1)
self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1)
# SolrError: "undefined field foo"
logger = logging.getLogger("pysolr")
logger.error("You should see an ERROR log message from pysolr here. "
"This indicates success, not an error in the test.")
with self.assertRaises(SolrError):
self.solr_conn.search("foo:300")
def test_nested_fields(self):
"""Test indexing fields that are sub-documents in MongoDB
The following fields are defined in the provided schema.xml:
<field name="person.address.street" type="string" ... />
<field name="person.address.state" type="string" ... />
<dynamicField name="numbers.*" type="string" ... />
<dynamicField name="characters.*" type="string" ... />
"""
self.solr_conn.delete(q='*:*')
# Connector is already running
self.conn["test"]["test"].insert({
"name": "Jeb",
"billing": {
"address": {
"street": "12345 Mariposa Street",
"state": "California"
}
}
})
self.conn["test"]["test"].insert({
"numbers": ["one", "two", "three"],
"characters": [
{"name": "Big Bird",
"color": "yellow"},
{"name": "Elmo",
"color": "red"},
"Cookie Monster"
]
})
self.assertTrue(wait_for(lambda: len(self.solr_conn.search("*:*")) > 0),
"documents should have been replicated to Solr")
# Search for first document
results = self.solr_conn.search(
"billing.address.street:12345\ Mariposa\ Street")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["billing.address.state"],
"California")
# Search for second document
results = self.solr_conn.search(
"characters.1.color:red")
self.assertEqual(len(results), 1)
self.assertEqual(next(iter(results))["numbers.2"], "three")
results = self.solr_conn.search("characters.2:Cookie\ Monster")
self.assertEqual(len(results), 1)
class TestSynchronizer(unittest.TestCase):
""" Tests the mongo instance
"""
def runTest(self):
""" Runs the tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
start_single_mongod_instance("30000", "/MC", "MC_log")
cls.mongo_doc = DocManager("localhost:30000")
cls.mongo_doc._remove()
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection("%s:%s" % (HOSTNAME, PORTS_ONE['MONGOS']),
replicaSet="demo-repl")
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_mongo_proc(HOSTNAME, 30000)
kill_all()
def tearDown(self):
self.connector.join()
def setUp(self):
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector(
address="%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]),
oplog_checkpoint=CONFIG,
target_url='%s:30000' % (HOSTNAME),
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py')
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
self.conn['test']['test'].remove(safe=True)
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
result_set_1 = self.mongo_doc._search()
self.assertEqual(sum(1 for _ in result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], result_set_2['_id'])
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
condition = lambda: self.conn['test']['test'].find_one(
{'name': 'paul'}) is not None
wait_for(condition)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
condition = lambda: admin.command("isMaster")['ismaster']
wait_for(condition)
time.sleep(5)
count = 0
while True:
try:
result_set_1 = self.conn['test']['test'].insert(
{'name': 'pauline'}, safe=True)
break
except OperationFailure:
time.sleep(1)
count += 1
if count >= 60:
sys.exit(1)
continue
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 2)
result_set_1 = list(self.mongo_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
wait_for(lambda: primary_conn['admin'].command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if (item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)},
safe=True)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda: admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)}, safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == self.conn[
'test']['test'].find().count())
result_set_1 = self.mongo_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], result_set_2['_id'])
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
db_admin = primary_conn['admin']
wait_for(lambda: db_admin.command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
search = self.mongo_doc._search
condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
result_set_1 = list(self.mongo_doc._search())
self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
class TestElastic(unittest.TestCase):
""" Tests the Elastic instance
"""
def runTest(self):
""" Runs the tests
"""
unittest.TestCase.__init__(self)
@classmethod
def setUpClass(cls):
""" Starts the cluster
"""
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
cls.elastic_doc = DocManager('localhost:9200',
auto_commit=False)
cls.elastic_doc._remove()
cls.flag = start_cluster()
if cls.flag:
cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']),
replicaSet="demo-repl")
import logging
logger = logging.getLogger()
loglevel = logging.INFO
logger.setLevel(loglevel)
@classmethod
def tearDownClass(cls):
""" Kills cluster instance
"""
kill_all()
def tearDown(self):
""" Ends the connector
"""
self.connector.doc_manager.auto_commit = False
time.sleep(2)
self.connector.join()
def setUp(self):
""" Starts a new connector for every test
"""
if not self.flag:
self.fail("Shards cannot be added to mongos")
self.connector = Connector(
address='%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']),
oplog_checkpoint=CONFIG,
target_url='localhost:9200',
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py'
)
self.connector.start()
while len(self.connector.shard_set) == 0:
pass
self.conn['test']['test'].remove(safe=True)
wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 0)
def test_shard_length(self):
"""Tests the shard_length to see if the shard set was recognized
properly
"""
self.assertEqual(len(self.connector.shard_set), 1)
def test_initial(self):
"""Tests search and assures that the databases are clear.
"""
self.conn['test']['test'].remove(safe=True)
self.assertEqual(self.conn['test']['test'].find().count(), 0)
self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0)
def test_insert(self):
"""Tests insert
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) > 0)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
result_set_2 = self.conn['test']['test'].find_one()
for item in result_set_1:
self.assertEqual(item['_id'], str(result_set_2['_id']))
self.assertEqual(item['name'], result_set_2['name'])
def test_remove(self):
"""Tests remove
"""
self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 1)
self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) != 1)
self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0)
def test_rollback(self):
"""Tests rollback. We force a rollback by adding a doc, killing the
primary, adding another doc, killing the new primary, and then
restarting both.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
condition1 = lambda : self.conn['test']['test'].find(
{'name': 'paul'}).count() == 1
condition2 = lambda : sum(1 for _ in self.elastic_doc._search()) == 1
wait_for(condition1)
wait_for(condition2)
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda : admin.command("isMaster")['ismaster'])
time.sleep(5)
count = 0
while True:
try:
self.conn['test']['test'].insert(
{'name': 'pauline'}, safe=True)
break
except OperationFailure:
time.sleep(1)
count += 1
if count >= 60:
sys.exit(1)
continue
wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 2)
result_set_1 = list(self.elastic_doc._search())
result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
self.assertEqual(len(result_set_1), 2)
#make sure pauline is there
for item in result_set_1:
if item['name'] == 'pauline':
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
time.sleep(2)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), 1)
for item in result_set_1:
self.assertEqual(item['name'], 'paul')
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stress(self):
"""Test stress by inserting and removing the number of documents
specified in global
variable
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
time.sleep(5)
search = self.elastic_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
for i in range(0, NUMBER_OF_DOC_DIRS):
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if(item['name'] == 'Paul' + str(i)):
self.assertEqual(item['_id'], item['_id'])
def test_stressed_rollback(self):
"""Test stressed rollback with number of documents equal to specified
in global variable. Strategy for rollback is the same as before.
"""
for i in range(0, NUMBER_OF_DOC_DIRS):
self.conn['test']['test'].insert({'name': 'Paul ' + str(i)},
safe=True)
search = self.elastic_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
wait_for(lambda : admin.command("isMaster")['ismaster'])
time.sleep(5)
count = -1
while count + 1 < NUMBER_OF_DOC_DIRS:
try:
count += 1
self.conn['test']['test'].insert(
{'name': 'Pauline ' + str(count)}, safe=True)
except (OperationFailure, AutoReconnect):
time.sleep(1)
wait_for(lambda : sum(1 for _ in self.elastic_doc._search())
== self.conn['test']['test'].find().count())
result_set_1 = self.elastic_doc._search()
for item in result_set_1:
if 'Pauline' in item['name']:
result_set_2 = self.conn['test']['test'].find_one(
{'name': item['name']})
self.assertEqual(item['_id'], str(result_set_2['_id']))
kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
db_admin = primary_conn["admin"]
wait_for(lambda : db_admin.command("isMaster")['ismaster'])
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
search = self.elastic_doc._search
condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS
wait_for(condition)
result_set_1 = list(self.elastic_doc._search())
self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS)
for item in result_set_1:
self.assertTrue('Paul' in item['name'])
find_cursor = retry_until_ok(self.conn['test']['test'].find)
self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
def test_non_standard_fields(self):
""" Tests ObjectIds, DBrefs, etc
"""
# This test can break if it attempts to insert before the dump takes
# place- this prevents it (other tests affected too actually)
while (self.connector.shard_set['demo-repl'].checkpoint is None):
time.sleep(1)
docs = [
{'foo': [1, 2]},
{'bar': {'hello': 'world'}},
{'code': Code("function x() { return 1; }")},
{'dbref': {'_ref': DBRef('simple',
ObjectId('509b8db456c02c5ab7e63c34'))}}
]
try:
self.conn['test']['test'].insert(docs)
except OperationFailure:
self.fail("Cannot insert documents into Elastic!")
search = self.elastic_doc._search
if not wait_for(lambda : sum(1 for _ in search()) == len(docs)):
self.fail("Did not get all expected documents")
self.assertIn("dbref", self.elastic_doc.get_last_doc())
