def test_connector_minimum_privileges(self): """Test the Connector works with a user with minimum privileges.""" if not (db_user and db_password): raise SkipTest("Need to set a user/password to test this.") client = self.repl_set.client() minimum_user = "******" minimum_pwd = "password" client.admin.add_user( minimum_user, minimum_pwd, roles=[ {"role": "read", "db": "test"}, {"role": "read", "db": "wildcard"}, {"role": "read", "db": "local"}, ], ) client.test.test.insert_one({"replicated": 1}) client.test.ignored.insert_one({"replicated": 0}) client.ignored.ignored.insert_one({"replicated": 0}) client.wildcard.test.insert_one({"replicated": 1}) conn = Connector( mongo_address=self.repl_set.primary.uri, auth_username=minimum_user, auth_key=minimum_pwd, namespace_options={"test.test": True, "wildcard.*": True}, ) conn.start() try: assert_soon(conn.doc_managers[0]._search) finally: conn.join()
def test_connector_minimum_privileges(self): """Test the Connector works with a user with minimum privileges.""" if not (db_user and db_password): raise SkipTest('Need to set a user/password to test this.') client = self.repl_set.client() minimum_user = '******' minimum_pwd = 'password' client.admin.add_user(minimum_user, minimum_pwd, roles=[{'role': 'read', 'db': 'test'}, {'role': 'read', 'db': 'wildcard'}, {'role': 'read', 'db': 'local'}]) client.test.test.insert_one({"replicated": 1}) client.test.ignored.insert_one({"replicated": 0}) client.ignored.ignored.insert_one({"replicated": 0}) client.wildcard.test.insert_one({"replicated": 1}) conn = Connector( mongo_address=self.repl_set.primary.uri, auth_username=minimum_user, auth_key=minimum_pwd, namespace_options={'test.test': True, 'wildcard.*': True} ) conn.start() try: assert_soon(conn.doc_managers[0]._search) finally: conn.join()
def test_start_with_auth(self): dm = DocManager() connector = Connector(mongo_address=self.cluster.uri, doc_managers=[dm], auth_username=db_user, auth_key=db_password) connector.start() # Insert some documents into the sharded cluster. These # should go to the DocManager, and the connector should not # have an auth failure. self.cluster.client().test.test.insert_one({'auth_failure': False}) assert_soon(lambda: len(dm._search()) > 0) connector.join()
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector(mongo_address=self.repl_set.uri, ns_set=['test.test'], **connector_opts) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_start_with_auth(self): dm = DocManager() connector = Connector( mongo_address=self.cluster.uri, doc_managers=[dm], auth_username=db_user, auth_key=db_password ) connector.start() # Insert some documents into the sharded cluster. These # should go to the DocManager, and the connector should not # have an auth failure. self.cluster.client().test.test.insert({'auth_failure': False}) assert_soon(lambda: len(dm._search()) > 0) connector.join()
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector(mongo_address=self.repl_set.uri, **connector_opts) conn.start() assert_soon(lambda: bool(conn.shard_set)) # Make sure get_mininum_mongodb_version returns the current version. self.assertEqual(Version.from_client(self.repl_set.client()), get_mininum_mongodb_version()) conn.join() # Make sure the connector is shutdown correctly self.assertFalse(conn.can_run) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector( mongo_address=self.repl_set.uri, **connector_opts ) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector(mongo_address=self.repl_set.uri, **connector_opts) conn.start() assert_soon(lambda: bool(conn.shard_set)) # Make sure get_mininum_mongodb_version returns the current version. self.assertEqual( Version.from_client(self.repl_set.client()), get_mininum_mongodb_version() ) conn.join() # Make sure the connector is shutdown correctly self.assertFalse(conn.can_run) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_connector(self): """Test whether the connector initiates properly """ if not self.flag: self.fail("Shards cannot be added to mongos") conn = Connector(MAIN_ADDRESS, CONFIG, None, ['test.test'], '_id', None, None) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_connector_minimum_privileges(self): """Test the Connector works with a user with minimum privileges.""" if not (db_user and db_password): raise SkipTest("Need to set a user/password to test this.") client = self.repl_set.client() minimum_user = "******" minimum_pwd = "password" client.admin.add_user( minimum_user, minimum_pwd, roles=[ { "role": "read", "db": "test" }, { "role": "read", "db": "wildcard" }, { "role": "read", "db": "local" }, ], ) client.test.test.insert_one({"replicated": 1}) client.test.ignored.insert_one({"replicated": 0}) client.ignored.ignored.insert_one({"replicated": 0}) client.wildcard.test.insert_one({"replicated": 1}) conn = Connector( mongo_address=self.repl_set.primary.uri, auth_username=minimum_user, auth_key=minimum_pwd, namespace_options={ "test.test": True, "wildcard.*": True }, ) conn.start() try: assert_soon(conn.doc_managers[0]._search) finally: conn.join()
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector(address='%s:%d' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
class TestConnectorSharded(unittest.TestCase): def setUp(self): if db_user and db_password: auth_args = dict(auth_username=db_user, auth_key=db_password) else: auth_args = {} self.cluster = ShardedClusterSingle().start() self.dm = DocManager() self.connector = Connector(mongo_address=self.cluster.uri, doc_managers=[self.dm], **auth_args) self.connector.start() def tearDown(self): self.connector.join() try: os.unlink('oplog.timestamp') except OSError: pass self.cluster.stop()
def test_connector(self): """Test whether the connector initiates properly """ conn = Connector( address='%s:%d' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None ) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
def test_connector(self): """Test whether the connector initiates properly """ if not self.flag: self.fail("Shards cannot be added to mongos") conn = Connector(address=MAIN_ADDRESS, oplog_checkpoint=CONFIG, target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
class MongoReplicaSetTestCase(MongoTestCase): def setUp(self): self.repl_set = self.replica_set_class().start() self.conn = self.repl_set.client() try: os.unlink("oplog.timestamp") except OSError: pass self._remove() self.connector = Connector( mongo_address=self.repl_set.uri, doc_managers=(self.mongo_doc,), namespace_options={ 'test.test': {'gridfs': True}, 'rename.me': 'new.target', 'rename.me2': 'new2.target2' }, **connector_opts ) self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def drop_all_databases(self): for name in self.mongo_conn.database_names(): if name not in ["local", "admin"]: self.mongo_conn.drop_database(name) for name in self.conn.database_names(): if name not in ["local", "admin"]: self.conn.drop_database(name) def tearDown(self): self.connector.join() self.drop_all_databases() self.repl_set.stop()
class TestConnectorSharded(unittest.TestCase): def setUp(self): if db_user and db_password: auth_args = dict(auth_username=db_user, auth_key=db_password) else: auth_args = {} self.cluster = ShardedClusterSingle().start() self.dm = DocManager() self.connector = Connector( mongo_address=self.cluster.uri, doc_managers=[self.dm], **auth_args ) self.connector.start() def tearDown(self): self.connector.join() try: os.unlink('oplog.timestamp') except OSError: pass self.cluster.stop()
class MongoReplicaSetTestCase(MongoTestCase): def setUp(self): self.repl_set = self.replica_set_class().start() self.conn = self.repl_set.client() try: os.unlink("oplog.timestamp") except OSError: pass self._remove() self.connector = Connector(mongo_address=self.repl_set.uri, doc_managers=(self.mongo_doc, ), namespace_options={ "test.test": { "gridfs": True }, "rename.me": "new.target", "rename.me2": "new2.target2", }, **connector_opts) self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def drop_all_databases(self): for name in self.mongo_conn.database_names(): if name not in ["local", "admin"]: self.mongo_conn.drop_database(name) for name in self.conn.database_names(): if name not in ["local", "admin"]: self.conn.drop_database(name) def tearDown(self): self.connector.join() self.drop_all_databases() self.repl_set.stop()
def test_connector(self): """Test whether the connector initiates properly """ if not self.flag: self.fail("Shards cannot be added to mongos") conn = Connector( address=MAIN_ADDRESS, oplog_checkpoint=CONFIG, target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None ) conn.start() while len(conn.shard_set) != 1: time.sleep(2) conn.join() self.assertFalse(conn.can_run) time.sleep(5) for thread in conn.shard_set.values(): self.assertFalse(thread.running)
class TestSolr(SolrTestCase): """ Tests Solr """ @classmethod def setUpClass(cls): SolrTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """ Kills cluster instance """ cls.repl_set.stop() def setUp(self): self._remove() try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() docman = DocManager('http://%s/solr' % solr_pair, auto_commit_interval=0) self.connector = Connector( mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman,), gridfs_set=['test.test'] ) retry_until_ok(self.conn.test.test.drop) retry_until_ok(self.conn.test.test.files.drop) retry_until_ok(self.conn.test.test.chunks.drop) self._remove() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) def tearDown(self): self.connector.join() def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0) result_set_1 = list(self.solr_conn.search('paulie')) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = "test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0) res = list(self.solr_conn.search('test_insert_file')) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], "test.txt") self.assertEqual(doc['_id'], str(id)) self.assertEqual(doc['content'][0].strip(), test_data.strip()) def test_remove_file(self): """Tests removing a gridfs file """ fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1) fs.delete(id) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0) def test_update(self): """Test update operations on Solr. Need to have the following defined in schema.xml: <field name="a" type="int" indexed="true" stored="true" /> <field name="b.0.c" type="int" indexed="true" stored="true" /> <field name="b.10.c" type="int" indexed="true" stored="true" /> <field name="b.0.e" type="int" indexed="true" stored="true" /> <field name="b.1.d" type="int" indexed="true" stored="true" /> <field name="b.1.f" type="int" indexed="true" stored="true" /> <field name="b.2.e" type="int" indexed="true" stored="true" /> """ docman = self.connector.doc_managers[0] # Use diabolical value for _id to test string escaping as well. self.conn.test.test.insert({"_id": u'+-&え|!(){}[]^"~*?:\\/', "a": 0}) assert_soon(lambda: sum(1 for _ in self._search("*:*")) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify( {"a": 0}, update_spec, new=True ) # Stringify _id to match what will be retrieved from Solr updated['_id'] = u(updated['_id']) # Flatten the MongoDB document to match Solr updated = docman._clean_doc(updated, 'dummy.namespace', 0) # Allow some time for update to propagate time.sleep(1) replicated = list(self._search("a:0"))[0] # Remove add'l fields until these are stored in a separate Solr core updated.pop('_ts') replicated.pop('_ts') updated.pop('ns') replicated.pop('ns') # Remove field added by Solr replicated.pop("_version_") self.assertEqual(replicated, updated) # Update by adding a field. # Note that Solr can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by replacing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = self.repl_set.primary.client() self.conn['test']['test'].insert({'name': 'paul'}) assert_soon( lambda: self.conn.test.test.find({'name': 'paul'}).count() == 1) assert_soon( lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2) result_set_1 = list(self.solr_conn.search('pauline')) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(sum(1 for _ in result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(sum(1 for _ in result_set_2), 1) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_valid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'popularity': 1}} ) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0) result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(sum(1 for _ in self._search( "name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_invalid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'break_this_test': 1}} ) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0) result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(sum(1 for _ in self._search( "name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert(match_first) self.conn["test"]["test"].insert(match_second) self.conn["test"]["test"].insert(match_none) # Should have documents in Solr now assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1) self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="person.address.street" type="string" ... /> <field name="person.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ # Connector is already running self.conn["test"]["test"].insert({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert({ "numbers": ["one", "two", "three"], "characters": [ {"name": "Big Bird", "color": "yellow"}, {"name": "Elmo", "color": "red"}, "Cookie Monster" ] }) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search( "characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase): """Integration tests for mongo-connector + Elasticsearch.""" @classmethod def setUpClass(cls): """Start the cluster.""" super(TestElastic, cls).setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """Kill the cluster.""" close_client(cls.conn) cls.repl_set.stop() def tearDown(self): """Stop the Connector thread.""" super(TestElastic, self).tearDown() self.connector.join() def setUp(self): """Start a new Connector for each test.""" super(TestElastic, self).setUp() try: os.unlink("oplog.timestamp") except OSError: pass self.connector = Connector( mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(self.elastic_doc,), gridfs_set=['test.test'] ) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: self._count() > 0) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].delete_one({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.assertEqual(self._count(), 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = b"test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: self._count() > 0) query = {"match": {"_all": "test_insert_file"}} res = list(self._search(query)) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], 'test.txt') self.assertEqual(doc['_id'], str(id)) self.assertEqual(base64.b64decode(doc['content']), test_data) def test_remove_file(self): fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: self._count() == 1) fs.delete(id) assert_soon(lambda: self._count() == 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert_one({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.command( SON([('findAndModify', 'test'), ('query', {"a": 0}), ('update', update_spec), ('new', True)]))['value'] # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) assert_soon(lambda: next(self._search()) == updated) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() # This doc can be picked up in the collection dump self.conn['test']['test'].insert_one({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) # This doc is definitely not picked up by collection dump self.conn['test']['test'].insert_one({'name': 'pauly'}) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'}) assert_soon(lambda: self._count() == 3) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 3) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 2) if result_set_1[0]['name'] == 'paul': self.assertEqual(result_set_1[1]['name'], 'pauly') elif result_set_1[0]['name'] == 'pauly': self.assertEqual(result_set_1[1]['name'], 'paul') else: self.assertTrue(0, 'Unknown document retrieved') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 2) def test_bad_int_value(self): self.conn.test.test.insert_one({ 'inf': float('inf'), 'nan': float('nan'), 'still_exists': True}) assert_soon(lambda: self._count() > 0) for doc in self._search(): self.assertNotIn('inf', doc) self.assertNotIn('nan', doc) self.assertTrue(doc['still_exists'])
class TestElastic(unittest.TestCase): """ Tests the Elastic instance """ def runTest(self): """ Runs the tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): """ Starts the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.elastic_doc = DocManager('localhost:9200') cls.elastic_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") import logging logger = logging.getLogger() loglevel = logging.INFO logger.setLevel(loglevel) @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_all() def tearDown(self): """ Ends the connector """ self.connector.join() def setUp(self): """ Starts a new connector for every test """ if not self.flag: self.fail("Shards cannot be added to mongos") self.connector = Connector( address='%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), oplog_checkpoint=CONFIG, target_url='localhost:9200', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py') self.connector.start() while len(self.connector.shard_set) == 0: pass self.conn['test']['test'].remove(safe=True) wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ self.conn['test']['test'].remove(safe=True) self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) > 0) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) condition1 = lambda: self.conn['test']['test'].find({ 'name': 'paul' }).count() == 1 condition2 = lambda: sum(1 for _ in self.elastic_doc._search()) == 1 wait_for(condition1) wait_for(condition2) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert({'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == 2) result_set_1 = list(self.elastic_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.elastic_doc._search() for item in result_set_1: if (item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) wait_for(lambda: sum(1 for _ in self.elastic_doc._search()) == self. conn['test']['test'].find().count()) result_set_1 = self.elastic_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) db_admin = primary_conn["admin"] wait_for(lambda: db_admin.command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS) def test_non_standard_fields(self): """ Tests ObjectIds, DBrefs, etc """ # This test can break if it attempts to insert before the dump takes # place- this prevents it (other tests affected too actually) while (self.connector.shard_set['demo-repl'].checkpoint is None): time.sleep(1) docs = [{ 'foo': [1, 2] }, { 'bar': { 'hello': 'world' } }, { 'code': Code("function x() { return 1; }") }, { 'dbref': { '_ref': DBRef('simple', ObjectId('509b8db456c02c5ab7e63c34')) } }] try: self.conn['test']['test'].insert(docs) except OperationFailure: self.fail("Cannot insert documents into Elastic!") search = self.elastic_doc._search if not wait_for(lambda: sum(1 for _ in search()) == len(docs)): self.fail("Did not get all expected documents") self.assertIn("dbref", self.elastic_doc.get_last_doc())
class TestElastic(unittest.TestCase): """ Tests the Elastic instance """ @classmethod def setUpClass(cls): """ Starts the cluster """ cls.elastic_doc = DocManager(elastic_pair, auto_commit=False) _, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-elastic') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_replica_set('test-elastic') def tearDown(self): """ Ends the connector """ self.connector.join() def setUp(self): """ Starts a new connector for every test """ try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=elastic_pair, ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py', auto_commit_interval=0 ) # Clean out test databases try: self.elastic_doc._remove() except OperationFailed: try: # Create test.test index if necessary client = Elasticsearch(hosts=[elastic_pair]) idx_client = IndicesClient(client) idx_client.create(index='test.test') except es_exceptions.TransportError: pass self.conn.test.test.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) > 0) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: sum(1 for _ in self.elastic_doc._search()) == 1 assert_soon(condition1) assert_soon(condition2) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 2) result_set_1 = list(self.elastic_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) for i in range(0, STRESS_COUNT): result_set_1 = self.elastic_doc._search() for item in result_set_1: if(item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.elastic_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn["admin"] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self.elastic_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSolr(SolrTestCase): """ Tests Solr """ @classmethod def setUpClass(cls): SolrTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """ Kills cluster instance """ cls.repl_set.stop() def setUp(self): self._remove() try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() docman = DocManager(solr_url, auto_commit_interval=0) self.connector = Connector(mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman, ), gridfs_set=['test.test']) retry_until_ok(self.conn.test.test.drop) retry_until_ok(self.conn.test.test.files.drop) retry_until_ok(self.conn.test.test.chunks.drop) self._remove() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) def tearDown(self): self.connector.join() def test_insert(self): """Tests insert """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0) result_set_1 = list(self.solr_conn.search('name:paulie')) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1) self.conn['test']['test'].delete_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = "test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0) res = list(self.solr_conn.search('content:*test_insert_file*')) if not res: res = list(self.solr_conn.search('_text_:*test_insert_file*')) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], "test.txt") self.assertEqual(doc['_id'], str(id)) content = doc.get('content', doc.get('_text_', None)) self.assertTrue(content) self.assertIn(test_data.strip(), content[0].strip()) def test_remove_file(self): """Tests removing a gridfs file """ fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1) fs.delete(id) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0) def test_update(self): """Test update operations on Solr. Need to have the following defined in schema.xml: <field name="a" type="int" indexed="true" stored="true" /> <field name="b.0.c" type="int" indexed="true" stored="true" /> <field name="b.10.c" type="int" indexed="true" stored="true" /> <field name="b.0.e" type="int" indexed="true" stored="true" /> <field name="b.1.d" type="int" indexed="true" stored="true" /> <field name="b.1.f" type="int" indexed="true" stored="true" /> <field name="b.2.e" type="int" indexed="true" stored="true" /> """ docman = self.connector.doc_managers[0] # Use diabolical value for _id to test string escaping as well. self.conn.test.test.insert_one({ "_id": u'+-&え|!(){}[]^"~*?:\\/', "a": 0 }) assert_soon(lambda: sum(1 for _ in self._search("*:*")) == 1) def check_update(update_spec): updated = self.conn.test.command( SON([('findAndModify', 'test'), ('query', { "a": 0 }), ('update', update_spec), ('new', True)]))['value'] # Stringify _id to match what will be retrieved from Solr updated[u('_id')] = u(updated['_id']) # Flatten the MongoDB document to match Solr updated = docman._clean_doc(updated, 'dummy.namespace', 0) # Allow some time for update to propagate time.sleep(3) replicated = list(self._search("a:0"))[0] # Remove add'l fields until these are stored in a separate Solr core updated.pop('_ts') replicated.pop('_ts') updated.pop('ns') replicated.pop('ns') # Remove field added by Solr replicated.pop("_version_") self.assertEqual(replicated, updated) # Update by adding a field. # Note that Solr can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by replacing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = self.repl_set.primary.client() self.conn['test']['test'].insert_one({'name': 'paul'}) assert_soon(lambda: self.conn.test.test.find({ 'name': 'paul' }).count() == 1) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2) result_set_1 = list(self.solr_conn.search('name:pauline')) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) result_set_1 = self.solr_conn.search('name:pauline') self.assertEqual(sum(1 for _ in result_set_1), 0) result_set_2 = self.solr_conn.search('name:paul') self.assertEqual(sum(1 for _ in result_set_2), 1) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert_one({ 'name': 'test_valid' }).inserted_id self.conn['test']['test'].update_one({'_id': inserted_obj}, {'$set': { 'popularity': 1 }}) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0) result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(sum(1 for _ in self._search("name:test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert_one({ 'name': 'test_invalid' }).inserted_id self.conn['test']['test'].update_one({'_id': inserted_obj}, {'$set': { 'break_this_test': 1 }}) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in self._search("*:*")) > 0) result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(sum(1 for _ in self._search("name:test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert_one(match_first) self.conn["test"]["test"].insert_one(match_second) self.conn["test"]["test"].insert_one(match_none) # Should have documents in Solr now assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1) self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="billing.address.street" type="string" ... /> <field name="billing.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ # Connector is already running self.conn["test"]["test"].insert_one({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert_one({ "numbers": ["one", "two", "three"], "characters": [{ "name": "Big Bird", "color": "yellow" }, { "name": "Elmo", "color": "red" }, "Cookie Monster"] }) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual( next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search("characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase): """Integration tests for mongo-connector + Elasticsearch.""" @classmethod def setUpClass(cls): """Start the cluster.""" super(TestElastic, cls).setUpClass() _, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-elastic') @classmethod def tearDownClass(cls): """Kill the cluster.""" kill_replica_set('test-elastic') def tearDown(self): """Stop the Connector thread.""" super(TestElastic, self).tearDown() self.connector.join() def setUp(self): """Start a new Connector for each test.""" super(TestElastic, self).setUp() try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=elastic_pair, ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py', auto_commit_interval=0 ) self.conn.test.test.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) def test_shard_length(self): self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() > 0) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.assertEqual(self._count(), 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify( {"a": 0}, update_spec, new=True ) # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) # Allow some time for update to propagate time.sleep(1) replicated = next(self._search()) self.assertEqual(replicated, updated) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Stress test for inserting and removing many documents.""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) self.assertEqual( set('Paul ' + str(i) for i in range(STRESS_COUNT)), set(item['name'] for item in self._search()) ) def test_stressed_rollback(self): """Stress test for a rollback with many documents.""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: self._count() == self.conn['test']['test'].find().count()) result_set_1 = self._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn["admin"] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase): """ Tests Solr """ @classmethod def setUpClass(cls): _, cls.secondary_p, cls.primary_p = start_replica_set('test-solr') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-solr') cls.solr_conn = Solr('http://%s/solr' % solr_pair) cls.solr_conn.delete(q='*:*') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_replica_set('test-solr') def setUp(self): try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url='http://localhost:8983/solr', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/solr_doc_manager.py', auto_commit_interval=0 ) self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) retry_until_ok(self.conn.test.test.remove) assert_soon(lambda: len(self.solr_conn.search('*:*')) == 0) def tearDown(self): self.connector.join() def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) while (len(self.solr_conn.search('*:*')) == 0): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: len(self.solr_conn.search("*:*")) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: len(self.solr_conn.search("*:*")) == 0) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1) def test_stress(self): """Test stress by inserting and removing a large amount of docs. """ #stress test for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT)) != STRESS_COUNT): time.sleep(5) for i in range(0, STRESS_COUNT): result_set_1 = self.solr_conn.search('Paul ' + str(i)) for item in result_set_1: self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}) while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT)) != STRESS_COUNT): time.sleep(1) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2, sort='_id asc' ) for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) while (len(self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2 ) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search( 'Paul', rows=STRESS_COUNT * 2 ) self.assertEqual(len(result_set_2), STRESS_COUNT) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_valid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'popularity': 1}} ) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(len(docman._search( "name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_invalid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'break_this_test': 1}} ) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(len(docman._search( "name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert(match_first) self.conn["test"]["test"].insert(match_second) self.conn["test"]["test"].insert(match_none) # Should have documents in Solr now assert_soon(lambda: len(self.solr_conn.search("*:*")) > 0, "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1) self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="person.address.street" type="string" ... /> <field name="person.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ # Connector is already running self.conn["test"]["test"].insert({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert({ "numbers": ["one", "two", "three"], "characters": [ {"name": "Big Bird", "color": "yellow"}, {"name": "Elmo", "color": "red"}, "Cookie Monster" ] }) assert_soon(lambda: len(self.solr_conn.search("*:*")) > 0, "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search( "characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestNeo4j(Neo4jTestCase): """Integration tests for mongo-connector + Neo4j.""" @classmethod def setUpClass(cls): """Start the cluster.""" Neo4jTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """Kill the cluster.""" cls.repl_set.stop() def tearDown(self): """Stop the Connector thread.""" super(TestNeo4j, self).tearDown() self.connector.join() time.sleep(10) def setUp(self): """Start a new Connector for each test.""" try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() docman = DocManager('http://localhost:7474/db/data', auto_commit_interval=0) self.connector = Connector(mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman, ), gridfs_set=['test.test']) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() self.neo4j_conn.delete_all() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) time.sleep(5) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) result_set_2 = self.conn['test']['test'].find_one() self.connector.doc_managers[0].upsert( { '_id': str(result_set_2['_id']), 'name': 'paulie' }, "test.test", 1) assert_soon(lambda: self._count() > 0) result_set_1 = self.neo4j_conn.find_one("test") self.assertNotEqual(result_set_1, None) self.assertEqual(result_set_1['_id'], str(result_set_2['_id'])) self.assertEqual(result_set_1['name'], result_set_2['name']) self.connector.doc_managers[0].graph.delete_all() def test_subdocument_with_id(self): """ Test inserting a document with a subdocument containing an _id property. In the current version of translating from document data model to property graph, the root level _id field is included in all subdocument nodes in the graph. This test verifies there is no error when inserting a document containing a subdocument with an _id property. See https://github.com/neo4j-contrib/neo4j_doc_manager/issues/56 """ doc = { '_id': 'root_level_object_id', 'name': 'Bob', 'location': { '_id': 'sub_document_object_id', 'city': 'Missoula', 'state': 'Montana' } } self.connector.doc_managers[0].upsert(doc, "test.test_id", 1) assert_soon(lambda: self._count() > 0) result_set = self.neo4j_conn.find_one("location") self.assertNotEqual(result_set, None) self.assertEqual( result_set['_id'], 'root_level_object_id') # expect subdocument _id to be ignored def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) result_set = self.conn['test']['test'].find_one() # self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.connector.doc_managers[0].remove(str(result_set['_id']), 'test.test', 1) self.assertEqual(self._count(), 0) self.connector.doc_managers[0].graph.delete_all() def test_update(self): """Test update operations.""" # Insert self.connector.doc_managers[0].graph.delete_all() self.conn['test']['updt'].insert({"a": 0}) result_set = self.conn['test']['updt'].find_one() self.connector.doc_managers[0].upsert( { '_id': str(result_set['_id']), 'a': '0' }, "test.updt", 1) assert_soon(lambda: self._count() == 1) def check_update(update_spec): updated = self.conn.test.updt.find_and_modify({"a": 0}, update_spec, new=True) # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) # Allow some time for update to propagate time.sleep(5) replicated = self.neo4j_conn.find_one("updt")['_id'] self.assertEqual(replicated, updated['_id']) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) self.connector.doc_managers[0].graph.delete_all() def test_bad_int_value(self): self.conn['test']['test'].insert({ 'inf': float('inf'), 'nan': float('nan'), 'still_exists': True }) result_set = self.conn['test']['test'].find_one() self.connector.doc_managers[0].upsert( { '_id': str(result_set['_id']), 'inf': float('inf'), 'nan': float('nan'), 'still_exists': True }, "test.test", 1) assert_soon(lambda: self._count() > 0) doc = self.neo4j_conn.find_one("test") self.assertNotIn('inf', doc) self.assertNotIn('nan', doc) self.assertTrue(doc['still_exists']) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() self.conn['test']['rollback'].insert({'name': 'paul'}) result_set = self.conn['test']['rollback'].find_one() self.connector.doc_managers[0].upsert( { '_id': str(result_set['_id']), 'name': 'paul' }, "test.rollback", 1) condition1 = lambda: self.conn['test']['rollback'].find({ 'name': 'paul' }).count() == 1 assert_soon(condition1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.rollback.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() > 0) result_set_2 = self.conn['test']['rollback'].find_one( {'name': 'pauline'}) self.connector.doc_managers[0].upsert( { '_id': str(result_set_2['_id']), 'name': 'pauline' }, "test.rollback", 1) result_set_1 = self.neo4j_conn.find_one("rollback", "name", "pauline") self.assertNotEqual(result_set_1, None) #make sure pauline is there if result_set_1['name'] == 'pauline': self.assertEqual(result_set_1['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) node = self.neo4j_conn.find_one("rollback") self.assertEqual(node['name'], 'paul') node = self.neo4j_conn.find_one("rollback", "name", "pauline") self.assertNotEqual(node, None) find_cursor = retry_until_ok(self.conn['test']['rollback'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestMongo(MongoTestCase): """ Tests the mongo instance """ @classmethod def setUpClass(cls): MongoTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """ Kills cluster instance """ MongoTestCase.tearDownClass() cls.repl_set.stop() def tearDown(self): self.connector.join() def setUp(self): try: os.unlink("oplog.timestamp") except OSError: pass self._remove() self.connector = Connector(mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(self.mongo_doc, ), gridfs_set=['test.test'], **connector_opts) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) result_set_1 = self._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) self.conn['test']['test'].delete_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) != 1) self.assertEqual(sum(1 for _ in self._search()), 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = b"test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self._search()) > 0) res = list(self._search()) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], 'test.txt') self.assertEqual(doc['_id'], id) self.assertEqual(doc['content'], test_data) def test_remove_file(self): fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self._search()) == 1) fs.delete(id) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert_one({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.command( SON([('findAndModify', 'test'), ('query', { "a": 0 }), ('update', update_spec), ('new', True)]))['value'] # Allow some time for update to propagate time.sleep(2) replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0}) self.assertEqual(replicated, updated) # Update by adding a field check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() self.conn['test']['test'].insert_one({'name': 'paul'}) condition = lambda: self.conn['test']['test'].find_one( {'name': 'paul'}) is not None assert_soon(condition) assert_soon(lambda: sum(1 for _ in self._search()) == 1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] condition = lambda: admin.command("isMaster")['ismaster'] assert_soon(lambda: retry_until_ok(condition)) retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self._search()) == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() assert_soon( lambda: primary_conn['admin'].command("isMaster")['ismaster']) self.repl_set.secondary.start() time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestSynchronizer(unittest.TestCase): """ Tests Solr """ @classmethod def setUpClass(cls): _, cls.secondary_p, cls.primary_p = start_replica_set('test-solr') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-solr') cls.solr_conn = Solr('http://%s/solr' % solr_pair) cls.solr_conn.delete(q='*:*') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_replica_set('test-solr') def setUp(self): try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url='http://localhost:8983/solr', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/solr_doc_manager.py', auto_commit_interval=0 ) self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) retry_until_ok(self.conn.test.test.remove) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 0) def tearDown(self): self.connector.join() def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search('*:*')) > 0) result_set_1 = list(self.solr_conn.search('paulie')) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) == 0) def test_update(self): """Test update operations on Solr. Need to have the following defined in schema.xml: <field name="a" type="int" indexed="true" stored="true" /> <field name="b.0.c" type="int" indexed="true" stored="true" /> <field name="b.0.e" type="int" indexed="true" stored="true" /> <field name="b.1.d" type="int" indexed="true" stored="true" /> <field name="b.1.f" type="int" indexed="true" stored="true" /> """ docman = self.connector.doc_managers[0] # Insert self.conn.test.test.insert({"a": 0}) assert_soon(lambda: sum(1 for _ in docman._search("*:*")) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify( {"a": 0}, update_spec, new=True ) # Stringify _id to match what will be retrieved from Solr updated['_id'] = str(updated['_id']) # Flatten the MongoDB document to match Solr updated = docman._clean_doc(updated) # Allow some time for update to propagate time.sleep(1) replicated = list(docman._search("a:0"))[0] # Remove add'l fields until these are stored in a separate Solr core replicated.pop("_ts") replicated.pop("ns") # Remove field added by Solr replicated.pop("_version_") self.assertEqual(replicated, docman._clean_doc(updated)) # Update by adding a field. # Note that Solr can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by replacing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) assert_soon( lambda: self.conn.test.test.find({'name': 'paul'}).count() == 1) assert_soon( lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon( lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2) result_set_1 = list(self.solr_conn.search('pauline')) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(sum(1 for _ in result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(sum(1 for _ in result_set_2), 1) def test_stress(self): """Test stress by inserting and removing a large amount of docs. """ #stress test for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) assert_soon( lambda: sum(1 for _ in self.solr_conn.search( '*:*', rows=STRESS_COUNT)) == STRESS_COUNT) for i in range(0, STRESS_COUNT): result_set_1 = self.solr_conn.search('Paul ' + str(i)) for item in result_set_1: self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}) assert_soon( lambda: sum(1 for _ in self.solr_conn.search( '*:*', rows=STRESS_COUNT)) == STRESS_COUNT) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) collection_size = self.conn['test']['test'].find().count() assert_soon( lambda: sum(1 for _ in self.solr_conn.search( '*:*', rows=STRESS_COUNT * 2)) == collection_size) result_set_1 = self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2, sort='_id asc' ) for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) assert_soon(lambda: sum(1 for _ in self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2)) == 0) result_set_1 = list(self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2 )) self.assertEqual(len(result_set_1), 0) result_set_2 = list(self.solr_conn.search( 'Paul', rows=STRESS_COUNT * 2 )) self.assertEqual(len(result_set_2), STRESS_COUNT) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_valid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'popularity': 1}} ) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in docman._search("*:*")) > 0) result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(sum(1 for _ in docman._search( "name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_invalid'}) self.conn['test']['test'].update( {'_id': inserted_obj}, {'$set': {'break_this_test': 1}} ) docman = self.connector.doc_managers[0] assert_soon(lambda: sum(1 for _ in docman._search("*:*")) > 0) result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(sum(1 for _ in docman._search( "name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert(match_first) self.conn["test"]["test"].insert(match_second) self.conn["test"]["test"].insert(match_none) # Should have documents in Solr now assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(sum(1 for _ in self.solr_conn.search("foo_i:100")), 1) self.assertEqual(sum(1 for _ in self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="person.address.street" type="string" ... /> <field name="person.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ # Connector is already running self.conn["test"]["test"].insert({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert({ "numbers": ["one", "two", "three"], "characters": [ {"name": "Big Bird", "color": "yellow"}, {"name": "Elmo", "color": "red"}, "Cookie Monster" ] }) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*")) > 0, "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search( "characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestSynchronizer(unittest.TestCase): """ Tests the mongo instance """ @classmethod def setUpClass(cls): try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() cls.standalone_port = start_mongo_proc(options=['--nojournal', '--noprealloc']) cls.mongo_doc = DocManager('%s:%d' % (mongo_host, cls.standalone_port)) cls.mongo_doc._remove() _, cls.secondary_p, cls.primary_p = start_replica_set('test-mongo') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-mongo') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(cls.standalone_port) kill_replica_set('test-mongo') def tearDown(self): self.connector.join() def setUp(self): self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint="config.txt", target_url='%s:%d' % (mongo_host, self.standalone_port), ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py' ) self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) self.conn['test']['test'].remove() assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) result_set_1 = self.mongo_doc._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert({"a": 0}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify( {"a": 0}, update_spec, new=True ) # Allow some time for update to propagate time.sleep(2) replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0}) self.assertEqual(replicated, updated) # Update by adding a field check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition = lambda: self.conn['test']['test'].find_one( {'name': 'paul'}) is not None assert_soon(condition) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] condition = lambda: admin.command("isMaster")['ismaster'] assert_soon(lambda: retry_until_ok(condition)) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 2) result_set_1 = list(self.mongo_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) assert_soon( lambda: primary_conn['admin'].command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) for i in range(0, STRESS_COUNT): result_set_1 = self.mongo_doc._search() for item in result_set_1: if(item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn['admin'] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase): """ Tests Solr """ def runTest(self): """ Runs tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']), replicaSet="demo-repl") # Creating a Solr object with an invalid URL # doesn't create an exception cls.solr_conn = Solr('http://localhost:8983/solr') try: cls.solr_conn.commit() except (SolrError, MissingSchema): cls.err_msg = "Cannot connect to Solr!" cls.flag = False if cls.flag: cls.solr_conn.delete(q='*:*') else: cls.err_msg = "Shards cannot be added to mongos" @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_all() def setUp(self): if not self.flag: self.fail(self.err_msg) self.connector = Connector(('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])), CONFIG, 'http://localhost:8983/solr', ['test.test'], '_id', None, 'mongo_connector/doc_managers/solr_doc_manager.py') self.connector.start() while len(self.connector.shard_set) == 0: time.sleep(1) count = 0 while (True): try: self.conn['test']['test'].remove(safe=True) break except (AutoReconnect, OperationFailure): time.sleep(1) count += 1 if count > 60: unittest.SkipTest('Call to remove failed too ' 'many times in setup') while (len(self.solr_conn.search('*:*')) != 0): time.sleep(1) def tearDown(self): self.connector.doc_manager.auto_commit = False time.sleep(2) self.connector.join() def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ while (True): try: self.conn['test']['test'].remove(safe=True) break except OperationFailure: continue self.solr_conn.delete(q='*:*') self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(len(self.solr_conn.search('*:*')), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 0): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 1): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 0) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too ' 'many times in test_rollback') time.sleep(1) continue while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1) def test_stress(self): """Test stress by inserting and removing a large amount of docs. """ #stress test for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(5) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.solr_conn.search('Paul ' + str(i)) for item in result_set_1: self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. The rollback is performed the same way as before but with more docs """ self.conn['test']['test'].remove() while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}, safe=True) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len(self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('Paul', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name':'test_valid'}) self.conn['test']['test'].update({'_id' : inserted_obj}, {'$set':{'popularity' : 1 }}) for _ in range(60): if len(self.connector.doc_manager._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = self.connector.doc_manager.get_last_doc() self.assertIn('popularity', result) self.assertEqual(len(self.connector.doc_manager._search( "name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name':'test_invalid'}) self.conn['test']['test'].update({'_id' : inserted_obj}, {'$set':{'break_this_test' : 1 }}) for _ in range(60): if len(self.connector.doc_manager._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = self.connector.doc_manager.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(len(self.connector.doc_manager._search( "name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following field in the supplied schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> """ inserted_obj = self.conn['test']['test'].insert({ 'name':'test_dynamic', 'foo_i':1, 'i_foo':1}) self.assertEqual(self.conn['test']['test'].find().count(), 1) for _ in range(60): if len(self.connector.doc_manager._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = self.connector.doc_manager.get_last_doc() self.assertIn('i_foo', result) self.assertIn('foo_i', result) self.assertEqual(len(self.connector.doc_manager._search( "i_foo:1")), 1) self.assertEqual(len(self.connector.doc_manager._search( "foo_i:1")), 1)
class TestElastic(ElasticsearchTestCase): """Integration tests for mongo-connector + Elasticsearch.""" @classmethod def setUpClass(cls): """Start the cluster.""" super(TestElastic, cls).setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """Kill the cluster.""" close_client(cls.conn) cls.repl_set.stop() def tearDown(self): """Stop the Connector thread.""" super(TestElastic, self).tearDown() self.connector.join() def setUp(self): """Start a new Connector for each test.""" super(TestElastic, self).setUp() try: os.unlink("oplog.timestamp") except OSError: pass docman = DocManager(elastic_pair) self.connector = Connector( mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman,), gridfs_set=['test.test'] ) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: self._count() > 0) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].delete_one({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.assertEqual(self._count(), 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = b"test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: self._count() > 0) query = {"match": {"_all": "test_insert_file"}} res = list(self._search(query)) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], 'test.txt') self.assertEqual(doc['_id'], str(id)) self.assertEqual(base64.b64decode(doc['content']), test_data) def test_remove_file(self): fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: self._count() == 1) fs.delete(id) assert_soon(lambda: self._count() == 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert_one({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.command( SON([('findAndModify', 'test'), ('query', {"a": 0}), ('update', update_spec), ('new', True)]))['value'] # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) assert_soon(lambda: next(self._search()) == updated) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() # This doc can be picked up in the collection dump self.conn['test']['test'].insert_one({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) # This doc is definitely not picked up by collection dump self.conn['test']['test'].insert_one({'name': 'pauly'}) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'}) assert_soon(lambda: self._count() == 3) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 3) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 2) if result_set_1[0]['name'] == 'paul': self.assertEqual(result_set_1[1]['name'], 'pauly') elif result_set_1[0]['name'] == 'pauly': self.assertEqual(result_set_1[1]['name'], 'paul') else: self.assertTrue(0, 'Unknown document retrieved') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 2) def test_bad_int_value(self): self.conn.test.test.insert_one({ 'inf': float('inf'), 'nan': float('nan'), 'still_exists': True}) assert_soon(lambda: self._count() > 0) for doc in self._search(): self.assertNotIn('inf', doc) self.assertNotIn('nan', doc) self.assertTrue(doc['still_exists'])
class TestElastic(ElasticsearchTestCase): """ Tests the Elastic instance """ @classmethod def setUpClass(cls): """ Starts the cluster """ super(TestElastic, cls).setUpClass() _, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-elastic') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_replica_set('test-elastic') def tearDown(self): """ Ends the connector """ super(TestElastic, self).tearDown() self.connector.join() def setUp(self): """ Starts a new connector for every test """ super(TestElastic, self).setUp() try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=elastic_pair, ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py', auto_commit_interval=0 ) self.conn.test.test.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() > 0) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.assertEqual(self._count(), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) self.assertEqual( set('Paul ' + str(i) for i in range(STRESS_COUNT)), set(item['name'] for item in self._search()) ) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: self._count() == self.conn['test']['test'].find().count()) result_set_1 = self._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn["admin"] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase): """ Tests the mongo instance """ def runTest(self): """ Runs the tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) start_single_mongod_instance("30000", "/MC", "MC_log") cls.mongo_doc = DocManager("localhost:30000") cls.mongo_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection("%s:%s" % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(HOSTNAME, 30000) kill_all() def tearDown(self): self.connector.join() def setUp(self): if not self.flag: self.fail("Shards cannot be added to mongos") self.connector = Connector("%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), CONFIG, '%s:30000' % (HOSTNAME), ['test.test'], '_id', None, 'mongo_connector/doc_managers/mongo_doc_manager.py') self.connector.start() while len(self.connector.shard_set) == 0: pass self.conn['test']['test'].remove(safe=True) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ self.conn['test']['test'].remove(safe=True) self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1) result_set_1 = self.mongo_doc._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) condition = lambda : self.conn['test']['test'].find_one( {'name': 'paul'}) is not None wait_for(condition) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] condition = lambda : admin.command("isMaster")['ismaster'] wait_for(condition) time.sleep(5) count = 0 while True: try: result_set_1 = self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == 2) result_set_1 = list(self.mongo_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) wait_for(lambda : primary_conn['admin'].command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.mongo_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.mongo_doc._search() for item in result_set_1: if(item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) search = self.mongo_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda : admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert({'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) db_admin = primary_conn['admin'] wait_for(lambda : db_admin.command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) search = self.mongo_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
class TestMongo(MongoTestCase): """ Tests the mongo instance """ @classmethod def setUpClass(cls): MongoTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """ Kills cluster instance """ MongoTestCase.tearDownClass() cls.repl_set.stop() def tearDown(self): self.connector.join() def setUp(self): try: os.unlink("oplog.timestamp") except OSError: pass self._remove() self.connector = Connector( mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(self.mongo_doc,), gridfs_set=['test.test'], **connector_opts ) self.conn.drop_database('test') self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) result_set_1 = self._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) self.conn['test']['test'].delete_one({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self._search()) != 1) self.assertEqual(sum(1 for _ in self._search()), 0) def test_insert_file(self): """Tests inserting a gridfs file """ fs = GridFS(self.conn['test'], 'test') test_data = b"test_insert_file test file" id = fs.put(test_data, filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self._search()) > 0) res = list(self._search()) self.assertEqual(len(res), 1) doc = res[0] self.assertEqual(doc['filename'], 'test.txt') self.assertEqual(doc['_id'], id) self.assertEqual(doc['content'], test_data) def test_remove_file(self): fs = GridFS(self.conn['test'], 'test') id = fs.put("test file", filename="test.txt", encoding='utf8') assert_soon(lambda: sum(1 for _ in self._search()) == 1) fs.delete(id) assert_soon(lambda: sum(1 for _ in self._search()) == 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert_one({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.command( SON([('findAndModify', 'test'), ('query', {"a": 0}), ('update', update_spec), ('new', True)]))['value'] def update_worked(): replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0}) return replicated == updated # Allow some time for update to propagate assert_soon(update_worked) # Update by adding a field check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() self.conn['test']['test'].insert_one({'name': 'paul'}) condition = lambda: self.conn['test']['test'].find_one( {'name': 'paul'}) is not None assert_soon(condition) assert_soon(lambda: sum(1 for _ in self._search()) == 1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] condition = lambda: admin.command("isMaster")['ismaster'] assert_soon(lambda: retry_until_ok(condition)) retry_until_ok(self.conn.test.test.insert_one, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self._search()) == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) # make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() assert_soon( lambda: primary_conn['admin'].command("isMaster")['ismaster']) self.repl_set.secondary.start() time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestNeo4j(Neo4jTestCase): """Integration tests for mongo-connector + Neo4j.""" @classmethod def setUpClass(cls): """Start the cluster.""" Neo4jTestCase.setUpClass() cls.repl_set = ReplicaSet().start() cls.conn = cls.repl_set.client() @classmethod def tearDownClass(cls): """Kill the cluster.""" cls.repl_set.stop() def tearDown(self): """Stop the Connector thread.""" super(TestNeo4j, self).tearDown() self.connector.join() time.sleep(10) def setUp(self): """Start a new Connector for each test.""" try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() docman = DocManager('http://localhost:7474/db/data',auto_commit_interval=0) self.connector = Connector( mongo_address=self.repl_set.uri, ns_set=['test.test'], doc_managers=(docman,), gridfs_set=['test.test'] ) self.conn.test.test.drop() self.conn.test.test.files.drop() self.conn.test.test.chunks.drop() self.connector.start() self.neo4j_conn.delete_all() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) time.sleep(5) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) result_set_2 = self.conn['test']['test'].find_one() self.connector.doc_managers[0].upsert({'_id': str(result_set_2['_id']),'name': 'paulie'}, "test.test", 1) assert_soon(lambda: self._count() > 0) result_set_1 = self.neo4j_conn.find_one("test") self.assertNotEqual(result_set_1, None) self.assertEqual(result_set_1['_id'], str(result_set_2['_id'])) self.assertEqual(result_set_1['name'], result_set_2['name']) self.connector.doc_managers[0].graph.delete_all() def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) result_set = self.conn['test']['test'].find_one() # self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paulie'}, "test.test", 1) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.connector.doc_managers[0].remove(str(result_set['_id']), 'test.test', 1) self.assertEqual(self._count(), 0) self.connector.doc_managers[0].graph.delete_all() def test_update(self): """Test update operations.""" # Insert self.connector.doc_managers[0].graph.delete_all() self.conn['test']['updt'].insert({"a": 0}) result_set = self.conn['test']['updt'].find_one() self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'a': '0'}, "test.updt", 1) assert_soon(lambda: self._count() == 1) def check_update(update_spec): updated = self.conn.test.updt.find_and_modify( {"a": 0}, update_spec, new=True ) # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) # Allow some time for update to propagate time.sleep(5) replicated = self.neo4j_conn.find_one("updt")['_id'] self.assertEqual(replicated, updated['_id']) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) self.connector.doc_managers[0].graph.delete_all() def test_bad_int_value(self): self.conn['test']['test'].insert({'inf': float('inf'), 'nan': float('nan'), 'still_exists': True}) result_set = self.conn['test']['test'].find_one() self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']), 'inf': float('inf'), 'nan': float('nan'), 'still_exists': True }, "test.test", 1) assert_soon(lambda: self._count() > 0) doc = self.neo4j_conn.find_one("test") self.assertNotIn('inf', doc) self.assertNotIn('nan', doc) self.assertTrue(doc['still_exists']) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = self.repl_set.primary.client() self.conn['test']['rollback'].insert({'name': 'paul'}) result_set = self.conn['test']['rollback'].find_one() self.connector.doc_managers[0].upsert({'_id': str(result_set['_id']),'name': 'paul'}, "test.rollback", 1) condition1 = lambda: self.conn['test']['rollback'].find( {'name': 'paul'}).count() == 1 assert_soon(condition1) self.repl_set.primary.stop(destroy=False) new_primary_conn = self.repl_set.secondary.client() admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.rollback.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() > 0) result_set_2 = self.conn['test']['rollback'].find_one({'name': 'pauline'}) self.connector.doc_managers[0].upsert({'_id': str(result_set_2['_id']),'name': 'pauline'}, "test.rollback", 1) result_set_1 = self.neo4j_conn.find_one("rollback", "name", "pauline") self.assertNotEqual(result_set_1, None) #make sure pauline is there if result_set_1['name'] == 'pauline': self.assertEqual(result_set_1['_id'], str(result_set_2['_id'])) self.repl_set.secondary.stop(destroy=False) self.repl_set.primary.start() while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) self.repl_set.secondary.start() time.sleep(2) node = self.neo4j_conn.find_one("rollback") self.assertEqual(node['name'], 'paul') node = self.neo4j_conn.find_one("rollback", "name", "pauline") self.assertNotEqual(node, None) find_cursor = retry_until_ok(self.conn['test']['rollback'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
class TestSynchronizer(unittest.TestCase): """ Tests the mongo instance """ @classmethod def setUpClass(cls): try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() cls.standalone_port = start_mongo_proc( options=['--nojournal', '--noprealloc']) cls.mongo_doc = DocManager('%s:%d' % (mongo_host, cls.standalone_port)) cls.mongo_doc._remove() _, cls.secondary_p, cls.primary_p = start_replica_set('test-mongo') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-mongo') @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(cls.standalone_port) kill_replica_set('test-mongo') def tearDown(self): self.connector.join() def setUp(self): self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint="config.txt", target_url='%s:%d' % (mongo_host, self.standalone_port), ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py') self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) self.conn['test']['test'].remove() assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) result_set_1 = self.mongo_doc._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert({"a": 0}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify({"a": 0}, update_spec, new=True) # Allow some time for update to propagate time.sleep(2) replicated = self.mongo_doc.mongo.test.test.find_one({"a": 0}) self.assertEqual(replicated, updated) # Update by adding a field check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition = lambda: self.conn['test']['test'].find_one( {'name': 'paul'}) is not None assert_soon(condition) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] condition = lambda: admin.command("isMaster")['ismaster'] assert_soon(lambda: retry_until_ok(condition)) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == 2) result_set_1 = list(self.mongo_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) assert_soon( lambda: primary_conn['admin'].command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) for i in range(0, STRESS_COUNT): result_set_1 = self.mongo_doc._search() for item in result_set_1: if (item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self. conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn['admin'] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase): """ Tests Solr """ def runTest(self): """ Runs tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']), replicaSet="demo-repl") # Creating a Solr object with an invalid URL # doesn't create an exception cls.solr_conn = Solr('http://localhost:8983/solr') try: cls.solr_conn.commit() except (SolrError, MissingSchema): cls.err_msg = "Cannot connect to Solr!" cls.flag = False if cls.flag: cls.solr_conn.delete(q='*:*') else: cls.err_msg = "Shards cannot be added to mongos" @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_all() def setUp(self): if not self.flag: self.fail(self.err_msg) self.connector = Connector( address=('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])), oplog_checkpoint=CONFIG, target_url='http://localhost:8983/solr', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/solr_doc_manager.py') self.connector.start() while len(self.connector.shard_set) == 0: time.sleep(1) count = 0 while (True): try: self.conn['test']['test'].remove(safe=True) break except (AutoReconnect, OperationFailure): time.sleep(1) count += 1 if count > 60: unittest.SkipTest('Call to remove failed too ' 'many times in setup') while (len(self.solr_conn.search('*:*')) != 0): time.sleep(1) def tearDown(self): self.connector.join() def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ while (True): try: self.conn['test']['test'].remove(safe=True) break except OperationFailure: continue self.solr_conn.delete(q='*:*') self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(len(self.solr_conn.search('*:*')), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 0): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 1): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 0) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert({'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too ' 'many times in test_rollback') time.sleep(1) continue while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1) def test_stress(self): """Test stress by inserting and removing a large amount of docs. """ #stress test for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(5) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.solr_conn.search('Paul ' + str(i)) for item in result_set_1: self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. The rollback is performed the same way as before but with more docs """ self.conn['test']['test'].remove() while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len( self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('Paul', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert({'name': 'test_valid'}) self.conn['test']['test'].update({'_id': inserted_obj}, {'$set': { 'popularity': 1 }}) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(len(docman._search("name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name': 'test_invalid'}) self.conn['test']['test'].update({'_id': inserted_obj}, {'$set': { 'break_this_test': 1 }}) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(len(docman._search("name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert(match_first) self.conn["test"]["test"].insert(match_second) self.conn["test"]["test"].insert(match_none) # Should have documents in Solr now self.assertTrue( wait_for(lambda: len(self.solr_conn.search("*:*")) > 0), "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1) self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="person.address.street" type="string" ... /> <field name="person.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ self.solr_conn.delete(q='*:*') # Connector is already running self.conn["test"]["test"].insert({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert({ "numbers": ["one", "two", "three"], "characters": [{ "name": "Big Bird", "color": "yellow" }, { "name": "Elmo", "color": "red" }, "Cookie Monster"] }) self.assertTrue( wait_for(lambda: len(self.solr_conn.search("*:*")) > 0), "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual( next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search("characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestElastic(ElasticsearchTestCase): """Integration tests for mongo-connector + Elasticsearch.""" @classmethod def setUpClass(cls): """Start the cluster.""" super(TestElastic, cls).setUpClass() _, cls.secondary_p, cls.primary_p = start_replica_set('test-elastic') cls.conn = MongoClient(mongo_host, cls.primary_p, replicaSet='test-elastic') @classmethod def tearDownClass(cls): """Kill the cluster.""" kill_replica_set('test-elastic') def tearDown(self): """Stop the Connector thread.""" super(TestElastic, self).tearDown() self.connector.join() def setUp(self): """Start a new Connector for each test.""" super(TestElastic, self).setUp() try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() self.connector = Connector( address='%s:%s' % (mongo_host, self.primary_p), oplog_checkpoint='config.txt', target_url=elastic_pair, ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py', auto_commit_interval=0 ) self.conn.test.test.drop() self.connector.start() assert_soon(lambda: len(self.connector.shard_set) > 0) assert_soon(lambda: self._count() == 0) def test_shard_length(self): self.assertEqual(len(self.connector.shard_set), 1) def test_insert(self): """Test insert operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() > 0) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove operations.""" self.conn['test']['test'].insert({'name': 'paulie'}) assert_soon(lambda: self._count() == 1) self.conn['test']['test'].remove({'name': 'paulie'}) assert_soon(lambda: self._count() != 1) self.assertEqual(self._count(), 0) def test_update(self): """Test update operations.""" # Insert self.conn.test.test.insert({"a": 0}) assert_soon(lambda: sum(1 for _ in self._search()) == 1) def check_update(update_spec): updated = self.conn.test.test.find_and_modify( {"a": 0}, update_spec, new=True ) # Stringify _id to match what will be retrieved from ES updated['_id'] = str(updated['_id']) # Allow some time for update to propagate time.sleep(1) replicated = next(self._search()) self.assertEqual(replicated, updated) # Update by adding a field. Note that ES can't mix types within an array check_update({"$set": {"b": [{"c": 10}, {"d": 11}]}}) # Update by setting an attribute of a sub-document beyond end of array. check_update({"$set": {"b.10.c": 42}}) # Update by changing a value within a sub-document (contains array) check_update({"$inc": {"b.0.c": 1}}) # Update by changing the value within an array check_update({"$inc": {"b.1.f": 12}}) # Update by adding new bucket to list check_update({"$push": {"b": {"e": 12}}}) # Update by changing an entire sub-document check_update({"$set": {"b.0": {"e": 4}}}) # Update by adding a sub-document check_update({"$set": {"b": {"0": {"c": 100}}}}) # Update whole document check_update({"a": 0, "b": {"1": {"d": 10000}}}) def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Stress test for inserting and removing many documents.""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) self.assertEqual( set('Paul ' + str(i) for i in range(STRESS_COUNT)), set(item['name'] for item in self._search()) ) def test_stressed_rollback(self): """Stress test for a rollback with many documents.""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) condition = lambda: self._count() == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: self._count() == self.conn['test']['test'].find().count()) result_set_1 = self._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn["admin"] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
class TestSynchronizer(unittest.TestCase): """ Tests Solr """ def runTest(self): """ Runs tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']), replicaSet="demo-repl") # Creating a Solr object with an invalid URL # doesn't create an exception cls.solr_conn = Solr('http://localhost:8983/solr') try: cls.solr_conn.commit() except (SolrError, MissingSchema): cls.err_msg = "Cannot connect to Solr!" cls.flag = False if cls.flag: cls.solr_conn.delete(q='*:*') else: cls.err_msg = "Shards cannot be added to mongos" @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_all() def setUp(self): if not self.flag: self.fail(self.err_msg) self.connector = Connector( address=('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN'])), oplog_checkpoint=CONFIG, target_url='http://localhost:8983/solr', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/solr_doc_manager.py' ) self.connector.start() while len(self.connector.shard_set) == 0: time.sleep(1) count = 0 while (True): try: self.conn['test']['test'].remove(safe=True) break except (AutoReconnect, OperationFailure): time.sleep(1) count += 1 if count > 60: unittest.SkipTest('Call to remove failed too ' 'many times in setup') while (len(self.solr_conn.search('*:*')) != 0): time.sleep(1) def tearDown(self): self.connector.join() def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ while (True): try: self.conn['test']['test'].remove(safe=True) break except OperationFailure: continue self.solr_conn.delete(q='*:*') self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(len(self.solr_conn.search('*:*')), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 0): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) while (len(self.solr_conn.search('*:*')) == 1): time.sleep(1) result_set_1 = self.solr_conn.search('paulie') self.assertEqual(len(result_set_1), 0) def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too ' 'many times in test_rollback') time.sleep(1) continue while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1) def test_stress(self): """Test stress by inserting and removing a large amount of docs. """ #stress test for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(5) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.solr_conn.search('Paul ' + str(i)) for item in result_set_1: self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. The rollback is performed the same way as before but with more docs """ self.conn['test']['test'].remove() while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}, safe=True) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len(self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('Paul', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS) def test_valid_fields(self): """ Tests documents with field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name':'test_valid'}) self.conn['test']['test'].update({'_id' : inserted_obj}, {'$set':{'popularity' : 1 }}) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertIn('popularity', result) self.assertEqual(len(docman._search( "name=test_valid")), 1) def test_invalid_fields(self): """ Tests documents without field definitions """ inserted_obj = self.conn['test']['test'].insert( {'name':'test_invalid'}) self.conn['test']['test'].update({'_id' : inserted_obj}, {'$set':{'break_this_test' : 1 }}) docman = self.connector.doc_managers[0] for _ in range(60): if len(docman._search("*:*")) != 0: break time.sleep(1) else: self.fail("Timeout when removing docs from Solr") result = docman.get_last_doc() self.assertNotIn('break_this_test', result) self.assertEqual(len(docman._search( "name=test_invalid")), 1) def test_dynamic_fields(self): """ Tests dynamic field definitions The following fields are supplied in the provided schema.xml: <dynamicField name="*_i" type="int" indexed="true" stored="true"/> <dynamicField name="i_*" type="int" indexed="true" stored="true"/> Cases: 1. Match on first definition 2. Match on second definition 3. No match """ self.solr_conn.delete(q='*:*') match_first = {"_id": 0, "foo_i": 100} match_second = {"_id": 1, "i_foo": 200} match_none = {"_id": 2, "foo": 300} # Connector is already running self.conn["test"]["test"].insert(match_first) self.conn["test"]["test"].insert(match_second) self.conn["test"]["test"].insert(match_none) # Should have documents in Solr now self.assertTrue(wait_for(lambda: len(self.solr_conn.search("*:*")) > 0), "Solr doc manager should allow dynamic fields") # foo_i and i_foo should be indexed, foo field should not exist self.assertEqual(len(self.solr_conn.search("foo_i:100")), 1) self.assertEqual(len(self.solr_conn.search("i_foo:200")), 1) # SolrError: "undefined field foo" logger = logging.getLogger("pysolr") logger.error("You should see an ERROR log message from pysolr here. " "This indicates success, not an error in the test.") with self.assertRaises(SolrError): self.solr_conn.search("foo:300") def test_nested_fields(self): """Test indexing fields that are sub-documents in MongoDB The following fields are defined in the provided schema.xml: <field name="person.address.street" type="string" ... /> <field name="person.address.state" type="string" ... /> <dynamicField name="numbers.*" type="string" ... /> <dynamicField name="characters.*" type="string" ... /> """ self.solr_conn.delete(q='*:*') # Connector is already running self.conn["test"]["test"].insert({ "name": "Jeb", "billing": { "address": { "street": "12345 Mariposa Street", "state": "California" } } }) self.conn["test"]["test"].insert({ "numbers": ["one", "two", "three"], "characters": [ {"name": "Big Bird", "color": "yellow"}, {"name": "Elmo", "color": "red"}, "Cookie Monster" ] }) self.assertTrue(wait_for(lambda: len(self.solr_conn.search("*:*")) > 0), "documents should have been replicated to Solr") # Search for first document results = self.solr_conn.search( "billing.address.street:12345\ Mariposa\ Street") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["billing.address.state"], "California") # Search for second document results = self.solr_conn.search( "characters.1.color:red") self.assertEqual(len(results), 1) self.assertEqual(next(iter(results))["numbers.2"], "three") results = self.solr_conn.search("characters.2:Cookie\ Monster") self.assertEqual(len(results), 1)
class TestSynchronizer(unittest.TestCase): """ Tests the mongo instance """ def runTest(self): """ Runs the tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) start_single_mongod_instance("30000", "/MC", "MC_log") cls.mongo_doc = DocManager("localhost:30000") cls.mongo_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection("%s:%s" % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(HOSTNAME, 30000) kill_all() def tearDown(self): self.connector.join() def setUp(self): if not self.flag: self.fail("Shards cannot be added to mongos") self.connector = Connector( address="%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), oplog_checkpoint=CONFIG, target_url='%s:30000' % (HOSTNAME), ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/mongo_doc_manager.py') self.connector.start() while len(self.connector.shard_set) == 0: pass self.conn['test']['test'].remove(safe=True) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ self.conn['test']['test'].remove(safe=True) self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) result_set_1 = self.mongo_doc._search() self.assertEqual(sum(1 for _ in result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], result_set_2['_id']) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.mongo_doc._search()), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) condition = lambda: self.conn['test']['test'].find_one( {'name': 'paul'}) is not None wait_for(condition) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] condition = lambda: admin.command("isMaster")['ismaster'] wait_for(condition) time.sleep(5) count = 0 while True: try: result_set_1 = self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == 2) result_set_1 = list(self.mongo_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) wait_for(lambda: primary_conn['admin'].command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.mongo_doc._search() for item in result_set_1: if (item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) wait_for(lambda: sum(1 for _ in self.mongo_doc._search()) == self.conn[ 'test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) db_admin = primary_conn['admin'] wait_for(lambda: db_admin.command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
class TestElastic(unittest.TestCase): """ Tests the Elastic instance """ def runTest(self): """ Runs the tests """ unittest.TestCase.__init__(self) @classmethod def setUpClass(cls): """ Starts the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.elastic_doc = DocManager('localhost:9200', auto_commit=False) cls.elastic_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") import logging logger = logging.getLogger() loglevel = logging.INFO logger.setLevel(loglevel) @classmethod def tearDownClass(cls): """ Kills cluster instance """ kill_all() def tearDown(self): """ Ends the connector """ self.connector.doc_manager.auto_commit = False time.sleep(2) self.connector.join() def setUp(self): """ Starts a new connector for every test """ if not self.flag: self.fail("Shards cannot be added to mongos") self.connector = Connector( address='%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), oplog_checkpoint=CONFIG, target_url='localhost:9200', ns_set=['test.test'], u_key='_id', auth_key=None, doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py' ) self.connector.start() while len(self.connector.shard_set) == 0: pass self.conn['test']['test'].remove(safe=True) wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 0) def test_shard_length(self): """Tests the shard_length to see if the shard set was recognized properly """ self.assertEqual(len(self.connector.shard_set), 1) def test_initial(self): """Tests search and assures that the databases are clear. """ self.conn['test']['test'].remove(safe=True) self.assertEqual(self.conn['test']['test'].find().count(), 0) self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0) def test_insert(self): """Tests insert """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) > 0) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) result_set_2 = self.conn['test']['test'].find_one() for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) self.assertEqual(item['name'], result_set_2['name']) def test_remove(self): """Tests remove """ self.conn['test']['test'].insert({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 1) self.conn['test']['test'].remove({'name': 'paulie'}, safe=True) wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) != 1) self.assertEqual(sum(1 for _ in self.elastic_doc._search()), 0) def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) condition1 = lambda : self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda : sum(1 for _ in self.elastic_doc._search()) == 1 wait_for(condition1) wait_for(condition2) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda : admin.command("isMaster")['ismaster']) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == 2) result_set_1 = list(self.elastic_doc._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1) def test_stress(self): """Test stress by inserting and removing the number of documents specified in global variable """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) time.sleep(5) search = self.elastic_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) for i in range(0, NUMBER_OF_DOC_DIRS): result_set_1 = self.elastic_doc._search() for item in result_set_1: if(item['name'] == 'Paul' + str(i)): self.assertEqual(item['_id'], item['_id']) def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) search = self.elastic_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda : admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) wait_for(lambda : sum(1 for _ in self.elastic_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.elastic_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) db_admin = primary_conn["admin"] wait_for(lambda : db_admin.command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) search = self.elastic_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) result_set_1 = list(self.elastic_doc._search()) self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS) def test_non_standard_fields(self): """ Tests ObjectIds, DBrefs, etc """ # This test can break if it attempts to insert before the dump takes # place- this prevents it (other tests affected too actually) while (self.connector.shard_set['demo-repl'].checkpoint is None): time.sleep(1) docs = [ {'foo': [1, 2]}, {'bar': {'hello': 'world'}}, {'code': Code("function x() { return 1; }")}, {'dbref': {'_ref': DBRef('simple', ObjectId('509b8db456c02c5ab7e63c34'))}} ] try: self.conn['test']['test'].insert(docs) except OperationFailure: self.fail("Cannot insert documents into Elastic!") search = self.elastic_doc._search if not wait_for(lambda : sum(1 for _ in search()) == len(docs)): self.fail("Did not get all expected documents") self.assertIn("dbref", self.elastic_doc.get_last_doc())