Exemplo n.º 1
0
 def setUp(self):
     self.c = Connector(
         'localhost:' + PORTS_ONE["MONGOS"], 'config.txt',
         'http://localhost:9200', ['test.test'], '_id', None,
         cmd_folder + '/../../doc_managers/elastic_doc_manager.py')
     self.c.start()
     while len(self.c.shard_set) == 0:
         pass
     conn['test']['test'].remove(safe=True)
     while (len(s._search()) != 0):
         time.sleep(1)
Exemplo n.º 2
0
    def test_write_oplog_progress(self):
        """Test write_oplog_progress under several circumstances
        """
        os.system('touch temp_config.txt')
        config_file_path = os.getcwd() + '/temp_config.txt'
        c = Connector(main_address, config_file_path, None, ['test.test'],
                      '_id', None, None)

        #test that None is returned if there is no config file specified.
        self.assertEqual(c.write_oplog_progress(), None)

        c.oplog_progress.get_dict()[1] = Timestamp(12, 34)
        #pretend to insert a thread/timestamp pair
        c.write_oplog_progress()

        data = json.load(open(config_file_path, 'r'))
        self.assertEqual(1, int(data[0]))
        self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(12, 34))

        #ensure the temp file was deleted
        self.assertFalse(os.path.exists(config_file_path + '~'))

        #ensure that updates work properly
        c.oplog_progress.get_dict()[1] = Timestamp(44, 22)
        c.write_oplog_progress()

        config_file = open(config_file_path, 'r')
        data = json.load(config_file)
        self.assertEqual(1, int(data[0]))
        self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(44, 22))

        os.system('rm ' + config_file_path)
        config_file.close()
        print("PASSED TEST WRITE OPLOG PROGRESS")
    def test_connector(self):
        """Test whether the connector initiates properly
        """
        if not self.flag:
            self.fail("Shards cannot be added to mongos")

        conn = Connector(MAIN_ADDRESS, 'config.txt', None, ['test.test'],
                      '_id', None, None)
        conn.start()

        while len(conn.shard_set) != 1:
            time.sleep(2)
        conn.join()

        self.assertFalse(conn.can_run)
        time.sleep(5)
        for thread in conn.shard_set.values():
            self.assertFalse(thread.running)
Exemplo n.º 4
0
 def setUp(self):
     self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
                        'config.txt', 'http://localhost:9200',
                        ['test.test'],
                        '_id', None, cmd_folder + '/../../doc_managers/elastic_doc_manager.py')
     self.c.start()
     while len(self.c.shard_set) == 0:
         pass
     conn['test']['test'].remove(safe=True)
     while(len(s._search()) != 0):
         time.sleep(1)
    def test_write_oplog_progress(self):
        """Test write_oplog_progress under several circumstances
        """
        os.system('touch temp_config.txt')
        config_file_path = os.getcwd() + '/temp_config.txt'
        conn = Connector(MAIN_ADDRESS, config_file_path, None, ['test.test'],
                      '_id', None, None)

        #test that None is returned if there is no config file specified.
        self.assertEqual(conn.write_oplog_progress(), None)

        conn.oplog_progress.get_dict()[1] = Timestamp(12, 34)
        #pretend to insert a thread/timestamp pair
        conn.write_oplog_progress()

        data = json.load(open(config_file_path, 'r'))
        self.assertEqual(1, int(data[0]))
        self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(12, 34))

        #ensure the temp file was deleted
        self.assertFalse(os.path.exists(config_file_path + '~'))

        #ensure that updates work properly
        conn.oplog_progress.get_dict()[1] = Timestamp(44, 22)
        conn.write_oplog_progress()

        config_file = open(config_file_path, 'r')
        data = json.load(config_file)
        self.assertEqual(1, int(data[0]))
        self.assertEqual(long_to_bson_ts(int(data[1])), Timestamp(44, 22))

        os.system('rm ' + config_file_path)
        config_file.close()
Exemplo n.º 6
0
 def setUp(self):
     if not self.flag:
         self.fail("Shards cannot be added to mongos")
     self.connector = Connector('localhost:' + PORTS_ONE["MONGOS"],
        'config.txt', 'localhost:30000',
        ['test.test'],
        '_id', None,
        '../mongo_connector/doc_managers/mongo_doc_manager.py')
     self.connector.start()
     while len(self.connector.shard_set) == 0:
         pass
     self.conn['test']['test'].remove(safe=True)
     while(len(self.mongo_doc._search()) != 0):
         time.sleep(1)
    def setUp(self):

        self.c = Connector(
            'localhost:' + PORTS_ONE["MAIN"], 'config.txt',
            'http://localhost:8080/solr', ['test.test'], '_id', None,
            cmd_folder + '/../../doc_managers/solr_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    string = 'Call to remove failed too many times'
                    string += ' in setUp'
                    logging.error(string)
                    sys.exit(1)
        while (len(s.search('*:*')) != 0):
            time.sleep(1)
Exemplo n.º 8
0
    def test_connector(self):
        """Test whether the connector initiates properly
        """

        c = Connector(main_address, 'config.txt', None, ['test.test'],
                      '_id', None, None)
        c.start()

        while len(c.shard_set) != 1:
            time.sleep(2)
        c.join()

        self.assertFalse(c.can_run)
        time.sleep(5)
        for thread in c.shard_set.values():
            self.assertFalse(thread.running)
Exemplo n.º 9
0
    def setUp(self):

        self.c = Connector('localhost:' + PORTS_ONE["MAIN"], 'config.txt',
                           'http://localhost:8080/solr', ['test.test'], '_id',
                           None, cmd_folder + '/../../doc_managers/solr_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    string = 'Call to remove failed too many times'
                    string += ' in setUp'
                    logging.error(string)
                    sys.exit(1)
        while (len(s.search('*:*')) != 0):
            time.sleep(1)
Exemplo n.º 10
0
    def setUp(self):
        if not self.flag:
            self.fail(self.err_msg)

        self.connector = Connector('localhost:' + PORTS_ONE["MAIN"], 
            'config.txt', 'http://localhost:8983/solr', ['test.test'], '_id',
            None, 
            '../mongo_connector/doc_managers/solr_doc_manager.py')
        self.connector.start()
        while len(self.connector.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    unittest.SkipTest('Call to remove failed too '
                    'many times in setup')
        while (len(self.solr_conn.search('*:*')) != 0):
            time.sleep(1)
Exemplo n.º 11
0
class TestSynchronizer(unittest.TestCase):

    c = None  # used for the connector

    def runTest(self):
        unittest.TestCase.__init__(self)

    def tearDown(self):
        self.c.doc_manager.auto_commit = False
        time.sleep(2)
        self.c.join()

    def setUp(self):
        self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
                           'config.txt', 'http://localhost:9200',
                           ['test.test'],
                           '_id', None, cmd_folder + '/../../doc_managers/elastic_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            pass
        conn['test']['test'].remove(safe=True)
        while(len(s._search()) != 0):
            time.sleep(1)

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
            properly
        """

        self.assertEqual(len(self.c.shard_set), 1)
        print("PASSED TEST SHARD LENGTH")

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        conn['test']['test'].remove(safe=True)
        self.assertEqual(conn['test']['test'].find().count(), 0)
        self.assertEqual(len(s._search()), 0)
        print("PASSED TEST INITIAL")

    def test_insert(self):
        """Tests insert
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(s._search()) == 0):
            time.sleep(1)
        a = s._search()
        self.assertEqual(len(a), 1)
        b = conn['test']['test'].find_one()
        for it in a:
            self.assertEqual(it['_id'], str(b['_id']))
            self.assertEqual(it['name'], b['name'])
        print("PASSED TEST INSERT")

    def test_remove(self):
        """Tests remove
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(s._search()) != 1):
            time.sleep(1)
        conn['test']['test'].remove({'name': 'paulie'}, safe=True)

        while(len(s._search()) == 1):
            time.sleep(1)
        a = s._search()
        self.assertEqual(len(a), 0)
        print("PASSED TEST REMOVE")

    def test_rollback(self):
        """Tests rollback. We force a rollback by adding a doc, killing the
            primary, adding another doc, killing the new primary, and then
            restarting both.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(s._search()) != 1:
            time.sleep(1)

        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                a = conn['test']['test'].insert({'name': 'pauline'}, safe=True)
                break
            except:
                time.sleep(1)
                count += 1
                if count >= 60:
                    sys.exit(1)
                continue
        while(len(s._search()) != 2):
            time.sleep(1)
        a = s._search()
        b = conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(a), 2)
                #make sure pauling is there
        for it in a:
            if it['name'] == 'pauline':
                self.assertEqual(it['_id'], str(b['_id']))
        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        a = s._search()
        self.assertEqual(len(a), 1)
        for it in a:
            self.assertEqual(it['name'], 'paul')
        find_cursor = retry_until_ok(conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), 1)
        print("PASSED TEST ROLLBACK")

    def test_stress(self):
        """Test stress by inserting and removing the number of documents
            specified in global
            variable
        """

        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while len(s._search()) != NUMBER_OF_DOCS:
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOCS):
            a = s._search()
            b = conn['test']['test'].find_one({'name': 'Paul ' + str(i)})
            for it in a:
                if(it['name'] == 'Paul' + str(i)):
                    self.assertEqual(it['_id'], it['_id'])
        print("PASSED TEST STRESS")

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
            in global variable. Strategy for rollback is the same as before.
        """

        while len(s._search()) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True)

        while len(s._search()) != NUMBER_OF_DOCS:
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOCS:
            try:
                count += 1
                conn['test']['test'].insert({'name': 'Pauline ' + str(count)},
                                            safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)
        while (len(s._search()) != conn['test']['test'].find().count()):
            time.sleep(1)
        a = s._search()
        for it in a:
            if 'Pauline' in it['name']:
                b = conn['test']['test'].find_one({'name': it['name']})
                self.assertEqual(it['_id'], str(b['_id']))

        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(1)
        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while(len(s._search()) != NUMBER_OF_DOCS):
            time.sleep(5)

        a = s._search()
        self.assertEqual(len(a), NUMBER_OF_DOCS)
        for it in a:
            self.assertTrue('Paul' in it['name'])
        find_cursor = retry_until_ok(conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOCS)

        print("PASSED TEST STRESSED ROLBACK")
Exemplo n.º 12
0
class TestElastic(unittest.TestCase):
    """ Tests the Elastic instance
    """

    def runTest(self):
        """ Runs the tests
        """
        unittest.TestCase.__init__(self)

    @classmethod
    def setUpClass(cls):    
        """ Starts the cluster
        """
        cls.elastic_doc = DocManager('http://localhost:9200', 
            auto_commit=False)
        cls.elastic_doc._remove()
        cls.flag = start_cluster()
        if cls.flag:
            cls.conn = Connection('localhost:' + PORTS_ONE['MONGOS'],
                        replicaSet="demo-repl")

    def tearDown(self):
        """ Ends the connector
        """
        self.connector.doc_manager.auto_commit = False
        time.sleep(2)
        self.connector.join()

    def setUp(self):
        """ Starts a new connector for every test
        """
        if not self.flag:
            self.fail("Shards cannot be added to mongos")
        self.connector = Connector('localhost:' + PORTS_ONE["MONGOS"],
            'config.txt', 'http://localhost:9200',
            ['test.test'],
            '_id', None,
            '../mongo_connector/doc_managers/elastic_doc_manager.py')
        self.connector.start()
        while len(self.connector.shard_set) == 0:
            pass
        self.conn['test']['test'].remove(safe=True)
        while(len(self.elastic_doc._search()) != 0):
            time.sleep(1)

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
            properly
        """

        self.assertEqual(len(self.connector.shard_set), 1)

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        self.conn['test']['test'].remove(safe=True)
        self.assertEqual(self.conn['test']['test'].find().count(), 0)
        self.assertEqual(len(self.elastic_doc._search()), 0)

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(self.elastic_doc._search()) == 0):
            time.sleep(1)
        result_set_1 = self.elastic_doc._search()
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
            self.assertEqual(item['name'], result_set_2['name'])

    def test_remove(self):
        """Tests remove
        """

        self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(self.elastic_doc._search()) != 1):
            time.sleep(1)
        self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)

        while(len(self.elastic_doc._search()) == 1):
            time.sleep(1)
        result_set_1 = self.elastic_doc._search()
        self.assertEqual(len(result_set_1), 0)

    def test_rollback(self):
        """Tests rollback. We force a rollback by adding a doc, killing the
            primary, adding another doc, killing the new primary, and then
            restarting both.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(self.elastic_doc._search()) != 1:
            time.sleep(1)

        kill_mongo_proc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                self.conn['test']['test'].insert(
                    {'name': 'pauline'}, safe=True)
                break
            except OperationFailure:
                time.sleep(1)
                count += 1
                if count >= 60:
                    sys.exit(1)
                continue
        while(len(self.elastic_doc._search()) != 2):
            time.sleep(1)
        result_set_1 = self.elastic_doc._search()
        result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(result_set_1), 2)
                #make sure pauling is there
        for item in result_set_1:
            if item['name'] == 'pauline':
                self.assertEqual(item['_id'], str(result_set_2['_id']))
        kill_mongo_proc('localhost', PORTS_ONE['SECONDARY'])

        start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        result_set_1 = self.elastic_doc._search()
        self.assertEqual(len(result_set_1), 1)
        for item in result_set_1:
            self.assertEqual(item['name'], 'paul')
        find_cursor = retry_until_ok(self.conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), 1)

    def test_stress(self):
        """Test stress by inserting and removing the number of documents
            specified in global
            variable
        """

        for i in range(0, NUMBER_OF_DOC_DIRS):
            self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while len(self.elastic_doc._search()) != NUMBER_OF_DOC_DIRS:
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOC_DIRS):
            result_set_1 = self.elastic_doc._search()
            for item in result_set_1:
                if(item['name'] == 'Paul' + str(i)):
                    self.assertEqual(item['_id'], item['_id'])

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
            in global variable. Strategy for rollback is the same as before.
        """

        while len(self.elastic_doc._search()) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOC_DIRS):
            self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, 
                safe=True)

        while len(self.elastic_doc._search()) != NUMBER_OF_DOC_DIRS:
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        kill_mongo_proc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOC_DIRS:
            try:
                count += 1
                self.conn['test']['test'].insert(
                    {'name': 'Pauline ' + str(count)}, safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)
        while (len(self.elastic_doc._search())
                != self.conn['test']['test'].find().count()):
            time.sleep(1)
        result_set_1 = self.elastic_doc._search()
        for item in result_set_1:
            if 'Pauline' in item['name']:
                result_set_2 = self.conn['test']['test'].find_one(
                    {'name': item['name']})
                self.assertEqual(item['_id'], str(result_set_2['_id']))

        kill_mongo_proc('localhost', PORTS_ONE['SECONDARY'])

        start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(1)
        start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while(len(self.elastic_doc._search()) != NUMBER_OF_DOC_DIRS):
            time.sleep(5)

        result_set_1 = self.elastic_doc._search()
        self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS)
        for item in result_set_1:
            self.assertTrue('Paul' in item['name'])
        find_cursor = retry_until_ok(self.conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
Exemplo n.º 13
0
class TestSynchronizer(unittest.TestCase):

    c = None  # used for the Connector

    def runTest(self):
        unittest.TestCase.__init__(self)

    def setUp(self):

        self.c = Connector('localhost:' + PORTS_ONE["MAIN"], 'config.txt',
                           'http://localhost:8080/solr', ['test.test'], '_id',
                           None, cmd_folder + '/../../doc_managers/solr_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    string = 'Call to remove failed too many times'
                    string += ' in setUp'
                    logging.error(string)
                    sys.exit(1)
        while (len(s.search('*:*')) != 0):
            time.sleep(1)

    def tearDown(self):
        self.c.doc_manager.auto_commit = False
        time.sleep(2)
        self.c.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.c.shard_set), 1)
        print("PASSED TEST SHARD LENGTH")

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except:
                continue

        s.delete(q='*:*')
        self.assertEqual(conn['test']['test'].find().count(), 0)
        self.assertEqual(len(s.search('*:*')), 0)
        print("PASSED TEST INITIAL")

    def test_insert(self):
        """Tests insert
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while (len(s.search('*:*')) == 0):
            time.sleep(1)
        a = s.search('paulie')
        self.assertEqual(len(a), 1)
        b = conn['test']['test'].find_one()
        for it in a:
            self.assertEqual(it['_id'], str(b['_id']))
            self.assertEqual(it['name'], b['name'])
        print("PASSED TEST INSERT")

    def test_remove(self):
        """Tests remove
        """

        conn['test']['test'].remove({'name': 'paulie'}, safe=True)
        while (len(s.search('*:*')) == 1):
            time.sleep(1)
        a = s.search('paulie')
        self.assertEqual(len(a), 0)
        print("PASSED TEST REMOVE")

    def test_rollback(self):
        """Tests rollback. We force a rollback by inserting one doc, killing
            primary, adding another doc, killing the new primary, and
            restarting both the servers.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(s.search('*:*')) != 1:
            time.sleep(1)
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']
        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                a = conn['test']['test'].insert({'name': 'pauline'}, safe=True)
                break
            except:
                count += 1
                if count > 60:
                    string = 'Call to insert failed too many times'
                    string += ' in test_rollback'
                    logging.error(string)
                    sys.exit(1)
                time.sleep(1)
                continue

        while (len(s.search('*:*')) != 2):
            time.sleep(1)

        a = s.search('pauline')
        b = conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(a), 1)
        for it in a:
            self.assertEqual(it['_id'], str(b['_id']))
        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        a = s.search('pauline')
        self.assertEqual(len(a), 0)
        a = s.search('paul')
        self.assertEqual(len(a), 1)
        print("PASSED TEST ROLLBACK")

    def test_stress(self):
        """Test stress by inserting and removing a large amount of docs.
        """
        #stress test
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != NUMBER_OF_DOCS:
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOCS):
            a = s.search('Paul ' + str(i))
            b = conn['test']['test'].find_one({'name': 'Paul ' + str(i)})
            for it in a:
                self.assertEqual(it['_id'], it['_id'])
        print("PASSED TEST STRESS")

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
        in global variable. The rollback is performed the same way as before
            but with more docs
        """

        conn['test']['test'].remove()
        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True)

        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != NUMBER_OF_DOCS:
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']

        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOCS:
            try:
                count += 1
                conn['test']['test'].insert({'name': 'Pauline ' + str(count)},
                                            safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)

        while (len(s.search('*:*', rows=NUMBER_OF_DOCS * 2)) !=
               conn['test']['test'].find().count()):
            time.sleep(1)
        a = s.search('Pauline', rows=NUMBER_OF_DOCS * 2, sort='_id asc')
        for it in a:
            b = conn['test']['test'].find_one({'name': it['name']})
            self.assertEqual(it['_id'], str(b['_id']))

        killMongoProc('localhost', PORTS_ONE['SECONDARY'])
        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while (len(s.search('Pauline', rows=NUMBER_OF_DOCS * 2)) != 0):
            time.sleep(15)
        a = s.search('Pauline', rows=NUMBER_OF_DOCS * 2)
        self.assertEqual(len(a), 0)
        a = s.search('Paul', rows=NUMBER_OF_DOCS * 2)
        self.assertEqual(len(a), NUMBER_OF_DOCS)
        print("PASSED TEST STRESSED ROLBACK")

    def abort_test(self):
        print("TEST FAILED")
        sys.exit(1)
Exemplo n.º 14
0
    def test_read_oplog_progress(self):
        """Test read_oplog_progress
        """

        c = Connector(main_address, None, None, ['test.test'], '_id',
                      None, None)

        #testing with no file
        self.assertEqual(c.read_oplog_progress(), None)

        os.system('touch temp_config.txt')
        config_file_path = os.getcwd() + '/temp_config.txt'
        c.oplog_checkpoint = config_file_path

        #testing with empty file
        self.assertEqual(c.read_oplog_progress(), None)

        oplog_dict = c.oplog_progress.get_dict()

        #add a value to the file, delete the dict, and then read in the value
        oplog_dict['oplog1'] = Timestamp(12, 34)
        c.write_oplog_progress()
        del oplog_dict['oplog1']

        self.assertEqual(len(oplog_dict), 0)

        c.read_oplog_progress()

        self.assertTrue('oplog1' in oplog_dict.keys())
        self.assertTrue(oplog_dict['oplog1'], Timestamp(12, 34))

        oplog_dict['oplog1'] = Timestamp(55, 11)

        #see if oplog progress dict is properly updated
        c.read_oplog_progress()
        self.assertTrue(oplog_dict['oplog1'], Timestamp(55, 11))

        os.system('rm ' + config_file_path)
        print("PASSED TEST READ OPLOG PROGRESS")
Exemplo n.º 15
0
class TestSynchronizer(unittest.TestCase):

    c = None  # used for the connector

    def runTest(self):
        unittest.TestCase.__init__(self)

    def tearDown(self):
        self.c.join()

    def setUp(self):
        self.c = Connector('localhost:' + PORTS_ONE["MONGOS"],
                           'config.txt', 'localhost:30000',
                           ['test.test'],
                           '_id', None, cmd_folder +
                           '/../../doc_managers/mongo_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            pass
        conn['test']['test'].remove(safe=True)
        while(len(doc_manager._search()) != 0):
            time.sleep(1)

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
            properly
        """

        self.assertEqual(len(self.c.shard_set), 1)
        print("PASSED TEST SHARD LENGTH")

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        conn['test']['test'].remove(safe=True)
        self.assertEqual(conn['test']['test'].find().count(), 0)
        self.assertEqual(len(doc_manager._search()), 0)
        print("PASSED TEST INITIAL")

    def test_insert(self):
        """Tests insert
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(doc_manager._search()) == 0):
            time.sleep(1)
        a = doc_manager._search()
        self.assertEqual(len(a), 1)
        b = conn['test']['test'].find_one()
        for it in a:
            self.assertEqual(it['_id'], b['_id'])
            self.assertEqual(it['name'], b['name'])
        print("PASSED TEST INSERT")

    def test_remove(self):
        """Tests remove
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while(len(doc_manager._search()) != 1):
            time.sleep(1)
        conn['test']['test'].remove({'name': 'paulie'}, safe=True)

        while(len(doc_manager._search()) == 1):
            time.sleep(1)
        a = doc_manager._search()
        self.assertEqual(len(a), 0)
        print("PASSED TEST REMOVE")

    def test_rollback(self):
        """Tests rollback. We force a rollback by adding a doc, killing the
            primary, adding another doc, killing the new primary, and then
            restarting both.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(doc_manager._search()) != 1:
            time.sleep(1)

        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                a = conn['test']['test'].insert({'name': 'pauline'}, safe=True)
                break
            except:
                time.sleep(1)
                count += 1
                if count >= 60:
                    sys.exit(1)
                continue
        while(len(doc_manager._search()) != 2):
            time.sleep(1)
        a = doc_manager._search()
        b = conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(a), 2)
                #make sure pauling is there
        for it in a:
            if it['name'] == 'pauline':
                self.assertEqual(it['_id'], b['_id'])
        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        a = doc_manager._search()
        self.assertEqual(len(a), 1)
        for it in a:
            self.assertEqual(it['name'], 'paul')
        find_cursor = retry_until_ok(conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), 1)
        print("PASSED TEST ROLLBACK")

    def test_stress(self):
        """Test stress by inserting and removing the number of documents
            specified in global
            variable
        """

        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while len(doc_manager._search()) != NUMBER_OF_DOCS:
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOCS):
            a = doc_manager._search()
            b = conn['test']['test'].find_one({'name': 'Paul ' + str(i)})
            for it in a:
                if(it['name'] == 'Paul' + str(i)):
                    self.assertEqual(it['_id'], it['_id'])
        print("PASSED TEST STRESS")

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
            in global variable. Strategy for rollback is the same as before.
        """

        while len(doc_manager._search()) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True)

        while len(doc_manager._search()) != NUMBER_OF_DOCS:
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))

        admin = new_primary_conn['admin']
        while admin.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOCS:
            try:
                count += 1
                conn['test']['test'].insert({'name': 'Pauline ' + str(count)},
                                            safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)
        while (len(doc_manager._search()) != conn['test']['test'].find().count()):
            time.sleep(1)
        a = doc_manager._search()
        for it in a:
            if 'Pauline' in it['name']:
                b = conn['test']['test'].find_one({'name': it['name']})
                self.assertEqual(it['_id'], b['_id'])

        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)
        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(1)
        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while(len(doc_manager._search()) != NUMBER_OF_DOCS):
            time.sleep(5)

        a = doc_manager._search()
        self.assertEqual(len(a), NUMBER_OF_DOCS)
        for it in a:
            self.assertTrue('Paul' in it['name'])
        find_cursor = retry_until_ok(conn['test']['test'].find)
        self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOCS)

        print("PASSED TEST STRESSED ROLBACK")

    def abort_test(self):
        print("TEST FAILED")
        sys.exit(1)
class TestSynchronizer(unittest.TestCase):

    c = None  # used for the Connector

    def runTest(self):
        unittest.TestCase.__init__(self)

    def setUp(self):

        self.c = Connector(
            'localhost:' + PORTS_ONE["MAIN"], 'config.txt',
            'http://localhost:8080/solr', ['test.test'], '_id', None,
            cmd_folder + '/../../doc_managers/solr_doc_manager.py')
        self.c.start()
        while len(self.c.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    string = 'Call to remove failed too many times'
                    string += ' in setUp'
                    logging.error(string)
                    sys.exit(1)
        while (len(s.search('*:*')) != 0):
            time.sleep(1)

    def tearDown(self):
        self.c.doc_manager.auto_commit = False
        time.sleep(2)
        self.c.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.c.shard_set), 1)
        print("PASSED TEST SHARD LENGTH")

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        while (True):
            try:
                conn['test']['test'].remove(safe=True)
                break
            except:
                continue

        s.delete(q='*:*')
        self.assertEqual(conn['test']['test'].find().count(), 0)
        self.assertEqual(len(s.search('*:*')), 0)
        print("PASSED TEST INITIAL")

    def test_insert(self):
        """Tests insert
        """

        conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while (len(s.search('*:*')) == 0):
            time.sleep(1)
        a = s.search('paulie')
        self.assertEqual(len(a), 1)
        b = conn['test']['test'].find_one()
        for it in a:
            self.assertEqual(it['_id'], str(b['_id']))
            self.assertEqual(it['name'], b['name'])
        print("PASSED TEST INSERT")

    def test_remove(self):
        """Tests remove
        """

        conn['test']['test'].remove({'name': 'paulie'}, safe=True)
        while (len(s.search('*:*')) == 1):
            time.sleep(1)
        a = s.search('paulie')
        self.assertEqual(len(a), 0)
        print("PASSED TEST REMOVE")

    def test_rollback(self):
        """Tests rollback. We force a rollback by inserting one doc, killing
            primary, adding another doc, killing the new primary, and
            restarting both the servers.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(s.search('*:*')) != 1:
            time.sleep(1)
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']
        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                a = conn['test']['test'].insert({'name': 'pauline'}, safe=True)
                break
            except:
                count += 1
                if count > 60:
                    string = 'Call to insert failed too many times'
                    string += ' in test_rollback'
                    logging.error(string)
                    sys.exit(1)
                time.sleep(1)
                continue

        while (len(s.search('*:*')) != 2):
            time.sleep(1)

        a = s.search('pauline')
        b = conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(a), 1)
        for it in a:
            self.assertEqual(it['_id'], str(b['_id']))
        killMongoProc('localhost', PORTS_ONE['SECONDARY'])

        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        a = s.search('pauline')
        self.assertEqual(len(a), 0)
        a = s.search('paul')
        self.assertEqual(len(a), 1)
        print("PASSED TEST ROLLBACK")

    def test_stress(self):
        """Test stress by inserting and removing a large amount of docs.
        """
        #stress test
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != NUMBER_OF_DOCS:
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOCS):
            a = s.search('Paul ' + str(i))
            b = conn['test']['test'].find_one({'name': 'Paul ' + str(i)})
            for it in a:
                self.assertEqual(it['_id'], it['_id'])
        print("PASSED TEST STRESS")

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
        in global variable. The rollback is performed the same way as before
            but with more docs
        """

        conn['test']['test'].remove()
        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOCS):
            conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True)

        while len(s.search('*:*', rows=NUMBER_OF_DOCS)) != NUMBER_OF_DOCS:
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        killMongoProc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']

        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOCS:
            try:
                count += 1
                conn['test']['test'].insert({'name': 'Pauline ' + str(count)},
                                            safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)

        while (len(s.search('*:*', rows=NUMBER_OF_DOCS * 2)) !=
               conn['test']['test'].find().count()):
            time.sleep(1)
        a = s.search('Pauline', rows=NUMBER_OF_DOCS * 2, sort='_id asc')
        for it in a:
            b = conn['test']['test'].find_one({'name': it['name']})
            self.assertEqual(it['_id'], str(b['_id']))

        killMongoProc('localhost', PORTS_ONE['SECONDARY'])
        startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while (len(s.search('Pauline', rows=NUMBER_OF_DOCS * 2)) != 0):
            time.sleep(15)
        a = s.search('Pauline', rows=NUMBER_OF_DOCS * 2)
        self.assertEqual(len(a), 0)
        a = s.search('Paul', rows=NUMBER_OF_DOCS * 2)
        self.assertEqual(len(a), NUMBER_OF_DOCS)
        print("PASSED TEST STRESSED ROLBACK")

    def abort_test(self):
        print("TEST FAILED")
        sys.exit(1)
Exemplo n.º 17
0
        print("TEST FAILED")
        sys.exit(1)


if __name__ == '__main__':
    os.system('rm config.txt; touch config.txt')
    parser = OptionParser()

    #-m is for the main address, which is a host:port pair, ideally of the
    #mongos. For non sharded clusters, it can be the primary.
    parser.add_option("-m", "--main", action="store", type="string",
                      dest="main_addr", default="27217")

    (options, args) = parser.parse_args()
    PORTS_ONE['MONGOS'] = options.main_addr
    c = Connector('localhost:' + PORTS_ONE["MONGOS"], 'config.txt', None,
                  ['test.test'], '_id', None, None)
    s = c.doc_manager
    if options.main_addr != "27217":
        start_cluster(use_mongos=False)
    else:
        start_cluster()
    conn = Connection('localhost:' + PORTS_ONE['MONGOS'],
                      replicaSet="demo-repl")
    t = Timer(60, abort_test)
    t.start()
    c.start()
    while len(c.shard_set) == 0:
        pass
    t.cancel()
    unittest.main(argv=[sys.argv[0]])
    c.join()
Exemplo n.º 18
0
if __name__ == '__main__':
    os.system('rm config.txt; touch config.txt')
    parser = OptionParser()

    #-m is for the main address, which is a host:port pair, ideally of the
    #mongos. For non sharded clusters, it can be the primary.
    parser.add_option("-m",
                      "--main",
                      action="store",
                      type="string",
                      dest="main_addr",
                      default="27217")

    (options, args) = parser.parse_args()
    PORTS_ONE['MONGOS'] = options.main_addr
    connection = Connector('localhost:' + PORTS_ONE["MONGOS"], 'config.txt',
                           None, ['test.test'], '_id', None, None)
    doc_manager = connection.doc_manager
    if options.main_addr != "27217":
        start_cluster(use_mongos=False)
    else:
        start_cluster()
    conn = Connection('localhost:' + PORTS_ONE['MONGOS'],
                      replicaSet="demo-repl")
    t = Timer(60, abort_test)
    t.start()
    connection.start()
    while len(connection.shard_set) == 0:
        pass
    t.cancel()
    unittest.main(argv=[sys.argv[0]])
    connection.join()
Exemplo n.º 19
0
        print("TEST FAILED")
        sys.exit(1)


if __name__ == '__main__':
    os.system('rm config.txt; touch config.txt')
    parser = OptionParser()

    #-m is for the main address, which is a host:port pair, ideally of the
    #mongos. For non sharded clusters, it can be the primary.
    parser.add_option("-m", "--main", action="store", type="string",
                      dest="main_addr", default="27217")

    (options, args) = parser.parse_args()
    PORTS_ONE['MONGOS'] = options.main_addr
    connection = Connector('localhost:' + PORTS_ONE["MONGOS"], 'config.txt', None,
                  ['test.test'], '_id', None, None)
    doc_manager = connection.doc_manager
    if options.main_addr != "27217":
        start_cluster(use_mongos=False)
    else:
        start_cluster()
    conn = Connection('localhost:' + PORTS_ONE['MONGOS'],
                      replicaSet="demo-repl")
    t = Timer(60, abort_test)
    t.start()
    connection.start()
    while len(connection.shard_set) == 0:
        pass
    t.cancel()
    unittest.main(argv=[sys.argv[0]])
    connection.join()
Exemplo n.º 20
0
        print ("PASSED INITIAL DUMP TEST")


def abort_test(self):
    print("TEST FAILED")
    sys.exit(1)

if __name__ == '__main__':
    os.system('rm config.txt; touch config.txt')
    parser = OptionParser()

    #-m is for the main address, which is a host:port pair, ideally of the
    #mongos. For non sharded clusters, it can be the primary.
    parser.add_option("-m", "--main", action="store", type="string",
                      dest="main_addr", default="27217")

    (options, args) = parser.parse_args()
    PORTS_ONE['MONGOS'] = options.main_addr
    connector = Connector('localhost:' + PORTS_ONE["MONGOS"], 'config.txt', None,
                  ['test.test'], '_id', None, None)
    doc_manager = connector.doc_manager
    if options.main_addr != "27217":
        start_cluster(use_mongos=False)
    else:
        start_cluster()
    conn = Connection('localhost:' + PORTS_ONE['MONGOS'],
                      replicaSet="demo-repl")
    unittest.main(argv=[sys.argv[0]])
    connector.join()
Exemplo n.º 21
0
class TestSynchronizer(unittest.TestCase):
    """ Tests Solr
    """

    def runTest(self):
        """ Runs tests
        """
        unittest.TestCase.__init__(self)

    @classmethod
    def setUpClass(cls):
        cls.flag = start_cluster()
        if cls.flag:
            cls.conn = Connection('localhost:' + PORTS_ONE['MAIN'],
                replicaSet="demo-repl")
            # Creating a Solr object with an invalid URL 
            # doesn't create an exception
            cls.solr_conn = Solr('http://localhost:8983/solr')
            try:
                cls.solr_conn.commit()
            except (SolrError, MissingSchema):
                cls.err_msg = "Cannot connect to Solr!"
                cls.flag = False
            if cls.flag:    
                cls.solr_conn.delete(q='*:*')
        else:
            cls.err_msg = "Shards cannot be added to mongos"        

    def setUp(self):
        if not self.flag:
            self.fail(self.err_msg)

        self.connector = Connector('localhost:' + PORTS_ONE["MAIN"], 
            'config.txt', 'http://localhost:8983/solr', ['test.test'], '_id',
            None, 
            '../mongo_connector/doc_managers/solr_doc_manager.py')
        self.connector.start()
        while len(self.connector.shard_set) == 0:
            time.sleep(1)
        count = 0
        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except (AutoReconnect, OperationFailure):
                time.sleep(1)
                count += 1
                if count > 60:
                    unittest.SkipTest('Call to remove failed too '
                    'many times in setup')
        while (len(self.solr_conn.search('*:*')) != 0):
            time.sleep(1)

    def tearDown(self):
        self.connector.doc_manager.auto_commit = False
        time.sleep(2)
        self.connector.join()

    def test_shard_length(self):
        """Tests the shard_length to see if the shard set was recognized
        """

        self.assertEqual(len(self.connector.shard_set), 1)

    def test_initial(self):
        """Tests search and assures that the databases are clear.
        """

        while (True):
            try:
                self.conn['test']['test'].remove(safe=True)
                break
            except OperationFailure:
                continue

        self.solr_conn.delete(q='*:*')
        self.assertEqual(self.conn['test']['test'].find().count(), 0)
        self.assertEqual(len(self.solr_conn.search('*:*')), 0)

    def test_insert(self):
        """Tests insert
        """

        self.conn['test']['test'].insert({'name': 'paulie'}, safe=True)
        while (len(self.solr_conn.search('*:*')) == 0):
            time.sleep(1)
        result_set_1 = self.solr_conn.search('paulie')
        self.assertEqual(len(result_set_1), 1)
        result_set_2 = self.conn['test']['test'].find_one()
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
            self.assertEqual(item['name'], result_set_2['name'])

    def test_remove(self):
        """Tests remove
        """

        self.conn['test']['test'].remove({'name': 'paulie'}, safe=True)
        while (len(self.solr_conn.search('*:*')) == 1):
            time.sleep(1)
        result_set_1 = self.solr_conn.search('paulie')
        self.assertEqual(len(result_set_1), 0)

    def test_rollback(self):
        """Tests rollback. We force a rollback by inserting one doc, killing
            primary, adding another doc, killing the new primary, and
            restarting both the servers.
        """

        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))

        self.conn['test']['test'].insert({'name': 'paul'}, safe=True)
        while self.conn['test']['test'].find({'name': 'paul'}).count() != 1:
            time.sleep(1)
        while len(self.solr_conn.search('*:*')) != 1:
            time.sleep(1)
        kill_mongo_proc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']
        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = 0
        while True:
            try:
                self.conn['test']['test'].insert(
                    {'name': 'pauline'}, safe=True)
                break
            except OperationFailure:
                count += 1
                if count > 60:
                    self.fail('Call to insert failed too ' 
                        'many times in test_rollback')
                time.sleep(1)
                continue

        while (len(self.solr_conn.search('*:*')) != 2):
            time.sleep(1)

        result_set_1 = self.solr_conn.search('pauline')
        result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'})
        self.assertEqual(len(result_set_1), 1)
        for item in result_set_1:
            self.assertEqual(item['_id'], str(result_set_2['_id']))
        kill_mongo_proc('localhost', PORTS_ONE['SECONDARY'])

        start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        time.sleep(2)
        result_set_1 = self.solr_conn.search('pauline')
        self.assertEqual(len(result_set_1), 0)
        result_set_2 = self.solr_conn.search('paul')
        self.assertEqual(len(result_set_2), 1)

    def test_stress(self):
        """Test stress by inserting and removing a large amount of docs.
        """
        #stress test
        for i in range(0, NUMBER_OF_DOC_DIRS):
            self.conn['test']['test'].insert({'name': 'Paul ' + str(i)})
        time.sleep(5)
        while  (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS))
                != NUMBER_OF_DOC_DIRS):
            time.sleep(5)
        for i in range(0, NUMBER_OF_DOC_DIRS):
            result_set_1 = self.solr_conn.search('Paul ' + str(i))
            for item in result_set_1:
                self.assertEqual(item['_id'], item['_id'])

    def test_stressed_rollback(self):
        """Test stressed rollback with number of documents equal to specified
        in global variable. The rollback is performed the same way as before
            but with more docs
        """

        self.conn['test']['test'].remove()
        while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0:
            time.sleep(1)
        for i in range(0, NUMBER_OF_DOC_DIRS):
            self.conn['test']['test'].insert(
                {'name': 'Paul ' + str(i)}, safe=True)

        while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) 
                != NUMBER_OF_DOC_DIRS):
            time.sleep(1)
        primary_conn = Connection('localhost', int(PORTS_ONE['PRIMARY']))
        kill_mongo_proc('localhost', PORTS_ONE['PRIMARY'])

        new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
        admin_db = new_primary_conn['admin']

        while admin_db.command("isMaster")['ismaster'] is False:
            time.sleep(1)
        time.sleep(5)
        count = -1
        while count + 1 < NUMBER_OF_DOC_DIRS:
            try:
                count += 1
                self.conn['test']['test'].insert(
                    {'name': 'Pauline ' + str(count)},
                                            safe=True)
            except (OperationFailure, AutoReconnect):
                time.sleep(1)

        while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) !=
               self.conn['test']['test'].find().count()):
            time.sleep(1)
        result_set_1 = self.solr_conn.search('Pauline', 
            rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc')
        for item in result_set_1:
            result_set_2 = self.conn['test']['test'].find_one(
                {'name': item['name']})
            self.assertEqual(item['_id'], str(result_set_2['_id']))

        kill_mongo_proc('localhost', PORTS_ONE['SECONDARY'])
        start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
                       "/replset1a.log", None)

        while primary_conn['admin'].command("isMaster")['ismaster'] is False:
            time.sleep(1)

        start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
                       "/replset1b.log", None)

        while (len(self.solr_conn.search('Pauline',
                rows=NUMBER_OF_DOC_DIRS * 2)) != 0):
            time.sleep(15)
        result_set_1 = self.solr_conn.search('Pauline',
            rows=NUMBER_OF_DOC_DIRS * 2)
        self.assertEqual(len(result_set_1), 0)
        result_set_2 = self.solr_conn.search('Paul', 
            rows=NUMBER_OF_DOC_DIRS * 2)
        self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)