Ejemplo n.º 1
0
 def migrate(self, instance_id=None):
     self.inst_id = instance_id
     if not self.inst_id:
         self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
     
     utils.check_server_status(self.inst_id, 
                               expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.SINGLE,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False)
     
     inst_db_info = utils.get_instance(self.inst_id, deleted= False)
     old_nova_server_id = inst_db_info.compute_instance_id
     
     ran_count = 45
     utils.generate_databases(self.inst_id, count = ran_count)
     _ret = rpc.call(utils.get_context(),
                     "taskmanager",
                     {"method": "migrate",
                      "args": {'migrate_id': inst_db_info.id}})
     
     utils.check_server_status(self.inst_id, 
                               expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.SINGLE,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False)
     
     new_nova_server_id = utils.get_instance(self.inst_id, deleted= False).compute_instance_id
     assert old_nova_server_id != new_nova_server_id
     
     utils.check_generated_databases(self.inst_id, count = ran_count)
     self.validate()
Ejemplo n.º 2
0
 def _failover(self, stop_mysqld = False, rm_mysql_data = False):
     self.inst_id = self._get_rid()
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False,timeout=10)
     self.vip_id = utils.get_vip_id(self.inst_id)
     _rr_server = utils.get_builtin_instance(self.inst_id)
     nova_instance = _rr_server.server
     self.group_id = _rr_server.db_info.group_id
     instance_id = self.inst_id
     
     ran_count = 56
     utils.generate_databases(self.master_id, count = ran_count)
     
     ip = utils.check_allocate_ip(nova_instance)
     if stop_mysqld:
         utils.stop_mysqld(ip)
     
     if rm_mysql_data:
         utils.mysql_data_lost(ip)
     
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
                               deleted=False,timeout=120)
     _ret = rpc.call(utils.get_context(),"taskmanager",
                     {
                      "method": "failover",
                      "args": {'instance_id':instance_id}
                      }
                     )
     
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False,timeout=120)
     
     self.master_id = utils.get_instance_id(self.group_id,DBInstanceType.MASTER)
     self.inst_id = self.inst_id
     
     utils.check_generated_databases(self.inst_id, count = ran_count)
     self.validate()
Ejemplo n.º 3
0
    def _override_master_to_point_in_time_after_operation(self, op_method):
        old_master_inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
        master = utils.get_builtin_instance(old_master_inst_id)
        master_vid = master.virtual_instance_id

        # Prepare data before resize.
        db_count = 100 
        utils.generate_databases(master_vid, count = db_count)
        utils.check_generated_databases(master_vid, count = db_count)
        time.sleep(3)
        dt1 = datetime.datetime.now()
        time1 = dt1.strftime('%Y-%m-%d %H:%M:%S')

        def _get_restorable_time(sleep = None):
            if sleep != None:
                time.sleep(sleep)
                
            restorable_time = utils.get_restorable_time(master_vid)
            return restorable_time
        
        before_rt = _get_restorable_time(30)
        op_method()
        after_op_master = utils.get_builtin_instance(master_vid)
        after_rt = _get_restorable_time(30)
        assert before_rt.begin == after_rt.begin and after_rt.end > before_rt.end, \
                                ("before: [%s, %s], after: [%s, %s]" % (before_rt.begin, before_rt.end, after_rt.begin, after_rt.end)) 
        
        # Clear data after resize.
        utils.clear_generated_databases(master_vid, count = db_count)
        utils.check_generated_databases(master_vid, count = 0)
        
        # Restore & check data correctness.
        utils.override_with_backup_byclient(master_vid, None, time1)
        utils.check_server_deleted(after_op_master.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
        utils.check_generated_databases(master_vid, count = db_count)
Ejemplo n.º 4
0
    def restore_to_point_in_time(self):
        self.create()
        old_master_inst = utils.get_builtin_instance(self.inst_id)
        master_vid = old_master_inst.virtual_instance_id
        
        db_count = 100
        utils.generate_databases(self.inst_id, count = db_count)        
        utils.check_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        time1 = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 
        utils.clear_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        dt2 = datetime.datetime.now()
        time2 = dt2.strftime('%Y-%m-%d %H:%M:%S')

        while True:
            restorable_time = utils.get_restorable_time(self.inst_id)
            dt = datetime.datetime.strptime(restorable_time.end, '%Y-%m-%d %H:%M:%S')
            if dt > dt2:
                break
            time.sleep(3)

        utils.restore_to_point_in_time_byclient(master_vid, time1) 
        time.sleep(10) # wait for compute instance appear
        new_inst1 = utils.get_restore_instance()
        utils.check_server_status(new_inst1, DBInstanceType.SINGLE, timeout=RESTORE_TIME_OUT) 
        utils.check_generated_databases(new_inst1, count = db_count)
        utils.delete_rds_byclient(new_inst1)
        utils.check_server_status(new_inst1, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.SINGLE,
                                  expected_svr_status=utils.ServiceStatuses.DELETED,
                                  deleted=True, timeout=CONF.trove_delete_timeout)
            
        # delete original instance
        utils.delete_rds_byclient(master_vid)
        utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.MASTER,
                                  expected_svr_status=utils.ServiceStatuses.DELETED,
                                  deleted=True, timeout=CONF.trove_delete_timeout)
 
        # restore to point in time after deleting
        utils.restore_to_point_in_time_byclient(master_vid, time1)
        time.sleep(10) # wait for compute instance appear
        new_inst2 = utils.get_restore_instance()
        utils.check_server_status(new_inst2, DBInstanceType.SINGLE, timeout=RESTORE_TIME_OUT)
        utils.check_generated_databases(new_inst2, count = db_count)
        utils.delete_rds_byclient(new_inst2)
        utils.check_server_status(new_inst2, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.SINGLE,
                                  expected_svr_status=utils.ServiceStatuses.DELETED,
                                  deleted=True, timeout=CONF.trove_delete_timeout)
Ejemplo n.º 5
0
    def override_to_point_in_time(self):
        
        self.create()
        old_master_inst = utils.get_builtin_instance(self.inst_id)
        master_vid = old_master_inst.virtual_instance_id

        db_count = 100
        utils.generate_databases(self.inst_id, count = db_count)        
        utils.check_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        time1 = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 
        utils.clear_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        dt2 = datetime.datetime.now()
        time2 = dt2.strftime('%Y-%m-%d %H:%M:%S')

        while True:
            restorable_time = utils.get_restorable_time(self.inst_id)
            dt = datetime.datetime.strptime(restorable_time.end, '%Y-%m-%d %H:%M:%S')
            if dt > dt2:
                break
            time.sleep(3)
            
        utils.override_with_backup_byclient(master_vid, None, time1)
        utils.check_server_deleted(old_master_inst.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
        new_master_inst1 = utils.get_builtin_instance(master_vid)
        self.inst_id = new_master_inst1.id
        utils.check_generated_databases(self.inst_id, count = db_count)
        
        time.sleep(3)
        
        utils.override_with_backup_byclient(master_vid, None, time2)
        utils.check_server_deleted(self.inst_id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
        utils.check_generated_databases(master_vid, count = 0)
        
        self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
        self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
        self.validate()
Ejemplo n.º 6
0
    def failover(self, instance_id=None, _strategy=None):
        self.inst_id = instance_id
        if not self.inst_id:
            self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
        _inst = utils.get_builtin_instance(self.inst_id)
        self.group_id = _inst.group_id
        self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
        
        utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.MASTER,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)
        
        utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.STANDBY,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)
        self.vip_id = utils.get_vip_id(self.inst_id)
        
        strategy = CONF.ha_failover_strategy
        virtual_instance_id = None
        if strategy == 'master':
            _ret = utils.get_builtin_instance(self.inst_id)
            nova_instance = _ret.server
            instance_id = _ret.id
            type = DBInstanceType.MASTER
            virtual_instance_id = _ret.virtual_instance_id
        elif strategy == 'standby':
            _ret = utils.get_builtin_instance(self.dbslave_id)
            nova_instance = _ret.server
            instance_id = _ret.id
            type = DBInstanceType.STANDBY
        else:
            raise Exception("not found ha_failover_strategy %s" % strategy)
            
        
        rancount = random.randint(50, 100)
        utils.generate_databases(self.inst_id, count = rancount)
        
        ip = utils.check_allocate_ip(nova_instance)
        utils.stop_mysqld(ip, stop_ga=True)
        utils.check_server_status(instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=type,
                                  expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
                                  deleted=False, timeout=120)
        rpc.call(utils.get_context(), "taskmanager",
                     {"method": "failover",
                       "args": {'instance_id':instance_id}})
        
        if strategy == 'master':
            origin_inst_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id)
            self.inst_id = origin_inst_id
            
            utils.check_server_status(origin_inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                      type=DBInstanceType.MASTER,
                                      expected_svr_status=utils.ServiceStatuses.RUNNING,
                                      deleted=False, timeout=120)

        new_slave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) 
        utils.check_server_status(new_slave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.STANDBY,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)

        self.dbslave_id = new_slave_id
        
        utils.check_generated_databases(self.inst_id, count = rancount)
        utils.clear_generated_databases(self.dbslave_id, count = rancount)
        self.validate()
Ejemplo n.º 7
0
 def migrate(self, instance_id=None, _strategy=None):
     self.inst_id = instance_id
     if not self.inst_id:
         self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     _master_inst = utils.get_instance(id = self.inst_id, deleted = 0)
     self.group_id = _master_inst.group_id
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=120)
     
     utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.STANDBY,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=120)
     
     self.vip_id = utils.get_vip_id(_master_inst.id)
     
     virtual_instance_id = None 
     strategy = CONF.migrate_strategy
     if strategy == 'master':
         _ret = utils.get_builtin_instance(_master_inst.id)
         nova_instance = _ret.server
         instance_id = _master_inst.id
         type = DBInstanceType.MASTER
         virtual_instance_id = _ret.virtual_instance_id
         
     elif strategy == 'standby':
         _ret = utils.get_builtin_instance(self.dbslave_id)
         nova_instance = _ret.server
         instance_id = self.dbslave_id
         type = DBInstanceType.STANDBY
     else:
         raise Exception("not found migrate_strategy ss%s" % strategy)
     
     ran_count = random.randint(50, 100)
     utils.generate_databases(self.inst_id, count = ran_count)    
     
     _ret = rpc.call(utils.get_context(), "taskmanager",
                  {"method": "migrate",
                   "args": {'migrate_id':instance_id}})
     
     if strategy == 'master':
         raw_instance_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id) 
         new_server_id = utils.get_builtin_instance(raw_instance_id).server_id
         utils.check_server_status(raw_instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                   type=DBInstanceType.MASTER,
                                   expected_svr_status=utils.ServiceStatuses.RUNNING,
                                   deleted=False, timeout=120)
         assert new_server_id != nova_instance.id
         self.inst_id = raw_instance_id
         
     elif strategy == "standby":
         utils.check_server_status(_ret['id'], expected_task=utils.tasks.InstanceTasks.NONE, 
                                   type = DBInstanceType.STANDBY, 
                                   expected_svr_status=utils.ServiceStatuses.RUNNING,
                                   deleted=False, timeout = 123)
         self.dbslave_id = _ret['id']
         
     utils.check_generated_databases(self.inst_id, count = ran_count)
     utils.check_generated_databases(self.dbslave_id, count = ran_count)
     utils.clear_generated_databases(self.inst_id, count = ran_count)
     self.validate()