Пример #1
0
    def override_with_backup(self, instance_id=None):
        self.inst_id = instance_id
        if not self.inst_id:
            self.create()
            self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)

        old_master_instance = utils.get_builtin_instance(self.inst_id)
        old_group_id = old_master_instance.group_id
        
        old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
        backup1 = utils.create_backup_byclient(self.inst_id)
        utils.check_backup_status(backup1.id)
                
        new_password = "******" + str(random.randint(0, 100))
        utils.reset_admin_user_byclient(self.inst_id, new_password)
        new_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
        assert old_adm_user_1['_password'] != new_adm_user_1['_password'], \
            (old_adm_user_1['_password'], new_adm_user_1['_password'])
        backup2 = utils.create_backup_byclient(self.inst_id)
        utils.check_backup_status(backup2.id)
        
        utils.override_with_backup_byclient(self.inst_id, backup1.id)
        utils.check_server_deleted(old_master_instance.id, type = DBInstanceType.PENDING, timeout = 600)


        self.inst_id = utils.get_instance_id(old_group_id, DBInstanceType.MASTER)
        new_master_instance = utils.get_builtin_instance(self.inst_id)
        self.dbslave_id = utils.get_instance_id(old_group_id, DBInstanceType.STANDBY)
        self.vip_id = utils.get_vip_id(self.inst_id)
        self.group_id = new_master_instance.group_id
        self.validate()
        
        new_adm_user_2 = utils.check_mysql_adminuser(self.inst_id)
        assert old_adm_user_1['_password'] == new_adm_user_2['_password'], \
            (old_adm_user_1['_password'], new_adm_user_2['_password'])
Пример #2
0
 def switch_master(self):
     self.create()
     inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     inst = utils.get_builtin_instance(inst_id)
     inst_standby_id = utils.get_instance_id(inst.group_id, DBInstanceType.STANDBY)
     utils.check_switch_master(inst.id, inst_standby_id)
     pass
Пример #3
0
 def create(self):
     inst_id = None
     try:
         inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     except Exception as e:
         print e
         pass
     self.inst_id = inst_id
     if not self.inst_id:
         self.inst_id = utils.create_rds_byclient("HA")
     
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     self.vip_id = utils.get_vip_id(self.inst_id) 
     _inst = utils.get_instance(self.inst_id, deleted = False)
     self.group_id = _inst.group_id
     origin_id = inst_utils.virtual_instid_2_origin_instid(self.inst_id)
     assert _inst.id == origin_id, (_inst.id, origin_id)
  
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
      
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     
     self.validate()
Пример #4
0
 def restart(self, instance_id=None):
     self.inst_id = instance_id
     if self.inst_id is None:
         self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     utils.restart_mysql(self.inst_id)
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.STANDBY,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     self.vip_id = utils.get_vip_id(self.inst_id)
     _inst = utils.get_builtin_instance(self.inst_id)
     self.group_id = _inst.group_id
     assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     
     utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.REBOOTING,
                               type=DBInstanceType.STANDBY,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     
     utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.STANDBY,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     self.validate()
Пример #5
0
 def configuration_patch(self):
     self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     if self.inst_id is None:
         raise Exception("not found HA instance")
     utils.patch_instance_config_and_check(self.inst_id)
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     self.vip_id = utils.get_vip_id(self.inst_id) 
     _inst = utils.get_builtin_instance(self.inst_id)
     self.group_id = _inst.group_id
     self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     self.validate()
     utils.check_instace_config_is_same(self.inst_id)
Пример #6
0
    def override_to_point_in_time(self):
        
        self.create()
        old_master_inst = utils.get_builtin_instance(self.inst_id)
        master_vid = old_master_inst.virtual_instance_id

        db_count = 100
        utils.generate_databases(self.inst_id, count = db_count)        
        utils.check_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        time1 = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') 
        utils.clear_generated_databases(self.inst_id, count = db_count)
        time.sleep(3)
        dt2 = datetime.datetime.now()
        time2 = dt2.strftime('%Y-%m-%d %H:%M:%S')

        while True:
            restorable_time = utils.get_restorable_time(self.inst_id)
            dt = datetime.datetime.strptime(restorable_time.end, '%Y-%m-%d %H:%M:%S')
            if dt > dt2:
                break
            time.sleep(3)
            
        utils.override_with_backup_byclient(master_vid, None, time1)
        utils.check_server_deleted(old_master_inst.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
        new_master_inst1 = utils.get_builtin_instance(master_vid)
        self.inst_id = new_master_inst1.id
        utils.check_generated_databases(self.inst_id, count = db_count)
        
        time.sleep(3)
        
        utils.override_with_backup_byclient(master_vid, None, time2)
        utils.check_server_deleted(self.inst_id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
        utils.check_generated_databases(master_vid, count = 0)
        
        self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
        self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
        self.validate()
Пример #7
0
 def _unlock(self):
     self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     utils.unlock_byclient(self.inst_id)
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                       type=DBInstanceType.MASTER,
                       expected_svr_status=utils.ServiceStatuses.RUNNING,
                       deleted=False, timeout=240)
     self.vip_id = utils.get_vip_id(self.inst_id) 
     _inst = utils.get_builtin_instance(self.inst_id)
     self.group_id = _inst.group_id
     assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     self.validate()
Пример #8
0
 def _create_with_backup(self, backup, admin_user=None, admin_password=None):
     self.inst_id = utils.create_rds_byclient("HA", backup=backup.id, admin_user=admin_user,
                                              admin_password=admin_password)
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=600)
     self.vip_id = utils.get_vip_id(self.inst_id)
     _inst = utils.get_builtin_instance(self.inst_id)
     self.group_id = _inst.group_id
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     self.validate()
     return self.inst_id
Пример #9
0
 def create_with_config(self):
     config = utils.get_config_bytenant()
     if not config:
         raise Exception("not found template config")
     self.inst_id = utils.create_rds_byclient("HA",config_id=config.id)
     
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=240)
     self.vip_id = utils.get_vip_id(self.inst_id) 
     _inst = utils.get_builtin_instance(self.inst_id)
     self.group_id = _inst.group_id
     self.inst_id =utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
  
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
      
     LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
     
     self.validate()
     
     utils.check_instancedb_mysql_variables(self.inst_id)
     utils.check_instancedb_mysql_variables(self.dbslave_id)
     utils.check_instace_config_is_same(self.inst_id)
Пример #10
0
 def _failover(self, stop_mysqld = False, rm_mysql_data = False):
     self.inst_id = self._get_rid()
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False,timeout=10)
     self.vip_id = utils.get_vip_id(self.inst_id)
     _rr_server = utils.get_builtin_instance(self.inst_id)
     nova_instance = _rr_server.server
     self.group_id = _rr_server.db_info.group_id
     instance_id = self.inst_id
     
     ran_count = 56
     utils.generate_databases(self.master_id, count = ran_count)
     
     ip = utils.check_allocate_ip(nova_instance)
     if stop_mysqld:
         utils.stop_mysqld(ip)
     
     if rm_mysql_data:
         utils.mysql_data_lost(ip)
     
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
                               deleted=False,timeout=120)
     _ret = rpc.call(utils.get_context(),"taskmanager",
                     {
                      "method": "failover",
                      "args": {'instance_id':instance_id}
                      }
                     )
     
     utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.READ_REPLI,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False,timeout=120)
     
     self.master_id = utils.get_instance_id(self.group_id,DBInstanceType.MASTER)
     self.inst_id = self.inst_id
     
     utils.check_generated_databases(self.inst_id, count = ran_count)
     self.validate()
Пример #11
0
    def failover(self, instance_id=None, _strategy=None):
        self.inst_id = instance_id
        if not self.inst_id:
            self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
        _inst = utils.get_builtin_instance(self.inst_id)
        self.group_id = _inst.group_id
        self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
        
        utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.MASTER,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)
        
        utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.STANDBY,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)
        self.vip_id = utils.get_vip_id(self.inst_id)
        
        strategy = CONF.ha_failover_strategy
        virtual_instance_id = None
        if strategy == 'master':
            _ret = utils.get_builtin_instance(self.inst_id)
            nova_instance = _ret.server
            instance_id = _ret.id
            type = DBInstanceType.MASTER
            virtual_instance_id = _ret.virtual_instance_id
        elif strategy == 'standby':
            _ret = utils.get_builtin_instance(self.dbslave_id)
            nova_instance = _ret.server
            instance_id = _ret.id
            type = DBInstanceType.STANDBY
        else:
            raise Exception("not found ha_failover_strategy %s" % strategy)
            
        
        rancount = random.randint(50, 100)
        utils.generate_databases(self.inst_id, count = rancount)
        
        ip = utils.check_allocate_ip(nova_instance)
        utils.stop_mysqld(ip, stop_ga=True)
        utils.check_server_status(instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=type,
                                  expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
                                  deleted=False, timeout=120)
        rpc.call(utils.get_context(), "taskmanager",
                     {"method": "failover",
                       "args": {'instance_id':instance_id}})
        
        if strategy == 'master':
            origin_inst_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id)
            self.inst_id = origin_inst_id
            
            utils.check_server_status(origin_inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                      type=DBInstanceType.MASTER,
                                      expected_svr_status=utils.ServiceStatuses.RUNNING,
                                      deleted=False, timeout=120)

        new_slave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) 
        utils.check_server_status(new_slave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                  type=DBInstanceType.STANDBY,
                                  expected_svr_status=utils.ServiceStatuses.RUNNING,
                                  deleted=False, timeout=120)

        self.dbslave_id = new_slave_id
        
        utils.check_generated_databases(self.inst_id, count = rancount)
        utils.clear_generated_databases(self.dbslave_id, count = rancount)
        self.validate()
Пример #12
0
 def migrate(self, instance_id=None, _strategy=None):
     self.inst_id = instance_id
     if not self.inst_id:
         self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
     _master_inst = utils.get_instance(id = self.inst_id, deleted = 0)
     self.group_id = _master_inst.group_id
     self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
     
     utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.MASTER,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=120)
     
     utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
                               type=DBInstanceType.STANDBY,
                               expected_svr_status=utils.ServiceStatuses.RUNNING,
                               deleted=False, timeout=120)
     
     self.vip_id = utils.get_vip_id(_master_inst.id)
     
     virtual_instance_id = None 
     strategy = CONF.migrate_strategy
     if strategy == 'master':
         _ret = utils.get_builtin_instance(_master_inst.id)
         nova_instance = _ret.server
         instance_id = _master_inst.id
         type = DBInstanceType.MASTER
         virtual_instance_id = _ret.virtual_instance_id
         
     elif strategy == 'standby':
         _ret = utils.get_builtin_instance(self.dbslave_id)
         nova_instance = _ret.server
         instance_id = self.dbslave_id
         type = DBInstanceType.STANDBY
     else:
         raise Exception("not found migrate_strategy ss%s" % strategy)
     
     ran_count = random.randint(50, 100)
     utils.generate_databases(self.inst_id, count = ran_count)    
     
     _ret = rpc.call(utils.get_context(), "taskmanager",
                  {"method": "migrate",
                   "args": {'migrate_id':instance_id}})
     
     if strategy == 'master':
         raw_instance_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id) 
         new_server_id = utils.get_builtin_instance(raw_instance_id).server_id
         utils.check_server_status(raw_instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
                                   type=DBInstanceType.MASTER,
                                   expected_svr_status=utils.ServiceStatuses.RUNNING,
                                   deleted=False, timeout=120)
         assert new_server_id != nova_instance.id
         self.inst_id = raw_instance_id
         
     elif strategy == "standby":
         utils.check_server_status(_ret['id'], expected_task=utils.tasks.InstanceTasks.NONE, 
                                   type = DBInstanceType.STANDBY, 
                                   expected_svr_status=utils.ServiceStatuses.RUNNING,
                                   deleted=False, timeout = 123)
         self.dbslave_id = _ret['id']
         
     utils.check_generated_databases(self.inst_id, count = ran_count)
     utils.check_generated_databases(self.dbslave_id, count = ran_count)
     utils.clear_generated_databases(self.inst_id, count = ran_count)
     self.validate()