def override_with_backup(self, instance_id=None):
self.inst_id = instance_id
if not self.inst_id:
self.create()
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
old_master_instance = utils.get_builtin_instance(self.inst_id)
old_group_id = old_master_instance.group_id
old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
backup1 = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup1.id)
new_password = "******" + str(random.randint(0, 100))
utils.reset_admin_user_byclient(self.inst_id, new_password)
new_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
assert old_adm_user_1['_password'] != new_adm_user_1['_password'], \
(old_adm_user_1['_password'], new_adm_user_1['_password'])
backup2 = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup2.id)
utils.override_with_backup_byclient(self.inst_id, backup1.id)
utils.check_server_deleted(old_master_instance.id, type = DBInstanceType.PENDING, timeout = 600)
self.inst_id = utils.get_instance_id(old_group_id, DBInstanceType.MASTER)
new_master_instance = utils.get_builtin_instance(self.inst_id)
self.dbslave_id = utils.get_instance_id(old_group_id, DBInstanceType.STANDBY)
self.vip_id = utils.get_vip_id(self.inst_id)
self.group_id = new_master_instance.group_id
self.validate()
new_adm_user_2 = utils.check_mysql_adminuser(self.inst_id)
assert old_adm_user_1['_password'] == new_adm_user_2['_password'], \
(old_adm_user_1['_password'], new_adm_user_2['_password'])
def switch_master(self):
self.create()
inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
inst = utils.get_builtin_instance(inst_id)
inst_standby_id = utils.get_instance_id(inst.group_id, DBInstanceType.STANDBY)
utils.check_switch_master(inst.id, inst_standby_id)
pass
def create(self):
inst_id = None
try:
inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
except Exception as e:
print e
pass
self.inst_id = inst_id
if not self.inst_id:
self.inst_id = utils.create_rds_byclient("HA")
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_instance(self.inst_id, deleted = False)
self.group_id = _inst.group_id
origin_id = inst_utils.virtual_instid_2_origin_instid(self.inst_id)
assert _inst.id == origin_id, (_inst.id, origin_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
def restart(self, instance_id=None):
self.inst_id = instance_id
if self.inst_id is None:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.restart_mysql(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.REBOOTING,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.validate()
def configuration_patch(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
if self.inst_id is None:
raise Exception("not found HA instance")
utils.patch_instance_config_and_check(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
utils.check_instace_config_is_same(self.inst_id)
def override_to_point_in_time(self):
self.create()
old_master_inst = utils.get_builtin_instance(self.inst_id)
master_vid = old_master_inst.virtual_instance_id
db_count = 100
utils.generate_databases(self.inst_id, count = db_count)
utils.check_generated_databases(self.inst_id, count = db_count)
time.sleep(3)
time1 = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
utils.clear_generated_databases(self.inst_id, count = db_count)
time.sleep(3)
dt2 = datetime.datetime.now()
time2 = dt2.strftime('%Y-%m-%d %H:%M:%S')
while True:
restorable_time = utils.get_restorable_time(self.inst_id)
dt = datetime.datetime.strptime(restorable_time.end, '%Y-%m-%d %H:%M:%S')
if dt > dt2:
break
time.sleep(3)
utils.override_with_backup_byclient(master_vid, None, time1)
utils.check_server_deleted(old_master_inst.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
new_master_inst1 = utils.get_builtin_instance(master_vid)
self.inst_id = new_master_inst1.id
utils.check_generated_databases(self.inst_id, count = db_count)
time.sleep(3)
utils.override_with_backup_byclient(master_vid, None, time2)
utils.check_server_deleted(self.inst_id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
utils.check_generated_databases(master_vid, count = 0)
self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
self.validate()
def _unlock(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.unlock_byclient(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
def _create_with_backup(self, backup, admin_user=None, admin_password=None):
self.inst_id = utils.create_rds_byclient("HA", backup=backup.id, admin_user=admin_user,
admin_password=admin_password)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=600)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
return self.inst_id
def create_with_config(self):
config = utils.get_config_bytenant()
if not config:
raise Exception("not found template config")
self.inst_id = utils.create_rds_byclient("HA",config_id=config.id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.inst_id =utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
utils.check_instancedb_mysql_variables(self.inst_id)
utils.check_instancedb_mysql_variables(self.dbslave_id)
utils.check_instace_config_is_same(self.inst_id)
def _failover(self, stop_mysqld = False, rm_mysql_data = False):
self.inst_id = self._get_rid()
utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.READ_REPLI,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False,timeout=10)
self.vip_id = utils.get_vip_id(self.inst_id)
_rr_server = utils.get_builtin_instance(self.inst_id)
nova_instance = _rr_server.server
self.group_id = _rr_server.db_info.group_id
instance_id = self.inst_id
ran_count = 56
utils.generate_databases(self.master_id, count = ran_count)
ip = utils.check_allocate_ip(nova_instance)
if stop_mysqld:
utils.stop_mysqld(ip)
if rm_mysql_data:
utils.mysql_data_lost(ip)
utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.READ_REPLI,
expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
deleted=False,timeout=120)
_ret = rpc.call(utils.get_context(),"taskmanager",
{
"method": "failover",
"args": {'instance_id':instance_id}
}
)
utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.READ_REPLI,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False,timeout=120)
self.master_id = utils.get_instance_id(self.group_id,DBInstanceType.MASTER)
self.inst_id = self.inst_id
utils.check_generated_databases(self.inst_id, count = ran_count)
self.validate()
def failover(self, instance_id=None, _strategy=None):
self.inst_id = instance_id
if not self.inst_id:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
self.vip_id = utils.get_vip_id(self.inst_id)
strategy = CONF.ha_failover_strategy
virtual_instance_id = None
if strategy == 'master':
_ret = utils.get_builtin_instance(self.inst_id)
nova_instance = _ret.server
instance_id = _ret.id
type = DBInstanceType.MASTER
virtual_instance_id = _ret.virtual_instance_id
elif strategy == 'standby':
_ret = utils.get_builtin_instance(self.dbslave_id)
nova_instance = _ret.server
instance_id = _ret.id
type = DBInstanceType.STANDBY
else:
raise Exception("not found ha_failover_strategy %s" % strategy)
rancount = random.randint(50, 100)
utils.generate_databases(self.inst_id, count = rancount)
ip = utils.check_allocate_ip(nova_instance)
utils.stop_mysqld(ip, stop_ga=True)
utils.check_server_status(instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=type,
expected_svr_status=utils.ServiceStatuses.SHUTDOWN,
deleted=False, timeout=120)
rpc.call(utils.get_context(), "taskmanager",
{"method": "failover",
"args": {'instance_id':instance_id}})
if strategy == 'master':
origin_inst_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id)
self.inst_id = origin_inst_id
utils.check_server_status(origin_inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
new_slave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
utils.check_server_status(new_slave_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
self.dbslave_id = new_slave_id
utils.check_generated_databases(self.inst_id, count = rancount)
utils.clear_generated_databases(self.dbslave_id, count = rancount)
self.validate()
def migrate(self, instance_id=None, _strategy=None):
self.inst_id = instance_id
if not self.inst_id:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
_master_inst = utils.get_instance(id = self.inst_id, deleted = 0)
self.group_id = _master_inst.group_id
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
self.vip_id = utils.get_vip_id(_master_inst.id)
virtual_instance_id = None
strategy = CONF.migrate_strategy
if strategy == 'master':
_ret = utils.get_builtin_instance(_master_inst.id)
nova_instance = _ret.server
instance_id = _master_inst.id
type = DBInstanceType.MASTER
virtual_instance_id = _ret.virtual_instance_id
elif strategy == 'standby':
_ret = utils.get_builtin_instance(self.dbslave_id)
nova_instance = _ret.server
instance_id = self.dbslave_id
type = DBInstanceType.STANDBY
else:
raise Exception("not found migrate_strategy ss%s" % strategy)
ran_count = random.randint(50, 100)
utils.generate_databases(self.inst_id, count = ran_count)
_ret = rpc.call(utils.get_context(), "taskmanager",
{"method": "migrate",
"args": {'migrate_id':instance_id}})
if strategy == 'master':
raw_instance_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id)
new_server_id = utils.get_builtin_instance(raw_instance_id).server_id
utils.check_server_status(raw_instance_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
assert new_server_id != nova_instance.id
self.inst_id = raw_instance_id
elif strategy == "standby":
utils.check_server_status(_ret['id'], expected_task=utils.tasks.InstanceTasks.NONE,
type = DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout = 123)
self.dbslave_id = _ret['id']
utils.check_generated_databases(self.inst_id, count = ran_count)
utils.check_generated_databases(self.dbslave_id, count = ran_count)
utils.clear_generated_databases(self.inst_id, count = ran_count)
self.validate()