def ha_to_sg(self):
self.create()
master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.check_ha_to_sg(master_id, True)
self.create()
master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.check_ha_to_sg(master_id, False)
pass
def create(self):
inst_id = None
try:
inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
except Exception as e:
print e
pass
self.inst_id = inst_id
if not self.inst_id:
self.inst_id = utils.create_rds_byclient("HA")
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_instance(self.inst_id, deleted = False)
self.group_id = _inst.group_id
origin_id = inst_utils.virtual_instid_2_origin_instid(self.inst_id)
assert _inst.id == origin_id, (_inst.id, origin_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
def _override_master_to_point_in_time_after_operation(self, op_method):
old_master_inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
master = utils.get_builtin_instance(old_master_inst_id)
master_vid = master.virtual_instance_id
# Prepare data before resize.
db_count = 100
utils.generate_databases(master_vid, count = db_count)
utils.check_generated_databases(master_vid, count = db_count)
time.sleep(3)
dt1 = datetime.datetime.now()
time1 = dt1.strftime('%Y-%m-%d %H:%M:%S')
def _get_restorable_time(sleep = None):
if sleep != None:
time.sleep(sleep)
restorable_time = utils.get_restorable_time(master_vid)
return restorable_time
before_rt = _get_restorable_time(30)
op_method()
after_op_master = utils.get_builtin_instance(master_vid)
after_rt = _get_restorable_time(30)
assert before_rt.begin == after_rt.begin and after_rt.end > before_rt.end, \
("before: [%s, %s], after: [%s, %s]" % (before_rt.begin, before_rt.end, after_rt.begin, after_rt.end))
# Clear data after resize.
utils.clear_generated_databases(master_vid, count = db_count)
utils.check_generated_databases(master_vid, count = 0)
# Restore & check data correctness.
utils.override_with_backup_byclient(master_vid, None, time1)
utils.check_server_deleted(after_op_master.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT)
utils.check_generated_databases(master_vid, count = db_count)
def override_with_backup(self, instance_id=None):
self.inst_id = instance_id
if not self.inst_id:
self.create()
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
old_master_instance = utils.get_builtin_instance(self.inst_id)
old_group_id = old_master_instance.group_id
old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
backup1 = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup1.id)
new_password = "******" + str(random.randint(0, 100))
utils.reset_admin_user_byclient(self.inst_id, new_password)
new_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
assert old_adm_user_1['_password'] != new_adm_user_1['_password'], \
(old_adm_user_1['_password'], new_adm_user_1['_password'])
backup2 = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup2.id)
utils.override_with_backup_byclient(self.inst_id, backup1.id)
utils.check_server_deleted(old_master_instance.id, type = DBInstanceType.PENDING, timeout = 600)
self.inst_id = utils.get_instance_id(old_group_id, DBInstanceType.MASTER)
new_master_instance = utils.get_builtin_instance(self.inst_id)
self.dbslave_id = utils.get_instance_id(old_group_id, DBInstanceType.STANDBY)
self.vip_id = utils.get_vip_id(self.inst_id)
self.group_id = new_master_instance.group_id
self.validate()
new_adm_user_2 = utils.check_mysql_adminuser(self.inst_id)
assert old_adm_user_1['_password'] == new_adm_user_2['_password'], \
(old_adm_user_1['_password'], new_adm_user_2['_password'])
def switch_master(self):
self.create()
inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
inst = utils.get_builtin_instance(inst_id)
inst_standby_id = utils.get_instance_id(inst.group_id, DBInstanceType.STANDBY)
utils.check_switch_master(inst.id, inst_standby_id)
pass
def restart(self, instance_id=None):
self.inst_id = instance_id
if self.inst_id is None:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.restart_mysql(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.REBOOTING,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.STANDBY,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.validate()
def create_with_backup(self, master_id=None, check_admin=True):
def _create_with_backup(self, backup, admin_user=None, admin_password=None):
self.inst_id = utils.create_rds_byclient("HA", backup=backup.id, admin_user=admin_user,
admin_password=admin_password)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=600)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
return self.inst_id
if master_id is not None:
self.inst_id = master_id
else:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
if self.inst_id is None:
raise Exception("not found HA instance")
#backup = utils.create_backup_byclient(self.inst_id, backup_type='snapshot')
backup = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup.id)
utils.check_backup_path(backup.id)
_create_with_backup(self, backup, admin_user=None, admin_password=None)
if check_admin:
old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
utils.check_admin(self, _create_with_backup, backup, old_adm_user_1)
def migrate(self, instance_id=None):
self.inst_id = instance_id
if not self.inst_id:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
utils.check_server_status(self.inst_id,
expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.SINGLE,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False)
inst_db_info = utils.get_instance(self.inst_id, deleted= False)
old_nova_server_id = inst_db_info.compute_instance_id
ran_count = 45
utils.generate_databases(self.inst_id, count = ran_count)
_ret = rpc.call(utils.get_context(),
"taskmanager",
{"method": "migrate",
"args": {'migrate_id': inst_db_info.id}})
utils.check_server_status(self.inst_id,
expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.SINGLE,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False)
new_nova_server_id = utils.get_instance(self.inst_id, deleted= False).compute_instance_id
assert old_nova_server_id != new_nova_server_id
utils.check_generated_databases(self.inst_id, count = ran_count)
self.validate()
def validate(self):
rids = self._get_rids()
if self.master_id == None:
self.master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
for rid in rids:
self._validate(self.master_id, rid)
def reset_adm_user(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
old_adm_user = utils.check_mysql_adminuser(self.inst_id)
new_password = "******" + str(random.randint(0, 100))
utils.reset_admin_user_byclient(self.inst_id, new_password)
new_adm_user = utils.check_mysql_adminuser(self.inst_id)
assert old_adm_user['_password'] != new_adm_user['_password'], (old_adm_user, new_adm_user)
def _lock(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.lock_byclient(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.LOCKED,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
utils.check_instance_is_lock(self.inst_id)
def attach_vip(self):
self.create()
instance_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
self.inst_id = utils.create_rds_byclient("SG")
self.validate()
attach_instance_id = self.inst_id
utils.check_vip_attach(instance_id, attach_instance_id)
pass
def resize_ha(self, instance_id=None):
self.inst_id = instance_id
if not self.inst_id:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
_inst = utils.get_instance(self.inst_id, deleted=0)
self.group_id = _inst.group_id
master = InstanceGroupItem.get_by_gid_type(utils.get_context(), self.group_id, DBInstanceType.MASTER)
standby = InstanceGroupItem.get_by_gid_type(utils.get_context(), self.group_id, DBInstanceType.STANDBY)
if master:
old_master_inst_id = master.instance_id
if standby:
old_standy_inst_id = standby.instance_id
old_master_inst = utils.get_builtin_instance(old_master_inst_id)
old_flavor_id = old_master_inst.flavor_id
old_virtual_instance_id = old_master_inst.virtual_instance_id
self.inst_id = old_master_inst.id
if old_flavor_id not in ['1', '2', '3', '4']:
raise Exception("It is not support to do resizing based on flavor id: %s, supported flavor_ids should be in (1,2,3,4)" % (old_flavor_id))
flavor = str(int(old_flavor_id) + 1)
LOG.info("old flavor : %s, new flavor : %s" % (old_flavor_id, flavor))
utils.resize_flavor_byclient(self.inst_id, flavor)
inst_ids = utils.check_resize_status(self.group_id)
if len(inst_ids) >= 2:
utils.check_rpl_delay(inst_ids[0])
utils.check_rpl_delay(inst_ids[1])
import time
time.sleep(60)
new_master = utils.get_builtin_instance(old_virtual_instance_id)
new_standby = InstanceGroupItem.get_by_gid_type(utils.get_context(), new_master.group_id, DBInstanceType.STANDBY)
if new_master.virtual_instance_id == old_virtual_instance_id and \
new_master.id != old_master_inst.id:
self.inst_id = new_master.id
self.dbslave_id = new_standby.instance_id
self.vip_id = utils.get_vip_id(self.inst_id)
else:
raise Exception("resize for ha failed,new_master.virtual_instance_id %s,"
" old_virtual_instance_id %s, new_master.id %s,"
" old_master_inst.id %s, new_standby_id %s,"
" old_standy_inst_id %s" % (
new_master.virtual_instance_id, old_virtual_instance_id,
new_master.id, old_master_inst.id,
new_standby.instance_id, old_standy_inst_id))
self.validate()
else:
raise Exception("the num of instance_id should be equal or greater than two after resize HA ")
def modify(self):
modify_name = 'modify_name' + str(random.randint(0, 100))
modify_admin_user = '******' + str(random.randint(0, 100))
modify_admin_password = str(random.randint(0, 100))
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
if self.inst_id is None:
raise Exception("not found SINGLE instance")
instance = utils.get_builtin_instance(self.inst_id)
utils.check_modify(self, instance, modify_name, modify_admin_user, modify_admin_password)
def delete_backup(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
_list = Backup.list_for_instance(self.inst_id)
_backup = None
for _item in _list:
_backup = _item
break
if _backup is None:
raise Exception("not found backup")
utils.delete_backup_byclient(_backup.id)
utils.check_backup_status(_backup.id, 'DELETED', deleted=True)
def create(self):
inst_id = None
try:
inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
except Exception as e:
LOG.error(e)
pass
self.inst_id = inst_id
if not self.inst_id:
self.inst_id = utils.create_rds_byclient("SG")
self.validate()
def delete(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
_master_vm_id = utils.get_instance(self.inst_id).compute_instance_id
utils.delete_rds_byclient(self.inst_id)
utils.check_vm_is_running(_master_vm_id, expect_status="DELETED", timeout=CONF.nova_delete_timeout)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
expected_svr_status=utils.ServiceStatuses.DELETED,
deleted=True, timeout=CONF.nova_delete_timeout)
utils.check_vip(self.inst_id, deleted=True)
def upgradeha(self,instance_id=None):
self.inst_id = instance_id
if not self.inst_id:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
self.group_id = utils.get_instance(self.inst_id).group_id
utils.upgrade_ha_byclient(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False)
_ha = HAInstance()
_ha.create()
def _unlock(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.unlock_byclient(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
def create_with_config(self):
self.master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.check_server_status(self.master_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False,timeout=10)
_inst = utils.get_builtin_instance(self.master_id)
self.group_id = _inst.group_id
config = utils.get_config_bytenant()
if not config:
raise Exception("not found template config")
self.inst_id = utils.create_rds_byclient("RR",instance_id = self.master_id, config_id = config.id)
self.validate()
utils.check_instancedb_mysql_variables(self.inst_id)
def create_with_master_config(self):
self.master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.check_server_status(self.master_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False,timeout=10)
_inst = utils.get_builtin_instance(self.master_id)
self.group_id = _inst.group_id
if not _inst.db_info.configuration_id:
raise Exception("master configuration_id is None")
self.inst_id = utils.create_rds_byclient("RR", instance_id=self.master_id,
config_id_from_master=_inst.db_info.configuration_id)
self.validate()
utils.check_instancedb_mysql_variables(self.inst_id)
def configuration_patch(self):
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
if self.inst_id is None:
raise Exception("not found HA instance")
utils.patch_instance_config_and_check(self.inst_id)
utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False, timeout=240)
self.vip_id = utils.get_vip_id(self.inst_id)
_inst = utils.get_builtin_instance(self.inst_id)
self.group_id = _inst.group_id
self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER)
self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY)
LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id))
self.validate()
utils.check_instace_config_is_same(self.inst_id)
def migrate(self):
rr_inst_id = self._get_rid()
old_rr_inst = utils.get_builtin_instance(rr_inst_id)
old_rr_id = old_rr_inst.id
rr_vid = old_rr_inst.virtual_instance_id
_ret = rpc.call(utils.get_context(), "taskmanager",
{"method": "migrate",
"args": {'migrate_id': old_rr_inst.id}})
utils.check_server_status(rr_vid, DBInstanceType.READ_REPLI, InstanceTasks.NONE, utils.ServiceStatuses.RUNNING, timeout = 600)
new_rr_inst = utils.get_builtin_instance(rr_vid)
assert new_rr_inst.virtual_instance_id == rr_vid and new_rr_inst.id != old_rr_id, (rr_vid, new_rr_inst.id)
master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
self._validate(master_id, rr_vid)
def force_migrate(self):
force_host = "rds_zone_1:rds-control-18-220.ksc.com"
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
_ret = rpc.call(utils.get_context(),
"taskmanager",
{"method": "migrate",
"args": {'migrate_id': self.inst_id, 'host' : force_host }})
self.inst_id = _ret['id']
self.validate()
new_inst = utils.get_instance(self.inst_id)
nova_id = new_inst.compute_instance_id
info = utils.get_nova_server_info(nova_id)
if info['OS-EXT-SRV-ATTR:hypervisor_hostname'] != force_host:
LOG.error("expected host is %s, finally host is %s" % (force_host, info['OS-EXT-SRV-ATTR:hypervisor_hostname']))
def create_with_backup(self, inst_id=None, check_admin=True):
def _create_with_backup(self, backup, admin_user=None, admin_password=None):
self.inst_id = utils.create_rds_byclient("SG", backup=backup.id, admin_user=admin_user,
admin_password=admin_password)
self.validate()
return self.inst_id
if inst_id != None:
self.inst_id = inst_id
else:
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.SINGLE)
backup = utils.create_backup_byclient(self.inst_id)
utils.check_backup_status(backup.id)
utils.check_backup_path(backup.id)
_create_with_backup(self, backup, admin_user=None, admin_password=None)
if check_admin:
old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id)
utils.check_admin(self, _create_with_backup, backup, old_adm_user_1)
def modify(self):
modify_name = 'modify_name' + str(random.randint(0, 100))
modify_admin_user = '******' + str(random.randint(0, 100))
modify_admin_password = str(random.randint(0, 32))
self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.READ_REPLI)
if self.inst_id is None:
raise Exception("not found RR instance")
instance = utils.get_builtin_instance(self.inst_id)
LOG.info("rr instance: '%s'" % instance.name)
utils.modify_byclient(self.inst_id, modify_name)
inst = utils.get_builtin_instance(self.inst_id)
assert inst.name == modify_name, (modify_name, inst.name)
try:
utils.modify_byclient(self.inst_id, modify_name, modify_admin_user, modify_admin_password)
assert False, ("Not allow to modify RR's admin_user and admin_password")
except:
pass
def _create_RRs(self, rr_count = 2, attach_same_vip = False):
self.master_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
utils.check_server_status(self.master_id,expected_task=utils.tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status=utils.ServiceStatuses.RUNNING,
deleted=False,timeout=10)
_inst = utils.get_builtin_instance(self.master_id)
self.group_id = _inst.group_id
rids = self._get_rids()
if len(rids) == 0:
vinst_id1 = utils.create_rds_byclient("RR", instance_id = self.master_id, attached_vip=None)
self._validate(self.master_id, vinst_id1)
origin_inst_id1 = inst_utils.virtual_instid_2_origin_instid(vinst_id1, get_deleted = False)
vip_id = utils.get_vip_id(vinst_id1, deleted = False)
left_to_create = rr_count - 1
while left_to_create > 0:
if attach_same_vip:
utils.create_rds_byclient("RR", instance_id = self.master_id, attached_vip=vip_id)
else:
utils.create_rds_byclient("RR", instance_id = self.master_id, attached_vip=None)
left_to_create -= 1
rids = self._get_rids()
map(lambda x: self._validate(self.master_id, x), rids)
if attach_same_vip:
utils.check_lbs_vip(vip_id, rids)
else:
utils.check_lbs_vip(vip_id, [origin_inst_id1])
else:
map(lambda x: self._validate(self.master_id, x), rids)
def attach_vip(self):
self.create()
instance_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER)
attach_instance_id = utils.get_instance_id_bytenant(DBInstanceType.READ_REPLI)
utils.check_vip_attach(instance_id, attach_instance_id)
def dettach_vip(self):
self.create()
instance_id = utils.get_instance_id_bytenant(DBInstanceType.READ_REPLI)
utils.check_vip_dettach(instance_id)
def allocate_vip(self):
self.create()
instance_id = utils.get_instance_id_bytenant(DBInstanceType.READ_REPLI)
utils.check_vip_allocate(instance_id)
