def override_with_backup(self, instance_id=None): self.inst_id = instance_id if not self.inst_id: self.create() self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) old_master_instance = utils.get_builtin_instance(self.inst_id) old_group_id = old_master_instance.group_id old_adm_user_1 = utils.check_mysql_adminuser(self.inst_id) backup1 = utils.create_backup_byclient(self.inst_id) utils.check_backup_status(backup1.id) new_password = "******" + str(random.randint(0, 100)) utils.reset_admin_user_byclient(self.inst_id, new_password) new_adm_user_1 = utils.check_mysql_adminuser(self.inst_id) assert old_adm_user_1['_password'] != new_adm_user_1['_password'], \ (old_adm_user_1['_password'], new_adm_user_1['_password']) backup2 = utils.create_backup_byclient(self.inst_id) utils.check_backup_status(backup2.id) utils.override_with_backup_byclient(self.inst_id, backup1.id) utils.check_server_deleted(old_master_instance.id, type = DBInstanceType.PENDING, timeout = 600) self.inst_id = utils.get_instance_id(old_group_id, DBInstanceType.MASTER) new_master_instance = utils.get_builtin_instance(self.inst_id) self.dbslave_id = utils.get_instance_id(old_group_id, DBInstanceType.STANDBY) self.vip_id = utils.get_vip_id(self.inst_id) self.group_id = new_master_instance.group_id self.validate() new_adm_user_2 = utils.check_mysql_adminuser(self.inst_id) assert old_adm_user_1['_password'] == new_adm_user_2['_password'], \ (old_adm_user_1['_password'], new_adm_user_2['_password'])
def switch_master(self): self.create() inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) inst = utils.get_builtin_instance(inst_id) inst_standby_id = utils.get_instance_id(inst.group_id, DBInstanceType.STANDBY) utils.check_switch_master(inst.id, inst_standby_id) pass
def create(self): inst_id = None try: inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) except Exception as e: print e pass self.inst_id = inst_id if not self.inst_id: self.inst_id = utils.create_rds_byclient("HA") utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_instance(self.inst_id, deleted = False) self.group_id = _inst.group_id origin_id = inst_utils.virtual_instid_2_origin_instid(self.inst_id) assert _inst.id == origin_id, (_inst.id, origin_id) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) self.validate()
def restart(self, instance_id=None): self.inst_id = instance_id if self.inst_id is None: self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) utils.restart_mysql(self.inst_id) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.REBOOTING, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.validate()
def configuration_patch(self): self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) if self.inst_id is None: raise Exception("not found HA instance") utils.patch_instance_config_and_check(self.inst_id) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) self.validate() utils.check_instace_config_is_same(self.inst_id)
def override_to_point_in_time(self): self.create() old_master_inst = utils.get_builtin_instance(self.inst_id) master_vid = old_master_inst.virtual_instance_id db_count = 100 utils.generate_databases(self.inst_id, count = db_count) utils.check_generated_databases(self.inst_id, count = db_count) time.sleep(3) time1 = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') utils.clear_generated_databases(self.inst_id, count = db_count) time.sleep(3) dt2 = datetime.datetime.now() time2 = dt2.strftime('%Y-%m-%d %H:%M:%S') while True: restorable_time = utils.get_restorable_time(self.inst_id) dt = datetime.datetime.strptime(restorable_time.end, '%Y-%m-%d %H:%M:%S') if dt > dt2: break time.sleep(3) utils.override_with_backup_byclient(master_vid, None, time1) utils.check_server_deleted(old_master_inst.id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT) new_master_inst1 = utils.get_builtin_instance(master_vid) self.inst_id = new_master_inst1.id utils.check_generated_databases(self.inst_id, count = db_count) time.sleep(3) utils.override_with_backup_byclient(master_vid, None, time2) utils.check_server_deleted(self.inst_id, DBInstanceType.PENDING, timeout = RESTORE_TIME_OUT) utils.check_generated_databases(master_vid, count = 0) self.inst_id = utils.get_instance_id(self.group_id, DBInstanceType.MASTER) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) self.validate()
def _unlock(self): self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) utils.unlock_byclient(self.inst_id) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id assert self.inst_id == _inst.virtual_instance_id, (self.inst_id, _inst.virtual_instance_id) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) self.validate()
def _create_with_backup(self, backup, admin_user=None, admin_password=None): self.inst_id = utils.create_rds_byclient("HA", backup=backup.id, admin_user=admin_user, admin_password=admin_password) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=600) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) self.validate() return self.inst_id
def create_with_config(self): config = utils.get_config_bytenant() if not config: raise Exception("not found template config") self.inst_id = utils.create_rds_byclient("HA",config_id=config.id) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=240) self.vip_id = utils.get_vip_id(self.inst_id) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id self.inst_id =utils.get_instance_id(self.group_id, DBInstanceType.MASTER) self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) LOG.info('master_id:%s slave_id:%s' % (self.inst_id, self.dbslave_id)) self.validate() utils.check_instancedb_mysql_variables(self.inst_id) utils.check_instancedb_mysql_variables(self.dbslave_id) utils.check_instace_config_is_same(self.inst_id)
def _failover(self, stop_mysqld = False, rm_mysql_data = False): self.inst_id = self._get_rid() utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.READ_REPLI, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False,timeout=10) self.vip_id = utils.get_vip_id(self.inst_id) _rr_server = utils.get_builtin_instance(self.inst_id) nova_instance = _rr_server.server self.group_id = _rr_server.db_info.group_id instance_id = self.inst_id ran_count = 56 utils.generate_databases(self.master_id, count = ran_count) ip = utils.check_allocate_ip(nova_instance) if stop_mysqld: utils.stop_mysqld(ip) if rm_mysql_data: utils.mysql_data_lost(ip) utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.READ_REPLI, expected_svr_status=utils.ServiceStatuses.SHUTDOWN, deleted=False,timeout=120) _ret = rpc.call(utils.get_context(),"taskmanager", { "method": "failover", "args": {'instance_id':instance_id} } ) utils.check_server_status(self.inst_id,expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.READ_REPLI, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False,timeout=120) self.master_id = utils.get_instance_id(self.group_id,DBInstanceType.MASTER) self.inst_id = self.inst_id utils.check_generated_databases(self.inst_id, count = ran_count) self.validate()
def failover(self, instance_id=None, _strategy=None): self.inst_id = instance_id if not self.inst_id: self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) _inst = utils.get_builtin_instance(self.inst_id) self.group_id = _inst.group_id self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) self.vip_id = utils.get_vip_id(self.inst_id) strategy = CONF.ha_failover_strategy virtual_instance_id = None if strategy == 'master': _ret = utils.get_builtin_instance(self.inst_id) nova_instance = _ret.server instance_id = _ret.id type = DBInstanceType.MASTER virtual_instance_id = _ret.virtual_instance_id elif strategy == 'standby': _ret = utils.get_builtin_instance(self.dbslave_id) nova_instance = _ret.server instance_id = _ret.id type = DBInstanceType.STANDBY else: raise Exception("not found ha_failover_strategy %s" % strategy) rancount = random.randint(50, 100) utils.generate_databases(self.inst_id, count = rancount) ip = utils.check_allocate_ip(nova_instance) utils.stop_mysqld(ip, stop_ga=True) utils.check_server_status(instance_id, expected_task=utils.tasks.InstanceTasks.NONE, type=type, expected_svr_status=utils.ServiceStatuses.SHUTDOWN, deleted=False, timeout=120) rpc.call(utils.get_context(), "taskmanager", {"method": "failover", "args": {'instance_id':instance_id}}) if strategy == 'master': origin_inst_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id) self.inst_id = origin_inst_id utils.check_server_status(origin_inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) new_slave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) utils.check_server_status(new_slave_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) self.dbslave_id = new_slave_id utils.check_generated_databases(self.inst_id, count = rancount) utils.clear_generated_databases(self.dbslave_id, count = rancount) self.validate()
def migrate(self, instance_id=None, _strategy=None): self.inst_id = instance_id if not self.inst_id: self.inst_id = utils.get_instance_id_bytenant(DBInstanceType.MASTER) _master_inst = utils.get_instance(id = self.inst_id, deleted = 0) self.group_id = _master_inst.group_id self.dbslave_id = utils.get_instance_id(self.group_id, DBInstanceType.STANDBY) utils.check_server_status(self.inst_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) utils.check_server_status(self.dbslave_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) self.vip_id = utils.get_vip_id(_master_inst.id) virtual_instance_id = None strategy = CONF.migrate_strategy if strategy == 'master': _ret = utils.get_builtin_instance(_master_inst.id) nova_instance = _ret.server instance_id = _master_inst.id type = DBInstanceType.MASTER virtual_instance_id = _ret.virtual_instance_id elif strategy == 'standby': _ret = utils.get_builtin_instance(self.dbslave_id) nova_instance = _ret.server instance_id = self.dbslave_id type = DBInstanceType.STANDBY else: raise Exception("not found migrate_strategy ss%s" % strategy) ran_count = random.randint(50, 100) utils.generate_databases(self.inst_id, count = ran_count) _ret = rpc.call(utils.get_context(), "taskmanager", {"method": "migrate", "args": {'migrate_id':instance_id}}) if strategy == 'master': raw_instance_id = inst_utils.virtual_instid_2_origin_instid(virtual_instance_id) new_server_id = utils.get_builtin_instance(raw_instance_id).server_id utils.check_server_status(raw_instance_id, expected_task=utils.tasks.InstanceTasks.NONE, type=DBInstanceType.MASTER, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout=120) assert new_server_id != nova_instance.id self.inst_id = raw_instance_id elif strategy == "standby": utils.check_server_status(_ret['id'], expected_task=utils.tasks.InstanceTasks.NONE, type = DBInstanceType.STANDBY, expected_svr_status=utils.ServiceStatuses.RUNNING, deleted=False, timeout = 123) self.dbslave_id = _ret['id'] utils.check_generated_databases(self.inst_id, count = ran_count) utils.check_generated_databases(self.dbslave_id, count = ran_count) utils.clear_generated_databases(self.inst_id, count = ran_count) self.validate()