def group_update(self, group_id,notify_when_delete=False):
LOG.debug("Making async send group_info %s to %s" % (group_id,self._get_routing_key()))
item_json = dict()
try:
group = InstanceGroup.get_by_groupid(self.context, group_id)
except exception.ModelNotFoundError:
if notify_when_delete:
group = InstanceGroup.get_by_groupid(self.context, group_id,deleted=True)
tenant_id = group.tenant_id
item_list = InstanceGroupItem.list_by_gid(self.context, group_id)
for item in item_list:
_id = item.instance_id
if self._get_running_instance(_id):
if item.type ==DBInstanceType.SINGLE or item.type == DBInstanceType.MASTER:
db_type = DBInstanceType.MASTER
else:
db_type = item.type
_data = {'id':item.instance_id,'role':item.type,'group_id':item.group_id}
if item.type == DBInstanceType.READ_REPLI:
_list = item_json.get(item.type,[])
_list.append(_data)
item_json[db_type] = _list
else:
item_json[db_type] = _data
if item_json or notify_when_delete:
ret_json = {"id":group_id}
ret_json.update({'tenant_id':tenant_id})
ret_json.update(item_json)
self._cast("group_update",{"value":ret_json})
def delete_async(self):
LOG.debug("prepare delete instance %s" % self.id)
deleted_at = utils.utcnow()
# Delete guest queue.
_item = InstanceGroupItem.get_by_instance_id(self.context,self.id)
group_id = _item.group_id
if _item.type == DBInstanceType.MASTER:
standby = None
try:
standby = InstanceGroupItem.get_by_gid_type(self.context, group_id,DBInstanceType.STANDBY)
except exception.ModelNotFoundError:
pass
if standby is not None:
standby.delete()
standby_inst_id = standby.instance_id
standby_tasks = self.load(self.context,standby_inst_id)
standby_tasks.update_db(deleted=True, deleted_at=deleted_at,
task_status=InstanceTasks.NONE)
standby_tasks.set_servicestatus_deleted()
standby_tasks._delete_instance_config()
item_list = InstanceGroupItem.list_by_gid(self.context,group_id)
if len(item_list)==1:
# Delete associated security group
if CONF.trove_security_groups_support:
try:
SecurityGroup.delete_for_group(self.group_id,
self.context)
except Exception as e:
LOG.error(utils.get_traceback_stack())
LOG.info("send notify to monitor when delete instance %s" % self.id)
group_rpcapi.API(self.context).group_update(group_id,notify_when_delete=True)
LOG.info("Delete group %s" % group_id)
InstanceGroup.delete(self.context,group_id)
if _item.type in [DBInstanceType.MASTER,DBInstanceType.SINGLE]:
try:
LOG.info("Delete autobackup_setting of group_id %s" % group_id)
AutoBackup.delete(self.context,group_id)
except:
LOG.error(utils.get_traceback_stack())
_item.delete()
self.update_db(deleted=True, deleted_at=deleted_at,
task_status=InstanceTasks.NONE)
self.set_servicestatus_deleted()
def delete_async(self, fake):
LOG.debug("prepare delete instance %s, fake: %s " % (self.id, fake))
modified_group_id = self.group_id
self._delete_resources(fake)
# Delete guest queue.
_item = InstanceGroupItem.get_by_instance_id(self.context, self.id)
group_id = _item.group_id
del_instance_type = _item.type
# if size of item_list equal 1,then we will delete last instance in group
item_list = InstanceGroupItem.list_by_gid(self.context, modified_group_id)
if len(item_list) == 1:
if CONF.trove_security_groups_support:
if fake is True:
LOG.debug("fake is True, %s skip delete secgroup rules", self.group_id)
else:
# Delete associated security group
self.update_db(task_status=InstanceTasks.DELETEING_SECURITY_GROUP)
try:
SecurityGroup.delete_for_group(modified_group_id, self.context)
except Exception as e:
LOG.error(utils.get_traceback_stack())
self.set_servicestatus_deleted()
# zs: configuration is needed for restore deleted instance, DO NOT DELETE!
# self._delete_instance_config()
LOG.info("Delete instance_group_item for instance %s" % self.id)
_type = self.type
InstanceGroupItem.delete(self.context, self.id)
deleted_at = utils.utcnow()
if fake is True and _type == DBInstanceType.MASTER:
LOG.debug("fake is True, %s is MASTER, set task_status :%s ", self.id, InstanceTasks.FAKE_DELETED)
self.update_db(deleted=True, deleted_at=deleted_at, task_status=InstanceTasks.FAKE_DELETED)
else:
self.update_db(deleted=True, deleted_at=deleted_at, task_status=InstanceTasks.NONE)
LOG.info("send notify to monitor when delete instance %s" % self.id)
group_rpcapi.API(self.context).group_update(group_id, notify_when_delete=True)
if len(item_list) == 1:
LOG.info("Delete group %s" % group_id)
InstanceGroup.delete(self.context, group_id)
self._send_usage_event(self.server, utils.utcnow())
def test_list_by_gid(self):
group_id = self.group_id
self._create(group_id, MASTER)
self._create(group_id, STANDBY)
_instance = fake()
task_status = fake()
task_status.action = "NONE"
_instance.task_status = task_status
service_status = fake()
service_status.status = models.ServiceStatuses.RUNNING
when(models.DBInstance).find_by(id = any()).thenReturn(_instance)
when(models.InstanceServiceStatus).find_by(instance_id = any()).thenReturn(service_status)
db_list = InstanceGroupItem.list_by_gid(self.context, group_id)
for group in db_list:
print group.data()
self.assertEqual(2, len(db_list))
def test_NULLlist_by_gid(self):
group_id = "not_found"
db_list = InstanceGroupItem.list_by_gid(self.context, group_id)
self.assertEqual(0,len(db_list))
def _failover_test(self, group_id, trigger_inst_id,
do_workload = False, do_prepare = False,
mysqld_killed = False,
host_rebooted = False,
remove_tmp_initsql = False,
mysql_data_lost = False,
check_vip = False,
check_rpl_consist = True,
check_binlog_range = False):
LOG.info("Doing Failover Test, group_id:%s, instance_id:%s, do_workload:%s, do_prepare:%s." %
(group_id, trigger_inst_id, do_workload, do_prepare))
before_group_items = InstanceGroupItem.list_by_gid(test_utils.get_context(), group_id, deleted = False)
before_items = set(map(lambda x: x.type + "_" + x.instance_id, before_group_items))
before_instance = test_utils.get_builtin_instance( trigger_inst_id)
before_rip = test_utils.check_allocate_ip(before_instance.server)
before_origin_instid = before_instance.id
rt_before = rt_after = None
if check_binlog_range:
rt_before = test_utils.get_restorable_time(trigger_inst_id)
if do_workload and before_instance.type == DBInstanceType.MASTER:
FAILOVERInstance.__run_workload(do_prepare = do_prepare)
if remove_tmp_initsql:
FAILOVERInstance.__trigger_vm_remove_tmp_sql_file(trigger_inst_id)
if mysqld_killed:
FAILOVERInstance.__trigger_mysqld_crash(trigger_inst_id)
test_utils.check_server_status(trigger_inst_id, expected_task=tasks.InstanceTasks.NONE,
type=before_instance.type, expected_svr_status=test_utils.ServiceStatuses.SHUTDOWN,
deleted=False, timeout=120)
if host_rebooted:
FAILOVERInstance.__trigger_host_reboot(trigger_inst_id)
# when host-machine rebooted, no guestagent update service's status.
# test_utils.check_server_status(trigger_inst_id, expected_task=tasks.InstanceTasks.NONE,
# type=before_instance.type, expected_svr_status=test_utils.ServiceStatuses.SHUTDOWN,
# deleted=False, timeout=120)
if mysql_data_lost:
FAILOVERInstance.__trigger_mysql_data_lost(trigger_inst_id)
rpc.call(test_utils.get_context(), "taskmanager",
{"method": "failover", "args": {'instance_id':before_origin_instid}}, timeout = 3600)
## check vip <--> rip mapping.
## vip should be changed in 10 seconds.
if before_instance.type == DBInstanceType.MASTER or before_instance.type == DBInstanceType.READ_REPLI:
after_instance = test_utils.get_builtin_instance( trigger_inst_id)
after_nova_inst = after_instance.server
after_rip = test_utils.check_allocate_ip(after_nova_inst)
assert after_instance.vip == before_instance.vip and before_rip != after_rip
if before_instance.type == DBInstanceType.MASTER:
test_utils.check_server_status(before_instance.id,
expected_task = tasks.InstanceTasks.NONE,
type=DBInstanceType.MASTER,
expected_svr_status = test_utils.ServiceStatuses.RUNNING,
deleted=False, timeout=120)
## check replication topo
after_group_items = InstanceGroupItem.list_by_gid(test_utils.get_context(), group_id, deleted = False)
after_items = set(map(lambda x: x.type + "_" + x.instance_id, after_group_items))
LOG.info("before " + str(before_items))
LOG.info("after " + str(after_items))
if check_rpl_consist:
diff_items = (before_items - after_items)
# assert len(diff_items) == 0
assert len(before_group_items) == len(after_group_items), "size of mysql cluster should be the same."
for group_item in after_group_items:
if group_item.type == DBInstanceType.STANDBY and group_item.instance_id == before_instance.id:
item = InstanceGroupItem.get_by_instance_id(test_utils.get_context(), group_item.instance_id, deleted = False)
assert item != None
continue
test_utils.check_server_status(group_item.instance_id,
expected_task = tasks.InstanceTasks.NONE,
type = group_item.type,
expected_svr_status = test_utils.ServiceStatuses.RUNNING,
deleted = False, timeout = 120)
if check_binlog_range:
rt_after = test_utils.get_restorable_time(trigger_inst_id)
assert rt_after.end > rt_before.end, (rt_after.end, rt_before.end)
time.sleep(60)
rt_after2 = test_utils.get_restorable_time(trigger_inst_id)
assert rt_after2.end > rt_after.end, (rt_after2.end, rt_after.end)