def _get_nc_port(src_host_id, dst_host_id):
'''
获取指定源主机上可用的端口
:param src_host_id:
:param dst_host_id:
:return:
'''
port_list = ins_m_s.InstanceMigrateService().get_host_using_nc_port(
src_host_id)
# 迁移的端口范围为9000 - 10000
for i in range(9000, 10000):
if i not in port_list:
nc_port = i
# 更新端口
update_data = {'nc_port': i}
where_data = {
'src_host_id': src_host_id,
'dst_host_id': dst_host_id,
'migrate_status': '0'
}
ret = ins_m_s.InstanceMigrateService(
).update_instance_migrate_info(update_data, where_data)
break
return nc_port
def x(host):
params = {
'WHERE_AND': {
'=': {
'dst_host_id': host['host_id'],
'migrate_status': '0'
}
},
}
migrate_num, migrate_host = ins_m_s.InstanceMigrateService(
).query_data(**params)
if migrate_num > 0:
return False
return True
def _check_has_migrate(host_id):
'''
检查目的物理机当前是否有正在迁入的VM
:param host_id:
:return:
'''
params = {
'WHERE_AND': {
'=': {
'dst_host_id': host_id,
'migrate_status': '0'
}
},
}
migrate_num, migrate_host = ins_m_s.InstanceMigrateService().query_data(
**params)
if migrate_num > 0:
return False
return True
def instance_hot_migrate(instance_id, host_id):
'''
虚拟机迁移
:param instance_id:
:param host_id:
:return:
'''
speed_limit = request.values.get('speed_limit')
if not instance_id or not host_id or not speed_limit or int(
speed_limit) < 0:
logging.info('the params is invalid when migrate instance')
return json_helper.format_api_resp(code=ErrorCode.PARAM_ERR,
msg="参数错误")
host_data_s = ins_s.get_host_of_instance(instance_id)
if not host_data_s:
logging.error(
'instance %s of host is not exist in db when migrate instance',
str(instance_id))
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
host_data_d = host_s.HostService().get_host_info(host_id)
if not host_data_d:
logging.error(
'target host %s is not exist in db when migrate instance',
str(instance_id))
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
if host_data_d['typestatus'] == HostTypeStatus.LOCK or host_data_d[
'typestatus'] == HostTypeStatus.MAINTAIN:
logging.error(
'target host %s is in invalid status %s when migrate instance',
host_id, host_data_d['typestatus'])
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg='不能迁移到锁定或维护的主机上')
ins_flavor_data = ins_s.get_flavor_of_instance(instance_id)
if not ins_flavor_data:
logging.error('hot migrate can not get instance %s flavor info' %
str(instance_id))
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg='无法获取被迁移虚拟机的基础配置信息')
ins_group = ins_g_s.InstanceGroupService().get_instance_group_info(
instance_id)
if not ins_group:
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg="无法获取虚拟机所在应用组信息")
data_disk_status, data_disk_size = ins_s.get_data_disk_size_of_instance(
instance_id)
if not data_disk_status:
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg="无法获取被迁移虚拟机磁盘配置信息")
instance_disk_size = int(
ins_flavor_data['root_disk_gb']) + int(data_disk_size)
# 获取物理机所在资源池可用物理机数量
all_hosts_nums, all_hosts_data = host_s.HostService(
).get_available_hosts_of_hostpool(host_data_d["hostpool_id"])
if all_hosts_nums < 1:
return False, '集群物理机资源不足，无法满足虚拟机迁移'
host_data_d_before_match = []
host_data_d_before_match.append(host_data_d)
# 过滤host
# 这里不核对host的cpu型号
hosts_after_filter = host_s_s.migrate_filter_hosts(
host_data_d_before_match, int(all_hosts_nums))
if len(hosts_after_filter) == 0:
logging.info('no available host when get migrate host')
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg="集群内其他物理机cpu、内存使用率超过阀值，暂时不能迁移")
# VM分配给HOST看是否满足迁移
vm = {
"vcpu": ins_flavor_data['vcpu'],
"mem_MB": ins_flavor_data['memory_mb'],
"disk_GB": instance_disk_size,
}
host_after_match = host_s_s.migrate_match_hosts(hosts_after_filter,
vm,
ins_group['group_id'],
least_host_num=1,
max_disk=2000)
if len(host_after_match) == 0:
logging.info('no available host when get migrate host')
return json_helper.format_api_resp(
code=ErrorCode.SYS_ERR, msg="集群内其他物理机cpu、内存资源不足或者应用互斥，暂时不能迁移")
# 不能本身
if host_data_s['id'] == host_id:
logging.error('no allow migrate to the same host %s', host_id)
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg="不能迁移到原主机上")
ins_data_s = ins_s.InstanceService().get_instance_info(instance_id)
if not ins_data_s:
logging.error('instance %s is not exist in db when migrate instance')
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
# 排除非法状态,vm只能在开机状态下才能热迁移
if ins_data_s['status'] != VMStatus.STARTUP or ins_data_s[
'typestatus'] != VMTypeStatus.NORMAL:
logging.error(
'instance status %s, typestatus %s is invalid when migrate instance',
ins_data_s['status'], ins_data_s['typestatus'])
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg='只能在关机状态下执行热迁移')
# 检测目标主机是否有迁入VM
if not _check_has_migrate(host_id):
logging.error(
'dest host %s has other migrating instance when migrate instance',
host_id)
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR,
msg='目标主机现有正在迁入的虚机，不能迁移到该主机')
# 将VM状态改为热迁移中
update_data = {
'status': VMStatus.MIGRATE, # 热迁移中
'updated_at': get_datetime_str()
}
where_data = {'id': instance_id}
ret = ins_s.InstanceService().update_instance_info(update_data, where_data)
if ret != 1:
logging.error(
'update instance status error when cold migrate, update_data:%s, where_data:%s',
update_data, where_data)
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
ins_data = ins_s.InstanceService().get_instance_info(instance_id)
# 新增一个instance_dsthost记录,dsthost为目标host的ip
# 迁移过程中失败则删除这条新增的Instance_dsthost记录，迁移成功之后则删除Instance_srchost记录
# 迁移过程中前端vm页面不会显示这条新增的记录
instance_host_data = {
'instance_id': instance_id,
'instance_name': ins_data['name'],
'host_id': host_id,
'host_name': host_data_d['name'],
'isdeleted': '0',
'created_at': get_datetime_str()
}
ret_h = ins_h_s.InstanceHostService().add_instance_host_info(
instance_host_data)
if ret_h.get('row_num') <= 0:
logging.error(
'add instance host info error when hot migrate, %instance_id,%host_id'
)
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
# 在instance_migrate表中增加一条数据
insert_data = {
'instance_id': instance_id,
'src_host_id': host_data_s['id'],
'dst_host_id': host_id,
'migrate_status': MigrateStatus.DOING,
'created_at': get_datetime_str()
}
ret_m = ins_m_s.InstanceMigrateService().add_instance_migrate_info(
insert_data)
if ret_m.get('row_num') <= 0:
logging.error(
'add instance migrate info error when cold migrate, insert_data:%s',
insert_data)
return json_helper.format_api_resp(code=ErrorCode.SYS_ERR)
# 发送异步消息到队列
data = {
"routing_key": "INSTANCE.HOTMIGRATE",
"send_time": get_datetime_str(),
"data": {
"request_id": ins_s.generate_req_id(),
"user_id": get_user()['user_id'],
"migrate_tab_id": ret_m.get('last_id'),
"task_id": ins_s.generate_task_id(),
"speed_limit": speed_limit,
"ins_data_s": {
"id": ins_data_s['id'],
"uuid": ins_data_s['uuid'],
"name": ins_data_s['name']
},
"host_data_d": {
"id": host_data_d['id'],
"name": host_data_d['name'],
"ipaddress": host_data_d['ipaddress']
},
"host_data_s": {
"id": host_data_s['id'],
"name": host_data_s['name'],
"ipaddress": host_data_s['ipaddress'],
"sn": host_data_s['sn']
}
}
}
# todo:这里有可能发送不成功
ret_kafka = send_async_msg(KAFKA_TOPIC_NAME, data)
return json_helper.format_api_resp(code=ErrorCode.SUCCESS)
def cold_migrate(msg_data):
'''
虚机冷迁移
:param msg_data:
:return:
'''
msg = json_helper.read(msg_data)
data = msg.get('data')
request_id = data.get('request_id')
# migarte表ID
migrate_tab_id = data.get('migrate_tab_id')
speed_limit = data.get('speed_limit')
ins_data_s = data.get('ins_data_s')
host_data_d = data.get('host_data_d')
host_data_s = data.get('host_data_s')
host_ip_d = host_data_d['ipaddress']
host_id_d = host_data_d['id']
host_name_d = host_data_d['name']
host_ip_s = host_data_s['ipaddress']
host_id_s = host_data_s['id']
host_name_s = host_data_s['name']
ins_id_s = ins_data_s['id']
ins_uuid_s = ins_data_s['uuid']
ins_name_s = ins_data_s['name']
user_id = data.get('user_id')
# todo:先判断该VM是否存在
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_CONFIRM_SPEED,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
m_flag, m_speed = _confirm_migrate_speed(speed_limit, host_data_s)
if not m_flag:
message = 'source host %s confirm migrate speed %s error' % (
host_id_s, speed_limit)
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CONFIRM_SPEED,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'source host %s confirm migrate speed %s successful' % (
host_id_s, speed_limit)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CONFIRM_SPEED,
ActionStatus.SUCCSESS, message)
ret_s = ansible_migrate_qos_speed(host_ip_s, host_ip_d, m_speed)
if not ret_s:
message = 'host %s ansible migrate qos speed %s Mbit/s error' % (
host_ip_d, m_speed)
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CONFIRM_SPEED,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s ansible migrate qos speed %s Mbit/s successful' % (
host_ip_d, m_speed)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CONFIRM_SPEED,
ActionStatus.SUCCSESS, message)
# 开始迁移前获取源端vol、xml文件md5
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_S,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ins_vol_md5_cmd = 'cd /app/image/' + ins_uuid_s + ';/bin/md5sum *'
ins_xml_md5_cmd = 'cd /etc/libvirt/qemu;/bin/md5sum ' + ins_data_s[
'name'] + '.xml'
ret_v_m, ins_vol_md5_s = ansible_migrate_md5_get(host_ip_s,
ins_vol_md5_cmd)
if not ret_v_m:
message = 'host %s get migrate vol md5 error' % host_id_s
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_S,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s get migrate vol md5 successful' % host_id_s
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_S,
ActionStatus.SUCCSESS, message)
ret_x_m, ins_xml_md5_s = ansible_migrate_md5_get(host_ip_s,
ins_xml_md5_cmd)
if not ret_x_m:
message = 'host %s get migrate xml md5 error' % host_id_s
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_S,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s get migrate xml md5 successful' % host_id_s
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_S,
ActionStatus.SUCCSESS, message)
# 在目标主机上新建虚拟机镜像存储目录
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_MKDIR_DIR,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_status, ret_middle_status, ret_msg = __ansible_remote_mkdir_instance_dir(
host_ip_d, ins_uuid_s)
if not ret_status:
message = 'host %s remote mkdir instance %s dir error' % (host_ip_d,
ins_uuid_s)
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_MKDIR_DIR,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s remote mkdir instance %s dir successful' % (
host_ip_d, ins_uuid_s)
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_MKDIR_DIR,
ActionStatus.SUCCSESS, message)
# 新建虚拟机池
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_STORAGE_POOL,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
logging.info('start create storage pool %s', ins_uuid_s)
ret_p = _create_storage_pool(host_ip_d, ins_uuid_s)
if not ret_p:
message = 'host %s create storage pool error' % host_ip_d
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_STORAGE_POOL,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s create storage pool successful' % host_ip_d
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_STORAGE_POOL,
ActionStatus.SUCCSESS, message)
# 拷贝vol和xml文件
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_COPY,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_c = _copy_vol_and_xml(host_id_s, host_ip_s, host_id_d, host_ip_d,
ins_uuid_s, ins_data_s['name'])
if not ret_c:
message = 'copy vol and xml from %s to %s error' % (host_ip_s,
host_ip_d)
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_COPY,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'copy vol and xml from %s to %s successful' % (host_ip_s,
host_ip_d)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_COPY,
ActionStatus.SUCCSESS, message)
# todo:先判断该VM是否存在
# 查看拷贝后目的端主机文件md5是否与源端一致
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_D,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_v_m, ins_vol_md5_d = ansible_migrate_md5_get(host_ip_d,
ins_vol_md5_cmd)
if not ret_v_m:
message = 'get migrate destination %s vol md5 error' % host_ip_d
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_D,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'get migrate destination %s vol md5 successful' % host_ip_d
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_D,
ActionStatus.SUCCSESS, message)
ret_x_m, ins_xml_md5_d = ansible_migrate_md5_get(host_ip_d,
ins_xml_md5_cmd)
if not ret_x_m:
message = 'get migrate destination %s xml md5 error' % host_ip_d
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_D,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'get migrate destination %s xml md5 successful' % host_ip_d
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_VOL_XML_MD5_D,
ActionStatus.SUCCSESS, message)
# 比对
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_MD5_MATCH,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
if ins_vol_md5_d != ins_vol_md5_s or ins_xml_md5_d != ins_xml_md5_s:
message = 'vol or xml md5 not match, vol_s:%s, vol_d:%s, xml_s:%s, xml_d:%s' % \
(ins_vol_md5_s, ins_vol_md5_d, ins_xml_md5_s, ins_xml_md5_d)
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_MD5_MATCH,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'vol or xml md5 match successful, vol_s:%s, vol_d:%s, xml_s:%s, xml_d:%s' % \
(ins_vol_md5_s, ins_vol_md5_d, ins_xml_md5_s, ins_xml_md5_d)
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_MD5_MATCH,
ActionStatus.SUCCSESS, message)
# 在目标主机上定义迁移后虚拟机
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_DEFINE_D,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_status, ret_middle_status, ret_msg = __ansible_migrate_instance_define(
host_ip_d, ins_name_s)
if not ret_status:
message = 'dest host %s define migrate instance error' % host_ip_d
logging.error(message)
_change_migrate_host(host_id_s, host_name_s, ins_id_s)
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_DEFINE_D,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'dest host %s define migrate instance successful' % host_ip_d
update_instance_actions(request_id,
InstaceActions.INSTANCE_COLD_MIGRATE_DEFINE_D,
ActionStatus.SUCCSESS, message)
# 这里之后的步骤都是在成功将VM迁移到目标host上后对源host的操作，所以接下来每一步如果出错都不将VM的host ip改回源host
# 将虚拟机和存储池在源宿主机上undefined
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_UNDEFINE_S,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_u = libvirt_instance_undefined(host_ip_s, ins_data_s)
if not ret_u:
message = 'source host %s undefined instance error' % host_id_s
logging.error(message)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_UNDEFINE_S,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'source host %s undefined instance successful' % host_id_s
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_UNDEFINE_S,
ActionStatus.SUCCSESS, message)
# 修改原虚拟机存储目录名字
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_BACKUP_NAME,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_d = ansible_change_migrate_dir(host_ip_s, host_ip_d, ins_uuid_s)
if not ret_d:
message = 'source host %s backup dir after migrate error' % host_id_s
logging.error(message)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_BACKUP_NAME,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'source host %s backup dir after migrate successful' % host_id_s
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_BACKUP_NAME,
ActionStatus.SUCCSESS, message)
# 迁移后取消限速
data = {
'action': InstaceActions.INSTANCE_COLD_MIGRATE_CANCEL_SPEED,
'instance_uuid': ins_uuid_s,
'request_id': request_id,
'user_id': user_id,
'message': 'start'
}
add_instance_actions(data)
ret_c = ansible_migrate_cancel_qos_speed(host_ip_s)
if not ret_c:
message = 'source host %s cancel set migrate speed error' % host_id_s
logging.error(message)
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CANCEL_SPEED,
ActionStatus.FAILD, message)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'source host %s cancel set migrate speed successful' % host_id_s
update_instance_actions(
request_id, InstaceActions.INSTANCE_COLD_MIGRATE_CANCEL_SPEED,
ActionStatus.SUCCSESS, message)
ret_u = ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.SUCCESS)
if ret_u != 1:
logging.error(
'update instance migrate info error when after cold migrate instance'
)
# 迁移后修改虚拟机状态为关机中
vm_cold_migrate_status = VMStatus.SHUTDOWN
_update_instance_status(ins_uuid_s, vm_cold_migrate_status)
def calc_dest_host(hostpool_id,vm_cpu,vm_mem,vm_disk,group_id):
'''
筛选host
:param hostpool_id:
:return:
'''
#入参完全性判断
vmcpu = vm_cpu
vmmem = vm_mem
vmdisk = int(vm_disk) + 80
if not hostpool_id or not vmcpu or not vmmem or not vmdisk :
logging.info('params are invalid or missing')
code = ErrorCode.PARAM_ERR
data = None
msg = '参数错误'
return code,data,msg
# 获取主机列表
all_hosts_nums, all_hosts_data = host_s.HostService().get_hosts_of_hostpool(hostpool_id)
# 可用物理机数量不足
least_host_num = hostpool_service.HostPoolService().get_least_host_num(hostpool_id)
if all_hosts_nums < least_host_num or all_hosts_nums < 1:
logging.info('not enough host in cluster')
code = ErrorCode.SYS_ERR
data = ''
msg = '集群不够资源，无法进行v2v操作'
return code, data, msg
# 取得host_list
hosts_after_filter = host_s_s.filter_hosts(all_hosts_data)
if len(hosts_after_filter) == 0:
logging.info('no host have free resource in cluster ')
code = ErrorCode.SYS_ERR
data = ''
msg = '没有适合的主机，无法进行v2v操作'
return code, data, msg
vm = {
"vcpu": vmcpu,
"mem_MB": vmmem,
"disk_GB": vmdisk,
"count":'1',
"group_id":group_id
}
host_list = host_s_s.match_hosts(hosts_after_filter, vm, least_host_num=least_host_num, max_disk=2000)
host_len = len(host_list)
if host_len == 0:
logging.info('no host have enough resouce for the target vm')
code = ErrorCode.SYS_ERR
data = ''
msg = '主机资源不满足v2v虚拟机，无法进行v2v操作'
return code, data, msg
else:
logging.info('the host resource has been worked out')
hosts_on_task_num,hosts_on_task = instance_migrate_service.InstanceMigrateService().get_host_on_task()
host_on_task_list = []
for h in hosts_on_task:
host_on_task_list.append(h['dst_host_id'])
for host in host_list:
if host['id'] in host_on_task_list:
host_list.pop(host)
if host_list[0] == '':
logging.info('the filted hosts are all having task for migrate or v2v,no free one')
code = ErrorCode.SYS_ERR
data = ''
msg = '匹配主机均在迁移或v2v任务中，无法进行v2v操作'
return code, data, msg
else:
dest_host_ip =host_list[0]['ipaddress']
logging.info('the suitable host for v2v has been workd out '+ dest_host_ip)
msg = "已找到匹配主机"
data = dest_host_ip
code = ErrorCode.SUCCESS
return code, data, msg
def hot_migrate(msg_data):
'''
虚机热迁移
:param msg_data:
:return:
'''
msg = json_helper.read(msg_data)
data = msg.get('data')
request_id = data.get('request_id')
user_id = data.get('user_id')
task_id = data.get('task_id')
migrate_tab_id = data.get('migrate_tab_id')
ins_data_s = data.get('ins_data_s')
uuid = data['ins_data_s'].get('uuid')
instance_id = data['ins_data_s'].get('id')
instance_name = data['ins_data_s'].get('name')
# speed_limit = data.get('speed_limit') 暂时不用这个字段
dst_host_ip = data['host_data_d'].get('ipaddress')
dst_host_name = data['host_data_d'].get('name')
dst_host_id = data['host_data_d'].get('id')
src_host_ip = data['host_data_s'].get('ipaddress')
src_host_name = data['host_data_s'].get('name')
src_host_id = data['host_data_s'].get('id')
pool_status = False
if dst_host_ip is None or request_id is None:
logging.error("empty input of host_ip or request_id")
_update_instance_status(uuid, VMStatus.STARTUP)
_change_migrate_host(dst_host_id, instance_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.SUCCESS)
return
# 开始热迁移前获取vm磁盘的大小、名称
# 热迁移过程中任意一步发生错误都执行以下操作:
# 1:将instance表中的status值改为运行中，
# 2:删除instance_host表中新增的instance_dsthost记录
# 3:将instance_actions表中操作status值改为2（失败）
# 4:将instance_migrate表中的migrate_status只改为2(失败)
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_DISK_INFO,
'start')
list_instance_status = get_instance_disks_size(src_host_ip, instance_name,
uuid)
if not list_instance_status:
message = 'instance %s get migrate stats error' % instance_name
logging.error(message)
_update_instance_status(uuid, VMStatus.STARTUP)
_change_migrate_host(dst_host_id, instance_id)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_DISK_INFO,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'instance %s get migrate stats success' % instance_name
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_DISK_INFO,
ActionStatus.SUCCSESS, message, task_id)
# 在源主机上的Hosts文件中添加目标主机的(IP和名称)记录
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_HOSTS_ADD,
'start')
logging.info('add dst_host record in src_host %s' % uuid)
retry_directory_create = 0
while retry_directory_create < 8:
# ret_d = ansible_remote_check_instance_dir(dst_host_ip, uuid)
ret_d = ansible_remote_mkdir_and_dns(src_host_ip, dst_host_ip,
dst_host_name)
if ret_d:
message = 'host %s add dst_host record successful %s' % (
dst_host_ip, uuid)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_HOSTS_ADD,
ActionStatus.SUCCSESS, message, task_id)
break
retry_directory_create += 1
time.sleep(5)
else:
message = "host %s add dst_host record error %s" % (src_host_ip, uuid)
logging.error(message)
_update_instance_status(uuid, VMStatus.STARTUP)
_change_migrate_host(dst_host_id, instance_id)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_HOSTS_ADD,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
# 在目标主机上新建虚拟机镜像存储目录
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_MKDIR_DIR,
'start')
logging.info('get instance disk info %s' % uuid)
# 设置重试次数8
retry_directory_create = 0
while retry_directory_create < 8:
try:
ret_d = ansible_remote_mkdir_instance_dir(dst_host_ip, uuid)
if ret_d:
message = 'host %s remote mkdir instance %s dir successful' % (
dst_host_ip, uuid)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_MKDIR_DIR,
ActionStatus.SUCCSESS, message, task_id)
break
except libvirt.libvirtError as err:
logging.error("vm %s create data disk in %s failed because %s" %
(instance_name, dst_host_ip, err))
logging.error(err)
retry_directory_create += 1
time.sleep(5)
else:
message = 'host %s remote mkdir instance %s dir error' % (dst_host_ip,
uuid)
logging.error(message)
_update_instance_status(uuid, VMStatus.STARTUP)
_change_migrate_host(dst_host_id, instance_id)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_MKDIR_DIR,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
# 在目标主机新建虚拟机池
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_STORAGE_POOL,
message)
logging.info('start create storage pool %s', uuid)
ret_p = _create_storage_pool(dst_host_ip, uuid)
if not ret_p:
message = 'host %s create storage pool error' % dst_host_ip
logging.error(message)
_change_migrate_host(dst_host_id, instance_id)
_update_instance_status(uuid, VMStatus.STARTUP)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_STORAGE_POOL,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
else:
message = 'host %s create storage pool successful' % dst_host_ip
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_STORAGE_POOL,
ActionStatus.SUCCSESS, message, task_id)
# 在目标host上创建同样大小、名称的磁盘
disks_dir_path = []
for instance_status in list_instance_status:
disk_name = instance_status['image']
dev_name = instance_status['dev']
disk_size = instance_status['disk_size']
# 记录创建的磁盘的路径
add_instance_actions(uuid, request_id, user_id,
"disk " + dev_name + " create", 'start')
logging.info('start create disk %s, %s', disk_name, uuid)
create_data_disk_ret, create_data_disk_msg, disk_dir_path = create_data_disk(
request_id, dst_host_ip, uuid, user_id, disk_name, dev_name,
int(disk_size / 1073741824), 'qcow2', task_id)
disks_dir_path.append(disk_dir_path)
if create_data_disk_ret is ActionStatus.FAILD:
logging.error("instance %s create date disk failed because %s" %
(uuid, create_data_disk_msg))
_change_migrate_host(dst_host_id, instance_id)
_update_instance_status(uuid, VMStatus.STARTUP)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_CHECK_DISK,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
return
# add_instance_actions(uuid,request_id,user_id,InstaceActions.INSTANCE_HOT_MIGRATE_CHECK_DISK,'start')
# if create_data_disk_ret is ActionStatus.FAILD:
# logging.error("instance %s create date disk failed because %s" % (uuid, create_data_disk_msg))
# _change_migrate_host(dst_host_id, instance_id)
# _update_instance_status(uuid, VMStatus.STARTUP)
# update_instance_actions(uuid, request_id, user_id, InstaceActions.INSTANCE_HOT_MIGRATE_CHECK_DISK,
# ActionStatus.FAILD, message, task_id)
# ins_m_s.InstanceMigrateService().change_migrate_status(migrate_tab_id, MigrateStatus.FAILED)
# return
else:
message = 'instance %s create disk successful' % (uuid)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_CHECK_DISK,
ActionStatus.SUCCSESS, message, task_id)
# libvirt执行实际迁移操作
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_START_MOVE,
'start')
logging.info('start hot migrate %s to destination host', uuid)
# 加入定时器，move_to_host执行超过40分钟时认定为迁移失败，终止迁移
move_timer = threading.Timer(7200,
cancel_move_to,
args=(src_host_ip, instance_name))
# setDaemon为False表示主线程会等待子线程move_timer执行完才继续执行
move_timer.setDaemon = False
move_timer.start()
# 执行迁移
move_to_host, move_to_msg = vmManager.instance_migrate_speed_limit(
src_host_ip, dst_host_ip, instance_name)
if move_to_host == True:
# 迁移成功返回True
message = 'instance %s moving to host %s successful' % (instance_name,
dst_host_ip)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_START_MOVE,
ActionStatus.SUCCSESS, message, task_id)
elif move_to_host == -1:
# 迁移超时被中止
# message = 'instance %s hot migrate canceled because overtime' % (instance_name)
_change_migrate_host(dst_host_id, instance_id)
_update_instance_status(uuid, VMStatus.STARTUP)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_START_MOVE,
ActionStatus.FAILD, move_to_msg, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
# 迁移失败，删除目标Host上刚创建的磁盘
connect_create = vmManager.libvirt_get_connect(dst_host_ip,
conn_type='create')
try:
for disk_dir_path in disks_dir_path:
connect_create.delete_volume(disk_dir_path)
except libvirt.libvirtError as err:
logging.error('host %s delete instance %s disk error because %s' %
(dst_host_ip, instance_name, err))
return
else:
# move_to_host有返回值时表示迁移失败
logging.error('instance %s moving to host %s error' %
(instance_name, dst_host_ip))
message = 'instance %s hot migrate moving faild' % (instance_name)
_change_migrate_host(dst_host_id, instance_id)
_update_instance_status(uuid, VMStatus.STARTUP)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_START_MOVE,
ActionStatus.FAILD, message, task_id)
ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.FAILED)
# 迁移失败，删除目标Host上刚创建的磁盘
connect_create = vmManager.libvirt_get_connect(dst_host_ip,
conn_type='create')
try:
for disk_dir_path in disks_dir_path:
connect_create.delete_volume(disk_dir_path)
except libvirt.libvirtError as err:
logging.error('host %s delete instance %s disk error, because %s' %
(dst_host_ip, instance_name, err))
return
# 迁移结束之后的操作，以下操作如果出错，都至在LOG中提示，而不作为热迁移失败处理
# 将存储池在源主机上undefined
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_UNDEFINE_S,
'start')
logging.info('source host %s undefined instance', src_host_ip)
ret_u = vmManager.libvirt_instance_undefined(src_host_ip, ins_data_s)
if not ret_u:
message = 'source host %s undefined instance error' % src_host_ip
logging.error(message)
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_UNDEFINE_S,
ActionStatus.FAILD, message, task_id)
return
else:
message = 'source host %s undefined instance successful' % src_host_ip
update_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_UNDEFINE_S,
ActionStatus.SUCCSESS, message, task_id)
# 修改原虚拟机存储目录名字
add_instance_actions(uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_BACKUP_NAME,
'start')
logging.info('start backup %s name in sourse host', uuid)
ret_d = ansible_change_migrate_dir(src_host_ip, dst_host_ip, uuid)
if not ret_d:
message = 'source host %s backup dir after migrate error' % src_host_ip
logging.error(message)
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_BACKUP_NAME,
ActionStatus.FAILD, message, task_id)
return
else:
message = 'source host %s backup dir after migrate successful' % src_host_ip
update_instance_actions(
uuid, request_id, user_id,
InstaceActions.INSTANCE_HOT_MIGRATE_BACKUP_NAME,
ActionStatus.SUCCSESS, message, task_id)
# 迁移后修改虚拟机状态为运行中
vm_hot_migrate_status = VMStatus.STARTUP
ret_v = _update_instance_status(uuid, vm_hot_migrate_status)
if ret_v != 1:
logging.error(
'update instance status error when after hot migrate instance')
# 迁移完成后删除instance_host表中vm对应源host的记录
ret_i = _change_migrate_host(src_host_id, instance_id)
if ret_i <= 0:
logging.error(
'delete instance_src_host error when after hot migrate instance'
)
# 修改instance_migrate中迁移状态为1，迁移完成
ret_u = ins_m_s.InstanceMigrateService().change_migrate_status(
migrate_tab_id, MigrateStatus.SUCCESS)
if ret_u != 1:
logging.error(
'update instance migrate info error when after hot migrate instance'
)