def sync_compute_post_to_ha_backup_with_network_info(command_data,
timeout=120):
"""
start InstanceHandler 和 create VoiHandler 这两个compute层接口需要提供网络信息。
当在备控上同步调用这两个接口时，必须将command_data中的network_info替换为备控的网络信息。
:param command_data:
:param timeout:
:return:
"""
try:
# 如果启用了HA，在备控上也同步执行对VOI模板的操作，未启用则不同步
# 其中，network_info必须替换为备控节点的
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
command_data_copy = copy.deepcopy(command_data)
command_data_copy["data"][
"network_info"] = get_voi_ha_backup_network_info(
None,
ha_info_obj.backup_uuid,
template_uuid=command_data_copy["data"]["instance"]
['uuid'])
vip_host_ip, peer_host_ip, peer_uuid = get_host_and_peer_ip(
ha_info_obj)
logger.info(
"sync_func: %s, backup node: %s, use_thread: %s, command_data: %s, timeout: %s",
compute_post, peer_host_ip, True, command_data_copy, timeout)
kwargs = {
"ipaddr": peer_host_ip,
"data": command_data_copy,
"timeout": timeout
}
task = Thread(target=compute_post, kwargs=kwargs)
task.start()
except Exception as e:
logger.exception("%s", str(e), exc_info=True)
def sync_func_to_ha_backup(func, *args, **kwargs):
"""
如果启用了HA，在备控上也同步执行对VOI模板的操作，未启用则不同步
:param func:
:param args:
:param kwargs:
:return:
"""
try:
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
vip_host_ip, peer_host_ip, peer_uuid = get_host_and_peer_ip(
ha_info_obj)
use_thread = kwargs.pop("use_thread", True)
logger.info(
"sync_func: %s, backup node: %s, use_thread: %s, args: %s, kwargs: %s",
func, peer_host_ip, use_thread, args, kwargs)
if use_thread:
new_args = [peer_host_ip]
new_args.extend(args)
task = Thread(target=func, args=new_args, kwargs=kwargs)
task.start()
else:
func(peer_host_ip, *args, **kwargs)
except Exception as e:
logger.exception("%s", str(e), exc_info=True)
def ha_sync_file(paths):
"""
如果启用了HA，把指定文件（例如：ISO库文件，数据库备份文件）同步给备控，未启用则不同步
:param paths:
:return:
"""
try:
# 如果启用了HA，把指定文件（例如：ISO库文件，数据库备份文件）同步给备控，未启用则不同步
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
vip_host_ip, peer_host_ip, peer_uuid = get_host_and_peer_ip(
ha_info_obj)
logger.info("sync_file: %s, backup node: %s, use_thread: %s",
paths, peer_host_ip, False)
notify_backup_sync_file(vip_host_ip, peer_host_ip, paths)
except Exception as e:
logger.exception("%s", str(e), exc_info=True)
def ha_sync_task():
"""
通知备控检查文件同步情况，下载缺少的，删除多余的
范围：iso库、数据库备份文件、VOI模板的实际启动盘、base盘、差异盘、种子文件
"""
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
paths = list()
for iso in db_api.get_item_with_all(models.YzyIso, {}):
paths.append({"path": iso.path, "md5": iso.md5_sum})
if paths:
notify_backup_sync_file(ha_info_obj.master_ip,
ha_info_obj.backup_ip,
paths,
check_path=os.path.dirname(
paths[0]["path"]))
backs = list()
for backup in db_api.get_item_with_all(models.YzyDatabaseBack, {}):
backs.append({"path": backup.path, "md5": backup.md5_sum})
if backs:
notify_backup_sync_file(ha_info_obj.master_ip,
ha_info_obj.backup_ip,
backs,
check_path=os.path.dirname(
backs[0]["path"]))
templates = list()
sys_base, data_base = get_template_storage_path(
ha_info_obj.master_uuid)
for dir_path in [
sys_base,
os.path.join(sys_base, constants.IMAGE_CACHE_DIRECTORY_NAME),
data_base,
os.path.join(data_base, constants.IMAGE_CACHE_DIRECTORY_NAME)
]:
find_file_to_sync(dir_path, templates)
find_xml_to_sync("/etc/libvirt/qemu/", templates)
if templates:
# 模板不删除多余的，容易导致正在刚创建的模板磁盘文件被删除
notify_backup_sync_file(ha_info_obj.master_ip,
ha_info_obj.backup_ip, templates)
def sync_voi_file_to_ha_backup_node(template_uuid,
sys_base,
data_base,
download_base_disk,
use_thread=True):
"""
如果启用了HA，把VOI模板磁盘文件同步给备控，未启用则不同步
:param template_uuid:
:param sys_base:
:param data_base:
:param download_base_disk:
:param use_thread:
:return:
"""
try:
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
voi_template_list = get_template_images(template_uuid,
"",
sys_base,
data_base,
download_base_disk,
download_torrent=False)
vip_host_ip, peer_host_ip, peer_uuid = get_host_and_peer_ip(
ha_info_obj)
logger.info("sync_file: %s, backup node: %s, use_thread: %s",
voi_template_list, peer_host_ip, use_thread)
if use_thread:
task = Thread(target=notify_backup_sync_voi,
args=(
vip_host_ip,
peer_host_ip,
[],
voi_template_list,
))
task.start()
else:
notify_backup_sync_voi(vip_host_ip, peer_host_ip, [],
voi_template_list)
except Exception as e:
logger.exception("%s", str(e), exc_info=True)
def sync_torrent_to_ha_backup_node(torrent_paths):
"""
如果启用了HA，把VOI模板的种子文件同步给备控，未启用则不同步
:param torrent_paths:
:return:
"""
try:
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
vip_host_ip, peer_host_ip, peer_uuid = get_host_and_peer_ip(
ha_info_obj)
logger.info("sync_torrent: %s, backup node: %s, use_thread: %s",
torrent_paths, peer_host_ip, True)
task = Thread(target=notify_backup_sync_voi,
args=(
vip_host_ip,
peer_host_ip,
torrent_paths,
))
task.start()
except Exception as e:
logger.exception("%s", str(e), exc_info=True)
def update_ha_master():
is_vip, is_master, is_backup = False, False, False
current_ip = None
ha_info_obj = db_api.get_ha_info_first()
if ha_info_obj:
# logger.info("ha_info before monitor update: %s" % ha_info_obj.dict())
# 获取本机所有启用网口的ip，查看本节点的ip在ha_info表中是master_ip还是backup_ip
code, out = cmdutils.run_cmd(
"""ip -br a |grep ' UP ' |grep -o '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*'""",
ignore_log=True)
if code != 0:
logger.error(out)
else:
ip_list = [ip for ip in out.strip('\n').split('\n')]
for ip in ip_list:
if ip == ha_info_obj.vip:
is_vip = True
elif ip == ha_info_obj.master_ip:
is_master = True
elif ip == ha_info_obj.backup_ip:
is_backup = True
if not is_vip:
logger.error("server running without vip[%s]" %
ha_info_obj.vip)
else:
if not is_master and not is_backup:
logger.error(
"server running without master_ip[%s], backup_ip[%s]" %
(ha_info_obj.master_ip, ha_info_obj.backup_ip))
elif is_master and is_backup:
logger.error(
"server running with both master_ip[%s], backup_ip[%s]"
% (ha_info_obj.master_ip, ha_info_obj.backup_ip))
elif is_master and not is_backup:
current_ip = ha_info_obj.master_ip
elif not is_master and is_backup:
# 如果发现本节点的ip在ha_info表中是backup_ip，说明notify.sh脚本中调用server服务/node/master接口去更新数据库的操作失败了
# 检查并修正ha_info表中的ip
current_ip = ha_info_obj.backup_ip
ha_info_obj.master_ip, ha_info_obj.backup_ip = ha_info_obj.backup_ip, ha_info_obj.master_ip
logger.info("update ha_info[%s] master_ip from %s to %s",
(ha_info_obj.uuid, ha_info_obj.backup_ip,
ha_info_obj.master_ip))
if current_ip:
# current_ip所在节点应该为master，检查并修正ha_info表中node_uuid、nic、nic_uuid
current_node_obj = db_api.get_node_by_ip(current_ip)
if current_node_obj.uuid == ha_info_obj.backup_uuid:
ha_info_obj.master_uuid, ha_info_obj.backup_uuid = ha_info_obj.backup_uuid, ha_info_obj.master_uuid
logger.info(
"update ha_info[%s] master_uuid from %s to %s",
(ha_info_obj.uuid, ha_info_obj.backup_uuid,
ha_info_obj.master_uuid))
current_ip_obj = db_api.get_nic_ip_by_ip(current_ip)
if current_ip_obj.nic_uuid == ha_info_obj.backup_nic_uuid:
ha_info_obj.master_nic_uuid, ha_info_obj.backup_nic_uuid = ha_info_obj.backup_nic_uuid, ha_info_obj.master_nic_uuid
logger.info(
"update ha_info[%s] master_nic_uuid from %s to %s",
(ha_info_obj.uuid, ha_info_obj.backup_nic_uuid,
ha_info_obj.master_nic_uuid))
if ha_info_obj.master_nic != ha_info_obj.backup_nic and current_ip_obj.name == ha_info_obj.backup_nic:
ha_info_obj.master_nic, ha_info_obj.backup_nic = ha_info_obj.backup_nic, ha_info_obj.master_nic
logger.info(
"update ha_info[%s] master_nic from %s to %s",
(ha_info_obj.uuid, ha_info_obj.backup_nic,
ha_info_obj.master_nic))
ha_info_obj.soft_update()
# 检查并修正backup_uuid节点的type
real_backup_node_obj = db_api.get_node_by_uuid(
ha_info_obj.backup_uuid)
if real_backup_node_obj.type not in [
constants.ROLE_SLAVE_AND_COMPUTE,
constants.ROLE_SLAVE
]:
wrong_type = real_backup_node_obj.type
if real_backup_node_obj.type in [
constants.ROLE_MASTER_AND_COMPUTE,
constants.ROLE_COMPUTE
]:
real_backup_node_obj.type = constants.ROLE_SLAVE_AND_COMPUTE
else:
real_backup_node_obj.type = constants.ROLE_SLAVE
real_backup_node_obj.soft_update()
logger.info(
"update real_backup_node[%s] role from %s to %s",
real_backup_node_obj.ip, wrong_type,
real_backup_node_obj.type)
# 检查并修正master_uuid节点的type
if current_node_obj.type not in [
constants.ROLE_MASTER,
constants.ROLE_MASTER_AND_COMPUTE
]:
wrong_type = current_node_obj.type
if wrong_type in [
constants.ROLE_SLAVE_AND_COMPUTE,
constants.ROLE_COMPUTE
]:
current_node_obj.type = constants.ROLE_MASTER_AND_COMPUTE
else:
current_node_obj.type = constants.ROLE_MASTER
current_node_obj.soft_update()
logger.info(
"update current_node[%s] role from %s to %s",
current_node_obj.ip, wrong_type,
current_node_obj.type)
# 检查并修正yzy_template、yzy_voi_template表的模板宿主机uuid
templates = db_api.get_template_with_all({})
for template in templates:
if constants.SYSTEM_DESKTOP == template.classify:
continue
if template.host_uuid != current_node_obj.uuid:
template.host_uuid = current_node_obj.uuid
template.soft_update()
logger.info("update template %s host_uuid to %s",
template.name, current_node_obj.uuid)
voi_templates = db_api.get_voi_template_with_all({})
for template in voi_templates:
if constants.SYSTEM_DESKTOP == template.classify:
continue
if template.host_uuid != current_node_obj.uuid:
template.host_uuid = current_node_obj.uuid
template.soft_update()
logger.info(
"update voi template %s host_uuid to %s",
template.name, current_node_obj.uuid)