def _main():
areas = VM_AREAS.keys()
idcs = utils.get_idcs()
content = u"""每个 area 和 idc 剩余资源 TOP100.<br/><br/>
可以访问下面 url 查看具体 area 和 idc 的资源:<br/><br/>
http://wdstack.internal.nosa.me/api/v1/vm/resources/?area=apps&idc=hlg01
<br/><br/><br/><br/>
"""
for area in areas:
for idc in idcs:
resouce = get_resource(area, idc)
if resouce == []:
continue
else:
content += mail.content_format(resouce,
main=[
"ignored", "area", "idc",
"vmh", "vcpu", "mem",
"space"
])
content += "<br/><br/>"
today = datetime.datetime.now().strftime('%Y%m%d')
subject = u"|wdstack 虚拟机| 每日资源统计 - {today}".format(today=today)
mail.send(None, subject, content)
def _send_email_to_uploader(testcase_id, to_email, content):
"""Send email to uploader when all the testcase tasks are finished."""
subject = 'Your testcase upload %d analysis is complete.' % testcase_id
content_with_footer = (
'%s\n\n'
'If you suspect that the result above is incorrect, '
'try re-doing that job on the testcase report page.') % content.strip()
html_content = content_with_footer.replace('\n', '<br>')
mail.send(to_email, subject, html_content)
def multi(common_data, email):
# 取一个列表的 type 和 version, 一批机器的 type 和 version 相同.
_type = common_data[0]["type"]
version = common_data[0]["version"]
# 因为手动安装是使用一个默认的配置文件, 如果多组机器同时安装, 配置文件需要在
# 所有的机器都安装完成之后删除, 所以用一个队列来保存正在安装的任务.
default_key = "default:{type}:{version}".format(type=type, version=version)
client.lpush(default_key, "")
# 拷贝默认配置文件.
# 不能同时安装两种类型的机器;
# 而且还只能是一个版本.
cmd = "sudo /bin/cp -f {path} /var/lib/tftpboot/pxelinux.cfg/default".format(
path=PXELINUX_CFGS[_type][version])
common_utils.shell(cmd)
# 执行安装任务.
pool = ThreadPool(MAX_THREAD_NUM)
result_data = pool.map(one, common_data)
pool.close()
pool.join()
# 安装完成出队列.
client.rpop(default_key)
# 队列为空时, 说明没有任务要执行了, 删除配置文件.
if len(client.lrange(default_key, 0, -1)) == 0:
cmd = r"sudo /bin/rm -f /var/lib/tftpboot/pxelinux.cfg/default"
common_utils.shell(cmd)
mail.send(email, u"|wdstack 物理机| 您提交的安装请求已经执行完毕", result_data)
logger.info(result_data)
code = 0
for data in result_data:
if data["code"] == 1:
code = 1
break
return_data = {
"code": code,
"error_message": None,
"common_data": common_data,
"result_data": result_data
}
return return_data
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_data()
except Exception, e:
self.logger.warning(traceback.format_exc())
self.code = -1
self.error_message = traceback.format_exc()
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "data"):
return_data["data"] = self.data
mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data))
return return_data
def multi(common_data, email):
pool = ThreadPool(MAX_THREAD_NUM)
result_data = pool.map(one, common_data)
pool.close()
pool.join()
mail.send(email, u"|wdstack 物理机| 您提交的安装请求已经执行完毕", result_data)
code = 0
for item in result_data:
if item["code"] == 1:
code = 1
break
return_data = {
"code": code,
"error_message": None,
"common_data": common_data,
"result_data": result_data
}
return return_data
def update_vmhs(email=True, is_ignore=True):
""" 更新 vmhs 到数据库.
"""
vmhs = get_vmhs_from_asset()
for vmh in vmhs:
# 根据主机名拿到 area.
vmh_prefix = get_prefix_from_hostname(vmh)
area = get_area_from_prefix(vmh_prefix)
if area is None:
continue
# 拿到 idc.
idc = get_idc_from_hostname(vmh)
# 如果 vmh 不在 redis, 则加入.
if vmh not in get_vmhs(area, idc):
add_vmh(area, idc, vmh)
# 获取 vmh 资源.
try:
update_info_from_rpc(vmh)
except Exception, e:
logger.info("get vmh:{vmh} error:{error}".format(vmh=vmh,
error=e))
del_vmh(area, idc, vmh)
continue
# 设置 vmh 的网络信息.
ip = asset_utils.get_ip_from_hostname(vmh)
network = get_network_from_ip(ip)
add_network(vmh, network)
# 记日志和发邮件.
logger.info("vmh:{vmh}, network:{network}".format(vmh=vmh,
network=network))
if email:
subject = u"|wdstack 宿主机| {vmh} 被添加到 {area}".format(vmh=vmh,
area=area)
mail.send(None, subject, "")
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_common_data()
self.get_vmhs()
self.get_unique_data()
self.logger.info(self.unique_data)
except Exception, e:
self.logger.warning(traceback.format_exc())
self.code = -1
self.error_message = traceback.format_exc()
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "common_data"):
return_data["common_data"] = self.common_data
mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕",
str(return_data))
return return_data
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_common_data() # unique_data 不依赖 common_data.
self.get_unique_data()
self.logger.info(self.unique_data)
except Exception, e:
self.logger.warning(str(e))
self.code = -1
self.error_message = str(e)
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "common_data"):
return_data["common_data"] = self.common_data
mail.send(
self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format(
oper_type_cn=self.oper_type_cn), str(return_data))
return return_data
def update_vmhs(email=True):
""" 更新 vmhs 到数据库.
"""
vmhs_all = [] # vmhs_all 是有效宿主机列表.
for area in VM_AREAS:
node_id = VM_AREAS[area]["node_id"]
vmhs = asset_utils.get_hostnames_from_node(node_id)
vmhs_all.extend(vmhs)
for vmh in vmhs:
idc = get_idc_from_hostname(vmh)
if vmh not in get_vmhs(area, idc):
add_vmh(area, idc, vmh)
# 获取 vmh 资源.
try:
update_info_from_rpc(vmh)
except Exception, e:
logger.info("get vmh:{vmh} error:{error}".format(vmh=vmh,
error=e))
del_vmh(area, idc, vmh)
continue
# 设置 vmh 的网络信息.
ip = asset_utils.get_ip_from_hostname(vmh)
network = get_network_from_ip(ip)
add_network(vmh, network)
# 记日志和发邮件.
logger.info("vmh:{vmh}, network:{network}".format(
vmh=vmh, network=network))
if email:
subject = u"|wdstack 宿主机| {vmh} 被添加到 {area}".format(
vmh=vmh, area=area)
mail.send(None, subject, "")
class Create(object):
""" 创建 wmi.
"""
def __init__(self, _type, wmi_name, hostname, vmh, vmname, excludes, email):
"""
data 任务的公共数据;
unique_data 每个宿主机任务的数据;
code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败;
error_message 表示失败信息.
"""
self.type = _type
self.wmi_name = wmi_name
self.hostname = hostname
self.vmh = vmh
self.vmname = vmname
self.excludes = excludes
self.email = email
self.logger = log.LogHandler().logger
self.client = redisoj.RedisClient().get(REDIS_DB_VM)
# 初始化任务的 code 和 error_message.
self.code = 0
self.error_message = None
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_data()
except Exception, e:
self.logger.warning(traceback.format_exc())
self.code = -1
self.error_message = traceback.format_exc()
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "data"):
return_data["data"] = self.data
mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data))
return return_data
try:
_os, _data, _partition = self.data["rpc_client"].wmi_create(
self.data["wmi_id"], self.data["wmi_name"],
self.data["vmname"], self.data["excludes"])
except Exception, e:
self.logger.warning(traceback.format_exc())
self.code = 1
self.error_message = traceback.format_exc()
return_data = {
"code": self.code,
"error_message": self.error_message,
"data": self.data
}
mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data))
return return_data
"os": _os,
"data": _data,
"partition": _partition # 仅在 raw 格式的宿主机上基于镜像创建虚拟机的时候才用到.
}
self.client.set("wmi:"+self.data["wmi_id"], json.dumps(wmi_data)) # key 以 wmi: 为前缀.
self.logger.info(str(wmi_data))
self.code = 0
return_data = {
"code": self.code,
"error_message": self.error_message,
"data": self.data,
"result_data": wmi_data
}
mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", [self.data])
return return_data
def check(self):
""" 参数检查.
"""
if self.type == "hostname":
if self.hostname is None:
raise Exception("hostname is missing")
elif self.type == "vmh":
if self.vmh is None or self.vmname is None:
raise Exception("vmh or vmname is missing")
else:
raise Exception("type not exists")
class Oper(object):
""" 操作 instance.
操作类型包括 删除, 重启, 关机.
"""
def __init__(self, oper_type, _type, unique_data, email):
"""
common_data 任务的公共数据;
unique_data 每个宿主机任务的数据;
code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败;
error_message 表示失败信息.
对于每个宿主机上的任务, 也有 code 和 error_message, 0 表示成功, 1 表示失败.
"""
self.oper_type = oper_type
self.type = _type
self.unique_data = unique_data
self.email = email
self.logger = log.LogHandler().logger
# 初始化任务的 code 和 error_message.
self.code = 0
self.error_message = None
# 对操作类型定义中文, 为了发邮件.
if self.oper_type == "delete":
self.oper_type_cn = u"删除"
elif self.oper_type == "reboot":
self.oper_type_cn = u"重启"
elif self.oper_type == "shutdown":
self.oper_type_cn = u"关机"
else:
raise Exception("oper_type not correct")
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_common_data() # unique_data 不依赖 common_data.
self.get_unique_data()
self.logger.info(self.unique_data)
except Exception, e:
self.logger.warning(str(e))
self.code = -1
self.error_message = str(e)
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "common_data"):
return_data["common_data"] = self.common_data
mail.send(
self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format(
oper_type_cn=self.oper_type_cn), str(return_data))
return return_data
pool = ThreadPool()
result_data = pool.map(self.one, self.unique_data)
pool.close()
pool.join()
for x in result_data:
if x["code"] != 0:
self.code = 1
break
return_data = {
"code": self.code,
"error_message": self.error_message,
"common_data": self.common_data,
"result_data": result_data
}
mail.send(
self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format(
oper_type_cn=self.oper_type_cn), self.unique_data)
return return_data
class Create(object):
""" 创建 instance.
创建 instance 有两种方式:
1. 通过 wmi 创建, 当 type 是 wmi 时是此种方式, 此时
需要指定 wmi_id 参数;
这里要注意两点:
1). 使用 wmi 创建虚拟机, 也需要指定 meal 和 data_size, 这点和 aws 一样;
2). 如果 os_size 或 data_size 小于 wmi 中的数据盘空间, 以 wmi 中的为准.
2. 通过 原始方式创建, 也就是用 virt-install 命令创建,
当 type 是 origin 时是这种方式, 默认是这种方式.
另外, 支持自动选取宿主机或是手动指定宿主机, 用变量 auto_vmhs
指定, 如果 auto_vmhs 为 False(默认为 True), 表示手动选择宿主机,
否则是自动选择宿主机, 当 auto_vmhs 是 False 时, 需要指定 vmhs
参数, 它是一个 list, vmhs 的长度就是 创建 instance 的个数,
所以这时候 num 参数无效了.
"""
def __init__(self, area, _type, wmi_id, auto_vmhs, vmhs, num, version,
vcpu, mem, data_size, idc, usage, user_data, email):
"""
common_data 任务的公共数据;
unique_data 每个宿主机任务的数据;
code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败;
error_message 表示失败信息.
对于每个宿主机上的任务, 也有 code 和 error_message, 0 表示成功, 1 表示失败.
"""
self.area = area
self.type = _type
self.wmi_id = wmi_id
self.auto_vmhs = auto_vmhs
self.vmhs = vmhs
self.num = num
self.version = version
self.vcpu = vcpu
self.mem = mem * 1024 # 这里以 M 计算.
self.data_size = data_size
self.idc = idc
self.usage = usage
self.user_data = user_data
self.email = email
self.vm_areas = VM_AREAS
self.os_size = OS_SIZE
self.locations = LOCATIONS
self.kss = KSS
self.network = NETMASK_DEFAULT
self.bridge = BRIDGE_DEFAULT
self.logger = log.LogHandler().logger
self.client = redisoj.RedisClient().get(REDIS_DB_VM)
self.client_user_data = redisoj.RedisClient().get(REDIS_DB_COMMON)
# 初始化任务的 code 和 error_message.
self.code = 0
self.error_message = None
def run(self):
""" 启动函数.
"""
try:
self.check()
self.get_common_data()
self.get_vmhs()
self.get_unique_data()
self.logger.info(self.unique_data)
except Exception, e:
self.logger.warning(traceback.format_exc())
self.code = -1
self.error_message = traceback.format_exc()
return_data = {
"code": self.code,
"error_message": self.error_message
}
if hasattr(self, "common_data"):
return_data["common_data"] = self.common_data
mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕",
str(return_data))
return return_data
pool = ThreadPool()
result_data = pool.map(self.one, self.unique_data)
pool.close()
pool.join()
for x in result_data:
if x["code"] != 0:
self.code = 1
break
return_data = {
"code": self.code,
"error_message": self.error_message,
"common_data": self.common_data,
"result_data": result_data
}
mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕",
self.unique_data)
return return_data
