def _main(): areas = VM_AREAS.keys() idcs = utils.get_idcs() content = u"""每个 area 和 idc 剩余资源 TOP100.<br/><br/> 可以访问下面 url 查看具体 area 和 idc 的资源:<br/><br/> http://wdstack.internal.nosa.me/api/v1/vm/resources/?area=apps&idc=hlg01 <br/><br/><br/><br/> """ for area in areas: for idc in idcs: resouce = get_resource(area, idc) if resouce == []: continue else: content += mail.content_format(resouce, main=[ "ignored", "area", "idc", "vmh", "vcpu", "mem", "space" ]) content += "<br/><br/>" today = datetime.datetime.now().strftime('%Y%m%d') subject = u"|wdstack 虚拟机| 每日资源统计 - {today}".format(today=today) mail.send(None, subject, content)
def _send_email_to_uploader(testcase_id, to_email, content): """Send email to uploader when all the testcase tasks are finished.""" subject = 'Your testcase upload %d analysis is complete.' % testcase_id content_with_footer = ( '%s\n\n' 'If you suspect that the result above is incorrect, ' 'try re-doing that job on the testcase report page.') % content.strip() html_content = content_with_footer.replace('\n', '<br>') mail.send(to_email, subject, html_content)
def multi(common_data, email): # 取一个列表的 type 和 version, 一批机器的 type 和 version 相同. _type = common_data[0]["type"] version = common_data[0]["version"] # 因为手动安装是使用一个默认的配置文件, 如果多组机器同时安装, 配置文件需要在 # 所有的机器都安装完成之后删除, 所以用一个队列来保存正在安装的任务. default_key = "default:{type}:{version}".format(type=type, version=version) client.lpush(default_key, "") # 拷贝默认配置文件. # 不能同时安装两种类型的机器; # 而且还只能是一个版本. cmd = "sudo /bin/cp -f {path} /var/lib/tftpboot/pxelinux.cfg/default".format( path=PXELINUX_CFGS[_type][version]) common_utils.shell(cmd) # 执行安装任务. pool = ThreadPool(MAX_THREAD_NUM) result_data = pool.map(one, common_data) pool.close() pool.join() # 安装完成出队列. client.rpop(default_key) # 队列为空时, 说明没有任务要执行了, 删除配置文件. if len(client.lrange(default_key, 0, -1)) == 0: cmd = r"sudo /bin/rm -f /var/lib/tftpboot/pxelinux.cfg/default" common_utils.shell(cmd) mail.send(email, u"|wdstack 物理机| 您提交的安装请求已经执行完毕", result_data) logger.info(result_data) code = 0 for data in result_data: if data["code"] == 1: code = 1 break return_data = { "code": code, "error_message": None, "common_data": common_data, "result_data": result_data } return return_data
def run(self): """ 启动函数. """ try: self.check() self.get_data() except Exception, e: self.logger.warning(traceback.format_exc()) self.code = -1 self.error_message = traceback.format_exc() return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "data"): return_data["data"] = self.data mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data)) return return_data
def multi(common_data, email): pool = ThreadPool(MAX_THREAD_NUM) result_data = pool.map(one, common_data) pool.close() pool.join() mail.send(email, u"|wdstack 物理机| 您提交的安装请求已经执行完毕", result_data) code = 0 for item in result_data: if item["code"] == 1: code = 1 break return_data = { "code": code, "error_message": None, "common_data": common_data, "result_data": result_data } return return_data
def update_vmhs(email=True, is_ignore=True): """ 更新 vmhs 到数据库. """ vmhs = get_vmhs_from_asset() for vmh in vmhs: # 根据主机名拿到 area. vmh_prefix = get_prefix_from_hostname(vmh) area = get_area_from_prefix(vmh_prefix) if area is None: continue # 拿到 idc. idc = get_idc_from_hostname(vmh) # 如果 vmh 不在 redis, 则加入. if vmh not in get_vmhs(area, idc): add_vmh(area, idc, vmh) # 获取 vmh 资源. try: update_info_from_rpc(vmh) except Exception, e: logger.info("get vmh:{vmh} error:{error}".format(vmh=vmh, error=e)) del_vmh(area, idc, vmh) continue # 设置 vmh 的网络信息. ip = asset_utils.get_ip_from_hostname(vmh) network = get_network_from_ip(ip) add_network(vmh, network) # 记日志和发邮件. logger.info("vmh:{vmh}, network:{network}".format(vmh=vmh, network=network)) if email: subject = u"|wdstack 宿主机| {vmh} 被添加到 {area}".format(vmh=vmh, area=area) mail.send(None, subject, "")
def run(self): """ 启动函数. """ try: self.check() self.get_common_data() self.get_vmhs() self.get_unique_data() self.logger.info(self.unique_data) except Exception, e: self.logger.warning(traceback.format_exc()) self.code = -1 self.error_message = traceback.format_exc() return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "common_data"): return_data["common_data"] = self.common_data mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕", str(return_data)) return return_data
def run(self): """ 启动函数. """ try: self.check() self.get_common_data() # unique_data 不依赖 common_data. self.get_unique_data() self.logger.info(self.unique_data) except Exception, e: self.logger.warning(str(e)) self.code = -1 self.error_message = str(e) return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "common_data"): return_data["common_data"] = self.common_data mail.send( self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format( oper_type_cn=self.oper_type_cn), str(return_data)) return return_data
def update_vmhs(email=True): """ 更新 vmhs 到数据库. """ vmhs_all = [] # vmhs_all 是有效宿主机列表. for area in VM_AREAS: node_id = VM_AREAS[area]["node_id"] vmhs = asset_utils.get_hostnames_from_node(node_id) vmhs_all.extend(vmhs) for vmh in vmhs: idc = get_idc_from_hostname(vmh) if vmh not in get_vmhs(area, idc): add_vmh(area, idc, vmh) # 获取 vmh 资源. try: update_info_from_rpc(vmh) except Exception, e: logger.info("get vmh:{vmh} error:{error}".format(vmh=vmh, error=e)) del_vmh(area, idc, vmh) continue # 设置 vmh 的网络信息. ip = asset_utils.get_ip_from_hostname(vmh) network = get_network_from_ip(ip) add_network(vmh, network) # 记日志和发邮件. logger.info("vmh:{vmh}, network:{network}".format( vmh=vmh, network=network)) if email: subject = u"|wdstack 宿主机| {vmh} 被添加到 {area}".format( vmh=vmh, area=area) mail.send(None, subject, "")
class Create(object): """ 创建 wmi. """ def __init__(self, _type, wmi_name, hostname, vmh, vmname, excludes, email): """ data 任务的公共数据; unique_data 每个宿主机任务的数据; code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败; error_message 表示失败信息. """ self.type = _type self.wmi_name = wmi_name self.hostname = hostname self.vmh = vmh self.vmname = vmname self.excludes = excludes self.email = email self.logger = log.LogHandler().logger self.client = redisoj.RedisClient().get(REDIS_DB_VM) # 初始化任务的 code 和 error_message. self.code = 0 self.error_message = None def run(self): """ 启动函数. """ try: self.check() self.get_data() except Exception, e: self.logger.warning(traceback.format_exc()) self.code = -1 self.error_message = traceback.format_exc() return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "data"): return_data["data"] = self.data mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data)) return return_data try: _os, _data, _partition = self.data["rpc_client"].wmi_create( self.data["wmi_id"], self.data["wmi_name"], self.data["vmname"], self.data["excludes"]) except Exception, e: self.logger.warning(traceback.format_exc()) self.code = 1 self.error_message = traceback.format_exc() return_data = { "code": self.code, "error_message": self.error_message, "data": self.data } mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", str(return_data)) return return_data
"os": _os, "data": _data, "partition": _partition # 仅在 raw 格式的宿主机上基于镜像创建虚拟机的时候才用到. } self.client.set("wmi:"+self.data["wmi_id"], json.dumps(wmi_data)) # key 以 wmi: 为前缀. self.logger.info(str(wmi_data)) self.code = 0 return_data = { "code": self.code, "error_message": self.error_message, "data": self.data, "result_data": wmi_data } mail.send(self.email, u"|wdstack 镜像| 您提交的创建请求已经执行完毕", [self.data]) return return_data def check(self): """ 参数检查. """ if self.type == "hostname": if self.hostname is None: raise Exception("hostname is missing") elif self.type == "vmh": if self.vmh is None or self.vmname is None: raise Exception("vmh or vmname is missing") else: raise Exception("type not exists")
class Oper(object): """ 操作 instance. 操作类型包括 删除, 重启, 关机. """ def __init__(self, oper_type, _type, unique_data, email): """ common_data 任务的公共数据; unique_data 每个宿主机任务的数据; code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败; error_message 表示失败信息. 对于每个宿主机上的任务, 也有 code 和 error_message, 0 表示成功, 1 表示失败. """ self.oper_type = oper_type self.type = _type self.unique_data = unique_data self.email = email self.logger = log.LogHandler().logger # 初始化任务的 code 和 error_message. self.code = 0 self.error_message = None # 对操作类型定义中文, 为了发邮件. if self.oper_type == "delete": self.oper_type_cn = u"删除" elif self.oper_type == "reboot": self.oper_type_cn = u"重启" elif self.oper_type == "shutdown": self.oper_type_cn = u"关机" else: raise Exception("oper_type not correct") def run(self): """ 启动函数. """ try: self.check() self.get_common_data() # unique_data 不依赖 common_data. self.get_unique_data() self.logger.info(self.unique_data) except Exception, e: self.logger.warning(str(e)) self.code = -1 self.error_message = str(e) return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "common_data"): return_data["common_data"] = self.common_data mail.send( self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format( oper_type_cn=self.oper_type_cn), str(return_data)) return return_data pool = ThreadPool() result_data = pool.map(self.one, self.unique_data) pool.close() pool.join() for x in result_data: if x["code"] != 0: self.code = 1 break return_data = { "code": self.code, "error_message": self.error_message, "common_data": self.common_data, "result_data": result_data } mail.send( self.email, u"|wdstack 虚拟机| 您提交的{oper_type_cn}请求已经执行完毕".format( oper_type_cn=self.oper_type_cn), self.unique_data) return return_data
class Create(object): """ 创建 instance. 创建 instance 有两种方式: 1. 通过 wmi 创建, 当 type 是 wmi 时是此种方式, 此时 需要指定 wmi_id 参数; 这里要注意两点: 1). 使用 wmi 创建虚拟机, 也需要指定 meal 和 data_size, 这点和 aws 一样; 2). 如果 os_size 或 data_size 小于 wmi 中的数据盘空间, 以 wmi 中的为准. 2. 通过 原始方式创建, 也就是用 virt-install 命令创建, 当 type 是 origin 时是这种方式, 默认是这种方式. 另外, 支持自动选取宿主机或是手动指定宿主机, 用变量 auto_vmhs 指定, 如果 auto_vmhs 为 False(默认为 True), 表示手动选择宿主机, 否则是自动选择宿主机, 当 auto_vmhs 是 False 时, 需要指定 vmhs 参数, 它是一个 list, vmhs 的长度就是 创建 instance 的个数, 所以这时候 num 参数无效了. """ def __init__(self, area, _type, wmi_id, auto_vmhs, vmhs, num, version, vcpu, mem, data_size, idc, usage, user_data, email): """ common_data 任务的公共数据; unique_data 每个宿主机任务的数据; code 装机任务返回值, 0 表示任务执行并成功, -1 表示任务未执行, 1表示任务执行但失败; error_message 表示失败信息. 对于每个宿主机上的任务, 也有 code 和 error_message, 0 表示成功, 1 表示失败. """ self.area = area self.type = _type self.wmi_id = wmi_id self.auto_vmhs = auto_vmhs self.vmhs = vmhs self.num = num self.version = version self.vcpu = vcpu self.mem = mem * 1024 # 这里以 M 计算. self.data_size = data_size self.idc = idc self.usage = usage self.user_data = user_data self.email = email self.vm_areas = VM_AREAS self.os_size = OS_SIZE self.locations = LOCATIONS self.kss = KSS self.network = NETMASK_DEFAULT self.bridge = BRIDGE_DEFAULT self.logger = log.LogHandler().logger self.client = redisoj.RedisClient().get(REDIS_DB_VM) self.client_user_data = redisoj.RedisClient().get(REDIS_DB_COMMON) # 初始化任务的 code 和 error_message. self.code = 0 self.error_message = None def run(self): """ 启动函数. """ try: self.check() self.get_common_data() self.get_vmhs() self.get_unique_data() self.logger.info(self.unique_data) except Exception, e: self.logger.warning(traceback.format_exc()) self.code = -1 self.error_message = traceback.format_exc() return_data = { "code": self.code, "error_message": self.error_message } if hasattr(self, "common_data"): return_data["common_data"] = self.common_data mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕", str(return_data)) return return_data pool = ThreadPool() result_data = pool.map(self.one, self.unique_data) pool.close() pool.join() for x in result_data: if x["code"] != 0: self.code = 1 break return_data = { "code": self.code, "error_message": self.error_message, "common_data": self.common_data, "result_data": result_data } mail.send(self.email, u"|wdstack 虚拟机| 您提交的安装请求已经执行完毕", self.unique_data) return return_data