def adjust_ability(dom=None, msg=None): extend_data = dict() try: assert isinstance(dom, libvirt.virDomain) assert isinstance(msg, dict) cpu = msg['cpu'].__str__() memory = msg['memory'].__str__() xml = ET.fromstring(dom.XMLDesc()) origin_ability = xml.find('vcpu').text + '核' + ( int(xml.find('memory').text) / 1024**2).__str__() + 'GiB' new_ability = cpu + '核' + memory + 'GiB' xml.find('vcpu').text = cpu xml.find('memory').set('unit', 'GiB') xml.find('memory').text = memory xml.find('currentMemory').set('unit', 'GiB') xml.find('currentMemory').text = memory xml_str = ET.tostring(xml, encoding='utf8', method='xml') if dom.isActive(): raise RuntimeError(u'虚拟机非关闭状态。') else: if dom.connect().defineXML(xml=xml_str): log = u' '.join([ u'域', dom.name(), u', UUID', dom.UUIDString(), u'配置从', origin_ability, '变更为', new_ability ]) log_emit.info(msg=log) else: raise RuntimeError(u'变更配置失败。') response_emit.success( _object=msg['_object'], action=msg['action'], uuid=msg['uuid'], data=extend_data, passback_parameters=msg.get('passback_parameters')) except: log_emit.error(traceback.format_exc()) response_emit.failure( _object=msg['_object'], action=msg.get('action'), uuid=msg.get('uuid'), data=extend_data, passback_parameters=msg.get('passback_parameters'))
def guest_state_report(guest): try: _uuid = guest.UUIDString() state, maxmem, mem, ncpu, cputime = guest.info() # state 参考链接: # http://libvirt.org/docs/libvirt-appdev-guide-python/en-US/html/libvirt_application_development_guide_using_python-Guest_Domains-Information-State.html # http://stackoverflow.com/questions/4986076/alternative-to-virsh-libvirt log = u' '.join([u'域', guest.name(), u', UUID', _uuid, u'的状态改变为']) if state == libvirt.VIR_DOMAIN_RUNNING: log += u' Running。' guest_event_emit.running(uuid=_uuid) # log += u' Booting。' # guest_event_emit.booting(uuid=_uuid) # q_booting_guest.put(guest) elif state == libvirt.VIR_DOMAIN_BLOCKED: log += u' Blocked。' guest_event_emit.blocked(uuid=_uuid) elif state == libvirt.VIR_DOMAIN_PAUSED: log += u' Paused。' guest_event_emit.paused(uuid=_uuid) elif state == libvirt.VIR_DOMAIN_SHUTDOWN: log += u' Shutdown。' guest_event_emit.shutdown(uuid=_uuid) elif state == libvirt.VIR_DOMAIN_SHUTOFF: log += u' Shutoff。' guest_event_emit.shutoff(uuid=_uuid) elif state == libvirt.VIR_DOMAIN_CRASHED: log += u' Crashed。' guest_event_emit.crashed(uuid=_uuid) elif state == libvirt.VIR_DOMAIN_PMSUSPENDED: log += u' PM_Suspended。' guest_event_emit.pm_suspended(uuid=_uuid) else: log += u' NO_State。' guest_event_emit.no_state(uuid=_uuid) logger.info(log) log_emit.info(log) except Exception as e: logger.error(e.message) log_emit.error(e.message)
def start_by_uuid(self, conn=None): try: domain = conn.lookupByUUIDString(uuidstr=self.uuid) domain.create() log = u' '.join([u'域', self.name, u', UUID', self.uuid, u'启动成功.']) logger.info(msg=log) log_emit.info(msg=log) except libvirt.libvirtError as e: logger.error(e.message) log_emit.error(e.message) return False return True
def guest_state_report(dom=None): try: _uuid = dom.UUIDString() state = Guest.get_state(dom=dom) log = u' '.join([u'域', dom.name(), u', UUID', _uuid, u'的状态改变为']) if state == GuestState.running.value: log += u' Running。' guest_event_emit.running(uuid=_uuid) elif state == GuestState.booting.value: log += u' Booting。' guest_event_emit.booting(uuid=_uuid) elif state == GuestState.blocked.value: log += u' Blocked。' guest_event_emit.blocked(uuid=_uuid) elif state == GuestState.paused.value: log += u' Paused。' guest_event_emit.paused(uuid=_uuid) elif state == GuestState.shutdown.value: log += u' Shutdown。' guest_event_emit.shutdown(uuid=_uuid) elif state == GuestState.shutoff.value: log += u' Shutoff。' guest_event_emit.shutoff(uuid=_uuid) elif state == GuestState.crashed.value: log += u' Crashed。' guest_event_emit.crashed(uuid=_uuid) elif state == GuestState.pm_suspended.value: log += u' PM_Suspended。' guest_event_emit.pm_suspended(uuid=_uuid) else: log += u' NO_State。' guest_event_emit.no_state(uuid=_uuid) log_emit.info(msg=log) except Exception as e: log_emit.warn(e.message)
def define_by_xml(self, conn=None): try: if conn.defineXML(xml=self.xml): log = u' '.join([u'域', self.name, u', UUID', self.uuid, u'定义成功.']) logger.info(msg=log) log_emit.info(msg=log) else: log = u' '.join([u'域', self.name, u', UUID', self.uuid, u'定义时未预期返回.']) logger.info(msg=log) log_emit.info(msg=log) return False except libvirt.libvirtError as e: logger.error(e.message) log_emit.error(e.message) return False return True
def guest_booting2running_report_engine(self): """ Guest 启动到运行状态上报 """ self.init_conn() list_booting_guest = list() def is_running(_guest): running = False try: exec_ret = libvirt_qemu.qemuAgentCommand(_guest, json.dumps({ 'execute': 'guest-ping', 'arguments': { } }), 3, libvirt_qemu.VIR_DOMAIN_QEMU_AGENT_COMMAND_NOWAIT) running = True except: logger.error(traceback.format_exc()) return running while True: if Utils.exit_flag: msg = 'Thread guest_booting2running_report_engine say bye-bye' print msg logger.info(msg=msg) return # noinspection PyBroadException try: try: payload = q_booting_guest.get(timeout=config['engine_cycle_interval']) list_booting_guest.append(payload) q_booting_guest.task_done() except Queue.Empty as e: time.sleep(config['engine_cycle_interval']) threads_status['guest_booting2running_report_engine'] = dict() threads_status['guest_booting2running_report_engine']['timestamp'] = ji.Common.ts() for i, uuid in enumerate(list_booting_guest): guest = self.conn.lookupByUUIDString(uuidstr=uuid) log = u' '.join([u'域', guest.name(), u', UUID', uuid, u'的状态改变为']) if guest is not None and guest.isActive() and is_running(_guest=guest): log += u' Running。' guest_event_emit.running(uuid=uuid) logger.info(log) log_emit.info(log) else: time.sleep(config['engine_cycle_interval']) Guest.guest_state_report(guest=guest) del list_booting_guest[i] except: logger.error(traceback.format_exc()) log_emit.error(traceback.format_exc())
def instruction_process_engine(self): self.init_conn() ps = r.pubsub(ignore_subscribe_messages=False) ps.subscribe(config['instruction_channel']) while True: if Utils.exit_flag: msg = 'Thread instruction_process_engine say bye-bye' print msg logger.info(msg=msg) return threads_status['instruction_process_engine'] = dict() threads_status['instruction_process_engine']['timestamp'] = ji.Common.ts() # noinspection PyBroadException try: msg = ps.get_message(timeout=config['engine_cycle_interval']) if msg is None or 'data' not in msg or not isinstance(msg['data'], basestring): continue try: msg = json.loads(msg['data']) if msg['action'] == 'pong': continue if msg['action'] == 'ping': # 通过 ping pong 来刷存在感。因为经过实际测试发现,当订阅频道长时间没有数据来往,那么订阅者会被自动退出。 r.publish(config['instruction_channel'], message=json.dumps({'action': 'pong'})) continue except ValueError as e: logger.error(e.message) log_emit.error(e.message) continue if 'node_id' in msg and int(msg['node_id']) != self.node_id: continue # 下列语句繁琐写法如 <code>if 'action' not in msg or 'uuid' not in msg:</code> if not all([key in msg for key in ['_object', 'action']]): continue extend_data = dict() if msg['_object'] == 'guest': self.refresh_guest_mapping() if msg['action'] not in ['create']: if msg['uuid'] not in self.guest_mapping_by_uuid: if config['DEBUG']: _log = u' '.join([u'uuid', msg['uuid'], u'在计算节点', self.hostname, u'中未找到.']) logger.debug(_log) log_emit.debug(_log) raise RuntimeError('The uuid ' + msg['uuid'] + ' not found in current domains list.') self.guest = self.guest_mapping_by_uuid[msg['uuid']] if not isinstance(self.guest, libvirt.virDomain): raise RuntimeError('Guest ' + msg['uuid'] + ' is not a domain.') if msg['action'] == 'create': t = threading.Thread(target=Guest.create, args=(self.conn, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'reboot': if self.guest.reboot() != 0: raise RuntimeError('Guest reboot failure.') elif msg['action'] == 'force_reboot': self.guest.destroy() self.guest.create() Guest.quota(guest=self.guest, msg=msg) elif msg['action'] == 'shutdown': if self.guest.shutdown() != 0: raise RuntimeError('Guest shutdown failure.') elif msg['action'] == 'force_shutdown': if self.guest.destroy() != 0: raise RuntimeError('Guest force shutdown failure.') elif msg['action'] == 'boot': if not self.guest.isActive(): if self.guest.create() != 0: raise RuntimeError('Guest boot failure.') Guest.quota(guest=self.guest, msg=msg) elif msg['action'] == 'suspend': if self.guest.suspend() != 0: raise RuntimeError('Guest suspend failure.') elif msg['action'] == 'resume': if self.guest.resume() != 0: raise RuntimeError('Guest resume failure.') elif msg['action'] == 'delete': root = ET.fromstring(self.guest.XMLDesc()) if self.guest.isActive(): self.guest.destroy() self.guest.undefine() system_disk = None for _disk in root.findall('devices/disk'): if 'vda' == _disk.find('target').get('dev'): system_disk = _disk if msg['storage_mode'] in [StorageMode.ceph.value, StorageMode.glusterfs.value]: # 签出系统镜像路径 path_list = system_disk.find('source').attrib['name'].split('/') if msg['storage_mode'] == StorageMode.glusterfs.value: Guest.dfs_volume = path_list[0] Guest.init_gfapi() try: Guest.gf.remove('/'.join(path_list[1:])) except OSError: pass elif msg['storage_mode'] in [StorageMode.local.value, StorageMode.shared_mount.value]: file_path = system_disk.find('source').attrib['file'] try: os.remove(file_path) except OSError: pass elif msg['action'] == 'reset_password': if self.guest.setUserPassword(msg['user'], msg['password']) != 0: raise RuntimeError('Guest reset password failure.') elif msg['action'] == 'attach_disk': if 'xml' not in msg: _log = u'添加磁盘缺少 xml 参数' raise KeyError(_log) flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG if self.guest.isActive(): flags |= libvirt.VIR_DOMAIN_AFFECT_LIVE # 添加磁盘成功返回时,ret值为0。可参考 Linux 命令返回值规范? if self.guest.attachDeviceFlags(xml=msg['xml'], flags=flags) != 0: raise RuntimeError('Attack disk failure.') Guest.quota(guest=self.guest, msg=msg) elif msg['action'] == 'detach_disk': if 'xml' not in msg: _log = u'分离磁盘缺少 xml 参数' raise KeyError(_log) flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG if self.guest.isActive(): flags |= libvirt.VIR_DOMAIN_AFFECT_LIVE if self.guest.detachDeviceFlags(xml=msg['xml'], flags=flags) != 0: raise RuntimeError('Detach disk failure.') elif msg['action'] == 'update_ssh_key': if not self.guest.isActive(): _log = u'欲更新 SSH-KEY 的目标虚拟机未处于活动状态。' logger.warning(_log) log_emit.warn(_log) continue ret = Guest.update_ssh_key(guest=self.guest, msg=msg) logger.info(json.dumps(ret, ensure_ascii=False)) elif msg['action'] == 'allocate_bandwidth': t = threading.Thread(target=Guest.allocate_bandwidth, args=(self.guest, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'adjust_ability': t = threading.Thread(target=Guest.adjust_ability, args=(self.conn, self.guest, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'migrate': # duri like qemu+ssh://destination_host/system if 'duri' not in msg: _log = u'迁移操作缺少 duri 参数' raise KeyError(_log) # https://rk4n.github.io/2016/08/10/qemu-post-copy-and-auto-converge-features/ flags = libvirt.VIR_MIGRATE_PERSIST_DEST | \ libvirt.VIR_MIGRATE_UNDEFINE_SOURCE | \ libvirt.VIR_MIGRATE_COMPRESSED | \ libvirt.VIR_MIGRATE_PEER2PEER | \ libvirt.VIR_MIGRATE_AUTO_CONVERGE root = ET.fromstring(self.guest.XMLDesc()) if msg['storage_mode'] == StorageMode.local.value: # 需要把磁盘存放路径加入到两边宿主机的存储池中 # 不然将会报 no storage pool with matching target path '/opt/Images' 错误 flags |= libvirt.VIR_MIGRATE_NON_SHARED_DISK flags |= libvirt.VIR_MIGRATE_LIVE if not self.guest.isActive(): _log = u'非共享存储不支持离线迁移。' logger.error(_log) log_emit.error(_log) raise RuntimeError('Nonsupport online migrate with storage of non sharing mode.') if self.init_ssh_client(hostname=msg['duri'].split('/')[2], user='******'): for _disk in root.findall('devices/disk'): _file_path = _disk.find('source').get('file') disk_info = Disk.disk_info_by_local(image_path=_file_path) disk_size = disk_info['virtual-size'] stdin, stdout, stderr = self.ssh_client.exec_command( ' '.join(['qemu-img', 'create', '-f', 'qcow2', _file_path, str(disk_size)])) for line in stdout: logger.info(line) log_emit.info(line) for line in stderr: logger.error(line) log_emit.error(line) elif msg['storage_mode'] in [StorageMode.shared_mount.value, StorageMode.ceph.value, StorageMode.glusterfs.value]: if self.guest.isActive(): flags |= libvirt.VIR_MIGRATE_LIVE flags |= libvirt.VIR_MIGRATE_TUNNELLED else: flags |= libvirt.VIR_MIGRATE_OFFLINE if self.guest.migrateToURI(duri=msg['duri'], flags=flags) == 0: if msg['storage_mode'] == StorageMode.local.value: for _disk in root.findall('devices/disk'): _file_path = _disk.find('source').get('file') if _file_path is not None: os.remove(_file_path) else: raise RuntimeError('Unknown storage mode.') elif msg['_object'] == 'disk': if msg['action'] == 'create': if msg['storage_mode'] == StorageMode.glusterfs.value: Guest.dfs_volume = msg['dfs_volume'] Guest.init_gfapi() if not Disk.make_qemu_image_by_glusterfs(gf=Guest.gf, dfs_volume=msg['dfs_volume'], image_path=msg['image_path'], size=msg['size']): raise RuntimeError('Create disk failure with glusterfs.') elif msg['storage_mode'] in [StorageMode.local.value, StorageMode.shared_mount.value]: if not Disk.make_qemu_image_by_local(image_path=msg['image_path'], size=msg['size']): raise RuntimeError('Create disk failure with local storage mode.') elif msg['action'] == 'delete': if msg['storage_mode'] == StorageMode.glusterfs.value: Guest.dfs_volume = msg['dfs_volume'] Guest.init_gfapi() if Disk.delete_qemu_image_by_glusterfs(gf=Guest.gf, image_path=msg['image_path']) \ is not None: raise RuntimeError('Delete disk failure with glusterfs.') elif msg['storage_mode'] in [StorageMode.local.value, StorageMode.shared_mount.value]: if Disk.delete_qemu_image_by_local(image_path=msg['image_path']) is not None: raise RuntimeError('Delete disk failure with local storage mode.') elif msg['action'] == 'resize': if 'size' not in msg: _log = u'添加磁盘缺少 disk 或 disk["size"] 参数' raise KeyError(_log) used = False if msg['guest_uuid'].__len__() == 36: used = True if used: self.refresh_guest_mapping() if msg['guest_uuid'] not in self.guest_mapping_by_uuid: if config['DEBUG']: _log = u' '.join([u'uuid', msg['uuid'], u'在计算节点', self.hostname, u'中未找到.']) logger.debug(_log) log_emit.debug(_log) raise RuntimeError('Resize disk failure, because the uuid ' + msg['guest_uuid'] + ' not found in current domains.') self.guest = self.guest_mapping_by_uuid[msg['guest_uuid']] if not isinstance(self.guest, libvirt.virDomain): raise RuntimeError('Resize disk failure, because the guest is not a domain.') # 在线磁盘扩容 if used and self.guest.isActive(): if 'device_node' not in msg: _log = u'添加磁盘缺少 disk 或 disk["device_node|size"] 参数' raise KeyError(_log) # 磁盘大小默认单位为KB,乘以两个 1024,使其单位达到GB msg['size'] = int(msg['size']) * 1024 * 1024 if self.guest.blockResize(disk=msg['device_node'], size=msg['size']) != 0: raise RuntimeError('Online resize disk failure in blockResize method.') Guest.quota(guest=self.guest, msg=msg) # 离线磁盘扩容 else: if not all([key in msg for key in ['storage_mode', 'dfs_volume', 'image_path']]): _log = u'添加磁盘缺少 disk 或 disk["storage_mode|dfs_volume|image_path|size"] 参数' raise KeyError(_log) if msg['storage_mode'] == StorageMode.glusterfs.value: if not Disk.resize_qemu_image_by_glusterfs(dfs_volume=msg['dfs_volume'], image_path=msg['image_path'], size=msg['size']): raise RuntimeError('Offline resize disk failure with glusterfs.') elif msg['storage_mode'] in [StorageMode.local.value, StorageMode.shared_mount.value]: if not Disk.resize_qemu_image_by_local(image_path=msg['image_path'], size=msg['size']): raise RuntimeError('Offline resize disk failure with local storage mode.') elif msg['action'] == 'quota': self.refresh_guest_mapping() if msg['guest_uuid'] not in self.guest_mapping_by_uuid: if config['DEBUG']: _log = u' '.join([u'uuid', msg['guest_uuid'], u'在计算节点', self.hostname, u'中未找到.']) logger.debug(_log) log_emit.debug(_log) raise RuntimeError('Disk quota failure, because the uuid ' + msg['guest_uuid'] + ' not found in current domains.') self.guest = self.guest_mapping_by_uuid[msg['guest_uuid']] if not isinstance(self.guest, libvirt.virDomain): raise RuntimeError('Disk quota failure, because the guest is not a domain.') if not self.guest.isActive(): _log = u'磁盘 ' + msg['uuid'] + u' 所属虚拟机未处于活动状态。' logger.warning(_log) log_emit.warn(_log) continue Guest.quota(guest=self.guest, msg=msg) elif msg['_object'] == 'snapshot': self.refresh_guest_mapping() if msg['uuid'] not in self.guest_mapping_by_uuid: if config['DEBUG']: _log = u' '.join([u'uuid', msg['uuid'], u'在计算节点', self.hostname, u'中未找到.']) logger.debug(_log) log_emit.debug(_log) raise RuntimeError('Snapshot ' + msg['action'] + ' failure, because the uuid ' + msg['uuid'] + ' not found in current domains.') self.guest = self.guest_mapping_by_uuid[msg['uuid']] if not isinstance(self.guest, libvirt.virDomain): raise RuntimeError('Snapshot ' + msg['action'] + ' failure, because the guest is not a domain.') if msg['action'] == 'create': t = threading.Thread(target=Guest.create_snapshot, args=(self.guest, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'delete': t = threading.Thread(target=Guest.delete_snapshot, args=(self.guest, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'revert': t = threading.Thread(target=Guest.revert_snapshot, args=(self.guest, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'convert': t = threading.Thread(target=Guest.convert_snapshot, args=(msg,)) t.setDaemon(False) t.start() continue elif msg['_object'] == 'os_template_image': if msg['action'] == 'delete': if msg['storage_mode'] == StorageMode.glusterfs.value: Guest.dfs_volume = msg['dfs_volume'] Guest.init_gfapi() try: Guest.gf.remove(msg['template_path']) except OSError: pass elif msg['storage_mode'] in [StorageMode.local.value, StorageMode.shared_mount.value]: try: os.remove(msg['template_path']) except OSError: pass elif msg['_object'] == 'global': if msg['action'] == 'refresh_guest_state': host_use_for_refresh_guest_state = Host() t = threading.Thread(target=host_use_for_refresh_guest_state.refresh_guest_state, args=()) t.setDaemon(False) t.start() continue else: _log = u'未支持的 _object:' + msg['_object'] logger.error(_log) log_emit.error(_log) response_emit.success(_object=msg['_object'], action=msg['action'], uuid=msg['uuid'], data=extend_data, passback_parameters=msg.get('passback_parameters')) except redis.exceptions.ConnectionError as e: logger.error(traceback.format_exc()) # 防止循环线程,在redis连接断开时,混水写入日志 time.sleep(5) except: # 防止循环线程,在redis连接断开时,混水写入日志 time.sleep(5) logger.error(traceback.format_exc()) log_emit.error(traceback.format_exc()) response_emit.failure(_object=msg['_object'], action=msg.get('action'), uuid=msg.get('uuid'), passback_parameters=msg.get('passback_parameters'))
def migrate(dom=None, msg=None): assert isinstance(dom, libvirt.virDomain) assert isinstance(msg, dict) # https://rk4n.github.io/2016/08/10/qemu-post-copy-and-auto-converge-features/ flags = libvirt.VIR_MIGRATE_PERSIST_DEST | \ libvirt.VIR_MIGRATE_UNDEFINE_SOURCE | \ libvirt.VIR_MIGRATE_COMPRESSED | \ libvirt.VIR_MIGRATE_PEER2PEER | \ libvirt.VIR_MIGRATE_AUTO_CONVERGE root = ET.fromstring(dom.XMLDesc()) if msg['storage_mode'] == StorageMode.local.value: # 需要把磁盘存放路径加入到两边宿主机的存储池中 # 不然将会报 no storage pool with matching target path '/opt/Images' 错误 flags |= libvirt.VIR_MIGRATE_NON_SHARED_DISK flags |= libvirt.VIR_MIGRATE_LIVE if not dom.isActive(): err = u'非共享存储不支持离线迁移。' log_emit.warn(err) raise RuntimeError( 'Nonsupport offline migrate with storage of non sharing mode.' ) ssh_client = Utils.ssh_client(hostname=msg['duri'].split('/')[2], user='******') for _disk in root.findall('devices/disk'): _file_path = _disk.find('source').get('file') disk_info = Storage.image_info_by_local(path=_file_path) disk_size = disk_info['virtual-size'] stdin, stdout, stderr = ssh_client.exec_command(' '.join([ 'qemu-img', 'create', '-f', 'qcow2', _file_path, str(disk_size) ])) for line in stdout: log_emit.info(line) for line in stderr: log_emit.error(line) elif msg['storage_mode'] in [ StorageMode.shared_mount.value, StorageMode.ceph.value, StorageMode.glusterfs.value ]: if dom.isActive(): flags |= libvirt.VIR_MIGRATE_LIVE flags |= libvirt.VIR_MIGRATE_TUNNELLED else: flags |= libvirt.VIR_MIGRATE_OFFLINE # duri like qemu+ssh://destination_host/system if dom.migrateToURI(duri=msg['duri'], flags=flags) == 0: if msg['storage_mode'] == StorageMode.local.value: for _disk in root.findall('devices/disk'): _file_path = _disk.find('source').get('file') if _file_path is not None: os.remove(_file_path) else: raise RuntimeError('Unknown storage mode.')
def create(conn, msg): try: guest = Guest(uuid=msg['uuid'], name=msg['name'], template_path=msg['template_path'], disk=msg['disks'][0], xml=msg['xml'], storage_mode=msg['storage_mode'], dfs_volume=msg['dfs_volume']) q_creating_guest.put({ 'storage_mode': guest.storage.storage_mode, 'dfs_volume': guest.storage.dfs_volume, 'uuid': guest.uuid, 'template_path': guest.template_path, 'system_image_path': guest.system_image_path }) guest.generate_system_image() dom = guest.define_by_xml(conn=conn) assert isinstance(dom, libvirt.virDomain) log = u' '.join( [u'域', guest.name, u', UUID', guest.uuid, u'定义成功.']) log_emit.info(msg=log) guest_event_emit.creating(uuid=guest.uuid, progress=92) disk_info = guest.storage.image_info(path=guest.system_image_path) # 由该线程最顶层的异常捕获机制,处理其抛出的异常 guest.execute_os_template_initialize_operates( dom=conn.lookupByUUIDString(uuidstr=guest.uuid), os_template_initialize_operates=msg[ 'os_template_initialize_operates'], os_type=msg['os_type']) extend_data = dict() extend_data.update({'disk_info': disk_info}) guest_event_emit.creating(uuid=guest.uuid, progress=97) dom.create() log = u' '.join( [u'域', guest.name, u', UUID', guest.uuid, u'启动成功.']) log_emit.info(msg=log) Guest.quota(dom=dom, msg=msg) response_emit.success( _object=msg['_object'], action=msg['action'], uuid=msg['uuid'], data=extend_data, passback_parameters=msg.get('passback_parameters')) except: log_emit.error(traceback.format_exc()) response_emit.failure( _object=msg['_object'], action=msg.get('action'), uuid=msg.get('uuid'), passback_parameters=msg.get('passback_parameters'))
def instruction_process_engine(self): ps = r.pubsub(ignore_subscribe_messages=False) ps.subscribe(config['instruction_channel']) while True: if Utils.exit_flag: msg = 'Thread instruction_process_engine say bye-bye' print msg logger.info(msg=msg) return threads_status['instruction_process_engine'] = { 'timestamp': ji.Common.ts() } msg = dict() extend_data = dict() try: msg = ps.get_message(timeout=config['engine_cycle_interval']) if msg is None or 'data' not in msg or not isinstance( msg['data'], basestring): continue try: msg = json.loads(msg['data']) if msg['action'] == 'pong': continue if msg['action'] == 'ping': # 通过 ping pong 来刷存在感。因为经过实际测试发现,当订阅频道长时间没有数据来往,那么订阅者会被自动退出。 r.publish(config['instruction_channel'], message=json.dumps({'action': 'pong'})) continue except ValueError as e: log_emit.error(e.message) continue if 'node_id' in msg and int(msg['node_id']) != self.node_id: continue # 下列语句繁琐写法如 <code>if '_object' not in msg or 'action' not in msg:</code> if not all([key in msg for key in ['_object', 'action']]): continue logger.info(msg=msg) if msg['_object'] == 'guest': self.refresh_dom_mapping() if msg['action'] not in ['create']: self.dom = self.dom_mapping_by_uuid[msg['uuid']] assert isinstance(self.dom, libvirt.virDomain) if msg['action'] == 'create': t = threading.Thread(target=Guest.create, args=(self.conn, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'reboot': Guest.reboot(dom=self.dom) elif msg['action'] == 'force_reboot': Guest.force_reboot(dom=self.dom, msg=msg) elif msg['action'] == 'shutdown': Guest.shutdown(dom=self.dom) elif msg['action'] == 'force_shutdown': Guest.force_shutdown(dom=self.dom) elif msg['action'] == 'boot': Guest.boot(dom=self.dom, msg=msg) elif msg['action'] == 'suspend': Guest.suspend(dom=self.dom) elif msg['action'] == 'resume': Guest.resume(dom=self.dom) elif msg['action'] == 'delete': Guest.delete(dom=self.dom, msg=msg) elif msg['action'] == 'reset_password': Guest.reset_password(dom=self.dom, msg=msg) elif msg['action'] == 'attach_disk': Guest.attach_disk(dom=self.dom, msg=msg) elif msg['action'] == 'detach_disk': Guest.detach_disk(dom=self.dom, msg=msg) elif msg['action'] == 'update_ssh_key': Guest.update_ssh_key(dom=self.dom, msg=msg) elif msg['action'] == 'allocate_bandwidth': t = threading.Thread(target=Guest.allocate_bandwidth, args=(self.dom, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'adjust_ability': t = threading.Thread(target=Guest.adjust_ability, args=(self.dom, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'migrate': Guest().migrate(dom=self.dom, msg=msg) elif msg['_object'] == 'disk': if msg['action'] == 'create': Storage(storage_mode=msg['storage_mode'], dfs_volume=msg['dfs_volume']).make_image( path=msg['image_path'], size=msg['size']) elif msg['action'] == 'delete': Storage(storage_mode=msg['storage_mode'], dfs_volume=msg['dfs_volume']).delete_image( path=msg['image_path']) elif msg['action'] == 'resize': mounted = True if msg['guest_uuid'].__len__( ) == 36 else False if mounted: self.refresh_dom_mapping() self.dom = self.dom_mapping_by_uuid[ msg['guest_uuid']] # 在线磁盘扩容 if mounted and self.dom.isActive(): # 磁盘大小默认单位为KB,乘以两个 1024,使其单位达到 GiB msg['size'] = int(msg['size']) * 1024 * 1024 # https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainBlockResize self.dom.blockResize(disk=msg['device_node'], size=msg['size']) Guest.quota(dom=self.dom, msg=msg) # 离线磁盘扩容 else: Storage(storage_mode=msg['storage_mode'], dfs_volume=msg['dfs_volume']).resize_image( path=msg['image_path'], size=msg['size']) elif msg['action'] == 'quota': self.refresh_dom_mapping() self.dom = self.dom_mapping_by_uuid[msg['guest_uuid']] Guest.quota(dom=self.dom, msg=msg) elif msg['_object'] == 'snapshot': self.refresh_dom_mapping() self.dom = self.dom_mapping_by_uuid[msg['uuid']] if msg['action'] == 'create': t = threading.Thread(target=Guest.create_snapshot, args=(self.dom, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'delete': t = threading.Thread(target=Guest.delete_snapshot, args=(self.dom, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'revert': t = threading.Thread(target=Guest.revert_snapshot, args=(self.dom, msg)) t.setDaemon(False) t.start() continue elif msg['action'] == 'convert': t = threading.Thread(target=Guest.convert_snapshot, args=(msg, )) t.setDaemon(False) t.start() continue elif msg['_object'] == 'os_template_image': if msg['action'] == 'delete': Storage(storage_mode=msg['storage_mode'], dfs_volume=msg['dfs_volume']).delete_image( path=msg['template_path']) elif msg['_object'] == 'global': if msg['action'] == 'refresh_guest_state': t = threading.Thread(target=Host().refresh_guest_state, args=()) t.setDaemon(False) t.start() continue if msg['action'] == 'upgrade': try: log = self.upgrade(msg['url']) log_emit.info(msg=log) except subprocess.CalledProcessError as e: log_emit.warn(e.output) self.rollback() continue log = self.restart() log_emit.info(msg=log) if msg['action'] == 'restart': log = self.restart() log_emit.info(msg=log) else: err = u'未支持的 _object:' + msg['_object'] log_emit.error(err) response_emit.success( _object=msg['_object'], action=msg['action'], uuid=msg['uuid'], data=extend_data, passback_parameters=msg.get('passback_parameters')) except KeyError as e: log_emit.warn(e.message) if msg['_object'] == 'guest': if msg['action'] == 'delete': response_emit.success( _object=msg['_object'], action=msg['action'], uuid=msg['uuid'], data=extend_data, passback_parameters=msg.get('passback_parameters')) except: # 防止循环线程,在redis连接断开时,混水写入日志 time.sleep(5) log_emit.error(traceback.format_exc()) response_emit.failure( _object=msg['_object'], action=msg.get('action'), uuid=msg.get('uuid'), passback_parameters=msg.get('passback_parameters'))