def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.caps = CapabilitiesModel() self.guests_stats_thread = BackgroundTask(GUESTS_STATS_INTERVAL, self._update_guests_stats) self.guests_stats_thread.start()
def start_scheduled_job(self): """ Scrape every 5 minutes in background and on a different thread """ background_job = BackgroundTask(5 * 60, self.scrapper.persist, bus=cherrypy.engine) background_job.start()
def __init__(self): self.routes = get_routes() for route in self.routes: route['time_table'] = explode_times(route['departure_times']) task = BackgroundTask(interval=1, function=self.track_minibuses, bus=cherrypy.engine) task.start() pass
def __init__(self, **kargs): self.host_stats = defaultdict(list) gbconfig = config.get('gingerbase', {}) self.statshistory_on = gbconfig.get('statshistory_on', True) # create thread to collect statistcs and cache values only if # statshistory_on is enabled in gingerbase.conf if self.statshistory_on: self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self.update_host_stats) self.host_stats_thread.start()
def start(self): ''' Starts Polling by initialising the CherryPy background task and the tester Class and ''' self.tester = OmniPingTester(interval=self.setup.interval) actual_interval = self.tester.interval - self.tester.timeout self.bgtask = BackgroundTask(actual_interval, self.testerCall, bus=cherrypy.engine) self.bgtask.start() self.running = True
def __init__(self): log = os.path.join(paths.state_dir, REQUEST_LOG_FILE) h = logging.handlers.WatchedFileHandler(log, 'a') h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start()
def __init__(self): log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) h = logging.handlers.RotatingFileHandler(log, 'a', maxBytes=MAX_FILE_SIZE, backupCount=NUM_BACKUP_FILES) h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start()
def start_updater(self): def predicate(x): try: return x.updater except (TypeError, AttributeError): return False for plugin in pluginmanager.get_instances(): try: for name, method in inspect.getmembers(plugin, predicate): worker = BackgroundTask(method.interval, method) self.worker.add(worker) worker.start() except (TypeError, AttributeError, StopIteration): pass
def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.caps = CapabilitiesModel(**kargs) self.guests_stats_thread = BackgroundTask(GUESTS_STATS_INTERVAL, self._update_guests_stats) self.guests_stats_thread.start()
class RequestLogger(object): def __init__(self): log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) h = logging.handlers.WatchedFileHandler(log, 'a') h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start() def cleanLogFiles(self): globexpr = "%s/*.txt" % get_log_download_path() remove_old_files(globexpr, LOG_DOWNLOAD_TIMEOUT)
class RequestLogger(object): def __init__(self): log = os.path.join(paths.state_dir, REQUEST_LOG_FILE) h = logging.handlers.WatchedFileHandler(log, 'a') h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start() def cleanLogFiles(self): globexpr = "%s/*.txt" % get_log_download_path() remove_old_files(globexpr, LOG_DOWNLOAD_TIMEOUT)
class RequestLogger(object): def __init__(self): log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) h = logging.handlers.RotatingFileHandler(log, 'a', maxBytes=MAX_FILE_SIZE, backupCount=NUM_BACKUP_FILES) h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start() def cleanLogFiles(self): globexpr = "%s/*.txt" % get_log_download_path() remove_old_files(globexpr, LOG_DOWNLOAD_TIMEOUT)
def __init__(self, libvirt_uri=None, objstore_loc=None): self.libvirt_uri = libvirt_uri or 'qemu:///system' self.conn = LibvirtConnection(self.libvirt_uri) self.objstore = ObjectStore(objstore_loc) self.graphics_ports = {} self.next_taskid = 1 self.stats = {} self.qemu_stream = False self.qemu_stream_dns = False self.libvirt_stream_protocols = [] # Subscribe function to set host capabilities to be run when cherrypy # server is up # It is needed because some features tests depends on the server cherrypy.engine.subscribe('start', self._set_capabilities) self.statsThread = BackgroundTask(STATS_INTERVAL, self._update_stats) self.statsThread.start() self.distros = self._get_distros() if 'qemu:///' in self.libvirt_uri: self._default_pool_check() self._default_network_check()
def __init__(self): log = os.path.join(config.get("logging", "log_dir"), REQUEST_LOG_FILE) h = logging.handlers.WatchedFileHandler(log, 'a') h.setFormatter(logging.Formatter('%(message)s')) self.handler = h self.logger = logging.getLogger(WOK_REQUEST_LOGGER) self.logger.setLevel(logging.INFO) self.logger.addHandler(self.handler) # start request log's downloadable temporary files removal task interval = LOG_DOWNLOAD_TIMEOUT * SECONDS_PER_HOUR self.clean_task = BackgroundTask(interval, self.cleanLogFiles) self.clean_task.start()
def start(ask_queue, data_queue, host=None, port=None, name=None): global server_ip global server_port global server_name import setproctitle setproctitle.setproctitle("aqi: web server") (machine, ipaddresses) = system_tools.get_host_info() cherrypy.log.screen = False if host == None: host = ipaddress[0] if port == None: port = 8080 if name == None: name = "localhost" server_ip = host server_port = port server_name = name config = { 'global': { 'server.socket_host': server_ip, 'server.socket_port': server_port, 'log.access_file': '', 'log.error_file': '', } } server = RawDataServer(ask_queue, data_queue) task = BackgroundTask(interval=1, function=server.read_queue, args=[], bus=cherrypy.engine) task.start() print("INFO: [aqi] web server starting: {}:{}".format( server_ip, server_port)) cherrypy.quickstart(server, '/', config) print("INFO: [aqi] web server finished")
def start_background_tasks(config): global database_cleanup_task, monitoring_aggregation_task if config["http_server"]["stale_session_cleanup_enabled"]: database_cleanup_task = BackgroundTask( config["http_server"]["stale_session_cleanup_interval"], clear_stale_sessions(config["database"]) ) database_cleanup_task.start() else: cherrypy.log( "Warning, stale_session_cleanup_enabled is set to false. The database may grow.", severity=logging.WARNING ) if config["http_server"]["monitoring_aggregation_enabled"]: monitoring_aggregation_task = BackgroundTask( config["http_server"]["monitoring_aggregation_interval"], monitoring_aggregation ) monitoring_aggregation_task.start() else: cherrypy.log( "Warning, monitoring_aggregation_enabled is set to false." + " There will be no useful monitoring data generated", severity=logging.WARNING, )
'entry 0', 'entry 1', 'entry 2', 'entry 3', 'entry 4', 'entry 5', 'entry 6', 'entry 7', 'entry 8', 'entry 9' ] # class Scraper(Thread): # def __init__(self): # Thread.__init__(self, delay) # self.delay = delay # def run(self): # while True: # time.sleep(self.delay) # nScrape() nScrape() BackgroundTask(300, nScrape, bus=cherrypy.engine).start() class HomePage: @cherrypy.expose def index(self): data_to_render = redis_inst tmpl = env.get_template('index.html') return tmpl.render(redis_data=data_to_render, entries=entries, keys=keys) root = HomePage() config = {
class VMsModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.caps = CapabilitiesModel() self.guests_stats_thread = BackgroundTask(GUESTS_STATS_INTERVAL, self._update_guests_stats) self.guests_stats_thread.start() def _update_guests_stats(self): vm_list = self.get_list() for name in vm_list: dom = VMModel.get_vm(name, self.conn) vm_uuid = dom.UUIDString() info = dom.info() state = DOM_STATE_MAP[info[0]] if state != 'running': stats[vm_uuid] = {} continue if stats.get(vm_uuid, None) is None: stats[vm_uuid] = {} timestamp = time.time() prevStats = stats.get(vm_uuid, {}) seconds = timestamp - prevStats.get('timestamp', 0) stats[vm_uuid].update({'timestamp': timestamp}) self._get_percentage_cpu_usage(vm_uuid, info, seconds) self._get_network_io_rate(vm_uuid, dom, seconds) self._get_disk_io_rate(vm_uuid, dom, seconds) def _get_percentage_cpu_usage(self, vm_uuid, info, seconds): prevCpuTime = stats[vm_uuid].get('cputime', 0) cpus = info[3] cpuTime = info[4] - prevCpuTime base = (((cpuTime) * 100.0) / (seconds * 1000.0 * 1000.0 * 1000.0)) percentage = max(0.0, min(100.0, base / cpus)) stats[vm_uuid].update({'cputime': info[4], 'cpu': percentage}) def _get_network_io_rate(self, vm_uuid, dom, seconds): prevNetRxKB = stats[vm_uuid].get('netRxKB', 0) prevNetTxKB = stats[vm_uuid].get('netTxKB', 0) currentMaxNetRate = stats[vm_uuid].get('max_net_io', 100) rx_bytes = 0 tx_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall('devices/interface/target'): dev = target.get('dev') io = dom.interfaceStats(dev) rx_bytes += io[0] tx_bytes += io[4] netRxKB = float(rx_bytes) / 1000 netTxKB = float(tx_bytes) / 1000 rx_stats = (netRxKB - prevNetRxKB) / seconds tx_stats = (netTxKB - prevNetTxKB) / seconds rate = rx_stats + tx_stats max_net_io = round(max(currentMaxNetRate, int(rate)), 1) stats[vm_uuid].update({ 'net_io': rate, 'max_net_io': max_net_io, 'netRxKB': netRxKB, 'netTxKB': netTxKB }) def _get_disk_io_rate(self, vm_uuid, dom, seconds): prevDiskRdKB = stats[vm_uuid].get('diskRdKB', 0) prevDiskWrKB = stats[vm_uuid].get('diskWrKB', 0) currentMaxDiskRate = stats[vm_uuid].get('max_disk_io', 100) rd_bytes = 0 wr_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall("devices/disk/target"): dev = target.get("dev") io = dom.blockStats(dev) rd_bytes += io[1] wr_bytes += io[3] diskRdKB = float(rd_bytes) / 1024 diskWrKB = float(wr_bytes) / 1024 rd_stats = (diskRdKB - prevDiskRdKB) / seconds wr_stats = (diskWrKB - prevDiskWrKB) / seconds rate = rd_stats + wr_stats max_disk_io = round(max(currentMaxDiskRate, int(rate)), 1) stats[vm_uuid].update({ 'disk_io': rate, 'max_disk_io': max_disk_io, 'diskRdKB': diskRdKB, 'diskWrKB': diskWrKB }) def create(self, params): conn = self.conn.get() t_name = template_name_from_uri(params['template']) vm_uuid = str(uuid.uuid4()) vm_list = self.get_list() name = get_vm_name(params.get('name'), t_name, vm_list) # incoming text, from js json, is unicode, do not need decode if name in vm_list: raise InvalidOperation("KCHVM0001E", {'name': name}) vm_overrides = dict() pool_uri = params.get('storagepool') if pool_uri: vm_overrides['storagepool'] = pool_uri vm_overrides['fc_host_support'] = self.caps.fc_host_support t = TemplateModel.get_template(t_name, self.objstore, self.conn, vm_overrides) if not self.caps.qemu_stream and t.info.get('iso_stream', False): raise InvalidOperation("KCHVM0005E") t.validate() # Store the icon for displaying later icon = t.info.get('icon') if icon: try: with self.objstore as session: session.store('vm', vm_uuid, {'icon': icon}) except Exception as e: # It is possible to continue Kimchi executions without store # vm icon info kimchi_log.error( 'Error trying to update database with guest ' 'icon information due error: %s', e.message) # If storagepool is SCSI, volumes will be LUNs and must be passed by # the user from UI or manually. vol_list = [] if t._get_storage_type() in ["iscsi", "scsi"]: vol_list = [] else: vol_list = t.fork_vm_storage(vm_uuid) graphics = params.get('graphics') stream_protocols = self.caps.libvirt_stream_protocols xml = t.to_vm_xml(name, vm_uuid, libvirt_stream_protocols=stream_protocols, qemu_stream_dns=self.caps.qemu_stream_dns, graphics=graphics, volumes=vol_list) try: conn.defineXML(xml.encode('utf-8')) except libvirt.libvirtError as e: if t._get_storage_type() not in READONLY_POOL_TYPE: for v in vol_list: vol = conn.storageVolLookupByPath(v['path']) vol.delete(0) raise OperationFailed("KCHVM0007E", { 'name': name, 'err': e.get_error_message() }) VMModel.vm_update_os_metadata(VMModel.get_vm(name, self.conn), t.info) return name def get_list(self): return self.get_vms(self.conn) @staticmethod def get_vms(conn): conn = conn.get() names = [dom.name().decode('utf-8') for dom in conn.listAllDomains(0)] return sorted(names, key=unicode.lower)
class HostStatsModel(object): __metaclass__ = Singleton def __init__(self, **kargs): self.host_stats = defaultdict(list) self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self._update_host_stats) self.host_stats_thread.start() def lookup(self, *name): return { "cpu_utilization": self.host_stats["cpu_utilization"][-1], "memory": self.host_stats["memory"][-1], "disk_read_rate": self.host_stats["disk_read_rate"][-1], "disk_write_rate": self.host_stats["disk_write_rate"][-1], "net_recv_rate": self.host_stats["net_recv_rate"][-1], "net_sent_rate": self.host_stats["net_sent_rate"][-1], } def _update_host_stats(self): preTimeStamp = self.host_stats["timestamp"] timestamp = time.time() # FIXME when we upgrade psutil, we can get uptime by psutil.uptime # we get uptime by float(open("/proc/uptime").readline().split()[0]) # and calculate the first io_rate after the OS started. seconds = timestamp - preTimeStamp if preTimeStamp else float(open("/proc/uptime").readline().split()[0]) self.host_stats["timestamp"] = timestamp self._get_host_disk_io_rate(seconds) self._get_host_network_io_rate(seconds) self._get_percentage_host_cpu_usage() self._get_host_memory_stats() # store only 60 stats (1 min) for key, value in self.host_stats.iteritems(): if isinstance(value, list): if len(value) == 60: self.host_stats[key] = value[10:] def _get_percentage_host_cpu_usage(self): # This is cpu usage producer. This producer will calculate the usage # at an interval of HOST_STATS_INTERVAL. # The psutil.cpu_percent works as non blocking. # psutil.cpu_percent maintains a cpu time sample. # It will update the cpu time sample when it is called. # So only this producer can call psutil.cpu_percent in kimchi. self.host_stats["cpu_utilization"].append(psutil.cpu_percent(None)) def _get_host_memory_stats(self): virt_mem = psutil.virtual_memory() # available: # the actual amount of available memory that can be given # instantly to processes that request more memory in bytes; this # is calculated by summing different memory values depending on # the platform (e.g. free + buffers + cached on Linux) memory_stats = { "total": virt_mem.total, "free": virt_mem.free, "cached": virt_mem.cached, "buffers": virt_mem.buffers, "avail": virt_mem.available, } self.host_stats["memory"].append(memory_stats) def _get_host_disk_io_rate(self, seconds): disk_read_bytes = self.host_stats["disk_read_bytes"] disk_write_bytes = self.host_stats["disk_write_bytes"] prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0 disk_io = psutil.disk_io_counters(False) read_bytes = disk_io.read_bytes write_bytes = disk_io.write_bytes rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) self.host_stats["disk_read_rate"].append(rd_rate) self.host_stats["disk_write_rate"].append(wr_rate) self.host_stats["disk_read_bytes"].append(read_bytes) self.host_stats["disk_write_bytes"].append(write_bytes) def _get_host_network_io_rate(self, seconds): net_recv_bytes = self.host_stats["net_recv_bytes"] net_sent_bytes = self.host_stats["net_sent_bytes"] prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 net_ios = psutil.network_io_counters(True) recv_bytes = 0 sent_bytes = 0 for key in set(netinfo.nics() + netinfo.wlans()) & set(net_ios.iterkeys()): recv_bytes = recv_bytes + net_ios[key].bytes_recv sent_bytes = sent_bytes + net_ios[key].bytes_sent rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) self.host_stats["net_recv_rate"].append(rx_rate) self.host_stats["net_sent_rate"].append(tx_rate) self.host_stats["net_recv_bytes"].append(recv_bytes) self.host_stats["net_sent_bytes"].append(sent_bytes)
class Model(object): dom_state_map = { 0: 'nostate', 1: 'running', 2: 'blocked', 3: 'paused', 4: 'shutdown', 5: 'shutoff', 6: 'crashed' } pool_state_map = { 0: 'inactive', 1: 'initializing', 2: 'active', 3: 'degraded', 4: 'inaccessible' } volume_type_map = {0: 'file', 1: 'block', 2: 'directory', 3: 'network'} def __init__(self, libvirt_uri=None, objstore_loc=None): self.libvirt_uri = libvirt_uri or 'qemu:///system' self.conn = LibvirtConnection(self.libvirt_uri) self.objstore = ObjectStore(objstore_loc) self.graphics_ports = {} self.next_taskid = 1 self.stats = {} self.qemu_stream = False self.qemu_stream_dns = False self.libvirt_stream_protocols = [] # Subscribe function to set host capabilities to be run when cherrypy # server is up # It is needed because some features tests depends on the server cherrypy.engine.subscribe('start', self._set_capabilities) self.statsThread = BackgroundTask(STATS_INTERVAL, self._update_stats) self.statsThread.start() self.distros = self._get_distros() if 'qemu:///' in self.libvirt_uri: self._default_pool_check() self._default_network_check() def _default_network_check(self): conn = self.conn.get() xml = """ <network> <name>default</name> <forward mode='nat'/> <bridge name='virbr0' stp='on' delay='0' /> <ip address='192.168.122.1' netmask='255.255.255.0'> <dhcp> <range start='192.168.122.2' end='192.168.122.254' /> </dhcp> </ip> </network> """ try: net = conn.networkLookupByName("default") except libvirt.libvirtError: try: net = conn.networkDefineXML(xml) except libvirt.libvirtError, e: cherrypy.log.error( "Fatal: Cannot create default network because of %s, exit kimchid" % e.message, severity=logging.ERROR) sys.exit(1) if net.isActive() == 0: try: net.create() except libvirt.libvirtError, e: cherrypy.log.error( "Fatal: Cannot activate default network because of %s, exit kimchid" % e.message, severity=logging.ERROR) sys.exit(1)
def __init__(self, *args, **kwargs): self.nse = NSE() # Start the cherrypy background cron-like task BackgroundTask(interval=5 * 60, function=self.store_now, bus=cherrypy.engine).start()
class OmniPingTestEng(): ''' This handles incoming requests to start/stop/reset polling As well as retuning the latest report ''' exposed = True content = [ '''When it comes to results if it says "Good" with a tick then obviously, things are good. If you see a cross and a row turns red then the status should explain what has happened. If you see "Incomplete" then the report has been requested whilst a test has not completed.''', '''The durations of the tests can vary; a failed ping takes long than a successful one. If you have any doubts then stopping polling should allow any outstanding tests to complete. If you find too many incomplete tasks you could try and reduce the interval between tests. Or try running less tests if possible.''', '''The Client Auto-refresh runs every 3 Seconds, so if you are polling devices every second you are more likely to ask for a report when some tests have not completed.''', '''When Running HTTP and HTTPS tests it is not obvious how best to treat any given HTTP error code, for example a 401 or 403 might be expected and hence be a good result showing the server is available. As such any HTTP response code is not considered a failure and subsequently Not flagged as a failure. However, the status will be highlighted yellow if it isn't good. Just so it stands out. Hope that makes sense.''' ] def __init__(self, setup): self.setup = setup self.running = False self.bgtask = False self.tester = False self.report = self.make_initial_report() @cherrypy.tools.json_out() def GET(self): report = self.make_jsonable_report() count = report.get('count', 'Error') report['message'] = f'Retrieved report ({count})' report['running'] = self.running report['content'] = self.content return report @cherrypy.tools.json_in() @cherrypy.tools.json_out() def POST(self): action = False valid_actions = ['start', 'stop', 'reset', 'restart_cp', 'clear'] if cherrypy.request.json.get('action', False): action = cherrypy.request.json['action'].lower() if action in valid_actions: update = 'No change' if action == 'start' and not self.running: if self.setup.pending_changes: self.report = self.make_initial_report() self.setup.pending_changes = False if len(self.report['tests']) != 1: self.start() update = f'Started Polling ({self.setup.interval} secs)' else: update = 'No Tests: Not starting' if action == 'stop' and self.running: self.stop() update = "Stopped Polling" if action == 'reset': update = "Report Cleared" if self.running: self.stop() update = "Report Cleared and Polling Stopped" self.report = self.make_initial_report() if action == 'clear': restart = False if self.running: self.stop() restart = True time.sleep(self.setup.interval) self.report = self.make_initial_report() if restart: self.start() update = "Report Counters Cleared" if action == 'restart_cp': self.stop() update = "Restarting CherryPy Server" cherrypy.engine.restart() cherrypy.log(f'[II] {update}') response = {} response['message'] = update response['polling'] = self.running return response mess = 'Invalid action requested' cherrypy.log(f'[EE] {mess}') raise cherrypy.HTTPError(400, f'{mess}') def testerCall(self): ''' the function refrenced by the CherryPy background task to update the report ''' self.report = self.tester.run_once(self.report) cherrypy.log( f'[II] Poll Count: {self.report["count"]} - Time {self.report["duration"]}' ) def start(self): ''' Starts Polling by initialising the CherryPy background task and the tester Class and ''' self.tester = OmniPingTester(interval=self.setup.interval) actual_interval = self.tester.interval - self.tester.timeout self.bgtask = BackgroundTask(actual_interval, self.testerCall, bus=cherrypy.engine) self.bgtask.start() self.running = True def stop(self): ''' Stops Polling by deleting the CherryPy background task and the tester. just to keep things clean (hopefully) ''' if self.bgtask: self.bgtask.cancel() self.bgtask = False self.tester = False self.running = False def make_initial_report(self): ''' Construct the report dictionary prior to tests running. ''' def make_test_dictionary(host, desc, test_type, pos): test_dict = {} test_dict['host'] = host test_dict['desc'] = desc test_dict['test'] = test_type.upper() test_dict['good'] = False test_dict['last_stat'] = '--' test_dict['status'] = '--' test_dict['rtt'] = '--' test_dict['total'] = 0 test_dict['total_successes'] = 0 test_dict['success_percent'] = "0.00 %" test_dict['last_good'] = '--' test_dict['last_bad'] = '--' test_dict['last_bad_status'] = '--' test_dict['pos'] = pos return test_dict report = {} report['started'] = False report['time'] = False report['count'] = 0 report['duration'] = 0 report['tests'] = [False] pos = 0 for host, desc, test_type, live in self.setup.tests: if live: report['tests'].append( make_test_dictionary(host, desc, test_type, pos)) pos += 1 return report def make_jsonable_report(self): ''' make sure all elements of the report dictionary are JSON serializable. prior to putting in the response. Also does some formating/ordering ''' response = self.report.copy() if response['duration'] != 0: response['duration'] = '{}'.format(str(response['duration'])[:-4]) if isinstance(response['started'], datetime): response['started'] = response['started'].strftime( '%a %d %b %Y %I:%M:%S %p') if isinstance(response['time'], datetime): response['time'] = response['time'].strftime( '%a %d %b %Y %I:%M:%S %p') new_tests = [{}] * (len(response['tests']) - 1) for test in response['tests']: if not test: continue new_tests[test['pos']] = test response['tests'] = new_tests return response
class VMsModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.caps = CapabilitiesModel(**kargs) self.guests_stats_thread = BackgroundTask(GUESTS_STATS_INTERVAL, self._update_guests_stats) self.guests_stats_thread.start() def _update_guests_stats(self): vm_list = self.get_list() for name in vm_list: try: dom = VMModel.get_vm(name, self.conn) vm_uuid = dom.UUIDString() info = dom.info() state = DOM_STATE_MAP[info[0]] if state != 'running': stats[vm_uuid] = {} continue if stats.get(vm_uuid, None) is None: stats[vm_uuid] = {} timestamp = time.time() prevStats = stats.get(vm_uuid, {}) seconds = timestamp - prevStats.get('timestamp', 0) stats[vm_uuid].update({'timestamp': timestamp}) self._get_percentage_cpu_usage(vm_uuid, info, seconds) self._get_network_io_rate(vm_uuid, dom, seconds) self._get_disk_io_rate(vm_uuid, dom, seconds) except Exception as e: # VM might be deleted just after we get the list. # This is OK, just skip. kimchi_log.debug('Error processing VM stats: %s', e.message) continue def _get_percentage_cpu_usage(self, vm_uuid, info, seconds): prevCpuTime = stats[vm_uuid].get('cputime', 0) cpus = info[3] cpuTime = info[4] - prevCpuTime base = (((cpuTime) * 100.0) / (seconds * 1000.0 * 1000.0 * 1000.0)) percentage = max(0.0, min(100.0, base / cpus)) stats[vm_uuid].update({'cputime': info[4], 'cpu': percentage}) def _get_network_io_rate(self, vm_uuid, dom, seconds): prevNetRxKB = stats[vm_uuid].get('netRxKB', 0) prevNetTxKB = stats[vm_uuid].get('netTxKB', 0) currentMaxNetRate = stats[vm_uuid].get('max_net_io', 100) rx_bytes = 0 tx_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall('devices/interface/target'): dev = target.get('dev') io = dom.interfaceStats(dev) rx_bytes += io[0] tx_bytes += io[4] netRxKB = float(rx_bytes) / 1000 netTxKB = float(tx_bytes) / 1000 rx_stats = (netRxKB - prevNetRxKB) / seconds tx_stats = (netTxKB - prevNetTxKB) / seconds rate = rx_stats + tx_stats max_net_io = round(max(currentMaxNetRate, int(rate)), 1) stats[vm_uuid].update({'net_io': rate, 'max_net_io': max_net_io, 'netRxKB': netRxKB, 'netTxKB': netTxKB}) def _get_disk_io_rate(self, vm_uuid, dom, seconds): prevDiskRdKB = stats[vm_uuid].get('diskRdKB', 0) prevDiskWrKB = stats[vm_uuid].get('diskWrKB', 0) currentMaxDiskRate = stats[vm_uuid].get('max_disk_io', 100) rd_bytes = 0 wr_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall("devices/disk/target"): dev = target.get("dev") io = dom.blockStats(dev) rd_bytes += io[1] wr_bytes += io[3] diskRdKB = float(rd_bytes) / 1024 diskWrKB = float(wr_bytes) / 1024 rd_stats = (diskRdKB - prevDiskRdKB) / seconds wr_stats = (diskWrKB - prevDiskWrKB) / seconds rate = rd_stats + wr_stats max_disk_io = round(max(currentMaxDiskRate, int(rate)), 1) stats[vm_uuid].update({'disk_io': rate, 'max_disk_io': max_disk_io, 'diskRdKB': diskRdKB, 'diskWrKB': diskWrKB}) def create(self, params): conn = self.conn.get() t_name = template_name_from_uri(params['template']) vm_uuid = str(uuid.uuid4()) vm_list = self.get_list() name = get_vm_name(params.get('name'), t_name, vm_list) # incoming text, from js json, is unicode, do not need decode if name in vm_list: raise InvalidOperation("KCHVM0001E", {'name': name}) vm_overrides = dict() pool_uri = params.get('storagepool') if pool_uri: vm_overrides['storagepool'] = pool_uri vm_overrides['fc_host_support'] = self.caps.fc_host_support t = TemplateModel.get_template(t_name, self.objstore, self.conn, vm_overrides) if not self.caps.qemu_stream and t.info.get('iso_stream', False): raise InvalidOperation("KCHVM0005E") t.validate() # Store the icon for displaying later icon = t.info.get('icon') if icon: try: with self.objstore as session: session.store('vm', vm_uuid, {'icon': icon}) except Exception as e: # It is possible to continue Kimchi executions without store # vm icon info kimchi_log.error('Error trying to update database with guest ' 'icon information due error: %s', e.message) # If storagepool is SCSI, volumes will be LUNs and must be passed by # the user from UI or manually. vol_list = [] if t._get_storage_type() not in ["iscsi", "scsi"]: vol_list = t.fork_vm_storage(vm_uuid) graphics = params.get('graphics', {}) stream_protocols = self.caps.libvirt_stream_protocols xml = t.to_vm_xml(name, vm_uuid, libvirt_stream_protocols=stream_protocols, qemu_stream_dns=self.caps.qemu_stream_dns, graphics=graphics, volumes=vol_list) try: conn.defineXML(xml.encode('utf-8')) except libvirt.libvirtError as e: if t._get_storage_type() not in READONLY_POOL_TYPE: for v in vol_list: vol = conn.storageVolLookupByPath(v['path']) vol.delete(0) raise OperationFailed("KCHVM0007E", {'name': name, 'err': e.get_error_message()}) VMModel.vm_update_os_metadata(VMModel.get_vm(name, self.conn), t.info, self.caps.metadata_support) return name def get_list(self): return self.get_vms(self.conn) @staticmethod def get_vms(conn): conn = conn.get() names = [dom.name().decode('utf-8') for dom in conn.listAllDomains(0)] return sorted(names, key=unicode.lower)
class HostStatsModel(object): __metaclass__ = Singleton def __init__(self, **kargs): self.host_stats = defaultdict(list) gbconfig = config.get('gingerbase', {}) self.statshistory_on = gbconfig.get('statshistory_on', True) # create thread to collect statistcs and cache values only if # statshistory_on is enabled in gingerbase.conf if self.statshistory_on: self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self.update_host_stats) self.host_stats_thread.start() def lookup(self, *name): if not self.statshistory_on: self.update_host_stats() return { 'cpu_utilization': self.host_stats['cpu_utilization'][-1], 'memory': self.host_stats['memory'][-1], 'disk_read_rate': self.host_stats['disk_read_rate'][-1], 'disk_write_rate': self.host_stats['disk_write_rate'][-1], 'net_recv_rate': self.host_stats['net_recv_rate'][-1], 'net_sent_rate': self.host_stats['net_sent_rate'][-1] } def update_host_stats(self): preTimeStamp = self.host_stats['timestamp'] timestamp = time.time() # FIXME when we upgrade psutil, we can get uptime by psutil.uptime # we get uptime by float(open("/proc/uptime").readline().split()[0]) # and calculate the first io_rate after the OS started. with open('/proc/uptime') as time_f: seconds = (timestamp - preTimeStamp if preTimeStamp else float( time_f.readline().split()[0])) self.host_stats['timestamp'] = timestamp self._get_host_disk_io_rate(seconds) self._get_host_network_io_rate(seconds) self._get_percentage_host_cpu_usage() self._get_host_memory_stats() # store only 60 stats (1 min) for key, value in self.host_stats.items(): if isinstance(value, list): if len(value) == 60: self.host_stats[key] = value[10:] def _get_percentage_host_cpu_usage(self): # This is cpu usage producer. This producer will calculate the usage # at an interval of HOST_STATS_INTERVAL. # The psutil.cpu_percent works as non blocking. # psutil.cpu_percent maintains a cpu time sample. # It will update the cpu time sample when it is called. # So only this producer can call psutil.cpu_percent in gingerbase. self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None)) def _get_host_memory_stats(self): virt_mem = psutil.virtual_memory() # available: # the actual amount of available memory that can be given # instantly to processes that request more memory in bytes; this # is calculated by summing different memory values depending on # the platform (e.g. free + buffers + cached on Linux) memory_stats = { 'total': virt_mem.total, 'free': virt_mem.free, 'cached': virt_mem.cached, 'buffers': virt_mem.buffers, 'avail': virt_mem.available } self.host_stats['memory'].append(memory_stats) def _get_host_disk_io_rate(self, seconds): disk_read_bytes = self.host_stats['disk_read_bytes'] disk_write_bytes = self.host_stats['disk_write_bytes'] prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0 disk_io = psutil.disk_io_counters(False) read_bytes = disk_io.read_bytes write_bytes = disk_io.write_bytes rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) self.host_stats['disk_read_rate'].append(rd_rate) self.host_stats['disk_write_rate'].append(wr_rate) self.host_stats['disk_read_bytes'].append(read_bytes) self.host_stats['disk_write_bytes'].append(write_bytes) def _get_host_network_io_rate(self, seconds): net_recv_bytes = self.host_stats['net_recv_bytes'] net_sent_bytes = self.host_stats['net_sent_bytes'] prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 net_ios = None if hasattr(psutil, 'net_io_counters'): net_ios = psutil.net_io_counters(True) elif hasattr(psutil, 'network_io_counters'): net_ios = psutil.network_io_counters(True) recv_bytes = 0 sent_bytes = 0 for key in set(self.nics() + self.wlans()) & set(net_ios.keys()): recv_bytes = recv_bytes + net_ios[key].bytes_recv sent_bytes = sent_bytes + net_ios[key].bytes_sent rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) self.host_stats['net_recv_rate'].append(rx_rate) self.host_stats['net_sent_rate'].append(tx_rate) self.host_stats['net_recv_bytes'].append(recv_bytes) self.host_stats['net_sent_bytes'].append(sent_bytes) def wlans(self): WLAN_PATH = '/sys/class/net/*/wireless' return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)] # FIXME if we do not want to list usb nic def nics(self): NIC_PATH = '/sys/class/net/*/device' return list( set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) - set(self.wlans()))
class VMsModel(object): def __init__(self, **kargs): self.conn = kargs['conn'] self.objstore = kargs['objstore'] self.caps = CapabilitiesModel() self.guests_stats_thread = BackgroundTask(GUESTS_STATS_INTERVAL, self._update_guests_stats) self.guests_stats_thread.start() def _update_guests_stats(self): vm_list = self.get_list() for name in vm_list: dom = VMModel.get_vm(name, self.conn) vm_uuid = dom.UUIDString() info = dom.info() state = DOM_STATE_MAP[info[0]] if state != 'running': stats[vm_uuid] = {} continue if stats.get(vm_uuid, None) is None: stats[vm_uuid] = {} timestamp = time.time() prevStats = stats.get(vm_uuid, {}) seconds = timestamp - prevStats.get('timestamp', 0) stats[vm_uuid].update({'timestamp': timestamp}) self._get_percentage_cpu_usage(vm_uuid, info, seconds) self._get_network_io_rate(vm_uuid, dom, seconds) self._get_disk_io_rate(vm_uuid, dom, seconds) def _get_percentage_cpu_usage(self, vm_uuid, info, seconds): prevCpuTime = stats[vm_uuid].get('cputime', 0) cpus = info[3] cpuTime = info[4] - prevCpuTime base = (((cpuTime) * 100.0) / (seconds * 1000.0 * 1000.0 * 1000.0)) percentage = max(0.0, min(100.0, base / cpus)) stats[vm_uuid].update({'cputime': info[4], 'cpu': percentage}) def _get_network_io_rate(self, vm_uuid, dom, seconds): prevNetRxKB = stats[vm_uuid].get('netRxKB', 0) prevNetTxKB = stats[vm_uuid].get('netTxKB', 0) currentMaxNetRate = stats[vm_uuid].get('max_net_io', 100) rx_bytes = 0 tx_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall('devices/interface/target'): dev = target.get('dev') io = dom.interfaceStats(dev) rx_bytes += io[0] tx_bytes += io[4] netRxKB = float(rx_bytes) / 1000 netTxKB = float(tx_bytes) / 1000 rx_stats = (netRxKB - prevNetRxKB) / seconds tx_stats = (netTxKB - prevNetTxKB) / seconds rate = rx_stats + tx_stats max_net_io = round(max(currentMaxNetRate, int(rate)), 1) stats[vm_uuid].update({'net_io': rate, 'max_net_io': max_net_io, 'netRxKB': netRxKB, 'netTxKB': netTxKB}) def _get_disk_io_rate(self, vm_uuid, dom, seconds): prevDiskRdKB = stats[vm_uuid].get('diskRdKB', 0) prevDiskWrKB = stats[vm_uuid].get('diskWrKB', 0) currentMaxDiskRate = stats[vm_uuid].get('max_disk_io', 100) rd_bytes = 0 wr_bytes = 0 tree = ElementTree.fromstring(dom.XMLDesc(0)) for target in tree.findall("devices/disk/target"): dev = target.get("dev") io = dom.blockStats(dev) rd_bytes += io[1] wr_bytes += io[3] diskRdKB = float(rd_bytes) / 1024 diskWrKB = float(wr_bytes) / 1024 rd_stats = (diskRdKB - prevDiskRdKB) / seconds wr_stats = (diskWrKB - prevDiskWrKB) / seconds rate = rd_stats + wr_stats max_disk_io = round(max(currentMaxDiskRate, int(rate)), 1) stats[vm_uuid].update({'disk_io': rate, 'max_disk_io': max_disk_io, 'diskRdKB': diskRdKB, 'diskWrKB': diskWrKB}) def create(self, params): conn = self.conn.get() t_name = template_name_from_uri(params['template']) vm_uuid = str(uuid.uuid4()) vm_list = self.get_list() name = get_vm_name(params.get('name'), t_name, vm_list) # incoming text, from js json, is unicode, do not need decode if name in vm_list: raise InvalidOperation("KCHVM0001E", {'name': name}) vm_overrides = dict() pool_uri = params.get('storagepool') if pool_uri: vm_overrides['storagepool'] = pool_uri t = TemplateModel.get_template(t_name, self.objstore, self.conn, vm_overrides) if not self.caps.qemu_stream and t.info.get('iso_stream', False): raise InvalidOperation("KCHVM0005E") t.validate() vol_list = t.fork_vm_storage(vm_uuid) # Store the icon for displaying later icon = t.info.get('icon') if icon: with self.objstore as session: session.store('vm', vm_uuid, {'icon': icon}) libvirt_stream = False if len(self.caps.libvirt_stream_protocols) == 0: libvirt_stream = True graphics = params.get('graphics') xml = t.to_vm_xml(name, vm_uuid, libvirt_stream=libvirt_stream, qemu_stream_dns=self.caps.qemu_stream_dns, graphics=graphics) try: conn.defineXML(xml.encode('utf-8')) except libvirt.libvirtError as e: for v in vol_list: vol = conn.storageVolLookupByPath(v['path']) vol.delete(0) raise OperationFailed("KCHVM0007E", {'name': name, 'err': e.get_error_message()}) return name def get_list(self): conn = self.conn.get() names = [dom.name().decode('utf-8') for dom in conn.listAllDomains(0)] return sorted(names, key=unicode.lower)
class HostStatsModel(object): __metaclass__ = Singleton def __init__(self, **kargs): self.host_stats = defaultdict(int) self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self._update_host_stats) self.host_stats_thread.start() def lookup(self, *name): return {'cpu_utilization': self.host_stats['cpu_utilization'], 'memory': self.host_stats.get('memory'), 'disk_read_rate': self.host_stats['disk_read_rate'], 'disk_write_rate': self.host_stats['disk_write_rate'], 'net_recv_rate': self.host_stats['net_recv_rate'], 'net_sent_rate': self.host_stats['net_sent_rate']} def _update_host_stats(self): preTimeStamp = self.host_stats['timestamp'] timestamp = time.time() # FIXME when we upgrade psutil, we can get uptime by psutil.uptime # we get uptime by float(open("/proc/uptime").readline().split()[0]) # and calculate the first io_rate after the OS started. seconds = (timestamp - preTimeStamp if preTimeStamp else float(open("/proc/uptime").readline().split()[0])) self.host_stats['timestamp'] = timestamp self._get_host_disk_io_rate(seconds) self._get_host_network_io_rate(seconds) self._get_percentage_host_cpu_usage() self._get_host_memory_stats() def _get_percentage_host_cpu_usage(self): # This is cpu usage producer. This producer will calculate the usage # at an interval of HOST_STATS_INTERVAL. # The psutil.cpu_percent works as non blocking. # psutil.cpu_percent maintains a cpu time sample. # It will update the cpu time sample when it is called. # So only this producer can call psutil.cpu_percent in kimchi. self.host_stats['cpu_utilization'] = psutil.cpu_percent(None) def _get_host_memory_stats(self): virt_mem = psutil.virtual_memory() # available: # the actual amount of available memory that can be given # instantly to processes that request more memory in bytes; this # is calculated by summing different memory values depending on # the platform (e.g. free + buffers + cached on Linux) memory_stats = {'total': virt_mem.total, 'free': virt_mem.free, 'cached': virt_mem.cached, 'buffers': virt_mem.buffers, 'avail': virt_mem.available} self.host_stats['memory'] = memory_stats def _get_host_disk_io_rate(self, seconds): prev_read_bytes = self.host_stats['disk_read_bytes'] prev_write_bytes = self.host_stats['disk_write_bytes'] disk_io = psutil.disk_io_counters(False) read_bytes = disk_io.read_bytes write_bytes = disk_io.write_bytes rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) self.host_stats.update({'disk_read_rate': rd_rate, 'disk_write_rate': wr_rate, 'disk_read_bytes': read_bytes, 'disk_write_bytes': write_bytes}) def _get_host_network_io_rate(self, seconds): prev_recv_bytes = self.host_stats['net_recv_bytes'] prev_sent_bytes = self.host_stats['net_sent_bytes'] net_ios = psutil.network_io_counters(True) recv_bytes = 0 sent_bytes = 0 for key in set(netinfo.nics() + netinfo.wlans()) & set(net_ios.iterkeys()): recv_bytes = recv_bytes + net_ios[key].bytes_recv sent_bytes = sent_bytes + net_ios[key].bytes_sent rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) self.host_stats.update({'net_recv_rate': rx_rate, 'net_sent_rate': tx_rate, 'net_recv_bytes': recv_bytes, 'net_sent_bytes': sent_bytes})
class HostStatsModel(object): __metaclass__ = Singleton def __init__(self, **kargs): self.host_stats = defaultdict(list) self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self._update_host_stats) self.host_stats_thread.start() def lookup(self, *name): return {'cpu_utilization': self.host_stats['cpu_utilization'][-1], 'memory': self.host_stats['memory'][-1], 'disk_read_rate': self.host_stats['disk_read_rate'][-1], 'disk_write_rate': self.host_stats['disk_write_rate'][-1], 'net_recv_rate': self.host_stats['net_recv_rate'][-1], 'net_sent_rate': self.host_stats['net_sent_rate'][-1]} def _update_host_stats(self): preTimeStamp = self.host_stats['timestamp'] timestamp = time.time() # FIXME when we upgrade psutil, we can get uptime by psutil.uptime # we get uptime by float(open("/proc/uptime").readline().split()[0]) # and calculate the first io_rate after the OS started. seconds = (timestamp - preTimeStamp if preTimeStamp else float(open("/proc/uptime").readline().split()[0])) self.host_stats['timestamp'] = timestamp self._get_host_disk_io_rate(seconds) self._get_host_network_io_rate(seconds) self._get_percentage_host_cpu_usage() self._get_host_memory_stats() # store only 60 stats (1 min) for key, value in self.host_stats.iteritems(): if isinstance(value, list): if len(value) == 60: self.host_stats[key] = value[10:] def _get_percentage_host_cpu_usage(self): # This is cpu usage producer. This producer will calculate the usage # at an interval of HOST_STATS_INTERVAL. # The psutil.cpu_percent works as non blocking. # psutil.cpu_percent maintains a cpu time sample. # It will update the cpu time sample when it is called. # So only this producer can call psutil.cpu_percent in kimchi. self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None)) def _get_host_memory_stats(self): virt_mem = psutil.virtual_memory() # available: # the actual amount of available memory that can be given # instantly to processes that request more memory in bytes; this # is calculated by summing different memory values depending on # the platform (e.g. free + buffers + cached on Linux) memory_stats = {'total': virt_mem.total, 'free': virt_mem.free, 'cached': virt_mem.cached, 'buffers': virt_mem.buffers, 'avail': virt_mem.available} self.host_stats['memory'].append(memory_stats) def _get_host_disk_io_rate(self, seconds): disk_read_bytes = self.host_stats['disk_read_bytes'] disk_write_bytes = self.host_stats['disk_write_bytes'] prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0 disk_io = psutil.disk_io_counters(False) read_bytes = disk_io.read_bytes write_bytes = disk_io.write_bytes rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) self.host_stats['disk_read_rate'].append(rd_rate) self.host_stats['disk_write_rate'].append(wr_rate) self.host_stats['disk_read_bytes'].append(read_bytes) self.host_stats['disk_write_bytes'].append(write_bytes) def _get_host_network_io_rate(self, seconds): net_recv_bytes = self.host_stats['net_recv_bytes'] net_sent_bytes = self.host_stats['net_sent_bytes'] prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 net_ios = psutil.network_io_counters(True) recv_bytes = 0 sent_bytes = 0 for key in set(netinfo.nics() + netinfo.wlans()) & set(net_ios.iterkeys()): recv_bytes = recv_bytes + net_ios[key].bytes_recv sent_bytes = sent_bytes + net_ios[key].bytes_sent rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) self.host_stats['net_recv_rate'].append(rx_rate) self.host_stats['net_sent_rate'].append(tx_rate) self.host_stats['net_recv_bytes'].append(recv_bytes) self.host_stats['net_sent_bytes'].append(sent_bytes)
class HostStatsModel(object): __metaclass__ = Singleton def __init__(self, **kargs): self.host_stats = defaultdict(list) self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self._update_host_stats) self.host_stats_thread.start() def lookup(self, *name): return {'cpu_utilization': self.host_stats['cpu_utilization'][-1], 'memory': self.host_stats['memory'][-1], 'disk_read_rate': self.host_stats['disk_read_rate'][-1], 'disk_write_rate': self.host_stats['disk_write_rate'][-1], 'net_recv_rate': self.host_stats['net_recv_rate'][-1], 'net_sent_rate': self.host_stats['net_sent_rate'][-1]} def _update_host_stats(self): preTimeStamp = self.host_stats['timestamp'] timestamp = time.time() # FIXME when we upgrade psutil, we can get uptime by psutil.uptime # we get uptime by float(open("/proc/uptime").readline().split()[0]) # and calculate the first io_rate after the OS started. with open("/proc/uptime") as time_f: seconds = (timestamp - preTimeStamp if preTimeStamp else float(time_f.readline().split()[0])) self.host_stats['timestamp'] = timestamp self._get_host_disk_io_rate(seconds) self._get_host_network_io_rate(seconds) self._get_percentage_host_cpu_usage() self._get_host_memory_stats() # store only 60 stats (1 min) for key, value in self.host_stats.iteritems(): if isinstance(value, list): if len(value) == 60: self.host_stats[key] = value[10:] def _get_percentage_host_cpu_usage(self): # This is cpu usage producer. This producer will calculate the usage # at an interval of HOST_STATS_INTERVAL. # The psutil.cpu_percent works as non blocking. # psutil.cpu_percent maintains a cpu time sample. # It will update the cpu time sample when it is called. # So only this producer can call psutil.cpu_percent in gingerbase. self.host_stats['cpu_utilization'].append(psutil.cpu_percent(None)) def _get_host_memory_stats(self): virt_mem = psutil.virtual_memory() # available: # the actual amount of available memory that can be given # instantly to processes that request more memory in bytes; this # is calculated by summing different memory values depending on # the platform (e.g. free + buffers + cached on Linux) memory_stats = {'total': virt_mem.total, 'free': virt_mem.free, 'cached': virt_mem.cached, 'buffers': virt_mem.buffers, 'avail': virt_mem.available} self.host_stats['memory'].append(memory_stats) def _get_host_disk_io_rate(self, seconds): disk_read_bytes = self.host_stats['disk_read_bytes'] disk_write_bytes = self.host_stats['disk_write_bytes'] prev_read_bytes = disk_read_bytes[-1] if disk_read_bytes else 0 prev_write_bytes = disk_write_bytes[-1] if disk_write_bytes else 0 disk_io = psutil.disk_io_counters(False) read_bytes = disk_io.read_bytes write_bytes = disk_io.write_bytes rd_rate = int(float(read_bytes - prev_read_bytes) / seconds + 0.5) wr_rate = int(float(write_bytes - prev_write_bytes) / seconds + 0.5) self.host_stats['disk_read_rate'].append(rd_rate) self.host_stats['disk_write_rate'].append(wr_rate) self.host_stats['disk_read_bytes'].append(read_bytes) self.host_stats['disk_write_bytes'].append(write_bytes) def _get_host_network_io_rate(self, seconds): net_recv_bytes = self.host_stats['net_recv_bytes'] net_sent_bytes = self.host_stats['net_sent_bytes'] prev_recv_bytes = net_recv_bytes[-1] if net_recv_bytes else 0 prev_sent_bytes = net_sent_bytes[-1] if net_sent_bytes else 0 net_ios = None if hasattr(psutil, 'net_io_counters'): net_ios = psutil.net_io_counters(True) elif hasattr(psutil, 'network_io_counters'): net_ios = psutil.network_io_counters(True) recv_bytes = 0 sent_bytes = 0 for key in set(self.nics() + self.wlans()) & set(net_ios.iterkeys()): recv_bytes = recv_bytes + net_ios[key].bytes_recv sent_bytes = sent_bytes + net_ios[key].bytes_sent rx_rate = int(float(recv_bytes - prev_recv_bytes) / seconds + 0.5) tx_rate = int(float(sent_bytes - prev_sent_bytes) / seconds + 0.5) self.host_stats['net_recv_rate'].append(rx_rate) self.host_stats['net_sent_rate'].append(tx_rate) self.host_stats['net_recv_bytes'].append(recv_bytes) self.host_stats['net_sent_bytes'].append(sent_bytes) def wlans(self): WLAN_PATH = '/sys/class/net/*/wireless' return [b.split('/')[-2] for b in glob.glob(WLAN_PATH)] # FIXME if we do not want to list usb nic def nics(self): NIC_PATH = '/sys/class/net/*/device' return list(set([b.split('/')[-2] for b in glob.glob(NIC_PATH)]) - set(self.wlans()))
def __init__(self, **kargs): self.host_stats = defaultdict(list) self.host_stats_thread = BackgroundTask(HOST_STATS_INTERVAL, self._update_host_stats) self.host_stats_thread.start()
class Model(object): dom_state_map = {0: 'nostate', 1: 'running', 2: 'blocked', 3: 'paused', 4: 'shutdown', 5: 'shutoff', 6: 'crashed'} pool_state_map = {0: 'inactive', 1: 'initializing', 2: 'active', 3: 'degraded', 4: 'inaccessible'} volume_type_map = {0: 'file', 1: 'block', 2: 'directory', 3: 'network'} def __init__(self, libvirt_uri=None, objstore_loc=None): self.libvirt_uri = libvirt_uri or 'qemu:///system' self.conn = LibvirtConnection(self.libvirt_uri) self.objstore = ObjectStore(objstore_loc) self.graphics_ports = {} self.next_taskid = 1 self.stats = {} self.qemu_stream = False self.qemu_stream_dns = False self.libvirt_stream_protocols = [] # Subscribe function to set host capabilities to be run when cherrypy # server is up # It is needed because some features tests depends on the server cherrypy.engine.subscribe('start', self._set_capabilities) self.statsThread = BackgroundTask(STATS_INTERVAL, self._update_stats) self.statsThread.start() self.distros = self._get_distros() if 'qemu:///' in self.libvirt_uri: self._default_pool_check() self._default_network_check() def _default_network_check(self): conn = self.conn.get() xml = """ <network> <name>default</name> <forward mode='nat'/> <bridge name='virbr0' stp='on' delay='0' /> <ip address='192.168.122.1' netmask='255.255.255.0'> <dhcp> <range start='192.168.122.2' end='192.168.122.254' /> </dhcp> </ip> </network> """ try: net = conn.networkLookupByName("default") except libvirt.libvirtError: try: net = conn.networkDefineXML(xml) except libvirt.libvirtError, e: cherrypy.log.error( "Fatal: Cannot create default network because of %s, exit kimchid" % e.message, severity=logging.ERROR) sys.exit(1) if net.isActive() == 0: try: net.create() except libvirt.libvirtError, e: cherrypy.log.error( "Fatal: Cannot activate default network because of %s, exit kimchid" % e.message, severity=logging.ERROR) sys.exit(1)
game = "Offline" elif resp.json( )['response']['players'][0]['personastate'] == 1 or resp.json( )['response']['players'][0]['personastate'] == 5 or resp.json( )['response']['players'][0]['personastate'] == 6: game = "Online" #c.execute("UPDATE games SET game = replace(game,'Away','Offline') WHERE game LIKE 'Away%'") c.execute("INSERT INTO games VALUES ('" + date + "','" + game + "')") conn.commit() conn.close() if __name__ == '__main__': cherrypy.server.socket_host = '0.0.0.0' cherrypy.server.socket_port = 80 if len(sys.argv) > 1: cherrypy.server.socket_port = int(sys.argv[1]) conf = { '/': { 'tools.staticdir.root': os.path.abspath(os.getcwd()) }, '/static': { 'tools.staticdir.on': True, 'tools.staticdir.dir': './public' } } StringGenerator()._task() BackgroundTask(59, StringGenerator()._task, bus=cherrypy.engine).start() cherrypy.quickstart(StringGenerator(), '/', conf)