def GetAllProcesses(): object = "Process" items, instances = win32pdh.EnumObjectItems(None, None, object, win32pdh.PERF_DETAIL_WIZARD) return instances
def get_processes_dict(self): """ returns a dictionary that contains all the processes PIDs as keys and names as values. :return: processes dictionary as described above. """ try: # Getting all processes name z, proc_name = win32pdh.EnumObjectItems( None, None, self.process_obj, win32pdh.PERF_DETAIL_WIZARD) # PERF_DETAIL_WIZARD = 400 instances = {} for instance in proc_name: if instance in instances: instances[instance] += 1 else: instances[instance] = 1 proc_pid_name = {} for instance, max_instances in instances.items(): for inum in xrange(max_instances + 1): try: hq = win32pdh.OpenQuery( ) # initializes the query handle path = win32pdh.MakeCounterPath( (None, self.process_obj, instance, None, inum, self.item)) counter_handle = win32pdh.AddCounter( hq, path) # convert counter path to counter handle win32pdh.CollectQueryData( hq) # collects data for the counter type, val = win32pdh.GetFormattedCounterValue( counter_handle, win32pdh.PDH_FMT_LONG) proc_pid_name[val] = [instance] win32pdh.CloseQuery(hq) except: raise OSError("Problem getting process id") return proc_pid_name except: try: from win32com.client import GetObject WMI = GetObject('winmgmts:') # COM object proc_instances = WMI.InstancesOf( 'Win32_Process') # WMI instanse proc_name = [ process.Properties_('Name').Value for process in proc_instances ] # Get the processess names proc_id = [ process.Properties_('ProcessId').Value for process in proc_instances ] # Get the processess names proc_pid_name = {} proc_id_counter = 0 for instance in range(len(proc_name)): proc_pid_name[proc_id[instance]] = [(proc_name[instance])] proc_id_counter += 1 return proc_pid_name except: raise OSError('Counldnt get the process list')
def __init__(self, class_name, counter_name, log, instance_name=None, machine_name=None): self._get_counter_dictionary() self._class_name = win32pdh.LookupPerfNameByIndex( None, int(WinPDHCounter.pdh_counter_dict[class_name])) self._counter_name = win32pdh.LookupPerfNameByIndex( None, int(WinPDHCounter.pdh_counter_dict[counter_name])) self._is_single_instance = False self.hq = win32pdh.OpenQuery() self.logger = log self.counterdict = {} counters, instances = win32pdh.EnumObjectItems( None, machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD) if instance_name is None and len(instances) > 0: for inst in instances: path = win32pdh.MakeCounterPath( (machine_name, self._class_name, inst, None, 0, self._counter_name)) try: self.counterdict[inst] = win32pdh.AddCounter(self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\%s" % (self._class_name, self._counter_name)) try: self.logger.debug("Path: %s\n" % unicode(path)) except: # noqa: E722 # some unicode characters are not translatable here. Don't fail just # because we couldn't log self.logger.debug("Failed to log path") pass else: if instance_name is not None: # check to see that it's valid if len(instances) <= 0: self.logger.error( "%s doesn't seem to be a multi-instance counter, but asked for specific instance %s" % (class_name, instance_name)) return if instance_name not in instances: self.logger.error("%s is not a counter instance in %s" % (instance_name, class_name)) return path = win32pdh.MakeCounterPath( (machine_name, self._class_name, instance_name, None, 0, self._counter_name)) try: self.logger.debug("Path: %s\n" % unicode(path)) except: # noqa: E722 # some unicode characters are not translatable here. Don't fail just # because we couldn't log self.logger.debug("Failed to log path") pass try: self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter( self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\%s" % (self._class_name, self._counter_name)) raise self._is_single_instance = True
def get_perfmon_counters(metrics_json) -> List[Metric]: collected_metrics: List[Metric] = [] with query_perfmon() as perfmon_conn: # A cache so we don't need to ask windows for the list of the instances multiple times instance_cache: Dict[str, List[str]] = {} perfmon_metrics = [metric for metric in metrics_json if "source" in metric] for metric_json in perfmon_metrics: result_type = "absolute" if "type" in metric_json["source"]: result_type = metric_json["source"]["type"] # This class will hold a list of it's instances metric = Metric( metric_json["timeseries"]["key"], metric_json["source"]["object"], metric_json["source"]["counter"], result_type, metric_json["source"]["process_name"], ) if metric.obj not in instance_cache: _, instances = win32pdh.EnumObjectItems( None, None, metric.obj, win32pdh.PERF_DETAIL_WIZARD ) else: instances = instance_cache[metric.obj] if instances: for instance in instances: try: # Add the query to perfmon, this does not do any metric collection path = win32pdh.MakeCounterPath( (None, metric.obj, instance, None, -1, metric.counter) ) metric.instances.append( Instance(instance, win32pdh.AddCounter(perfmon_conn, path)) ) except Exception as e: logger.exception( f"Could not add the counter {metric.obj} - {metric.counter}: {e}" ) else: path = win32pdh.MakeCounterPath( (None, metric.obj, None, None, -1, metric.counter) ) metric.instances.append( Instance("Null", win32pdh.AddCounter(perfmon_conn, path)) ) collected_metrics.append(metric) # We do this because of % and rate metrics need to be collected twice # This needs to be reviewed, we cannot collect a subset of datapoints for a 60 second interval # We cannot also keep the perfmon connection opened across multiple runs (maybe we can?) try: win32pdh.CollectQueryData(perfmon_conn) time.sleep(1) win32pdh.CollectQueryData(perfmon_conn) except Exception as e: logger.exception(f"Failed to collect query data: {e}") for metric in collected_metrics: for instance in metric.instances: _, value = win32pdh.GetFormattedCounterValue(instance.counter_query, win32pdh.PDH_FMT_DOUBLE) instance.value = value return collected_metrics
def __init__(self, class_name, counter_name, log, instance_name=None, machine_name=None, precision=None): self.logger = log self._get_counter_dictionary() class_name_index_list = WinPDHCounter.pdh_counter_dict[class_name] if len(class_name_index_list) == 0: self.logger.warn( "Class %s was not in counter name list, attempting english counter" % class_name) self._class_name = class_name else: if len(class_name_index_list) > 1: self.logger.warn( "Class %s had multiple (%d) indices, using first" % (class_name, len(class_name_index_list))) self._class_name = win32pdh.LookupPerfNameByIndex( None, int(class_name_index_list[0])) self._is_single_instance = False self.hq = win32pdh.OpenQuery() self.counterdict = {} if precision is None: self._precision = win32pdh.PDH_FMT_DOUBLE else: self._precision = precision counters, instances = win32pdh.EnumObjectItems( None, machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD) if instance_name is None and len(instances) > 0: for inst in instances: path = self._make_counter_path(machine_name, counter_name, inst, counters) try: self.counterdict[inst] = win32pdh.AddCounter(self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\%s" % (self._class_name, self._counter_name)) try: self.logger.debug("Path: %s\n" % unicode(path)) except: # noqa: E722 # some unicode characters are not translatable here. Don't fail just # because we couldn't log self.logger.debug("Failed to log path") pass else: if instance_name is not None: # check to see that it's valid if len(instances) <= 0: self.logger.error( "%s doesn't seem to be a multi-instance counter, but asked for specific instance %s" % (class_name, instance_name)) return if instance_name not in instances: self.logger.error("%s is not a counter instance in %s" % (instance_name, class_name)) return path = self._make_counter_path(machine_name, counter_name, instance_name, counters) try: self.logger.debug("Path: %s\n" % unicode(path)) except: # noqa: E722 # some unicode characters are not translatable here. Don't fail just # because we couldn't log self.logger.debug("Failed to log path") pass try: self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter( self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\%s" % (self._class_name, self._counter_name)) raise self._is_single_instance = True
def objlist(self): _object = "Process" items, instances = win32pdh.EnumObjectItems(None,None,_object, win32pdh.PERF_DETAIL_WIZARD) return instances
def extract(self): def getQueryValue(data,key): if key in data: return data[key] else: return None paths = [ ('cpu-total','Processor','_Total','% Processor Time'), ('cpu-us','Processor','_Total','% User Time'), ('cpu-sy','Processor','_Total','% Privileged Time'), ('cpu-r','System',None,'Processor Queue Length'), ('mem-free','Memory',None,'Available Mbytes'), ('mem-pages-ps','Memory',None,'Pages/sec'), ('disk-readtime','PhysicalDisk','_Total','% Disk Read Time'), ('disk-writetime','PhysicalDisk','_Total','% Disk Write Time'), ('disk-queue','PhysicalDisk','_Total','Current Disk Queue Length'), ('disk-rbps','PhysicalDisk','_Total','Disk Read Bytes/sec'), ('disk-wbps','PhysicalDisk','_Total','Disk Write Bytes/sec'), ('disk-rps','PhysicalDisk','_Total','Disk Reads/sec'), ('disk-wps','PhysicalDisk','_Total','Disk Writes/sec'), ('sql-qps','SQLServer:SQL Statistics',None,'Batch Requests/sec'), ('sql-tps','SQLServer:Databases','_Total','Transactions/sec'), ('sql-connections','SQLServer:General Statistics',None,'User Connections'), ('sql-fullscans','SQLServer:Access Methods',None,'Full Scans/sec'), ('sql-targetmemory','SQLServer:Memory Manager',None,'Target Server Memory (KB)'), ('sql-totalmemory','SQLServer:Memory Manager',None,'Total Server Memory (KB)'), ('sql-dataspace','SQLServer:Databases','_Total','Data File(s) Size (KB)'), ('sql-logspace','SQLServer:Databases','_Total','Log File(s) Size (KB)'), ('sql-cachehitratio','SQLServer:Buffer Manager',None,'Buffer cache hit ratio'), ('ws-bps','Web Service','_Total','Bytes Total/sec'), ('ws-rps','Web Service','_Total','Total Method Requests/sec'), ('ws-connections','Web Service','_Total','Current Connections') #('ni-bps','Network Interface','_Total','Bytes Total/sec') ] counters, instances = win32pdh.EnumObjectItems(None, None, 'Network Interface', win32pdh.PERF_DETAIL_WIZARD) #print instances for i, n in enumerate(instances): paths.append( ('ni-bps' + str(i),'Network Interface',n,'Bytes Total/sec') ) #print paths counters = {} base = win32pdh.OpenQuery() #print base for path in paths: counterPath = win32pdh.MakeCounterPath( (None,path[1],path[2], None, -1, path[3]) ) if win32pdh.ValidatePath(counterPath) == 0: #print path counter = win32pdh.AddCounter(base, counterPath) #print counter counters[path[0]] = counter #else: #print path[0], '------path is not valid' # collect the data for the query object. We need to collect the query data # twice to be able to calculate the % Processor Time win32pdh.CollectQueryData(base) set1 = self.getProcess() time.sleep(1) win32pdh.CollectQueryData(base) queryData = {} nibps = 0 # Get the formatted value of the counter for key in counters.keys(): if key.find('ni-bps') == 0: nibps += win32pdh.GetFormattedCounterValue(counters[key],win32pdh.PDH_FMT_LONG)[1] else: queryData[key] = win32pdh.GetFormattedCounterValue(counters[key],win32pdh.PDH_FMT_LONG)[1] queryData['ni-bps'] = nibps win32pdh.CloseQuery(base) info = {} cpu = {} cpu["us"] = getQueryValue(queryData,"cpu-us") cpu["sy"] = getQueryValue(queryData,"cpu-sy") cpu["id"] = 100 - float(getQueryValue(queryData,"cpu-total")) cpu["r"] = getQueryValue(queryData,"cpu-r") memory = {} memory["free"] = getQueryValue(queryData,"mem-free") memory["pages-ps"] = getQueryValue(queryData,"mem-pages-ps") disk = {} disk["readtime"] = getQueryValue(queryData,"disk-readtime") disk["writetime"] = getQueryValue(queryData,"disk-writetime") disk["queue"] = getQueryValue(queryData,"disk-queue") disk["rbps"] = getQueryValue(queryData,"disk-rbps") disk["wbps"] = getQueryValue(queryData,"disk-wbps") disk["bps"] = disk["rbps"] + disk["wbps"] disk["rps"] = getQueryValue(queryData,"disk-rps") disk["wps"] = getQueryValue(queryData,"disk-wps") disk["iops"] = disk["rps"] + disk["wps"] freespace,disksize = self.getDiskSize() #print freespace,disksize if(disksize > 0): disk["size"] = disksize/(1024*1024) disk["used"] = (disksize - freespace) / (1024*1024) disk["use"] = disk["used"] * 100 / disk["size"] if("sql-qps" in queryData): database = {} sqlserver = {} sqlserver["qps"] = getQueryValue(queryData,"sql-qps") sqlserver["tps"] = getQueryValue(queryData,"sql-tps") sqlserver["connections"] = getQueryValue(queryData,"sql-connections") sqlserver["fullscans"] = getQueryValue(queryData,"sql-fullscans") sqlserver["targetmemory"] = getQueryValue(queryData,"sql-targetmemory") sqlserver["totalmemory"] = getQueryValue(queryData,"sql-totalmemory") sqlserver["dataspace"] = getQueryValue(queryData,"sql-dataspace") sqlserver["logspace"] = getQueryValue(queryData,"sql-logspace") sqlserver["cachehitratio"] = getQueryValue(queryData,"sql-cachehitratio") database["sqlserver"] = sqlserver info["database"] = database webserver = {} iis = {} if("ws-bps" in queryData): iis["bps"] = getQueryValue(queryData,"ws-bps") iis["rps"] = getQueryValue(queryData,"ws-rps") iis["connections"] = getQueryValue(queryData,"ws-connections") #iis["requestqueued"] = float(result[col+3]) #+ float(result[col+4]) webserver['iis'] = iis info["webserver"] = webserver network = {} network["bps"] = getQueryValue(queryData,"ni-bps") process = [] set2 = self.getProcess() #self.logger.debug(set1) #self.logger.debug(set2) totaltime = set2['total'] - set1['total'] for s in set1: if s == 'total' : continue if(set2.has_key(s)): proc = {} proc["pid"] = s if totaltime > 0: proc["cpu"] = (sum(set2[s][0:2]) - sum(set1[s][0:2]) )* 100 / totaltime else: proc["cpu"] = 0 proc["mem"] = set2[s][2] proc["mem2"] = set2[s][3] proc["name"] = set2[s][4] proc["command"] = set2[s][5] process.append(proc) process = sorted(process, key=lambda d: d["cpu"]) if len(process) > 10: process = process[-11:] info["infoid"] = str(uuid.uuid1()) info["machineid"] = str(uuid.getnode()) info["timestamp"] = time.strftime("%Y%m%d%H%M%S", time.localtime()) info["os"] = "windows" info["appkey"] = self.AppKey info["cpu"] = cpu info["memory"] = memory info["disk"] = disk info["network"] = network info["process"] = process self.val["info"] = info #pythoncom.CoUninitialize() return self.val
def collect_counters(self): counters, instances = win32pdh.EnumObjectItems( None, self._machine_name, self.class_name, win32pdh.PERF_DETAIL_WIZARD) if self._instance_name is None and len(instances) > 0: all_instances = set() for inst in instances: path = self._make_counter_path(self._machine_name, self._counter_name, inst, counters) if not path: continue all_instances.add(inst) try: if inst not in self.counterdict: self.logger.debug('Adding instance `%s`', inst) self.counterdict[inst] = win32pdh.AddCounter( self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\\%s" % (self.class_name, self._counter_name)) expired_instances = set(self.counterdict) - all_instances for inst in expired_instances: self.logger.debug('Removing expired instance `%s`', inst) del self.counterdict[inst] else: if self._instance_name is not None: # check to see that it's valid if len(instances) <= 0: self.logger.error( "%s doesn't seem to be a multi-instance counter, but asked for specific instance %s", self.class_name, self._instance_name, ) raise AttributeError("%s is not a multi-instance counter" % self.class_name) if self._instance_name not in instances: self.logger.error("%s is not a counter instance in %s", self._instance_name, self.class_name) raise AttributeError( "%s is not an instance of %s" % (self._instance_name, self.class_name)) path = self._make_counter_path(self._machine_name, self._counter_name, self._instance_name, counters) if not path: self.logger.warning("Empty path returned") elif win32pdh.ValidatePath(path) != 0: # Multi-instance counter with no instances presently pass else: try: if SINGLE_INSTANCE_KEY not in self.counterdict: self.logger.debug( 'Adding single instance for path `%s`', path) self.counterdict[ SINGLE_INSTANCE_KEY] = win32pdh.AddCounter( self.hq, path) except: # noqa: E722 self.logger.fatal( "Failed to create counter. No instances of %s\\%s" % (self.class_name, self._counter_name)) raise self._is_single_instance = True
# # Ensures there are more than N processes currently running on the system (default: 50), Python # Ensures at least N processes running on the system (defaults to 50) # Module written by Brandon Arvanaghi # Website: arvanaghi.com # Twitter: @arvanaghi # import sys import win32pdh MinimumNumberOfProcesses = 50 if len(sys.argv) == 2: MinimumNumberOfProcesses = sys.argv[1] _, runningProcesses = win32pdh.EnumObjectItems(None,None,'process', win32pdh.PERF_DETAIL_WIZARD) if len(runningProcesses) >= int(MinimumNumberOfProcesses): print("There are {0} processes running on the system, which satisfies the minimum you set of {1}. Proceed!".format(len(runningProcesses), MinimumNumberOfProcesses)) else: print("Only {0} processes are running on the system, which is less than the minimum you set of {1}. Do not proceed.".format(len(runningProcesses), MinimumNumberOfProcesses))
# get the instance and instance number for each thread of this process id threads = self.__win32GetThreads(pid=self.pid, bRefresh=1) # log the stats in a separate thread t = threading.Thread(target=self.__win32LogProfile, args=(instance, inum, threads, self.interval, self.file)) t.start() def stop(self): """Stop the process monitor. """ self.active = 0 if __name__ == "__main__": items, instances = win32pdh.EnumObjectItems(None, None, "System", -1) print("System - available counters are; ") for item in items: print(item) print() items, instances = win32pdh.EnumObjectItems(None, None, "Processor", -1) print("Processor - available counters are; ") for item in items: print(item) print() items, instances = win32pdh.EnumObjectItems(None, None, "Process", -1) print("Process - available counters are; ") for item in items: print(item) print() items, instances = win32pdh.EnumObjectItems(None, None, "Thread", -1)