def GetAllProcesses():
object = "Process"
items, instances = win32pdh.EnumObjectItems(None, None, object,
win32pdh.PERF_DETAIL_WIZARD)
return instances
def get_processes_dict(self):
"""
returns a dictionary that contains all the processes PIDs as keys and names as values.
:return: processes dictionary as described above.
"""
try:
# Getting all processes name
z, proc_name = win32pdh.EnumObjectItems(
None, None, self.process_obj,
win32pdh.PERF_DETAIL_WIZARD) # PERF_DETAIL_WIZARD = 400
instances = {}
for instance in proc_name:
if instance in instances:
instances[instance] += 1
else:
instances[instance] = 1
proc_pid_name = {}
for instance, max_instances in instances.items():
for inum in xrange(max_instances + 1):
try:
hq = win32pdh.OpenQuery(
) # initializes the query handle
path = win32pdh.MakeCounterPath(
(None, self.process_obj, instance, None, inum,
self.item))
counter_handle = win32pdh.AddCounter(
hq, path) # convert counter path to counter handle
win32pdh.CollectQueryData(
hq) # collects data for the counter
type, val = win32pdh.GetFormattedCounterValue(
counter_handle, win32pdh.PDH_FMT_LONG)
proc_pid_name[val] = [instance]
win32pdh.CloseQuery(hq)
except:
raise OSError("Problem getting process id")
return proc_pid_name
except:
try:
from win32com.client import GetObject
WMI = GetObject('winmgmts:') # COM object
proc_instances = WMI.InstancesOf(
'Win32_Process') # WMI instanse
proc_name = [
process.Properties_('Name').Value
for process in proc_instances
] # Get the processess names
proc_id = [
process.Properties_('ProcessId').Value
for process in proc_instances
] # Get the processess names
proc_pid_name = {}
proc_id_counter = 0
for instance in range(len(proc_name)):
proc_pid_name[proc_id[instance]] = [(proc_name[instance])]
proc_id_counter += 1
return proc_pid_name
except:
raise OSError('Counldnt get the process list')
def __init__(self,
class_name,
counter_name,
log,
instance_name=None,
machine_name=None):
self._get_counter_dictionary()
self._class_name = win32pdh.LookupPerfNameByIndex(
None, int(WinPDHCounter.pdh_counter_dict[class_name]))
self._counter_name = win32pdh.LookupPerfNameByIndex(
None, int(WinPDHCounter.pdh_counter_dict[counter_name]))
self._is_single_instance = False
self.hq = win32pdh.OpenQuery()
self.logger = log
self.counterdict = {}
counters, instances = win32pdh.EnumObjectItems(
None, machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD)
if instance_name is None and len(instances) > 0:
for inst in instances:
path = win32pdh.MakeCounterPath(
(machine_name, self._class_name, inst, None, 0,
self._counter_name))
try:
self.counterdict[inst] = win32pdh.AddCounter(self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\%s" %
(self._class_name, self._counter_name))
try:
self.logger.debug("Path: %s\n" % unicode(path))
except: # noqa: E722
# some unicode characters are not translatable here. Don't fail just
# because we couldn't log
self.logger.debug("Failed to log path")
pass
else:
if instance_name is not None:
# check to see that it's valid
if len(instances) <= 0:
self.logger.error(
"%s doesn't seem to be a multi-instance counter, but asked for specific instance %s"
% (class_name, instance_name))
return
if instance_name not in instances:
self.logger.error("%s is not a counter instance in %s" %
(instance_name, class_name))
return
path = win32pdh.MakeCounterPath(
(machine_name, self._class_name, instance_name, None, 0,
self._counter_name))
try:
self.logger.debug("Path: %s\n" % unicode(path))
except: # noqa: E722
# some unicode characters are not translatable here. Don't fail just
# because we couldn't log
self.logger.debug("Failed to log path")
pass
try:
self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter(
self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\%s" %
(self._class_name, self._counter_name))
raise
self._is_single_instance = True
def get_perfmon_counters(metrics_json) -> List[Metric]:
collected_metrics: List[Metric] = []
with query_perfmon() as perfmon_conn:
# A cache so we don't need to ask windows for the list of the instances multiple times
instance_cache: Dict[str, List[str]] = {}
perfmon_metrics = [metric for metric in metrics_json if "source" in metric]
for metric_json in perfmon_metrics:
result_type = "absolute"
if "type" in metric_json["source"]:
result_type = metric_json["source"]["type"]
# This class will hold a list of it's instances
metric = Metric(
metric_json["timeseries"]["key"],
metric_json["source"]["object"],
metric_json["source"]["counter"],
result_type,
metric_json["source"]["process_name"],
)
if metric.obj not in instance_cache:
_, instances = win32pdh.EnumObjectItems(
None, None, metric.obj, win32pdh.PERF_DETAIL_WIZARD
)
else:
instances = instance_cache[metric.obj]
if instances:
for instance in instances:
try:
# Add the query to perfmon, this does not do any metric collection
path = win32pdh.MakeCounterPath(
(None, metric.obj, instance, None, -1, metric.counter)
)
metric.instances.append(
Instance(instance, win32pdh.AddCounter(perfmon_conn, path))
)
except Exception as e:
logger.exception(
f"Could not add the counter {metric.obj} - {metric.counter}: {e}"
)
else:
path = win32pdh.MakeCounterPath(
(None, metric.obj, None, None, -1, metric.counter)
)
metric.instances.append(
Instance("Null", win32pdh.AddCounter(perfmon_conn, path))
)
collected_metrics.append(metric)
# We do this because of % and rate metrics need to be collected twice
# This needs to be reviewed, we cannot collect a subset of datapoints for a 60 second interval
# We cannot also keep the perfmon connection opened across multiple runs (maybe we can?)
try:
win32pdh.CollectQueryData(perfmon_conn)
time.sleep(1)
win32pdh.CollectQueryData(perfmon_conn)
except Exception as e:
logger.exception(f"Failed to collect query data: {e}")
for metric in collected_metrics:
for instance in metric.instances:
_, value = win32pdh.GetFormattedCounterValue(instance.counter_query, win32pdh.PDH_FMT_DOUBLE)
instance.value = value
return collected_metrics
def __init__(self,
class_name,
counter_name,
log,
instance_name=None,
machine_name=None,
precision=None):
self.logger = log
self._get_counter_dictionary()
class_name_index_list = WinPDHCounter.pdh_counter_dict[class_name]
if len(class_name_index_list) == 0:
self.logger.warn(
"Class %s was not in counter name list, attempting english counter"
% class_name)
self._class_name = class_name
else:
if len(class_name_index_list) > 1:
self.logger.warn(
"Class %s had multiple (%d) indices, using first" %
(class_name, len(class_name_index_list)))
self._class_name = win32pdh.LookupPerfNameByIndex(
None, int(class_name_index_list[0]))
self._is_single_instance = False
self.hq = win32pdh.OpenQuery()
self.counterdict = {}
if precision is None:
self._precision = win32pdh.PDH_FMT_DOUBLE
else:
self._precision = precision
counters, instances = win32pdh.EnumObjectItems(
None, machine_name, self._class_name, win32pdh.PERF_DETAIL_WIZARD)
if instance_name is None and len(instances) > 0:
for inst in instances:
path = self._make_counter_path(machine_name, counter_name,
inst, counters)
try:
self.counterdict[inst] = win32pdh.AddCounter(self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\%s" %
(self._class_name, self._counter_name))
try:
self.logger.debug("Path: %s\n" % unicode(path))
except: # noqa: E722
# some unicode characters are not translatable here. Don't fail just
# because we couldn't log
self.logger.debug("Failed to log path")
pass
else:
if instance_name is not None:
# check to see that it's valid
if len(instances) <= 0:
self.logger.error(
"%s doesn't seem to be a multi-instance counter, but asked for specific instance %s"
% (class_name, instance_name))
return
if instance_name not in instances:
self.logger.error("%s is not a counter instance in %s" %
(instance_name, class_name))
return
path = self._make_counter_path(machine_name, counter_name,
instance_name, counters)
try:
self.logger.debug("Path: %s\n" % unicode(path))
except: # noqa: E722
# some unicode characters are not translatable here. Don't fail just
# because we couldn't log
self.logger.debug("Failed to log path")
pass
try:
self.counterdict[SINGLE_INSTANCE_KEY] = win32pdh.AddCounter(
self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\%s" %
(self._class_name, self._counter_name))
raise
self._is_single_instance = True
def objlist(self):
_object = "Process"
items, instances = win32pdh.EnumObjectItems(None,None,_object, win32pdh.PERF_DETAIL_WIZARD)
return instances
def extract(self):
def getQueryValue(data,key):
if key in data:
return data[key]
else:
return None
paths = [
('cpu-total','Processor','_Total','% Processor Time'),
('cpu-us','Processor','_Total','% User Time'),
('cpu-sy','Processor','_Total','% Privileged Time'),
('cpu-r','System',None,'Processor Queue Length'),
('mem-free','Memory',None,'Available Mbytes'),
('mem-pages-ps','Memory',None,'Pages/sec'),
('disk-readtime','PhysicalDisk','_Total','% Disk Read Time'),
('disk-writetime','PhysicalDisk','_Total','% Disk Write Time'),
('disk-queue','PhysicalDisk','_Total','Current Disk Queue Length'),
('disk-rbps','PhysicalDisk','_Total','Disk Read Bytes/sec'),
('disk-wbps','PhysicalDisk','_Total','Disk Write Bytes/sec'),
('disk-rps','PhysicalDisk','_Total','Disk Reads/sec'),
('disk-wps','PhysicalDisk','_Total','Disk Writes/sec'),
('sql-qps','SQLServer:SQL Statistics',None,'Batch Requests/sec'),
('sql-tps','SQLServer:Databases','_Total','Transactions/sec'),
('sql-connections','SQLServer:General Statistics',None,'User Connections'),
('sql-fullscans','SQLServer:Access Methods',None,'Full Scans/sec'),
('sql-targetmemory','SQLServer:Memory Manager',None,'Target Server Memory (KB)'),
('sql-totalmemory','SQLServer:Memory Manager',None,'Total Server Memory (KB)'),
('sql-dataspace','SQLServer:Databases','_Total','Data File(s) Size (KB)'),
('sql-logspace','SQLServer:Databases','_Total','Log File(s) Size (KB)'),
('sql-cachehitratio','SQLServer:Buffer Manager',None,'Buffer cache hit ratio'),
('ws-bps','Web Service','_Total','Bytes Total/sec'),
('ws-rps','Web Service','_Total','Total Method Requests/sec'),
('ws-connections','Web Service','_Total','Current Connections')
#('ni-bps','Network Interface','_Total','Bytes Total/sec')
]
counters, instances = win32pdh.EnumObjectItems(None, None, 'Network Interface', win32pdh.PERF_DETAIL_WIZARD)
#print instances
for i, n in enumerate(instances):
paths.append( ('ni-bps' + str(i),'Network Interface',n,'Bytes Total/sec') )
#print paths
counters = {}
base = win32pdh.OpenQuery()
#print base
for path in paths:
counterPath = win32pdh.MakeCounterPath( (None,path[1],path[2], None, -1, path[3]) )
if win32pdh.ValidatePath(counterPath) == 0:
#print path
counter = win32pdh.AddCounter(base, counterPath)
#print counter
counters[path[0]] = counter
#else:
#print path[0], '------path is not valid'
# collect the data for the query object. We need to collect the query data
# twice to be able to calculate the % Processor Time
win32pdh.CollectQueryData(base)
set1 = self.getProcess()
time.sleep(1)
win32pdh.CollectQueryData(base)
queryData = {}
nibps = 0
# Get the formatted value of the counter
for key in counters.keys():
if key.find('ni-bps') == 0:
nibps += win32pdh.GetFormattedCounterValue(counters[key],win32pdh.PDH_FMT_LONG)[1]
else:
queryData[key] = win32pdh.GetFormattedCounterValue(counters[key],win32pdh.PDH_FMT_LONG)[1]
queryData['ni-bps'] = nibps
win32pdh.CloseQuery(base)
info = {}
cpu = {}
cpu["us"] = getQueryValue(queryData,"cpu-us")
cpu["sy"] = getQueryValue(queryData,"cpu-sy")
cpu["id"] = 100 - float(getQueryValue(queryData,"cpu-total"))
cpu["r"] = getQueryValue(queryData,"cpu-r")
memory = {}
memory["free"] = getQueryValue(queryData,"mem-free")
memory["pages-ps"] = getQueryValue(queryData,"mem-pages-ps")
disk = {}
disk["readtime"] = getQueryValue(queryData,"disk-readtime")
disk["writetime"] = getQueryValue(queryData,"disk-writetime")
disk["queue"] = getQueryValue(queryData,"disk-queue")
disk["rbps"] = getQueryValue(queryData,"disk-rbps")
disk["wbps"] = getQueryValue(queryData,"disk-wbps")
disk["bps"] = disk["rbps"] + disk["wbps"]
disk["rps"] = getQueryValue(queryData,"disk-rps")
disk["wps"] = getQueryValue(queryData,"disk-wps")
disk["iops"] = disk["rps"] + disk["wps"]
freespace,disksize = self.getDiskSize()
#print freespace,disksize
if(disksize > 0):
disk["size"] = disksize/(1024*1024)
disk["used"] = (disksize - freespace) / (1024*1024)
disk["use"] = disk["used"] * 100 / disk["size"]
if("sql-qps" in queryData):
database = {}
sqlserver = {}
sqlserver["qps"] = getQueryValue(queryData,"sql-qps")
sqlserver["tps"] = getQueryValue(queryData,"sql-tps")
sqlserver["connections"] = getQueryValue(queryData,"sql-connections")
sqlserver["fullscans"] = getQueryValue(queryData,"sql-fullscans")
sqlserver["targetmemory"] = getQueryValue(queryData,"sql-targetmemory")
sqlserver["totalmemory"] = getQueryValue(queryData,"sql-totalmemory")
sqlserver["dataspace"] = getQueryValue(queryData,"sql-dataspace")
sqlserver["logspace"] = getQueryValue(queryData,"sql-logspace")
sqlserver["cachehitratio"] = getQueryValue(queryData,"sql-cachehitratio")
database["sqlserver"] = sqlserver
info["database"] = database
webserver = {}
iis = {}
if("ws-bps" in queryData):
iis["bps"] = getQueryValue(queryData,"ws-bps")
iis["rps"] = getQueryValue(queryData,"ws-rps")
iis["connections"] = getQueryValue(queryData,"ws-connections")
#iis["requestqueued"] = float(result[col+3]) #+ float(result[col+4])
webserver['iis'] = iis
info["webserver"] = webserver
network = {}
network["bps"] = getQueryValue(queryData,"ni-bps")
process = []
set2 = self.getProcess()
#self.logger.debug(set1)
#self.logger.debug(set2)
totaltime = set2['total'] - set1['total']
for s in set1:
if s == 'total' : continue
if(set2.has_key(s)):
proc = {}
proc["pid"] = s
if totaltime > 0:
proc["cpu"] = (sum(set2[s][0:2]) - sum(set1[s][0:2]) )* 100 / totaltime
else:
proc["cpu"] = 0
proc["mem"] = set2[s][2]
proc["mem2"] = set2[s][3]
proc["name"] = set2[s][4]
proc["command"] = set2[s][5]
process.append(proc)
process = sorted(process, key=lambda d: d["cpu"])
if len(process) > 10:
process = process[-11:]
info["infoid"] = str(uuid.uuid1())
info["machineid"] = str(uuid.getnode())
info["timestamp"] = time.strftime("%Y%m%d%H%M%S", time.localtime())
info["os"] = "windows"
info["appkey"] = self.AppKey
info["cpu"] = cpu
info["memory"] = memory
info["disk"] = disk
info["network"] = network
info["process"] = process
self.val["info"] = info
#pythoncom.CoUninitialize()
return self.val
def collect_counters(self):
counters, instances = win32pdh.EnumObjectItems(
None, self._machine_name, self.class_name,
win32pdh.PERF_DETAIL_WIZARD)
if self._instance_name is None and len(instances) > 0:
all_instances = set()
for inst in instances:
path = self._make_counter_path(self._machine_name,
self._counter_name, inst,
counters)
if not path:
continue
all_instances.add(inst)
try:
if inst not in self.counterdict:
self.logger.debug('Adding instance `%s`', inst)
self.counterdict[inst] = win32pdh.AddCounter(
self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\\%s" %
(self.class_name, self._counter_name))
expired_instances = set(self.counterdict) - all_instances
for inst in expired_instances:
self.logger.debug('Removing expired instance `%s`', inst)
del self.counterdict[inst]
else:
if self._instance_name is not None:
# check to see that it's valid
if len(instances) <= 0:
self.logger.error(
"%s doesn't seem to be a multi-instance counter, but asked for specific instance %s",
self.class_name,
self._instance_name,
)
raise AttributeError("%s is not a multi-instance counter" %
self.class_name)
if self._instance_name not in instances:
self.logger.error("%s is not a counter instance in %s",
self._instance_name, self.class_name)
raise AttributeError(
"%s is not an instance of %s" %
(self._instance_name, self.class_name))
path = self._make_counter_path(self._machine_name,
self._counter_name,
self._instance_name, counters)
if not path:
self.logger.warning("Empty path returned")
elif win32pdh.ValidatePath(path) != 0:
# Multi-instance counter with no instances presently
pass
else:
try:
if SINGLE_INSTANCE_KEY not in self.counterdict:
self.logger.debug(
'Adding single instance for path `%s`', path)
self.counterdict[
SINGLE_INSTANCE_KEY] = win32pdh.AddCounter(
self.hq, path)
except: # noqa: E722
self.logger.fatal(
"Failed to create counter. No instances of %s\\%s" %
(self.class_name, self._counter_name))
raise
self._is_single_instance = True
#
# Ensures there are more than N processes currently running on the system (default: 50), Python
# Ensures at least N processes running on the system (defaults to 50)
# Module written by Brandon Arvanaghi
# Website: arvanaghi.com
# Twitter: @arvanaghi
#
import sys
import win32pdh
MinimumNumberOfProcesses = 50
if len(sys.argv) == 2:
MinimumNumberOfProcesses = sys.argv[1]
_, runningProcesses = win32pdh.EnumObjectItems(None,None,'process', win32pdh.PERF_DETAIL_WIZARD)
if len(runningProcesses) >= int(MinimumNumberOfProcesses):
print("There are {0} processes running on the system, which satisfies the minimum you set of {1}. Proceed!".format(len(runningProcesses), MinimumNumberOfProcesses))
else:
print("Only {0} processes are running on the system, which is less than the minimum you set of {1}. Do not proceed.".format(len(runningProcesses), MinimumNumberOfProcesses))
# get the instance and instance number for each thread of this process id
threads = self.__win32GetThreads(pid=self.pid, bRefresh=1)
# log the stats in a separate thread
t = threading.Thread(target=self.__win32LogProfile, args=(instance, inum, threads, self.interval, self.file))
t.start()
def stop(self):
"""Stop the process monitor.
"""
self.active = 0
if __name__ == "__main__":
items, instances = win32pdh.EnumObjectItems(None, None, "System", -1)
print("System - available counters are; ")
for item in items: print(item)
print()
items, instances = win32pdh.EnumObjectItems(None, None, "Processor", -1)
print("Processor - available counters are; ")
for item in items: print(item)
print()
items, instances = win32pdh.EnumObjectItems(None, None, "Process", -1)
print("Process - available counters are; ")
for item in items: print(item)
print()
items, instances = win32pdh.EnumObjectItems(None, None, "Thread", -1)