def _query_counters(self):
# Refresh the list of performance objects, see:
# https://docs.microsoft.com/en-us/windows/win32/api/pdh/nf-pdh-pdhenumobjectitemsa#remarks
try:
# https://docs.microsoft.com/en-us/windows/win32/api/pdh/nf-pdh-pdhenumobjectsa
# https://mhammond.github.io/pywin32/win32pdh__EnumObjects_meth.html
win32pdh.EnumObjects(None, self._connection.server,
win32pdh.PERF_DETAIL_WIZARD, True)
except pywintypes.error as error:
message = 'Error refreshing performance objects: {}'.format(
error.strerror)
self.submit_health_check(self.CRITICAL, message=message)
self.log.error(message)
return
# Avoid collection of performance objects that failed to refresh
collection_queue = []
for perf_object in self.perf_objects:
self.log.debug('Refreshing counters for performance object: %s',
perf_object.name)
try:
perf_object.refresh()
except ConfigurationError as e:
# Counters are lazily configured and any errors should prevent check execution
exception_class = type(e)
message = str(e)
self.check_initializations.append(
lambda: raise_exception(exception_class, message))
return
except Exception as e:
self.log.error(
'Error refreshing counters for performance object `%s`: %s',
perf_object.name, e)
else:
collection_queue.append(perf_object)
try:
# https://docs.microsoft.com/en-us/windows/win32/api/pdh/nf-pdh-pdhcollectquerydata
# https://mhammond.github.io/pywin32/win32pdh__CollectQueryData_meth.html
win32pdh.CollectQueryData(self._connection.query_handle)
except pywintypes.error as error:
message = 'Error querying performance counters: {}'.format(
error.strerror)
self.submit_health_check(self.CRITICAL, message=message)
self.log.error(message)
return
for perf_object in collection_queue:
self.log.debug('Collecting query data for performance object: %s',
perf_object.name)
try:
perf_object.collect()
except Exception as e:
self.log.error(
'Error collecting query data for performance object `%s`: %s',
perf_object.name, e)
self.submit_health_check(self.OK)
def __init__(self,
ffprocess,
process,
counters=None,
childProcess="plugin-container"):
self.ffprocess = ffprocess
self.childProcess = childProcess
self.registeredCounters = {}
self.registerCounters(counters)
# PDH might need to be "refreshed" if it has been queried while the browser
# is closed
win32pdh.EnumObjects(None, None, 0, 1)
# Add the counter path for the default process.
for counter in self.registeredCounters:
path = win32pdh.MakeCounterPath(
(None, 'process', process, None, -1, counter))
hq = win32pdh.OpenQuery()
try:
hc = win32pdh.AddCounter(hq, path)
except:
win32pdh.CloseQuery(hq)
#assume that this is a memory counter for the system, not a process counter
path = win32pdh.MakeCounterPath(
(None, 'Memory', None, None, -1, counter))
hq = win32pdh.OpenQuery()
try:
hc = win32pdh.AddCounter(hq, path)
except:
win32pdh.CloseQuery(hq)
self.registeredCounters[counter] = [hq, [(hc, path)]]
self.updateCounterPathsForChildProcesses(counter)
def FindPerformanceAttributesByName(instanceName, object=None,
counter=None,
format=win32pdh.PDH_FMT_LONG,
machine=None, bRefresh=0):
"""Find peformance attributes by (case insensitive) instance name.
Given a process name, return a list with the requested attributes.
Most useful for returning a tuple of PIDs given a process name.
"""
if object is None: object = find_pdh_counter_localized_name("Process", machine)
if counter is None: counter = find_pdh_counter_localized_name("ID Process", machine)
if bRefresh: # PDH docs say this is how you do a refresh.
win32pdh.EnumObjects(None, machine, 0, 1)
instanceName = instanceName.lower()
items, instances = win32pdh.EnumObjectItems(None, None, object, -1)
# Track multiple instances.
instance_dict = {}
for instance in instances:
try:
instance_dict[instance] = instance_dict[instance] + 1
except KeyError:
instance_dict[instance] = 0
ret = []
for instance, max_instances in instance_dict.items():
for inum in range(max_instances + 1):
if instance.lower() == instanceName:
ret.append(GetPerformanceAttributes(object, counter,
instance, inum, format,
machine))
return ret
def __init__(self, process, counters=None):
self.process = process
self.registeredCounters = {}
self.registerCounters(counters)
# PDH might need to be "refreshed" if it has been queried while Firefox
# is closed
win32pdh.EnumObjects(None, None, 0, 1)
def GetProcesses():
win32pdh.EnumObjects(None, None, win32pdh.PERF_DETAIL_WIZARD)
junk, instances = win32pdh.EnumObjectItems(None,None,'Process', win32pdh.PERF_DETAIL_WIZARD )
proc_dict = {}
for instance in instances:
if proc_dict.has_key(instance):
proc_dict[instance] = proc_dict[instance] + 1
else:
proc_dict[instance]=0
proc_ids = []
for instance, max_instances in proc_dict.items():
for inum in xrange(max_instances+1):
hq = win32pdh.OpenQuery() # initializes the query handle
try:
path = win32pdh.MakeCounterPath( (None, 'Process', instance, None, inum, 'ID Process') )
counter_handle=win32pdh.AddCounter(hq, path) #convert counter path to counter handle
try:
win32pdh.CollectQueryData(hq) #collects data for the counter
type, val = win32pdh.GetFormattedCounterValue(counter_handle, win32pdh.PDH_FMT_LONG)
proc_ids.append((instance, val))
except win32pdh.error, e:
print e
win32pdh.RemoveCounter(counter_handle)
except win32pdh.error, e:
print e
win32pdh.CloseQuery (hq)
def __win32GetInstance(self, pid, bRefresh=0):
if bRefresh: win32pdh.EnumObjects(None, None, 0, 1)
# get the list of processes running
items, instances = win32pdh.EnumObjectItems(None, None, "Process", -1)
# convert to a dictionary of process instance, to number of instances
instanceDict = {}
for i in instances:
try: instanceDict[i] = instanceDict[i] + 1
except KeyError: instanceDict[i] = 0
# loop through to locate the instance and inum of the supplied pid
instance = None
inum = -1
for instance, numInstances in list(instanceDict.items()):
for inum in range(numInstances+1):
try:
value = self.__win32getProfileAttribute("Process", instance, inum, "ID Process")
if value == pid:
return instance, inum
except Exception:
pass
return instance, inum
def updateCounterPathsForChildProcesses(self, counter):
# Create a counter path for each instance of the child process that
# is running. If any of these paths are not in our counter list,
# add them to our counter query and append them to the counter list,
# so that we'll begin tracking their statistics. We don't need to
# worry about removing invalid paths from the list, as getCounterValue()
# will generate a value of 0 for those.
hq = self.registeredCounters[counter][0]
win32pdh.EnumObjects(None, None, 0, 1)
counterListLength = len(self.registeredCounters[counter][1])
try:
expandedCounterPaths = \
win32pdh.ExpandCounterPath('\\process(%s*)\\%s' % (self.childProcess, counter))
except:
return
for expandedPath in expandedCounterPaths:
alreadyInCounterList = False
for singleCounter in self.registeredCounters[counter][1]:
if expandedPath == singleCounter[1]:
alreadyInCounterList = True
if not alreadyInCounterList:
try:
counterHandle = win32pdh.AddCounter(hq, expandedPath)
self.registeredCounters[counter][1].append(
(counterHandle, expandedPath))
except:
continue
if counterListLength != len(self.registeredCounters[counter][1]):
try:
win32pdh.CollectQueryData(hq)
except:
return
def getpidsforprocess(processname):
import win32pdh, win32pdhutil
win32pdh.EnumObjects(None, None, 0, 1) # refresh internal cache
pids = win32pdhutil.FindPerformanceAttributesByName(processname,
"Process",
"ID Process")
return pids
def list_objects():
'''
Get a list of available counter objects on the system
Returns:
list: A list of counter objects
'''
return sorted(win32pdh.EnumObjects(None, None, -1, 0))
def _GetMozillaPIDs(exename):
if sys.platform.startswith("win"):
win32pdh.EnumObjects(None, None, 0, 1) # refresh internal cache
pids = win32pdhutil.FindPerformanceAttributesByName(
exename, "Process", "ID Process")
return pids
else:
raise "Don't know how to list the mozilla PIDs yet on this OS."
def getProcessInstance(self, pid):
'''
Get the process instance name using pid.
'''
hq = None
counter_handle = None
try:
win32pdh.EnumObjects(None, None, win32pdh.PERF_DETAIL_WIZARD)
junk, instances = win32pdh.EnumObjectItems(
None, None, 'Process', win32pdh.PERF_DETAIL_WIZARD)
proc_dict = {}
for instance in instances:
if proc_dict.has_key(instance):
proc_dict[instance] = proc_dict[instance] + 1
else:
proc_dict[instance] = 0
proc_ids = []
for instance, max_instances in proc_dict.items():
for inum in xrange(max_instances + 1):
hq = win32pdh.OpenQuery(
) # initializes the query handle
try:
path = win32pdh.MakeCounterPath(
(None, 'Process', instance, None, inum,
'ID Process'))
counter_handle = win32pdh.AddCounter(
hq,
path) #convert counter path to counter handle
try:
win32pdh.CollectQueryData(
hq) #collects data for the counter
type, val = win32pdh.GetFormattedCounterValue(
counter_handle, win32pdh.PDH_FMT_LONG)
proc_ids.append((instance, val))
if val == pid:
return "%s#%d" % (instance, inum)
except win32pdh.error, e:
#print e
pass
win32pdh.RemoveCounter(counter_handle)
counter_handle = None
except win32pdh.error, e:
#print e
pass
def startMonitor(self):
# PDH might need to be "refreshed" if it has been queried while Firefox
# is closed
win32pdh.EnumObjects(None, None, 0, 1)
for counter in self.registeredCounters:
path = win32pdh.MakeCounterPath(
(None, 'process', self.process, None, -1, counter))
hq = win32pdh.OpenQuery()
try:
hc = win32pdh.AddCounter(hq, path)
except:
win32pdh.CloseQuery(hq)
self.registeredCounters[counter] = [hq, hc]
def TerminateAllProcesses(self, *process_names):
"""Helper function to terminate all processes with the given process name
Args:
process_name: String or strings containing the process name, i.e. "firefox"
"""
for process_name in process_names:
# Get all the process ids of running instances of this process, and terminate them.
try:
# refresh list of processes
win32pdh.EnumObjects(None, None, 0, 1)
pids = win32pdhutil.FindPerformanceAttributesByName(process_name, counter="ID Process")
for pid in pids:
self.TerminateProcess(pid)
except:
# Might get an exception if there are no instances of the process running.
continue
def ProcessesWithNameExist(self, *process_names):
"""Returns true if there are any processes running with the
given name. Useful to check whether a Browser process is still running
Args:
process_name: String or strings containing the process name, i.e. "firefox"
Returns:
True if any processes with that name are running, False otherwise.
"""
for process_name in process_names:
try:
# refresh list of processes
win32pdh.EnumObjects(None, None, 0, 1)
pids = win32pdhutil.FindPerformanceAttributesByName(process_name, counter="ID Process")
if len(pids) > 0:
return True
except:
# Might get an exception if there are no instances of the process running.
continue
return False
def __win32GetThreads(self, pid, bRefresh=0):
if bRefresh: win32pdh.EnumObjects(None, None, 0, 1)
# get the list of threads running
items, instances = win32pdh.EnumObjectItems(None, None, "Thread", -1)
# convert to a dictionary of thread instance, to number of instances
instanceNum = []
instanceDict = {}
for i in instances:
try: instanceDict[i] = instanceDict[i] + 1
except KeyError: instanceDict[i] = 0
instanceNum.append(instanceDict[i])
# loop through to locate the instance and inum of each thread for the supplied process id
threads=[]
for i in range(0, len(instances)):
try:
value = self.__win32getProfileAttribute("Thread", instances[i], instanceNum[i], "ID Process")
if value == pid: threads.append((instances[i], instanceNum[i]))
except Exception:
pass
return threads
def browse(counterset):
"""
Explore performance counters.
You'll need to install pywin32 manually beforehand.
"""
# Leave imports in function to not add the dependencies
import win32pdh
if not counterset:
echo_info('Searching for available countersets:')
countersets = sorted(
win32pdh.EnumObjects(None, None, win32pdh.PERF_DETAIL_WIZARD,
True))
for name in countersets:
echo_info(name)
return
description_prefix = ' Description: '
description_indent = ' ' * len(description_prefix)
def display_counter(handle):
counter_info = win32pdh.GetCounterInfo(handle, True)
counter_description = counter_info[-1]
counter_type = counter_info[0]
if counter_type in COUNTER_TYPES:
counter_type_name = COUNTER_TYPES[counter_type][0]
else:
counter_type_name = 'unknown'
echo_info(f'--> {counter}')
echo_info(f' Type: {counter_type_name}')
echo_info(description_prefix, nl=False)
echo_info(
textwrap.indent(textwrap.fill(counter_description),
description_indent).lstrip())
query_handle = win32pdh.OpenQuery()
try:
header = f'<<< {counterset} >>>'
echo_info(header)
echo_info('-' * len(header))
echo_info('')
counters, instances = win32pdh.EnumObjectItems(
None, None, counterset, win32pdh.PERF_DETAIL_WIZARD)
counters.sort()
if instances:
header = 'Instances'
echo_info(header)
echo_info('=' * len(header))
instance_index = defaultdict(int)
for instance in instances:
instance_index[instance] += 1
echo_info(instance)
echo_info('')
header = 'Counters'
echo_info(header)
echo_info('=' * len(header))
for counter in counters:
for instance, num_instances in instance_index.items():
for index in range(num_instances):
path = win32pdh.MakeCounterPath(
(None, counterset, instance, None, index, counter))
counter_handle = win32pdh.AddCounter(
query_handle, path)
display_counter(counter_handle)
# Only need information from one instance
break
break
else:
header = 'Counters'
echo_info(header)
echo_info('=' * len(header))
for counter in counters:
path = win32pdh.MakeCounterPath(
(None, counterset, None, None, 0, counter))
if win32pdh.ValidatePath(path) != 0:
echo_info(f'--> {counter}')
echo_info(' Error: no current instances')
continue
counter_handle = win32pdh.AddCounter(query_handle, path)
display_counter(counter_handle)
finally:
win32pdh.CloseQuery(query_handle)
def RunPltTests(source_profile_dir, profile_configs, num_cycles, counters,
resolution):
"""Runs the Page Load tests with profiles created from the given
base diectory and list of configuations.
Args:
source_profile_dir: Full path to base directory to copy profile from.
profile_configs: Array of configuration options for each profile.
These are of the format:
[{prefname:prevalue,prefname2:prefvalue2},
{extensionguid:path_to_extension}],[{prefname...
num_cycles: Number of times to cycle through all the urls on each test.
counters: Array of counters ("% Processor Time", "Working Set", etc)
See http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
for a list of available counters and their descriptions.
resolution: Time (in seconds) between collecting counters
Returns:
A tuple containing:
An array of plt results for each run. For example:
["mean: 150.30\nstdd:34.2", "mean 170.33\nstdd:22.4"]
An array of counter data from each run. For example:
[{"counter1": [1, 2, 3], "counter2":[4,5,6]},
{"counter1":[1,3,5], "counter2":[2,4,6]}]
"""
counter_data = []
plt_results = []
for config in profile_configs:
# Create the new profile
profile_dir = ffprofile.CreateTempProfileDir(source_profile_dir,
config[0], config[1])
# Run Firefox once with new profile so initializing it doesn't cause
# a performance hit, and the second Firefox that gets created is properly
# terminated.
ffprofile.InitializeNewProfile(config[2], profile_dir)
ffprocess.SyncAndSleep()
# Run the plt test for this profile
timeout = 300
total_time = 0
output = ''
url = paths.TP_URL + '?cycles=' + str(num_cycles)
command_line = ffprocess.GenerateFirefoxCommandLine(
config[2], profile_dir, url)
handle = os.popen(command_line)
# PDH might need to be "refreshed" if it has been queried while
# Firefox is closed.
win32pdh.EnumObjects(None, None, 0, 1)
# Initialize counts
counts = {}
counter_handles = {}
for counter in counters:
counts[counter] = []
counter_handles[counter] = AddCounter(counter)
while total_time < timeout:
# Sleep for [resolution] seconds
time.sleep(resolution)
total_time += resolution
# Get the output from all the possible counters
for count_type in counters:
val = GetCounterValue(counter_handles[count_type][0],
counter_handles[count_type][1])
if (val):
# Sometimes the first sample can be None, or PLT test will have
# closed Firefox by this point. Only count real values.
counts[count_type].append(val)
# Check to see if page load times were outputted
(bytes,
current_output) = ffprocess.NonBlockingReadProcessOutput(handle)
output += current_output
match = TP_REGEX.search(output)
if match:
plt_results.append(match.group(1))
break
# Cleanup counters
for counter in counters:
CleanupCounter(counter_handles[counter][0],
counter_handles[counter][1])
# Firefox should have exited cleanly, but close it if it doesn't
# after 2 seconds.
time.sleep(2)
ffprocess.TerminateAllProcesses("firefox")
ffprocess.SyncAndSleep()
# Delete the temp profile directory Make it writeable first,
# because every once in a while Firefox seems to drop a read-only
# file into it.
ffprofile.MakeDirectoryContentsWritable(profile_dir)
shutil.rmtree(profile_dir)
counter_data.append(counts)
return (plt_results, counter_data)
