def test_as_dict(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
monitor.begin_phase('phase1')
monitor.record_event('foo')
time.sleep(0.1)
monitor.begin_phase('phase2')
monitor.record_event('bar')
time.sleep(0.2)
monitor.finish_phase('phase1')
time.sleep(0.2)
monitor.finish_phase('phase2')
time.sleep(0.4)
monitor.stop()
d = monitor.as_dict()
self.assertEqual(d['version'], 2)
self.assertEqual(len(d['events']), 2)
self.assertEqual(len(d['phases']), 2)
self.assertIn('system', d)
self.assertIsInstance(d['system'], dict)
self.assertIsInstance(d['overall'], dict)
self.assertIn('duration', d['overall'])
self.assertIn('cpu_times', d['overall'])
def test_as_dict(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
monitor.begin_phase('phase1')
monitor.record_event('foo')
time.sleep(0.1)
monitor.begin_phase('phase2')
monitor.record_event('bar')
time.sleep(0.2)
monitor.finish_phase('phase1')
time.sleep(0.2)
monitor.finish_phase('phase2')
time.sleep(0.4)
monitor.stop()
d = monitor.as_dict()
self.assertEqual(d['version'], 2)
self.assertEqual(len(d['events']), 2)
self.assertEqual(len(d['phases']), 2)
self.assertIn('system', d)
self.assertIsInstance(d['system'], dict)
self.assertIsInstance(d['overall'], dict)
self.assertIn('duration', d['overall'])
self.assertIn('cpu_times', d['overall'])
def test_as_dict(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
monitor.begin_phase("phase1")
monitor.record_event("foo")
time.sleep(0.1)
monitor.begin_phase("phase2")
monitor.record_event("bar")
time.sleep(0.2)
monitor.finish_phase("phase1")
time.sleep(0.2)
monitor.finish_phase("phase2")
time.sleep(0.4)
monitor.stop()
d = monitor.as_dict()
self.assertEqual(d["version"], 2)
self.assertEqual(len(d["events"]), 2)
self.assertEqual(len(d["phases"]), 2)
self.assertIn("system", d)
self.assertIsInstance(d["system"], dict)
self.assertIsInstance(d["overall"], dict)
self.assertIn("duration", d["overall"])
self.assertIn("cpu_times", d["overall"])
def test_as_dict(self):
monitor = SystemResourceMonitor(poll_interval=0.25)
monitor.start()
time.sleep(0.1)
monitor.begin_phase('phase1')
monitor.record_event('foo')
time.sleep(0.1)
monitor.begin_phase('phase2')
monitor.record_event('bar')
time.sleep(0.2)
monitor.finish_phase('phase1')
time.sleep(0.2)
monitor.finish_phase('phase2')
time.sleep(0.4)
monitor.stop()
d = monitor.as_dict()
self.assertEqual(d['version'], 1)
self.assertEqual(len(d['events']), 2)
self.assertEqual(len(d['phases']), 2)
class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=3.1.1', method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.3',
method='pip', optional=True)
self.register_virtualenv_module('jsonschema==2.5.1',
method='pip')
# explicitly install functools32, because some slaves aren't using
# a version of pip recent enough to install it automatically with
# jsonschema (which depends on it)
# https://github.com/Julian/jsonschema/issues/233
self.register_virtualenv_module('functools32==3.2.3-2',
method='pip')
self._resource_monitor = None
# 2-tuple of (name, options) to assign Perfherder resource monitor
# metrics to. This needs to be assigned by a script in order for
# Perfherder metrics to be reported.
self.resource_monitor_perfherder_id = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
# Upload a JSON file containing the raw resource data.
try:
upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
if not os.path.exists(upload_dir):
os.makedirs(upload_dir)
with open(os.path.join(upload_dir, 'resource-usage.json'), 'wb') as fh:
json.dump(self._resource_monitor.as_dict(), fh,
sort_keys=True, indent=4)
except (AttributeError, KeyError):
self.exception('could not upload resource usage JSON',
level=WARNING)
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
# Delay import because not available until virtualenv is populated.
import jsonschema
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
swap_in = sum(m.swap.sin for m in rm.measurements)
swap_out = sum(m.swap.sout for m in rm.measurements)
return cpu_percent, cpu_times, io, (swap_in, swap_out)
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(
prefix=prefix, duration=duration,
cpu_percent=cpu_percent_str, io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes, io_read_time=io.read_time,
io_write_time=io.write_time
)
)
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None)
duration = rm.end_time - rm.start_time
# Write out Perfherder data if configured.
if self.resource_monitor_perfherder_id:
perfherder_name, perfherder_options = self.resource_monitor_perfherder_id
suites = []
overall = []
if cpu_percent:
overall.append({
'name': 'cpu_percent',
'value': cpu_percent,
})
overall.extend([
{'name': 'io_write_bytes', 'value': io.write_bytes},
{'name': 'io.read_bytes', 'value': io.read_bytes},
{'name': 'io_write_time', 'value': io.write_time},
{'name': 'io_read_time', 'value': io.read_time},
])
suites.append({
'name': '%s.overall' % perfherder_name,
'extraOptions': perfherder_options + self.perfherder_resource_options(),
'subtests': overall,
})
for phase in rm.phases.keys():
phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
subtests = [
{
'name': 'time',
'value': phase_duration,
}
]
cpu_percent = rm.aggregate_cpu_percent(phase=phase,
per_cpu=False)
if cpu_percent is not None:
subtests.append({
'name': 'cpu_percent',
'value': rm.aggregate_cpu_percent(phase=phase,
per_cpu=False),
})
# We don't report I/O during each step because measured I/O
# is system I/O and that I/O can be delayed (e.g. writes will
# buffer before being flushed and recorded in our metrics).
suites.append({
'name': '%s.%s' % (perfherder_name, phase),
'subtests': subtests,
})
data = {
'framework': {'name': 'job_resource_usage'},
'suites': suites,
}
schema_path = os.path.join(external_tools_path,
'performance-artifact-schema.json')
with open(schema_path, 'rb') as fh:
schema = json.load(fh)
# this will throw an exception that causes the job to fail if the
# perfherder data is not valid -- please don't change this
# behaviour, otherwise people will inadvertently break this
# functionality
self.info('Validating Perfherder data against %s' % schema_path)
jsonschema.validate(data, schema)
self.info('PERFHERDER_DATA: %s' % json.dumps(data))
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
# Print special messages so usage shows up in Treeherder.
if cpu_percent:
self._tinderbox_print('CPU usage<br/>{:,.1f}%'.format(
cpu_percent))
self._tinderbox_print('I/O read bytes / time<br/>{:,} / {:,}'.format(
io.read_bytes, io.read_time))
self._tinderbox_print('I/O write bytes / time<br/>{:,} / {:,}'.format(
io.write_bytes, io.write_time))
# Print CPU components having >1%. "cpu_times" is a data structure
# whose attributes are measurements. Ideally we'd have an API that
# returned just the measurements as a dict or something.
cpu_attrs = []
for attr in sorted(dir(cpu_times)):
if attr.startswith('_'):
continue
if attr in ('count', 'index'):
continue
cpu_attrs.append(attr)
cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs)
for attr in cpu_attrs:
value = getattr(cpu_times, attr)
# cpu_total can be 0.0. Guard against division by 0.
percent = value / cpu_total * 100.0 if cpu_total else 0.0
if percent > 1.00:
self._tinderbox_print('CPU {}<br/>{:,.1f} ({:,.1f}%)'.format(
attr, value, percent))
# Swap on Windows isn't reported by psutil.
if not self._is_windows():
self._tinderbox_print('Swap in / out<br/>{:,} / {:,}'.format(
swap_in, swap_out))
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=3.1.1',
method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.3',
method='pip',
optional=True)
self.register_virtualenv_module('jsonschema==2.5.1', method='pip')
# explicitly install functools32, because some slaves aren't using
# a version of pip recent enough to install it automatically with
# jsonschema (which depends on it)
# https://github.com/Julian/jsonschema/issues/233
self.register_virtualenv_module('functools32==3.2.3-2', method='pip')
self._resource_monitor = None
# 2-tuple of (name, options) to assign Perfherder resource monitor
# metrics to. This needs to be assigned by a script in order for
# Perfherder metrics to be reported.
self.resource_monitor_perfherder_id = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning(
'Resource monitoring will not be enabled! Python 2.7+ required.'
)
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
# Upload a JSON file containing the raw resource data.
try:
upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
if not os.path.exists(upload_dir):
os.makedirs(upload_dir)
with open(os.path.join(upload_dir, 'resource-usage.json'),
'wb') as fh:
json.dump(self._resource_monitor.as_dict(),
fh,
sort_keys=True,
indent=4)
except (AttributeError, KeyError):
self.exception('could not upload resource usage JSON',
level=WARNING)
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
# Delay import because not available until virtualenv is populated.
import jsonschema
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
swap_in = sum(m.swap.sin for m in rm.measurements)
swap_out = sum(m.swap.sout for m in rm.measurements)
return cpu_percent, cpu_times, io, (swap_in, swap_out)
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(
cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(prefix=prefix,
duration=duration,
cpu_percent=cpu_percent_str,
io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes,
io_read_time=io.read_time,
io_write_time=io.write_time))
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None)
duration = rm.end_time - rm.start_time
# Write out Perfherder data if configured.
if self.resource_monitor_perfherder_id:
perfherder_name, perfherder_options = self.resource_monitor_perfherder_id
suites = []
overall = []
if cpu_percent:
overall.append({
'name': 'cpu_percent',
'value': cpu_percent,
})
overall.extend([
{
'name': 'io_write_bytes',
'value': io.write_bytes
},
{
'name': 'io.read_bytes',
'value': io.read_bytes
},
{
'name': 'io_write_time',
'value': io.write_time
},
{
'name': 'io_read_time',
'value': io.read_time
},
])
suites.append({
'name':
'%s.overall' % perfherder_name,
'extraOptions':
perfherder_options + self.perfherder_resource_options(),
'subtests':
overall,
})
for phase in rm.phases.keys():
phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
subtests = [{
'name': 'time',
'value': phase_duration,
}]
cpu_percent = rm.aggregate_cpu_percent(phase=phase,
per_cpu=False)
if cpu_percent is not None:
subtests.append({
'name':
'cpu_percent',
'value':
rm.aggregate_cpu_percent(phase=phase, per_cpu=False),
})
# We don't report I/O during each step because measured I/O
# is system I/O and that I/O can be delayed (e.g. writes will
# buffer before being flushed and recorded in our metrics).
suites.append({
'name': '%s.%s' % (perfherder_name, phase),
'subtests': subtests,
})
data = {
'framework': {
'name': 'job_resource_usage'
},
'suites': suites,
}
schema_path = os.path.join(external_tools_path,
'performance-artifact-schema.json')
with open(schema_path, 'rb') as fh:
schema = json.load(fh)
# this will throw an exception that causes the job to fail if the
# perfherder data is not valid -- please don't change this
# behaviour, otherwise people will inadvertently break this
# functionality
self.info('Validating Perfherder data against %s' % schema_path)
jsonschema.validate(data, schema)
self.info('PERFHERDER_DATA: %s' % json.dumps(data))
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
# Print special messages so usage shows up in Treeherder.
if cpu_percent:
self._tinderbox_print('CPU usage<br/>{:,.1f}%'.format(cpu_percent))
self._tinderbox_print('I/O read bytes / time<br/>{:,} / {:,}'.format(
io.read_bytes, io.read_time))
self._tinderbox_print('I/O write bytes / time<br/>{:,} / {:,}'.format(
io.write_bytes, io.write_time))
# Print CPU components having >1%. "cpu_times" is a data structure
# whose attributes are measurements. Ideally we'd have an API that
# returned just the measurements as a dict or something.
cpu_attrs = []
for attr in sorted(dir(cpu_times)):
if attr.startswith('_'):
continue
if attr in ('count', 'index'):
continue
cpu_attrs.append(attr)
cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs)
for attr in cpu_attrs:
value = getattr(cpu_times, attr)
# cpu_total can be 0.0. Guard against division by 0.
percent = value / cpu_total * 100.0 if cpu_total else 0.0
if percent > 1.00:
self._tinderbox_print('CPU {}<br/>{:,.1f} ({:,.1f}%)'.format(
attr, value, percent))
# Swap on Windows isn't reported by psutil.
if not self._is_windows():
self._tinderbox_print('Swap in / out<br/>{:,} / {:,}'.format(
swap_in, swap_out))
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
class ResourceMonitoringMixin(object):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=0.7.1',
method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.0.0',
method='pip',
optional=True)
self._resource_monitor = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning(
'Resource monitoring will not be enabled! Python 2.7+ required.'
)
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
return cpu_percent, cpu_times, io
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(
cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(prefix=prefix,
duration=duration,
cpu_percent=cpu_percent_str,
io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes,
io_read_time=io.read_time,
io_write_time=io.write_time))
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io = resources(None)
duration = rm.end_time - rm.start_time
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
class ResourceMonitoringMixin(object):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=0.7.1', method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.0.0',
method='pip', optional=True)
self._resource_monitor = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
return cpu_percent, cpu_times, io
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(
prefix=prefix, duration=duration,
cpu_percent=cpu_percent_str, io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes, io_read_time=io.read_time,
io_write_time=io.write_time
)
)
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io = resources(None)
duration = rm.end_time - rm.start_time
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
class ResourceMonitoringMixin(object):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=3.1.1', method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.3',
method='pip', optional=True)
self._resource_monitor = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
# Upload a JSON file containing the raw resource data.
try:
upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
with open(os.path.join(upload_dir, 'resource-usage.json'), 'wb') as fh:
json.dump(self._resource_monitor.as_dict(), fh,
sort_keys=True, indent=4)
except (AttributeError, KeyError):
self.exception('could not upload resource usage JSON',
level=WARNING)
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
swap_in = sum(m.swap.sin for m in rm.measurements)
swap_out = sum(m.swap.sout for m in rm.measurements)
return cpu_percent, cpu_times, io, (swap_in, swap_out)
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(
prefix=prefix, duration=duration,
cpu_percent=cpu_percent_str, io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes, io_read_time=io.read_time,
io_write_time=io.write_time
)
)
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None)
duration = rm.end_time - rm.start_time
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
# Print special messages so usage shows up in Treeherder.
if cpu_percent:
self._tinderbox_print('CPU usage<br/>{:,.1f}%'.format(
cpu_percent))
self._tinderbox_print('I/O read bytes / time<br/>{:,} / {:,}'.format(
io.read_bytes, io.read_time))
self._tinderbox_print('I/O write bytes / time<br/>{:,} / {:,}'.format(
io.write_bytes, io.write_time))
# Print CPU components having >1%. "cpu_times" is a data structure
# whose attributes are measurements. Ideally we'd have an API that
# returned just the measurements as a dict or something.
cpu_attrs = []
for attr in sorted(dir(cpu_times)):
if attr.startswith('_'):
continue
if attr in ('count', 'index'):
continue
cpu_attrs.append(attr)
cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs)
for attr in cpu_attrs:
value = getattr(cpu_times, attr)
percent = value / cpu_total * 100.0
if percent > 1.00:
self._tinderbox_print('CPU {}<br/>{:,.1f} ({:,.1f}%)'.format(
attr, value, percent))
# Swap on Windows isn't reported by psutil.
if not self._is_windows():
self._tinderbox_print('Swap in / out<br/>{:,} / {:,}'.format(
swap_in, swap_out))
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
class ResourceMonitoringMixin(object):
"""Provides resource monitoring capabilities to scripts.
When this class is in the inheritance chain, resource usage stats of the
executing script will be recorded.
This class requires the VirtualenvMixin in order to install a package used
for recording resource usage.
While we would like to record resource usage for the entirety of a script,
since we require an external package, we can only record resource usage
after that package is installed (as part of creating the virtualenv).
That's just the way things have to be.
"""
def __init__(self, *args, **kwargs):
super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
self.register_virtualenv_module('psutil>=3.1.1',
method='pip',
optional=True)
self.register_virtualenv_module('mozsystemmonitor==0.3',
method='pip',
optional=True)
self._resource_monitor = None
@PostScriptAction('create-virtualenv')
def _start_resource_monitoring(self, action, success=None):
self.activate_virtualenv()
# Resource Monitor requires Python 2.7, however it's currently optional.
# Remove when all machines have had their Python version updated (bug 711299).
if sys.version_info[:2] < (2, 7):
self.warning(
'Resource monitoring will not be enabled! Python 2.7+ required.'
)
return
try:
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
self.info("Starting resource monitoring.")
self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
self._resource_monitor.start()
except Exception:
self.warning("Unable to start resource monitor: %s" %
traceback.format_exc())
@PreScriptAction
def _resource_record_pre_action(self, action):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.begin_phase(action)
@PostScriptAction
def _resource_record_post_action(self, action, success=None):
# Resource monitor isn't available until after create-virtualenv.
if not self._resource_monitor:
return
self._resource_monitor.finish_phase(action)
@PostScriptRun
def _resource_record_post_run(self):
if not self._resource_monitor:
return
# This should never raise an exception. This is a workaround until
# mozsystemmonitor is fixed. See bug 895388.
try:
self._resource_monitor.stop()
self._log_resource_usage()
# Upload a JSON file containing the raw resource data.
try:
upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
with open(os.path.join(upload_dir, 'resource-usage.json'),
'wb') as fh:
json.dump(self._resource_monitor.as_dict(),
fh,
sort_keys=True,
indent=4)
except (AttributeError, KeyError):
self.exception('could not upload resource usage JSON',
level=WARNING)
except Exception:
self.warning("Exception when reporting resource usage: %s" %
traceback.format_exc())
def _log_resource_usage(self):
rm = self._resource_monitor
if rm.start_time is None:
return
def resources(phase):
cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
io = rm.aggregate_io(phase=phase)
swap_in = sum(m.swap.sin for m in rm.measurements)
swap_out = sum(m.swap.sout for m in rm.measurements)
return cpu_percent, cpu_times, io, (swap_in, swap_out)
def log_usage(prefix, duration, cpu_percent, cpu_times, io):
message = '{prefix} - Wall time: {duration:.0f}s; ' \
'CPU: {cpu_percent}; ' \
'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
'Read time: {io_read_time}; Write time: {io_write_time}'
# XXX Some test harnesses are complaining about a string being
# being fed into a 'f' formatter. This will help diagnose the
# issue.
cpu_percent_str = str(round(
cpu_percent)) + '%' if cpu_percent else "Can't collect data"
try:
self.info(
message.format(prefix=prefix,
duration=duration,
cpu_percent=cpu_percent_str,
io_read_bytes=io.read_bytes,
io_write_bytes=io.write_bytes,
io_read_time=io.read_time,
io_write_time=io.write_time))
except ValueError:
self.warning("Exception when formatting: %s" %
traceback.format_exc())
cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None)
duration = rm.end_time - rm.start_time
log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
# Print special messages so usage shows up in Treeherder.
if cpu_percent:
self._tinderbox_print('CPU usage<br/>{:,.1f}%'.format(cpu_percent))
self._tinderbox_print('I/O read bytes / time<br/>{:,} / {:,}'.format(
io.read_bytes, io.read_time))
self._tinderbox_print('I/O write bytes / time<br/>{:,} / {:,}'.format(
io.write_bytes, io.write_time))
# Print CPU components having >1%. "cpu_times" is a data structure
# whose attributes are measurements. Ideally we'd have an API that
# returned just the measurements as a dict or something.
cpu_attrs = []
for attr in sorted(dir(cpu_times)):
if attr.startswith('_'):
continue
if attr in ('count', 'index'):
continue
cpu_attrs.append(attr)
cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs)
for attr in cpu_attrs:
value = getattr(cpu_times, attr)
percent = value / cpu_total * 100.0
if percent > 1.00:
self._tinderbox_print('CPU {}<br/>{:,.1f} ({:,.1f}%)'.format(
attr, value, percent))
# Swap on Windows isn't reported by psutil.
if not self._is_windows():
self._tinderbox_print('Swap in / out<br/>{:,} / {:,}'.format(
swap_in, swap_out))
for phase in rm.phases.keys():
start_time, end_time = rm.phases[phase]
cpu_percent, cpu_times, io, swap = resources(phase)
log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
def _tinderbox_print(self, message):
self.info('TinderboxPrint: %s' % message)
