def check_record_diagnoser(phase_record):
self.assertEqual(test_record.PhaseOutcome.FAIL,
phase_record.outcome)
self.assertEqual(
htf.Measurement('pass_measure',
outcome=measurements.Outcome.PASS,
measured_value=measurements.MeasuredValue(
'pass_measure',
is_value_set=True,
stored_value=True,
cached_value=True),
cached=mock.ANY),
phase_record.measurements['pass_measure'])
self.assertEqual(
htf.Measurement('fail_measure',
outcome=measurements.Outcome.FAIL,
measured_value=measurements.MeasuredValue(
'fail_measure',
is_value_set=True,
stored_value=False,
cached_value=False),
validators=[is_true],
cached=mock.ANY),
phase_record.measurements['fail_measure'])
return None
def get_criteria(criterion):
criteria_dict = {
"dc_gain": htf.Measurement('dc_gain').in_range(minimum=50),
"unity_bandwidth":
htf.Measurement('unity_bandwidth').in_range(minimum=1e6)
}
return criteria_dict[criterion]
def test_precision(self):
"""Check that with_precision does what it says on the tin."""
m = htf.Measurement('meas_with_precision').with_precision(3)
m.measured_value.set(1.2345)
self.assertAlmostEqual(m.measured_value.value, 1.234)
m.measured_value.set(1.2346)
self.assertAlmostEqual(m.measured_value.value, 1.235)
m = htf.Measurement('meas_with_precision_and_dims').with_precision(
3).with_dimensions('x')
m.measured_value[42] = 1.2346
self.assertAlmostEqual(m.measured_value[42], 1.235)
def get_criteria(criterion):
criteria_dict = {
"dc_gain" : htf.Measurement('dc_gain').in_range(minimum=50),
"phase_margin" : htf.Measurement('phase_margin').in_range(minimum=50),
"gain_margin" : htf.Measurement('gain_margin').in_range(minimum=20),
# "current_regulation_temp" : htf.Measurement('current_regulation_temp').with_validator(validator_relative_variation),
# "current_regulation_process" : htf.Measurement('current_regulation_process').with_validator(validator_absolute_variation)
"current_regulation_temp" : htf.Measurement('current_regulation_temp').relative_variation(allowed_variation=0.01),
"current_regulation_process" : htf.Measurement('current_regulation_process').absolute_variation(expected_value=5e-6, allowed_variation=0.22)
}
return criteria_dict[criterion]
def test_bad_precision(self):
"""Creating a measurement with invalid precision should raise."""
m = htf.Measurement('bad_precision')
with self.assertRaises(TypeError):
m.with_precision(1.1)
with self.assertRaises(TypeError):
m.with_precision('1')
class DimensionPivotTest(htf_test.TestCase):
"""Tests validators.DimensionPivot. Used with dimensioned measurements."""
_test_value = 10
_sub_validator = validators.in_range(0, _test_value)
_test_measurement = htf.Measurement('pivot').with_dimensions(
'height', 'width').dimension_pivot_validate(_sub_validator)
@htf_test.yields_phases
def testPasses(self):
@htf.measures(self._test_measurement)
def phase(test):
test.measurements.pivot[10, 10] = self._test_value - 2
test.measurements.pivot[11, 10] = self._test_value - 1
phase_record = yield phase
self.assertMeasurementPass(phase_record, 'pivot')
@htf_test.yields_phases
def testFails(self):
@htf.measures(self._test_measurement)
def phase(test):
test.measurements.pivot[11, 12] = self._test_value - 1
test.measurements.pivot[14, 12] = self._test_value + 1
phase_record = yield phase
self.assertMeasurementFail(phase_record, 'pivot')
def test_to_dataframe(self, units=True):
measurement = htf.Measurement('test_multidim')
measurement.with_dimensions('ms', 'assembly', htf.Dimension('my_zone'))
if units:
measurement.with_units('°C')
measure_column_name = 'degree Celsius'
else:
measure_column_name = 'value'
for t in range(5):
for assembly in ['A', 'B', 'C']:
for zone in range(3):
temp = zone + t
dims = (t, assembly, zone)
measurement.measured_value[dims] = temp
measurement.outcome = measurements.Outcome.PASS
df = measurement.to_dataframe()
coordinates = (1, 'A', 2)
query = '(ms == %s) & (assembly == "%s") & (my_zone == %s)' % (
coordinates)
self.assertEqual(measurement.measured_value[coordinates],
df.query(query)[measure_column_name].values[0])
class ConsistentEndDimensionPivotTest(htf_test.TestCase):
"""Tests validators.ConsistentEndRange. Similar to DimensionPivot."""
_sub_validator = validators.in_range(minimum=5)
_test_measurement = htf.Measurement('pivot').with_dimensions(
'time').consistent_end_dimension_pivot_validate(_sub_validator)
@htf_test.yields_phases
def testPasses(self):
@htf.measures(self._test_measurement)
def phase(test):
test.measurements.pivot[0] = 0
test.measurements.pivot[1] = 2
test.measurements.pivot[2] = 6
test.measurements.pivot[3] = 8
phase_record = yield phase
self.assertMeasurementPass(phase_record, 'pivot')
@htf_test.yields_phases
def testFails(self):
@htf.measures(self._test_measurement)
def phase(test):
test.measurements.pivot[0] = 3
test.measurements.pivot[1] = 4
test.measurements.pivot[2] = 6
test.measurements.pivot[3] = 4
phase_record = yield phase
self.assertMeasurementFail(phase_record, 'pivot')
def test_bad_validator(self):
measurement = htf.Measurement('bad_measure')
measurement.with_dimensions('a')
measurement.with_validator(bad_validator)
measurement.measured_value['A'] = 1
measurement.measured_value['B'] = 2
with self.assertRaises(BadValidatorError):
measurement.validate()
def testStringFromMeasurement_SuccessfullyConvertsPassMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a')
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(10)
measurement.notify_value_set()
self.assertEqual(text.StringFromMeasurement(measurement),
'| text_measurement_a: 10')
def test_bad_transform_fn(self):
"""Bad functions or setting multiple functions should raise."""
m = htf.Measurement('transform')
with self.assertRaises(TypeError):
m.with_transform(None)
with self.assertRaises(TypeError):
m.with_transform('int')
with self.assertRaises(ValueError):
m.with_transform(abs).with_transform(int)
def test_phase_diagnoser__access_to_phase_record(self):
def is_true(value):
return value
@htf.PhaseDiagnoser(OkayResult)
def check_record_diagnoser(phase_record):
self.assertEqual(test_record.PhaseOutcome.FAIL, phase_record.outcome)
self.assertEqual(
htf.Measurement(
'pass_measure',
outcome=measurements.Outcome.PASS,
measured_value=measurements.MeasuredValue(
'pass_measure',
is_value_set=True,
stored_value=True,
_cached_value=True),
_cached=mock.ANY), phase_record.measurements['pass_measure'])
self.assertEqual(
htf.Measurement(
'fail_measure',
outcome=measurements.Outcome.FAIL,
measured_value=measurements.MeasuredValue(
'fail_measure',
is_value_set=True,
stored_value=False,
_cached_value=False),
validators=[is_true],
_cached=mock.ANY), phase_record.measurements['fail_measure'])
return None
@htf.diagnose(check_record_diagnoser)
@htf.measures(
htf.Measurement('pass_measure'),
htf.Measurement('fail_measure').with_validator(is_true))
def phase(test):
test.measurements.pass_measure = True
test.measurements.fail_measure = False
phase_rec = yield phase
self.assertPhaseContinue(phase_rec)
self.assertPhaseOutcomeFail(phase_rec)
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurementColorized(
self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
self.assertEqual(
text.StringFromMeasurement(measurement, colorize_text=True).count(
text._BRIGHT_RED_STYLE), 1)
def test_diagnose_decorator_later(self):
def phase_func():
pass
phase = htf.measures('m')(
htf.diagnose(basic_wrapper_phase_diagnoser)(phase_func))
self.assertEqual(
htf.PhaseDescriptor(phase_func,
measurements=[htf.Measurement('m')],
diagnosers=[basic_wrapper_phase_diagnoser]),
phase)
def test_multi_dimension_correct(self):
measurement = htf.Measurement('measure')
measurement.with_dimensions('dimension1', 'dimension2')
dimension_vals = (
'dim val 1',
1234,
)
try:
measurement.measured_value[dimension_vals] = 42
except measurements.InvalidDimensionsError:
self.fail('measurement.DimensionedMeasuredValue.__setitem__ '
'raised error unexpectedly.')
def test_conditional_measurement__not_run_different_results(self):
@htf.measures(
htf.Measurement('validator_not_run').validate_on(
{OkayResult.OKAY: self._make_validator()}))
def phase(test):
test.measurements.validator_not_run = True
phase_rec = yield phase
self.assertPhaseContinue(phase_rec)
self.assertPhaseOutcomePass(phase_rec)
self.assertEqual([], self.validator_values)
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
output = text.StringFromMeasurement(measurement)
self.assertEqual(
output,
"| text_measurement_a failed because 5 failed these checks: ['x <= 3']"
)
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def test_cache_same_object(self):
m = htf.Measurement('measurement')
basetypes0 = m.as_base_types()
self.assertEqual({
'name': 'measurement',
'outcome': 'UNSET',
}, basetypes0)
basetypes1 = m.as_base_types()
self.assertIs(basetypes0, basetypes1)
m.measured_value.set(1)
m.notify_value_set()
basetypes2 = m.as_base_types()
self.assertEqual(
{
'name': 'measurement',
'outcome': 'PASS',
'measured_value': 1,
}, basetypes2)
self.assertIs(basetypes0, basetypes2)
def test_multi_dimension_not_enough_error(self):
measurement = htf.Measurement('measure')
measurement.with_dimensions('dimension1', 'dimension2')
with self.assertRaises(measurements.InvalidDimensionsError):
measurement.measured_value['dim val'] = 42
class DiagTestError(Exception):
pass
@htf.PhaseOptions()
def basic_phase():
pass
@htf.PhaseOptions()
def exception_phase():
raise PhaseError('it broke')
@htf.measures(htf.Measurement('bad').equals(1))
def fail_measurement_phase(test):
test.measurements.bad = 0
@htf.measures(htf.Measurement('good'))
def pass_measurement_phase(test):
test.measurements.good = 'good'
class BadResult(htf.DiagResultEnum):
ONE = 'bad_one'
TWO = 'bad_two'
@htf.PhaseDiagnoser(BadResult, name='my_bad_result')
class PingDnsA(PingPlug):
host = '8.8.8.8'
class PingDnsB(PingPlug):
host = '8.8.4.4'
# Note: phase name and total_time measurement use {} formatting with args
# passed into the phase so each phase has a unique name.
@htf.PhaseOptions(name='Ping-{pinger.host}-{count}')
@htf.plug(pinger=PingPlug.placeholder)
@htf.measures('total_time_{pinger.host}_{count}',
htf.Measurement('retcode').equals('{expected_retcode}',
type=int))
def test_ping(test, pinger, count, expected_retcode):
"""This tests that we can ping a host.
The plug, pinger, is expected to be replaced at test creation time, so the
placeholder property was used instead of the class directly.
"""
del expected_retcode # Not used in the phase, only used by a measurement
start = time.time()
retcode = pinger.run(count)
elapsed = time.time() - start
test.measurements['total_time_%s_%s' % (pinger.host, count)] = elapsed
test.measurements.retcode = retcode
if __name__ == '__main__':
import random
import openhtf as htf
# Import this output mechanism as it's the specific one we want to use.
from openhtf.output.callbacks import json_factory
# You won't normally need to import this, see validators.py example for
# more details. It's used for the inline measurement declaration example
# below, but normally you'll only import it when you want to define custom
# measurement validators.
from openhtf.util import validators
# Simple example of measurement use, similar to hello_world.py usage.
@htf.measures(htf.Measurement('hello_world_measurement'))
def hello_phase(test):
test.measurements.hello_world_measurement = 'Hello!'
# An alternative simpler syntax that creates the Measurement for you.
@htf.measures('hello_again_measurement')
def again_phase(test):
test.measurements.hello_again_measurement = 'Again!'
# Multiple measurements can be specified in a single decorator, using either of
# the above syntaxes. Technically, these syntaxes can be mixed and matched, but
# as a matter of convention you should always use one or the other within a
# single decorator call. You'll also note that you can stack multiple
# decorations on a single phase. This is useful if you have a handful of simple
def test_single_dimension_float(self):
measurement = htf.Measurement('measure')
measurement.with_dimensions('dimension')
val = measurement.measured_value[42.42] = 'measurement'
self.assertEqual(val, 'measurement')
def test_single_dimension_mutable_obj_error(self):
measurement = htf.Measurement('measure')
measurement.with_dimensions('dimension')
with self.assertRaises(measurements.InvalidDimensionsError):
measurement.measured_value[['dim val']] = 42
def test_transform_fn(self):
"""Check that the transform_fn is working."""
m = htf.Measurement('abs_transform').with_transform(abs)
m.measured_value.set(-1.234)
self.assertAlmostEqual(m.measured_value.value, 1.234)
# Import openhtf with an abbreviated name, as we'll be using a bunch of stuff
# from it throughout our test scripts. See __all__ at the top of
# openhtf/__init__.py for details on what's in top-of-module namespace.
import openhtf as htf
# Import this output mechanism as it's the specific one we want to use.
from openhtf.output.callbacks import json_factory
from openhtf.plugs import user_input
# The @htf.measures annotation notifies the OpenHTF framework that this test
# phase will be taking a measurement that we'd like to call
# 'hello_world_measurement'. Measurements can be accessed and set via
# the 'test' object, always passed as the first argument to test phases.
@htf.measures(htf.Measurement('hello_world_measurement'))
def hello_world(test):
"""A hello world test phase."""
# At the heart of an OpenHTF test script are the test phases, such as
# this one. Any callable can be used as a test phase, so long as it
# accepts a single argument that is the 'test' object. This test object
# is how you will interact with the OpenHTF test framework once a test is
# running. See other examples for more complex cases, but here is a good
# example of the sort of minimal functionality you're likely to use.
# The test.logger attribute is a Python logger instance that is configured
# to log to the test record we will output at the end of the test. This
# is the recommended way to do any logging within test phases (this is also
# how to get logs to show up in the frontend).
test.logger.info('Hello World!')
# limitations under the License.
"""Example OpenHTF test demonstrating use of checkpoints and measurements."""
import time
import openhtf as htf
from openhtf.output.callbacks import console_summary
from openhtf.output.callbacks import json_factory
from openhtf.util import checkpoints
from examples import measurements as measurements_example
@htf.measures(
htf.Measurement('fixed_time').in_range(
0, 10).doc('This is going to fail validation.').with_units(
htf.units.SECOND))
def failing_phase(test):
# The 'outcome' of this measurement in the test_record result will be a FAIL
# because its value fails the validator specified (0 <= 5 <= 10).
test.measurements.fixed_time = 20
def long_running_phase(test):
# A long running phase could be something like a hardware burn-in. This
# phase should not run if previous phases have failed, so we make sure
# checkpoint phase is run right before this phase.
for i in range(10):
test.logger.info('Still running....')
time.sleep(10)
test.logger.info('Done with long_running_phase')
def test_multi_dimension_too_many_error(self):
measurement = htf.Measurement('measure')
measurement.with_dimensions('dimension1', 'dimension2')
dimension_vals = ('dim val 1', 2, 3, 4)
with self.assertRaises(measurements.InvalidDimensionsError):
measurement.measured_value[dimension_vals] = 42
from openhtf.plugs import user_input
from openhtf.output import callbacks
from openhtf.output.callbacks import json_factory
import example_plugs
@htf.plug(example=example_plugs.ExamplePlug)
@htf.plug(frontend_aware=example_plugs.ExampleFrontendAwarePlug)
def example_monitor(example, frontend_aware):
time.sleep(.2)
frontend_aware.increment()
return example.increment()
@htf.measures(htf.Measurement('unset_meas'),
htf.Measurement('widget_type').matches_regex(r'.*Widget$').doc(
'''This measurement tracks the type of widgets.'''),
htf.Measurement('widget_color').doc('Color of the widget'),
htf.Measurement('widget_size').in_range(1,
4).doc('Size of widget'))
@htf.measures('specified_as_args',
docstring='Helpful docstring',
units=units.HERTZ,
validators=[util.validators.matches_regex('Measurement')])
@htf.plug(example=example_plugs.ExamplePlug)
@htf.plug(prompts=user_input.UserInput)
def hello_world(test, example, prompts):
"""A hello world test phase."""
test.logger.info('Hello World!')
test.measurements.widget_type = prompts.prompt('What\'s the widget type?',
def test_unit_enforcement(self):
"""Creating a measurement with invalid units should raise."""
self.assertRaises(TypeError,
htf.Measurement('bad_units').with_units, 1701)