def test_save_load_checkpoint(self):
model = delay_model_pb2.DelayModel()
# Maps an op name to the set of bit configurations we've run that op with.
data_points = collections.defaultdict(set)
# Set up some dummy data.
ops = ["op_a", "op_b", "op_c", "op_d", "op_e"]
bit_configs = ["3, 1, 1", "4, 1, 2", "5, 2, 1", "6, 2, 2"]
for op in ops:
data_points[op] = set()
for bit_config in bit_configs:
data_points[op].add(bit_config)
result = delay_model_pb2.DataPoint()
result.operation.op = op
for elem in bit_config.split(",")[1:]:
operand = delay_model_pb2.Operation.Operand()
operand.bit_count = int(elem)
result.operation.operands.append(operand)
result.operation.bit_count = int(bit_config[0])
result.delay = 5
model.data_points.append(result)
tf = tempfile.NamedTemporaryFile()
client.save_checkpoint(model, tf.name)
loaded_data_points, loaded_model = client.init_data(tf.name)
self.assertEqual(model, loaded_model)
# Fancy equality checking so we get clearer error messages on
# mismatch.
for op in ops:
self.assertIn(op, loaded_data_points)
loaded_op = loaded_data_points[op]
for bit_config in bit_configs:
self.assertIn(bit_config, loaded_op)
self.assertIn(bit_config, data_points[op])
self.assertEqual(data_points, loaded_data_points)
def test_regression_estimator_cross_validation_passes(self):
def gen_operation(result_bit_count, operand_bit_count):
return 'op: "kFoo" bit_count: %d operands { } operands { bit_count: %d }' % (
result_bit_count, operand_bit_count)
def gen_data_point(result_bit_count, operand_bit_count, delay):
return 'data_points{{ operation {{ {} }} delay: {} delay_offset: 0}}'.format(
gen_operation(result_bit_count, operand_bit_count), delay)
data_points_str = [
gen_data_point(1, 2, 1),
gen_data_point(2, 2, 2),
gen_data_point(4, 1, 4),
gen_data_point(5, 111, 5),
gen_data_point(7, 13, 7),
gen_data_point(8, 2, 8),
gen_data_point(10, 12, 10),
gen_data_point(15, 6, 15),
gen_data_point(20, 40, 20),
gen_data_point(30, 15, 30),
gen_data_point(31, 2, 31),
gen_data_point(35, 2, 35),
gen_data_point(40, 30, 40),
gen_data_point(45, 9, 45),
gen_data_point(50, 4, 50),
gen_data_point(55, 400, 55),
gen_data_point(70, 10, 70),
gen_data_point(100, 50, 100),
gen_data_point(125, 15, 125),
gen_data_point(150, 100, 150),
]
proto_text = """
op_models {
op: "kFoo"
estimator {
regression {
expressions {
factor {
source: RESULT_BIT_COUNT
}
}
kfold_validator {
max_data_point_error: 0.15
max_fold_geomean_error: 0.075
}
}
}
}
"""
proto_text = proto_text + '\n'.join(data_points_str)
delay_model.DelayModel(
text_format.Parse(proto_text, delay_model_pb2.DelayModel()))
def run_characterization(
stub: synthesis_service_pb2_grpc.SynthesisServiceStub) -> None:
"""Run characterization with the given synthesis service."""
model = delay_model_pb2.DelayModel()
for ops, runner in OPS_RUNNERS:
for op in ops:
runner(op, model, stub)
print('# proto-file: xls/delay_model/delay_model.proto')
print('# proto-message: xls.delay_model.DelayModel')
print(model)
def test_regression_estimator_cross_validation_insufficient_data_for_folds(
self):
def gen_operation(result_bit_count, operand_bit_count):
return 'op: "kFoo" bit_count: %d operands { } operands { bit_count: %d }' % (
result_bit_count, operand_bit_count)
def gen_data_point(result_bit_count, operand_bit_count, delay):
return 'data_points{{ operation {{ {} }} delay: {} delay_offset: 0}}'.format(
gen_operation(result_bit_count, operand_bit_count), delay)
data_points_str = [
gen_data_point(1, 2, 100),
gen_data_point(4, 1, 125),
gen_data_point(4, 6, 150),
gen_data_point(7, 13, 175),
gen_data_point(10, 12, 200),
gen_data_point(30, 15, 400),
]
proto_text = """
op_models {
op: "kFoo"
estimator {
regression {
expressions {
factor {
source: OPERAND_BIT_COUNT
operand_number: 1
}
}
expressions {
factor {
source: RESULT_BIT_COUNT
}
}
kfold_validator {
num_cross_validation_folds: 8
max_data_point_error: 99.0
max_fold_geomean_error: 99.0
}
}
}
}
"""
proto_text = proto_text + '\n'.join(data_points_str)
with self.assertRaises(delay_model.Error) as e:
delay_model.DelayModel(
text_format.Parse(proto_text, delay_model_pb2.DelayModel()))
self.assertEqualIgnoringWhitespaceAndFloats(
'kFoo: Too few data points to cross '
'validate: 6 data points, 8 folds', str(e.exception))
def init_data(
checkpoint_path: str
) -> Tuple[Dict[str, Set[str]], delay_model_pb2.DelayModel]:
"""Return new state, loading data from a checkpoint, if available."""
data_points = {}
model = delay_model_pb2.DelayModel()
if checkpoint_path:
filesystem.parse_text_proto_file(checkpoint_path, model)
for data_point in model.data_points:
op = data_point.operation
if op.op not in data_points:
data_points[op.op] = set()
types_str = ', '.join([str(op.bit_count)] +
[str(x.bit_count) for x in op.operands])
data_points[op.op].add(types_str)
return data_points, model
def main(argv):
if len(argv) > 2:
raise app.UsageError('Too many command-line arguments.')
with open(argv[1], 'rb') as f:
contents = f.read()
dm = delay_model.DelayModel(
text_format.Parse(contents, delay_model_pb2.DelayModel()))
for op in dm.ops():
op_model = dm.op_model(op)
maybe_plot_op_model(op_model.estimator)
for specialization_kind, estimator in op_model.specializations.items():
maybe_plot_op_model(
estimator,
delay_model_pb2.SpecializationKind.Name(specialization_kind))
def main(argv):
if len(argv) > 2:
raise app.UsageError('Too many command-line arguments.')
with open(argv[1], 'rb') as f:
contents = f.read()
dm = delay_model.DelayModel(
text_format.Parse(contents, delay_model_pb2.DelayModel()))
env = jinja2.Environment(undefined=jinja2.StrictUndefined)
tmpl_text = runfiles.get_contents_as_text(
'xls/delay_model/generate_delay_lookup.tmpl')
template = env.from_string(tmpl_text)
rendered = template.render(
delay_model=dm,
name=FLAGS.model_name,
precedence=FLAGS.precedence,
camel_case_name=''.join(
s.capitalize() for s in FLAGS.model_name.split('_')))
print('// DO NOT EDIT: this file is AUTOMATICALLY GENERATED and should not '
'be changed.')
print(rendered)
def test_regression_estimator_cross_validation_data_point_exceeds_geomean_error(
self):
def gen_operation(result_bit_count, operand_bit_count):
return 'op: "kFoo" bit_count: %d operands { } operands { bit_count: %d }' % (
result_bit_count, operand_bit_count)
def gen_data_point(result_bit_count, operand_bit_count, delay):
return 'data_points{{ operation {{ {} }} delay: {} delay_offset: 0}}'.format(
gen_operation(result_bit_count, operand_bit_count), delay)
data_points_str = [
gen_data_point(1, 2, 1),
gen_data_point(2, 2, 2),
gen_data_point(4, 1, 4),
gen_data_point(5, 111, 5),
gen_data_point(7, 13, 7),
gen_data_point(8, 2, 8),
gen_data_point(10, 12, 10),
gen_data_point(15, 6, 15),
gen_data_point(20, 40, 20),
gen_data_point(30, 15, 30),
gen_data_point(31, 2, 31),
gen_data_point(35, 2, 35),
gen_data_point(40, 30, 40),
gen_data_point(45, 9, 45),
gen_data_point(50, 4, 50),
gen_data_point(55, 400, 55),
gen_data_point(70, 10, 70),
gen_data_point(100, 50, 100),
gen_data_point(125, 15, 125),
gen_data_point(150, 100, 150),
]
proto_text = """
op_models {
op: "kFoo"
estimator {
regression {
expressions {
factor {
source: OPERAND_BIT_COUNT
operand_number: 1
}
}
kfold_validator {
max_fold_geomean_error: 0.1
}
}
}
}
"""
proto_text = proto_text + '\n'.join(data_points_str)
# Build regression model with operand_bit_count (uncorrelated with delay)
# as only factor.
with self.assertRaises(delay_model.Error) as e:
delay_model.DelayModel(
text_format.Parse(proto_text, delay_model_pb2.DelayModel()))
self.assertEqualIgnoringWhitespaceAndFloats(
'kFoo: Regression model failed '
'k-fold cross validation for test set with geomean error 0.0 > max 0.0',
str(e.exception))
self.assertIn('> max 0.1', str(e.exception))
def test_regression_estimator_cross_validation_data_point_exceeds_max_error(
self):
def gen_operation(result_bit_count, operand_bit_count):
return 'op: "kFoo" bit_count: %d operands { } operands { bit_count: %d }' % (
result_bit_count, operand_bit_count)
def gen_data_point(result_bit_count, operand_bit_count, delay):
return 'data_points{{ operation {{ {} }} delay: {} delay_offset: 0}}'.format(
gen_operation(result_bit_count, operand_bit_count), delay)
data_points_str = [
gen_data_point(1, 2, 1),
gen_data_point(2, 2, 2),
gen_data_point(4, 1, 4),
gen_data_point(5, 111, 5),
gen_data_point(7, 13, 7),
gen_data_point(8, 2, 8),
gen_data_point(10, 12, 10),
gen_data_point(15, 6, 15),
gen_data_point(20, 40, 20),
# Outlier
gen_data_point(30, 15, 50),
#
gen_data_point(31, 2, 31),
gen_data_point(35, 2, 35),
gen_data_point(40, 30, 40),
gen_data_point(45, 9, 45),
gen_data_point(50, 4, 50),
gen_data_point(55, 400, 55),
gen_data_point(70, 10, 70),
gen_data_point(100, 50, 100),
gen_data_point(125, 15, 125),
gen_data_point(150, 100, 150),
]
proto_text = """
op_models {
op: "kFoo"
estimator {
regression {
expressions {
factor {
source: RESULT_BIT_COUNT
}
}
kfold_validator {
max_data_point_error: 0.3
}
}
}
}
"""
proto_text = proto_text + '\n'.join(data_points_str)
with self.assertRaises(delay_model.Error) as e:
delay_model.DelayModel(
text_format.Parse(proto_text, delay_model_pb2.DelayModel()))
self.assertEqualIgnoringWhitespaceAndFloats(
'kFoo: Regression model failed k-fold '
'cross validation for data point (30, 50) with absolute error 0.0'
' > max 0.0', str(e.exception))
self.assertIn('> max 0.3', str(e.exception))
def run_characterization(
stub: synthesis_service_pb2_grpc.SynthesisServiceStub) -> None:
"""Runs characterization via 'stub', DelayModel to stdout as prototext."""
model = delay_model_pb2.DelayModel()
# Bin ops
_run_linear_bin_op_and_add('add', 'kAdd', model, stub)
_run_linear_bin_op_and_add('sub', 'kSub', model, stub)
# Observed shift data is noisy.
_run_linear_bin_op_and_add('shll', 'kShll', model, stub)
_run_linear_bin_op_and_add('shrl', 'kShrl', model, stub)
_run_linear_bin_op_and_add('shra', 'kShra', model, stub)
_run_quadratic_bin_op_and_add('sdiv', 'kSDiv', model, stub, signed=True)
_run_quadratic_bin_op_and_add('smod', 'kSMod', model, stub, signed=True)
_run_quadratic_bin_op_and_add('udiv', 'kUDiv', model, stub)
_run_quadratic_bin_op_and_add('umod', 'kUMod', model, stub)
# Unary ops
_run_unary_op_and_add('neg', 'kNeg', model, stub, signed=True)
_run_unary_op_and_add('not', 'kNot', model, stub)
# Nary ops
_run_nary_op_and_add('and', 'kAnd', model, stub)
_run_nary_op_and_add('nand', 'kNand', model, stub)
_run_nary_op_and_add('nor', 'kNor', model, stub)
_run_nary_op_and_add('or', 'kOr', model, stub)
_run_nary_op_and_add('xor', 'kXor', model, stub)
# Reduction ops
_run_reduction_op_and_add('and_reduce', 'kAndReduce', model, stub)
_run_reduction_op_and_add('or_reduce', 'kOrReduce', model, stub)
_run_reduction_op_and_add('xor_reduce', 'kXorReduce', model, stub)
# Comparison ops
_run_comparison_op_and_add('eq', 'kEq', model, stub)
_run_comparison_op_and_add('ne', 'kNe', model, stub)
# Note: Could optimize for sign - accuracy gains from
# sign have been marginal so far, though. These ops
# also cost less than smul / sdiv anyway.
_run_comparison_op_and_add('sge', 'kSGe', model, stub)
_run_comparison_op_and_add('sgt', 'kSGt', model, stub)
_run_comparison_op_and_add('sle', 'kSLe', model, stub)
_run_comparison_op_and_add('slt', 'kSLt', model, stub)
_run_comparison_op_and_add('uge', 'kUGe', model, stub)
_run_comparison_op_and_add('ugt', 'kUGt', model, stub)
_run_comparison_op_and_add('ule', 'kULe', model, stub)
_run_comparison_op_and_add('ult', 'kULt', model, stub)
# Select ops
# For functions only called for 1 op, could just encode
# op and kOp into function. However, perfer consistency
# and readability of passing them in as args.
# Note: Select op observed data is really weird, see _run_select_op_and_add
_run_select_op_and_add('sel', 'kSel', model, stub)
_run_one_hot_select_op_and_add('one_hot_sel', 'kOneHotSel', model, stub)
# Encode ops
_run_encode_op_and_add('encode', 'kEncode', model, stub)
_run_decode_op_and_add('decode', 'kDecode', model, stub)
# Dynamic bit slice op
_run_dynamic_bit_slice_op_and_add('dynamic_bit_slice', 'kDynamicBitSlice',
model, stub)
# One hot op
_run_one_hot_op_and_add('one_hot', 'kOneHot', model, stub)
# Mul ops
# Note: Modeling smul w/ sign bit as with sdiv decreases accuracy.
_run_mul_op_and_add('smul', 'kSMul', model, stub)
_run_mul_op_and_add('umul', 'kUMul', model, stub)
# Array ops
_run_array_index_op_and_add('array_index', 'kArrayIndex', model, stub)
_run_array_update_op_and_add('array_update', 'kArrayUpdate', model, stub)
# Add free ops.
for free_op in FREE_OPS:
entry = model.op_models.add(op=free_op)
entry.estimator.fixed = 0
# Final validation
delay_model.DelayModel(model)
print('# proto-file: xls/delay_model/delay_model.proto')
print('# proto-message: xls.delay_model.DelayModel')
print(model)