def _assign(self, variable: Union[c_ast.ID, c_ast.ArrayRef],
expression: ExprType, node: c_ast.Node):
"""T-Asgn rule, which can be re-used both in Assignment node and Decl node"""
# get new distance from the assignment expression (T-Asgn)
variable_name = variable.name if isinstance(
variable, c_ast.ID) else variable.name.name
var_aligned, var_shadow, *_ = self._type_system.get_types(
variable_name)
aligned, shadow = DistanceGenerator(
self._type_system).visit(expression)
if self._loop_level == 0:
# insert x^align = n^align if x^aligned is *
if var_aligned == '*' or aligned != '0':
self._insert_at(
node,
parse(
f'{constants.ALIGNED_DISTANCE}_{variable_name} = {aligned}'
),
after=True)
if self._enable_shadow:
# generate x^shadow = x + x^shadow - e according to (T-Asgn)
if self._pc:
if isinstance(variable, c_ast.ID):
shadow_distance = c_ast.ID(
name=f'{constants.SHADOW_DISTANCE}_{variable_name}'
)
elif isinstance(variable, c_ast.ArrayRef):
shadow_distance = c_ast.ArrayRef(
name=f'{constants.SHADOW_DISTANCE}_{variable_name}',
subscript=variable.subscript)
else:
raise NotImplementedError(
f'Assigned value type not supported {type(variable)}'
)
# insert x^shadow = x + x^shadow - e;
insert_node = c_ast.Assignment(
op='=',
lvalue=shadow_distance,
rvalue=c_ast.BinaryOp(op='-',
left=c_ast.BinaryOp(
op='+',
left=variable,
right=shadow_distance),
right=expression))
self._insert_at(node, insert_node, after=False)
# insert x^shadow = n^shadow if n^shadow is not 0
elif var_shadow == '*' or shadow != '0':
self._insert_at(
node,
parse(
f'{constants.SHADOW_DISTANCE}_{variable_name} = {shadow}'
),
after=True)
shadow_distance = '*' if self._pc or shadow != '0' or var_shadow == '*' else '0'
aligned_distance = '*' if aligned != '0' or var_aligned == '*' else '0'
self._type_system.update_distance(variable_name, aligned_distance,
shadow_distance)
def process(self, node: Union[c_ast.FileAST, c_ast.FuncDef]) \
-> Tuple[c_ast.FuncDef, TypeSystem, Iterable[str], Iterable[str], str]:
if not isinstance(node, (c_ast.FileAST, c_ast.FuncDef)):
raise TypeError('node must be of type Union(FileAST, FuncDef)')
processed = self.visit(node)
# scale up the random noise scales due to limitations of symbolic executor KLEE only supporting integers
all_scales = [
f'1 / ({generate(func_call.args.exprs[0])})'
for func_call in self._random_scales
]
lcm = sp.lcm(
tuple(
map(lambda scale: sp.fraction(scale)[1],
all_scales + [self._goal])))
# TODO: use less hackery method to tackle the Pow operation in sympy
# see also https://stackoverflow.com/questions/14264431/expanding-algebraic-powers-in-python-sympy
if str(lcm) != '1':
logger.warning(
f'Scaling down the noise scales by {lcm} due to limitations of symbolic executor KLEE '
f'only supporting integers, therefore the cost calculations will not contain divisions.'
)
for scale in self._random_scales:
scale.args.exprs[0] = parse(
expr_simplify(
f'({generate(scale.args.exprs[0])}) / ({lcm})'))
logger.warning(
f'Scaling up the final goal by {lcm} as well due to the cost scale-ups.'
)
self._goal = expr_simplify(f'({self._goal}) * ({lcm})')
return processed, self._type_system, self._preconditions, self._hole_preconditions, self._goal
def transform(code: str, enable_shadow: bool = False):
node = parse(code)
# first preprocess the node, extract the annotations and do sanity checks
logger.info('Transformation starts')
logger.info('Preprocess starts')
preprocessed, type_system, preconditions, hole_preconditions, goal = Preprocessor(
).process(node)
# TODO: here we use a simple regex to find custom hole variables
holes = list(set(re.findall(f'{constants.HOLE}_\\d+', code)))
# update the type system with the custom hole variables
for hole in holes:
type_system.update_base_type(hole, 'int', False)
# update the type system with the symbolic cost variables
type_system.update_base_type(constants.SYMBOLIC_COST, 'int', True)
type_system.update_distance(constants.SYMBOLIC_COST, '0', '0')
logger.info('Preprocess finished')
logger.debug(f'Initial type system : {type_system}')
logger.debug(f'Extracted preconditions : {preconditions}')
logger.debug(f'Final goal to check : {goal}')
logger.info('Core transformation starts')
transformed, type_system = Transformer(
type_system, enable_shadow).transform(preprocessed)
templates, alignment_array_types = RandomDistanceGenerator(
type_system, enable_shadow).generate_macros(transformed)
logger.debug(f'alignment array types: {alignment_array_types}')
logger.info('Core transformation finishes')
logger.info('Postprocess starts')
postprocessed, sample_array_size_func = PostProcessor(
type_system, custom_variables=holes).process(transformed)
logger.info('Postprocess finishes')
code_template = Template(type_system, postprocessed, templates, goal,
alignment_array_types, sample_array_size_func,
preconditions, holes, hole_preconditions)
return code_template
def test_multiple_functions():
source = f"{plain_function} {'int b() { return 1; }'}"
node = parse(source)
with pytest.raises(ValueError):
Preprocessor().process(node)
# add a target function, should not raise any error
Preprocessor(target_function='a').process(node)
def transform(self,
node: c_ast.FuncDef) -> Tuple[c_ast.FuncDef, TypeSystem]:
if not isinstance(node, c_ast.FuncDef):
raise TypeError(
'Input node must have type c_ast.FuncDef, try to preprocess the node first.'
)
transformed = self.visit_FuncDef(node)
# add returning the final cost variable
transformed.body.block_items.append(
parse(f'return {constants.V_EPSILON};'))
return transformed, self._type_system
def visit_FuncCall(self, node: c_ast.FuncCall):
"""T-Return rule, which adds assertion after the OUTPUT command."""
if self._loop_level == 0 and node.name.name == constants.OUTPUT:
distance_generator = DistanceGenerator(self._type_system)
# add assertion of the output distance == 0
aligned_distance = distance_generator.visit(node.args.exprs[0])[0]
# there is no need to add assertion if the distance is obviously 0
if aligned_distance != '0':
self._insert_at(
node,
parse(f'{constants.ASSERT}({aligned_distance} == 0)'))
self.generic_visit(node)
def assert_templates(name: str, templates: Dict[str, Tuple[Set[str], Set[str]]], enable_shadow=False):
# remove comments
pattern = re.compile(r'\/\/.*|\/\*.*\*\/')
with open(example_folder / Path(name).with_suffix('.c')) as f:
node = parse(pattern.sub('', f.read()))
preprocessed, type_system, preconditions, hole_preconditions, goal = Preprocessor().process(node)
transformed, type_system = Transformer(type_system, enable_shadow=enable_shadow).transform(preprocessed)
generate_templates = RandomDistanceGenerator(type_system).generate(transformed)
for name, (conditions, variables) in generate_templates.items():
assert name in templates, f'Template for {name} is generated but not specified'
specified_conditions, specified_variables = templates[name]
assert specified_conditions == conditions
assert specified_variables == variables
def visit_FuncDef(self, node: c_ast.FuncDef) -> c_ast.FuncDef:
# the start of the transformation
logger.info(f'Start transforming function {node.decl.name} ...')
# make a deep copy and transform on the copied node
node = deepcopy(node)
self._parameters = tuple(decl.name
for decl in node.decl.type.args.params)
logger.debug(f'Params: {self._parameters}')
# visit children
self.generic_visit(node)
insert_statements = [
# insert float CHECKDP_v_epsilon = 0;
parse(f'float {constants.V_EPSILON} = 0'),
# insert int SAMPLE_INDEX = 0;
parse(f'int {constants.SAMPLE_INDEX} = 0')
]
# add declarations of distance variables for dynamically tracked local variables
for name, *distances, _, _ in filter(
lambda variable: variable[0] not in self._parameters,
self._type_system.variables()):
for version, distance in zip(
(constants.ALIGNED_DISTANCE, constants.SHADOW_DISTANCE),
distances):
# skip shadow generation if enable_shadow is not specified
if version == constants.SHADOW_DISTANCE and not self._enable_shadow:
continue
if distance == '*' or distance == f'{version}_{name}':
insert_statements.append(
parse(f'float {version}_{name} = 0'))
# prepend the inserted statements
node.body.block_items[:0] = insert_statements
return node
def visit_FuncDef(self, node: c_ast.FuncDef) -> c_ast.FuncDef:
self._parameters = tuple(decl.name for decl in node.decl.type.args.params)
query = self._parameters[0]
# if it is a dynamically tracked parameter, add new parameters
# add distance variables for dynamically tracked parameters
for name in self._parameters:
*distances, _, _ = self._type_system.get_types(name)
for index, distance in enumerate(distances):
version = constants.ALIGNED_DISTANCE if index == 0 else constants.SHADOW_DISTANCE
if distance == '*':
_, _, plain_type, is_array = self._type_system.get_types(name)
distance_variable = f'{plain_type} {version}_{name}'
distance_variable = distance_variable + '[]' if is_array else distance_variable
node.decl.type.args.params.append(parse(distance_variable))
# add the plain type to the type system
self._type_system.update_base_type(f'{version}_{name}', plain_type, is_array)
if name == query:
# only generate aligned distance for query variable
break
# add sample array variable
node.decl.type.args.params.append(parse(f'int {constants.SAMPLE_ARRAY}[]'))
self._type_system.update_base_type(f'{constants.SAMPLE_ARRAY}', 'int', True)
# add alignment array
node.decl.type.args.params.append(parse(f'int {constants.ALIGNMENT_ARRAY}[]'))
self._type_system.update_base_type(f'{constants.ALIGNMENT_ARRAY}', 'int', True)
# add custom variables
for hole in self._custom_varaibles:
node.decl.type.args.params.append(parse(f'int {hole}'))
# change the return type to int
node.decl.type.type.type.names = ['int']
self.generic_visit(node)
return node
def random_distance(self, alignments_values):
alignments_values = alignments_values[-1][constants.ALIGNMENT_ARRAY]
alignments = {}
for match in re.finditer(
f'#define\\s+{constants.RANDOM_DISTANCE}_([a-zA-Z_][a-zA-Z0-9_]+)\\s+(.*)',
self._random_distances):
alignment = match.group(2)
for index, value in enumerate(alignments_values):
alignment = alignment.replace(
f'{constants.ALIGNMENT_ARRAY}[{index}]', str(value))
# try to simplify the expression (mostly eliminating the terms with coefficient 0)
try:
alignment = _simplify_node(parse(alignment))
finally:
alignments[match.group(1)] = alignment
return alignments
def generate_macros(
self, node: c_ast.FuncDef) -> Tuple[str, List[AlignmentIndexType]]:
"""Generate C-style macros for random variable templates"""
self.visit(node)
logger.debug(f'The generated templates: {self._templates}')
# iterate through all collected templates and insert macros
# e.g. #define CHECKDP_RANDOM_DISTANCE_eta (condition_set ? variable_set)
distance_generator = DistanceGenerator(self._type_system)
inserted, alignment_array_types = [], []
for random_variable, (conditions,
variables) in self._templates.items():
distance_variables = (distance_generator.visit(parse(variable))[0]
for variable in variables)
if self._enable_shadow:
# generate template for selectors
if len(conditions) == 0:
template = constants.SELECT_ALIGNED
else:
template = _generate_random_distance(tuple(conditions),
tuple(),
alignment_array_types,
is_selector=True)
logger.debug(
f'Generated selector template for {random_variable}: {template}'
)
inserted.append(
f'#define {constants.SELECTOR}_{random_variable} ({template})'
)
# convert the sets to tuples since our naive recursive implementation requires orders
template = _generate_random_distance(tuple(conditions),
tuple(distance_variables),
alignment_array_types)
logger.debug(
f'Generated alignment template for {random_variable}: {template}'
)
inserted.append(
f'#define {constants.RANDOM_DISTANCE}_{random_variable} ({template})'
)
logger.debug(
f'Final alignment array size: {len(alignment_array_types)}')
return '\n'.join(inserted), alignment_array_types
def _instrument(self, type_system_1: TypeSystem, type_system_2: TypeSystem,
pc: bool) -> Sequence[c_ast.Assignment]:
inserted_statement = []
for name in set(type_system_1.names()).intersection(
type_system_2.names()):
for version, distance_1, distance_2 in zip(
(constants.ALIGNED_DISTANCE, constants.SHADOW_DISTANCE),
type_system_1.get_types(name),
type_system_2.get_types(name)):
if distance_1 is None or distance_2 is None:
continue
# do not instrument shadow statements if pc = True or enable_shadow is not specified
if not self._enable_shadow or (
version == constants.SHADOW_DISTANCE and pc):
continue
if distance_1 != '*' and distance_2 == '*':
inserted_statement.append(
parse(f'{version}_{name} = {distance_1}'))
return inserted_statement
def visit_Decl(self, node: c_ast.Decl):
logger.debug(f'Line {str(node.coord.line)}: {generate(node)}')
# ignore the FuncDecl node since it's already preprocessed
if isinstance(node.type, c_ast.FuncDecl):
return
# TODO - Enhancement: Array Declaration support
elif not isinstance(node.type, c_ast.TypeDecl):
raise NotImplementedError(
f'Declaration type {node.type} currently not supported for statement: {generate(node)}'
)
# if declarations are in function body, store distance into type system
assert isinstance(node.type, c_ast.TypeDecl)
# if no initial value is given, default to (0, 0)
if not node.init:
self._type_system.update_distance(node.name, '0', '0')
# else update the distance to the distance of initial value (T-Asgn)
elif isinstance(
node.init,
(c_ast.Constant, c_ast.BinaryOp, c_ast.BinaryOp, c_ast.UnaryOp)):
self._assign(c_ast.ID(name=node.name), node.init, node)
# if it is random variable declaration (T-Laplace)
elif isinstance(node.init, c_ast.FuncCall):
if self._enable_shadow and self._pc:
raise ValueError(
'Cannot have random variable assignment in shadow-diverging branches'
)
self._random_variables.add(node.name)
logger.debug(f'Random variables: {self._random_variables}')
# set the random variable distance
self._type_system.update_distance(node.name, '*', '0')
if self._enable_shadow:
# since we have to dynamically switch (the aligned distances) to shadow version, we have to guard the
# switch with the selector
shadow_type_system = deepcopy(self._type_system)
for name, _, shadow_distance, _, _ in shadow_type_system.variables(
):
# skip the distance of custom holes
if constants.HOLE in name:
continue
shadow_type_system.update_distance(name, shadow_distance,
shadow_distance)
self._type_system.merge(shadow_type_system)
if self._loop_level == 0:
to_inserts = []
if self._enable_shadow:
# insert distance updates for normal variables
distance_update_statements = []
for name, align, shadow, _, _ in self._type_system.variables(
):
if align == '*' and name not in self._parameters and name != node.name:
shadow_distance = f'{constants.SHADOW_DISTANCE}_{name}' if shadow == '*' else shadow
distance_update_statements.append(
parse(
f'{constants.ALIGNED_DISTANCE}_{name} = {shadow_distance};'
))
distance_update = c_ast.If(cond=parse(
f'{constants.SELECTOR}_{node.name} == {constants.SELECT_SHADOW}'
),
iftrue=c_ast.Compound(
block_items=
distance_update_statements),
iffalse=None)
to_inserts.append(distance_update)
# insert distance template for the variable
distance = parse(
f'{constants.ALIGNED_DISTANCE}_{node.name} = {constants.RANDOM_DISTANCE}_{node.name}'
)
to_inserts.append(distance)
# insert cost variable update statement
scale = generate(node.init.args.exprs[0])
cost = expr_simplify(
f'(Abs({constants.ALIGNED_DISTANCE}_{node.name}) * (1 / ({scale})))'
)
# calculate v_epsilon by combining normal cost and sampling cost
if self._enable_shadow:
previous_cost = \
f'(({constants.SELECTOR}_{node.name} == {constants.SELECT_ALIGNED}) ? {constants.V_EPSILON} : 0)'
else:
previous_cost = constants.V_EPSILON
v_epsilon = parse(
f'{constants.V_EPSILON} = {previous_cost} + {cost}')
to_inserts.append(v_epsilon)
# transform sampling command to havoc command
node.init = parse(
f'{constants.SAMPLE_ARRAY}[{constants.SAMPLE_INDEX}]')
to_inserts.append(
parse(
f'{constants.SAMPLE_INDEX} = {constants.SAMPLE_INDEX} + 1;'
))
self._insert_at(node, to_inserts)
else:
raise NotImplementedError(
f'Initial value currently not supported: {node.init}')
logger.debug(f'types: {self._type_system}')
def test_sensitivities():
for sensitivity in ('ALL_DIFFER', 'ONE_DIFFER', 'INCREASING',
'DECREASING'):
node = parse(plain_function.replace('ALL_DIFFER', sensitivity))
_, _, preconditions, _, _ = Preprocessor().process(node)
assert sensitivity in preconditions
