def _shift_and_fix_registers(registers, offset, addresses, min_address):
"""Shift register contents."""
sregisters = []
for register in registers:
register = register.copy()
if not register.name.startswith("GPR"):
# Do not shift not GPR registers
sregisters.append(register)
continue
# if GPR value is a negative one, do not shift it
value = twocs_to_int(register.value, 64)
if value < 0:
sregisters.append(register)
continue
if not ((addresses[0] <= register.value < addresses[1]) and
register.value > min_address):
sregisters.append(register)
continue
distance = min((abs(register.value - addr) for addr in addresses[2]))
if distance > _FIX_ADDRESS_TOLERANCE:
sregisters.append(register)
continue
print_info(
"Shift '%s' from '0x%016X' to '0x%016X'" %
(register.name,
register.value,
register.value + offset),
)
register.value = register.value + offset
fmt_str = "{0:064b}"
value_coded = fmt_str.format(register.value)
if len(value_coded) > 64:
print_warning(
"Overflow during shifting. Cropping of"
" register '%s'" % register.name
)
register.value = int(value_coded[-64:], 2)
sregisters.append(register)
return sregisters
def dump_mpt(input_file_fd, target, arguments):
"""
:param input_file_fd:
:type input_file_fd:
:param target:
:type target:
:param arguments:
:type arguments:
"""
try:
contents = input_file_fd.read()
if six.PY3 and not isinstance(contents, str):
contents = contents.decode()
except KeyboardInterrupt:
print_info("No input data provided. Exiting...")
exit(1)
print_info("Parsing input file...")
print_info("Sections to parse: %s" % arguments['sections'])
var_defs, req_defs, instr_defs = \
interpret_objdump(contents, target,
strict=arguments.get('strict', False),
sections=arguments['sections'],
start_address=arguments['from_address'],
end_address=arguments['to_address'])
print_info("Input file parsed")
print_info(
"%d instructions processed from the input file" % len(instr_defs)
)
if var_defs != []:
print_info(
"Variables referenced and detected in the dump: %s" %
','.join([var.name for var in var_defs])
)
if req_defs != []:
print_warning(
"Variables referenced and *NOT* detected in the dump: %s" %
','.join([var.name for var in req_defs])
)
print_warning(
"You might need to edit the generated MPT to fix the"
" declaration of such variables"
)
print_info("Generating the MPT contents...")
mpt_config = mpt_configuration_factory()
if 'default_code_address' in arguments:
mpt_config.set_default_code_address(arguments['default_code_address'])
else:
mpt_config.set_default_code_address(instr_defs[0].address.displacement)
if 'default_data_address' in arguments:
mpt_config.set_default_data_address(arguments['default_data_address'])
else:
mpt_config.set_default_data_address(0)
if arguments.get('elf_abi', False):
kwargs = {}
if "stack_name" in arguments:
kwargs["stack_name"] = arguments["stack_name"]
if "stack_address" in arguments:
kwargs["stack_address"] = Address(
base_address="code",
displacement=arguments["stack_address"]
)
variables, instructions = target.elf_abi(
arguments["stack_size"], arguments.get(
"start_symbol", None
), **kwargs
)
for variable in variables:
req_defs.append(variable_to_test_definition(variable))
address = instr_defs[0].address
for instr in reversed(instructions):
instr_defs = [instruction_to_definition(instr)] + instr_defs
address -= instr.architecture_type.format.length
if address.displacement < 0:
print_error(
"Default code address is below zero after"
" adding the initialization code."
)
print_error(
"Check/modify the objdump provided or do not use"
" the elf_abi flag."
)
exit(-1)
mpt_config.set_default_code_address(address.displacement)
instr = None
if "end_branch_to_itself" in arguments:
instr = target.branch_to_itself()
elif arguments.get('elf_abi', False):
instr = target.nop()
if instr is not None:
instr.set_label("ELF_ABI_EXIT")
instr_defs.append(instruction_to_definition(instr))
for var in var_defs + req_defs:
mpt_config.register_variable_definition(var)
mpt_config.register_instruction_definitions(instr_defs)
print_info("Dumping MPT to '%s'" % arguments['output_mpt_file'])
mpt_parser = mpt_parser_factory()
mpt_parser.dump_mpt_config(mpt_config, arguments['output_mpt_file'])
def _compute_reset_code(target, test_def, args):
instructions = interpret_asm(
test_def.code, target, [var.name for var in test_def.variables],
show_progress=True,
)
# TODO: This can be done in parallel or look for speed up the process
instructions = [
instruction_from_definition(instr) for instr in instructions
]
instruction_dict = {}
address = test_def.default_code_address
progress = Progress(
len(test_def.roi_memory_access_trace),
msg="Building instruction dictionary",
)
for instr in instructions:
progress()
if instr.address is not None:
if instr.address.base_address == "code":
address = test_def.default_code_address + \
instr.address.displacement
instr.set_address(address)
else:
address = address + instr.architecture_type.format.length
instr.set_address(address)
instruction_dict[instr.address] = instr
free_regs = []
written_after_read_regs = []
read_regs = []
level = 0
dynamic_count = 0
progress = Progress(
len(test_def.roi_memory_access_trace),
msg="Evaluating register usage",
)
reset_regs = set()
for access in test_def.roi_memory_access_trace:
progress()
if access.data_type == "D":
continue
dynamic_count += 1
try:
instr = instruction_dict[access.address]
uses = instruction_dict[access.address].uses()
sets = instruction_dict[access.address].sets()
except KeyError:
print_error(
"Access to from instruction at address "
"0x%016X registered but such instruction is not"
" present in the definition." % access.address,
)
exit(1)
# Calls
if instr.mnemonic == "BL":
level += 1
elif instr.mnemonic == "BCL":
level += 1
elif instr.mnemonic == "BCCTRL":
if instr.operands()[2].value in [0, 3]:
level += 1
# Returns
if instr.mnemonic == "BCLR":
if (((instr.operands()[0].value & 0b10100) == 20) and
(instr.operands()[2].value == 0)):
level -= 1
# TODO: this should include Z and RISCV instructions for call
# and return, but currently we do not have memory access traces
# for such platforms
for reg in uses:
if reg not in read_regs:
read_regs.append(reg)
for reg in sets:
if reg in free_regs:
continue
elif reg not in read_regs:
free_regs.append(reg)
elif reg not in written_after_read_regs:
written_after_read_regs.append(reg)
reset_regs = set(read_regs).intersection(
set(written_after_read_regs),
)
reset_regs = sorted(reset_regs)
assert len(free_regs) == len(set(free_regs))
assert len(set(free_regs).intersection(set(reset_regs))) == 0
if len(test_def.roi_memory_access_trace) == 0:
# We do not have memory access trace, assume calling conventions
reset_regs = target.volatile_registers
reset_regs = [
reg for reg in reset_regs if reg in target.volatile_registers]
if len(reset_regs) == 0 and len(test_def.roi_memory_access_trace) == 0:
print_info(
"No memory access trace found. Resetting volatile registers."
)
reset_regs = target.volatile_registers
unused_regs = sorted(
(reg for reg in target.registers.values() if reg not in read_regs),
)
#
# Make sure scratch registers are reset last
#
for reg in target.scratch_registers:
if reg in reset_regs:
reset_regs.remove(reg)
reset_regs.append(reg)
free_regs = unused_regs + free_regs
# Know which ones are not used (or written) and which ones are used
# Use them as base / temporal registers for addresses
# Check addresses
conflict_addresses = {}
new_ins = []
progress = Progress(
len(test_def.roi_memory_access_trace),
msg="Evaluating memory usage",
)
for access in test_def.roi_memory_access_trace:
progress()
if access.data_type == "I":
continue
val = conflict_addresses.get(
access.address,
[access.length, access.access_type],
)
if access.access_type not in val[1]:
val[1] += access.access_type
val[0] = max(val[0], access.length)
conflict_addresses[access.address] = val
fix_addresses = []
for address in conflict_addresses:
value = conflict_addresses[address]
if value[1] == "RW":
wvalue = None
for var in test_def.variables:
if var.var_type.upper() in ["CHAR", "UINT8_T"]:
elem_size = 1
else:
raise NotImplementedError
end_address = var.address + var.num_elements * elem_size
if var.address <= address <= end_address:
offset = int((address - var.address) / elem_size)
svalue = var.init_value[
offset:offset + int(value[0] / elem_size)
]
svalue = "".join(["%02X" % tval for tval in svalue])
wvalue = int(svalue, 16)
break
if wvalue is None:
print_error(
"Unable to restore original value for address 0x%X" %
address,
)
exit(1)
if value[0] <= 8:
fix_addresses.append((address, value[0], wvalue))
else:
for selem in range(0, value[0]//8):
sfmt = "%%0%dX" % (2*value[0])
nvalue = sfmt % wvalue
nvalue = int(nvalue[selem*16:(selem+1)*16], 16)
fix_addresses.append(
(address + selem * 8,
8,
nvalue)
)
reset_steps = []
context = Context()
context.set_symbolic(True)
if len(fix_addresses) > 0:
# TODO: This can be optimized. Reduce the number of instructions to
# be added by sorting the reset code (shared values or similar
# addresses)
# TODO: This can be optimized for use vector registers when
# needed
#
print_info("Adding instructions to reset memory state")
reset_register = [
reg
for reg in free_regs
if reg.type.used_for_address_arithmetic and
reg.name != "GPR0"
][0]
for address, length, value in fix_addresses:
address_obj = Address(base_address="data", displacement=address)
new_instructions = target.set_register(
reset_register, value, context, opt=False,
)
for ins in new_instructions:
ins.add_comment(
"Reset code. Setting %s to 0X%016X" %
(reset_register.name, value),
)
reset_steps.append([new_instructions[:], reset_register, value])
context.set_register_value(reset_register, value)
try:
store_ins = target.store_integer(
reset_register, address_obj, length * 8, context,
)
new_instructions += store_ins
reset_steps.append(
[store_ins, reset_register, address_obj, length],
)
except MicroprobeCodeGenerationError:
areg = [
reg for reg in free_regs
if reg.type.used_for_address_arithmetic and reg.name !=
"GPR0"
][1]
set_ins = target.set_register(
areg, address, context, opt=False,
)
new_instructions += set_ins
reset_steps.append([set_ins, areg, address_obj])
context.set_register_value(areg, address_obj)
store_ins = target.store_integer(
reset_register, address_obj, length * 8, context,
)
new_instructions += store_ins
reset_steps.append(
[store_ins, reset_register, address_obj, length],
)
for ins in set_ins:
ins.add_comment(
"Reset code. Setting %s to 0X%016X" %
(areg.name, address),
)
for ins in store_ins:
ins.add_comment(
"Reset code. Setting mem content in 0X%016X" % (address),
)
new_ins.extend(new_instructions)
# Reset contents of used registers
for reset_register in reset_regs:
try:
value = [
reg for reg in test_def.registers if reg.name ==
reset_register.name
][0].value
except IndexError:
continue
new_instructions = target.set_register(
reset_register, value, context, opt=False,
)
reset_steps.append([new_instructions, reset_register, value])
context.set_register_value(reset_register, value)
for ins in new_instructions:
ins.add_comment(
"Reset code. Setting %s to 0X%016X" %
(reset_register.name, value),
)
new_ins.extend(new_instructions)
try:
overhead = (((len(new_ins) * 1.0) / dynamic_count) * 100)
except ZeroDivisionError:
print_warning("Unable to compute overhead. Zero dynamic instruction "
"count")
overhead = 0
print_info(
"%03.2f%% overhead added by resetting code" % overhead,
)
if overhead > args['wrap_endless_threshold']:
print_error(
"Instructions added: %d" % len(new_ins),
)
print_error(
"Total instructions: %d" % dynamic_count,
)
print_error(
"Reset code above --wrap-endless-threshold. Stopping generation.",
)
exit(1)
return new_ins, overhead, reset_steps
def dump_mpt(input_file, target, init_data, arguments):
"""
:param input_file:
:type input_file:
:param target:
:type target:
:param init_data:
:type init_data:
:param arguments:
:type arguments:
"""
input_file_fd = io.open(input_file, 'r')
contents = input_file_fd.read()
if six.PY2:
contents = contents.encode("ascii")
elif six.PY3:
pass
print_info("Parsing input file...")
var_defs, req_defs, instr_defs = \
interpret_objdump(contents, target,
strict=arguments.get('strict', False),
sections=["microprobe.text"])
print_info("Input file parsed")
print_info("%d instructions processed from the input file" %
len(instr_defs))
if var_defs != []:
print_info("Variables referenced and detected in the dump: %s" %
','.join([var.name for var in var_defs]))
if req_defs != []:
print_warning(
"Variables referenced and *NOT* detected in the dump: %s" %
','.join([var.name for var in req_defs]))
print_warning("You might need to edit the generated MPT to fix the"
" declaration of such variables")
print_info("Generating the MPT contents...")
mpt_config = mpt_configuration_factory()
mpt_config.set_default_code_address(arguments['default_code_address'])
mpt_config.set_default_data_address(arguments['default_data_address'])
kwargs = {}
if "stack_name" in arguments:
kwargs["stack_name"] = arguments["stack_name"]
if "stack_address" in arguments:
kwargs["stack_address"] = Address(
base_address="code", displacement=arguments["stack_address"])
variables, instructions = target.elf_abi(arguments["stack_size"],
"c2mpt_function", **kwargs)
for variable in variables:
req_defs.append(variable_to_test_definition(variable))
address = instr_defs[0].address
for instr in reversed(instructions):
instr_defs = [instruction_to_definition(instr)] + instr_defs
address -= instr.architecture_type.format.length
if address.displacement < 0:
print_error("Default code address is below zero after"
" adding the initialization code.")
print_error("Check/modify the objdump provided or do not use"
" the elf_abi flag.")
_exit(-1)
if "end_branch_to_itself" in arguments:
instr = target.branch_to_itself()
else:
instr = target.nop()
instr.set_label("ELF_ABI_EXIT")
instr_defs.append(instruction_to_definition(instr))
mpt_config.set_default_code_address(address.displacement)
initialized_variables = {}
mindisplacement = None
for data in init_data.split('\n'):
if data.strip() == "":
continue
if data.startswith("WARNING"):
print_warning(data.split(":")[1].strip())
continue
name = data.split("=")[0].strip()
values = ast.literal_eval(data.split("=")[1].strip())
initialized_variables[name] = values
if mindisplacement is None:
mindisplacement = values[2]
# TODO: this is unsafe. We should compute the real size
maxdisplacement = values[2] + (values[1] * 8)
else:
mindisplacement = min(values[2], mindisplacement)
maxdisplacement = max(values[2] + (values[1] * 8), maxdisplacement)
if "host_displacement" in arguments:
mindisplacement = arguments.get("host_displacement", mindisplacement)
for name, values in initialized_variables.items():
values[2] = values[2] - mindisplacement + \
arguments['default_data_address']
if 'fix_displacement' in arguments:
for name, values in initialized_variables.items():
if "*" in values[0] and isinstance(values[4], list):
new_values = []
for value in values[4]:
if value <= maxdisplacement and value >= mindisplacement:
value = value - mindisplacement + \
arguments['default_data_address']
new_values.append(value)
values[4] = new_values
elif "*" in values[0] and isinstance(values[4], int):
if (values[4] <= maxdisplacement
and values[4] >= mindisplacement):
values[4] = values[4] - mindisplacement + \
arguments['default_data_address']
elif values[0] == "uint8_t" and isinstance(values[4], list):
if len(values[4]) > 8:
new_values = "".join(["%02x" % elem for elem in values[4]])
# Enable this for testing switched endianness
# new_values = [new_values[idx:idx + 2]
# for idx in range(0, len(new_values), 2)]
# new_values = new_values[::-1]
# new_values = "".join(new_values)
new_values = _fix_displacement(new_values, mindisplacement,
maxdisplacement, arguments)
values[4] = [
int(new_values[idx:idx + 2], 16)
for idx in range(0, len(new_values), 2)
]
for name, values in initialized_variables.items():
mpt_config.register_variable_definition(
MicroprobeTestVariableDefinition(name, *values))
print_info("Init values for variable '%s' parsed" % name)
for var in var_defs + req_defs:
if var.name in list(initialized_variables.keys()):
continue
if var.name != arguments["stack_name"]:
print_warning("Variable '%s' not registered in the C file "
"using the macros provided!" % var.name)
mpt_config.register_variable_definition(var)
mpt_config.register_instruction_definitions(instr_defs)
print_info("Dumping MPT to '%s'" % arguments['output_mpt_file'])
mpt_parser = mpt_parser_factory()
mpt_parser.dump_mpt_config(mpt_config, arguments['output_mpt_file'])
def dump_objdump(target, arguments):
"""
:param target:
:type target:
:param arguments:
:type arguments:
"""
cmd = "'%s' -Ax -tx1 -v '%s'" % (arguments['od_bin'],
arguments['input_bin_file'])
text_string = run_cmd_output(cmd)
bintext = []
for line in text_string.split('\n'):
if line == "":
continue
bintext.append("".join(line.split(' ')[1:]))
instrs = interpret_bin("".join(bintext), target)
print("")
print("%s:\tfile format raw %s" %
(os.path.basename(arguments['input_bin_file']), target.isa.name))
print("")
print("")
print("Disassembly of section .raw:")
print("")
maxlen = max(len(instr.asm[2:]) for instr in instrs)
if maxlen % 2 != 0:
maxlen = maxlen + 1
maxlen += maxlen // 2
counter = arguments['start_address']
label_dict = RejectingOrderedDict()
instr_dict = RejectingOrderedDict()
for instr_def in instrs:
instr = instruction_from_definition(instr_def, fix_relative=False)
asm = instr.assembly().lower()
relative = None
absolute = None
if instr.branch:
for memoperand in instr.memory_operands():
if not memoperand.descriptor.is_branch_target:
continue
for operand in memoperand.operands:
if operand.type.address_relative:
relative = operand.value
if operand.type.address_absolute:
absolute = operand.value
masm = instr_def.asm[2:]
if len(masm) % 2 != 0:
masm = "0" + masm
binary = " ".join([str(masm)[i:i + 2] for i in range(0, len(masm), 2)])
label = None
if counter == 0:
label = ".raw"
label_dict[counter] = label
elif counter in label_dict:
label = label_dict[counter]
rtarget = None
atarget = None
if relative is not None or absolute is not None:
if relative is not None:
assert absolute is None
if isinstance(relative, six.integer_types):
target_addr = counter + relative
rtarget = relative
elif isinstance(relative, Address):
target_addr = counter + relative.displacement
rtarget = relative.displacement
else:
raise NotImplementedError
if absolute is not None:
assert relative is None
if isinstance(absolute, six.integer_types):
target_addr = absolute
atarget = absolute
elif isinstance(absolute, Address):
target_addr = absolute.displacement
atarget = absolute.displacement
else:
raise NotImplementedError
if target_addr not in label_dict:
label_dict[target_addr] = "branch_%x" % target_addr
if target_addr in instr_dict:
instr_dict[target_addr][2] = label_dict[target_addr]
instr_dict[counter] = [binary, asm, label, rtarget, atarget]
counter = counter + len(masm) // 2
str_format = "%8s:\t%-" + str(maxlen) + "s\t%s"
addresses = []
for counter, values in instr_dict.items():
binary, asm, label, rtarget, atarget = values
if label is not None:
print("%016x <%s>:" % (counter, label))
cformat = str_format
if rtarget is not None:
cformat = str_format + "\t <%s>" % (label_dict[counter + rtarget])
elif atarget is not None:
cformat = str_format + "\t <%s>" % (label_dict[atarget])
print(cformat % (hex(counter)[2:], binary, asm))
addresses.append(counter)
error = False
for key in label_dict.keys():
if key not in addresses:
print_warning("Target address '%s' not in the objdump" % hex(key))
error = True
if error and arguments['strict']:
print_error("Check target addresses of relative branches")
exit(-1)
def generate(test_definition, outputfile, target, **kwargs):
"""
Microbenchmark generation policy
:param test_definition: Test definition object
:type test_definition: :class:`MicroprobeTestDefinition`
:param outputfile: Outputfile name
:type outputfile: :class:`str`
:param target: Target definition object
:type target: :class:`Target`
"""
variables = test_definition.variables
if 'raw_bin' in kwargs:
print_info("Interpreting RAW dump...")
sequence = []
raw_dict = {}
current_address = 0
for entry in test_definition.code:
if (not entry.assembly.upper().startswith("0X")
and not entry.assembly.upper().startswith("0B")):
raise MicroprobeMPTFormatError(
"This is not a RAW dump as it contains "
"assembly (%s)" % entry.assembly)
if entry.label is not None:
raise MicroprobeMPTFormatError(
"This is not a RAW dump as it contains "
"labels (%s)" % entry.label)
if entry.decorators not in ['', ' ', None, []]:
raise MicroprobeMPTFormatError(
"This is not a RAW dump as it contains "
"decorators (%s)" % entry.decorators)
if entry.comments not in ['', ' ', None, []]:
raise MicroprobeMPTFormatError(
"This is not a RAW dump as it contains "
"comments (%s)" % entry.comments)
if entry.address is not None:
current_address = entry.address
if current_address not in raw_dict:
raw_dict[current_address] = ""
# Assume that raw dump use a 4 bytes hex dump
if len(entry.assembly) != 10:
raise MicroprobeMPTFormatError(
"This is not a RAW 4-byte dump as it contains "
"lines with other formats (%s)" % entry.assembly)
raw_dict[current_address] += entry.assembly[2:]
if len(raw_dict) > 1:
address_ant = sorted(raw_dict.keys())[0]
len_ant = len(raw_dict[address_ant]) // 2
# Assume that raw dump use a 4 bytes hex dump
assert len_ant % 4 == 0
for address in sorted(raw_dict.keys())[1:]:
if address_ant + len_ant == address:
raw_dict[address_ant] += raw_dict[address]
len_ant = len(raw_dict[address_ant]) // 2
raw_dict.pop(address)
else:
len_ant = len(raw_dict[address]) // 2
address_ant = address
# Assume that raw dump use a 4 bytes hex dump
assert len_ant % 4 == 0
sequence = []
for address in sorted(raw_dict.keys()):
# Endianess will be big endian, because we are concatenating
# full words resulting in the higher bits being encoded first
code = interpret_bin(raw_dict[address],
target,
safe=True,
little_endian=False,
word_length=4)
code[0].address = address
sequence.extend(code)
else:
print_info("Interpreting assembly...")
sequence = interpret_asm(test_definition.code, target,
[var.name for var in variables])
if len(sequence) < 1:
raise MicroprobeMPTFormatError(
"No instructions found in the 'instructions' entry of the MPT"
" file. Check the input file.")
registers = test_definition.registers
raw = test_definition.raw
for instruction_def in sequence:
if instruction_def.address is not None:
print_warning("Instruction address not needed in '%s'" %
instruction_def.asm)
if test_definition.default_data_address is not None:
print_warning("Default data address not needed")
if test_definition.default_code_address is not None:
print_warning("Default code address not needed")
wrapper_name = "CWrapper"
if kwargs.get("endless", False):
print_warning("Using endless C wrapper")
wrapper_name = "CInfGen"
print_info("Creating benchmark synthesizer...")
try:
cwrapper = microprobe.code.get_wrapper(wrapper_name)
except MicroprobeValueError as exc:
raise MicroprobeException(
"Wrapper '%s' not available. Check if you have the wrappers "
"of the target installed or set up an appropriate "
"MICROPROBEWRAPPERS environment variable. Original error was: %s" %
(wrapper_name, str(exc)))
wrapper_kwargs = {}
wrapper = cwrapper(**wrapper_kwargs)
synth = microprobe.code.Synthesizer(target, wrapper, extra_raw=raw)
if len(registers) >= 0:
print_info("Add register initialization pass")
synth.add_pass(
microprobe.passes.initialization.InitializeRegistersPass(
registers, skip_unknown=True, warn_unknown=True))
synth.add_pass(
microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence)))
synth.add_pass(microprobe.passes.variable.DeclareVariablesPass(variables))
synth.add_pass(
microprobe.passes.instruction.ReproduceSequencePass(sequence))
if kwargs.get("fix_indirect_branches", False):
print_info("Fix indirect branches: On")
synth.add_pass(
microprobe.passes.address.UpdateInstructionAddressesPass())
synth.add_pass(microprobe.passes.branch.FixIndirectBranchPass())
if kwargs.get("fix_branch_next", False):
print_info("Force branch to next: On")
synth.add_pass(
microprobe.passes.address.UpdateInstructionAddressesPass())
synth.add_pass(microprobe.passes.branch.BranchNextPass(force=True))
if kwargs.get("fix_memory_registers", False):
kwargs["fix_memory_references"] = True
if kwargs.get("fix_memory_references", False):
synth.add_pass(
microprobe.passes.memory.FixMemoryReferencesPass(
reset_registers=kwargs.get("fix_memory_registers", False)))
if kwargs.get("fix_memory_registers", False):
print_info("Fix memory registers: On")
synth.add_pass(microprobe.passes.register.NoHazardsAllocationPass())
print_info("Synthesizing...")
bench = synth.synthesize()
# Save the microbenchmark
synth.save(outputfile, bench=bench)
return
def generate_genetic(compname, ipc):
"""Generate a microbenchmark stressing compname at the given ipc."""
comps = []
bcomps = []
any_comp = False
if compname.find("FXU") >= 0:
comps.append(TARGET.elements["FXU0_Core0_SCM_Processor"])
if compname.find("VSU") >= 0:
comps.append(TARGET.elements["VSU0_Core0_SCM_Processor"])
if len(comps) == 2:
any_comp = True
elif compname.find("noLSU") >= 0:
bcomps.append(TARGET.elements["LSU0_Core0_SCM_Processor"])
elif compname.find("LSU") >= 0:
comps.append(TARGET.elements["LSU_Core0_SCM_Processor"])
if (len(comps) == 1 and ipc > 2) or (len(comps) == 2 and ipc > 4):
return True
for elem in os.listdir(DIRECTORY):
if not elem.endswith(".c"):
continue
if elem.startswith("%s:IPC:%.2f:DIST" % (compname, ipc)):
print_info("Already generated: %s %d" % (compname, ipc))
return True
print_info("Going for IPC: %f and Element: %s" % (ipc, compname))
def generate(name, *args):
"""Benchmark generation function.
First argument is name, second the dependency distance and the
third is the average instruction latency.
"""
dist, latency = args
wrapper = microprobe.code.get_wrapper("CInfPpc")
synth = microprobe.code.Synthesizer(TARGET, wrapper())
synth.add_pass(
microprobe.passes.initialization.InitializeRegistersPass(
value=RNDINT))
synth.add_pass(
microprobe.passes.structure.SimpleBuildingBlockPass(BENCHMARK_SIZE)
)
synth.add_pass(
microprobe.passes.instruction.SetInstructionTypeByElementPass(
TARGET,
comps,
{},
block=bcomps,
avelatency=latency,
any_comp=any_comp))
synth.add_pass(
microprobe.passes.register.DefaultRegisterAllocationPass(
dd=dist))
bench = synth.synthesize()
synth.save(name, bench=bench)
# Set the genetic algorithm parameters
ga_params = []
ga_params.append((0, 20, 0.05)) # Average dependency distance design space
ga_params.append((2, 8, 0.05)) # Average instruction latency design space
# Set up the search driver
driver = microprobe.driver.genetic.ExecCmdDriver(
generate, 20, 30, 30, "'%s' %f " %
(COMMAND, ipc), ga_params)
starttime = runtime.time()
print_info("Start search...")
driver.run(1)
print_info("Search end")
endtime = runtime.time()
print_info("Genetic time::%s" % (
datetime.timedelta(seconds=endtime - starttime))
)
# Check if we found a solution
ga_params = driver.solution()
score = driver.score()
print_info("IPC found: %f, score: %f" % (ipc, score))
if score < 20:
print_warning("Unable to find an optimal solution with IPC: %f:" % ipc)
print_info("Generating the closest solution...")
generate(
"%s/%s:IPC:%.2f:DIST:%.2f:LAT:%.2f-check" %
(DIRECTORY, compname, ipc, ga_params[0], ga_params[1]),
ga_params[0], ga_params[1]
)
print_info("Closest solution generated")
else:
print_info(
"Solution found for %s and IPC %f -> dist: %f , "
"latency: %f " %
(compname, ipc, ga_params[0], ga_params[1]))
print_info("Generating solution...")
generate("%s/%s:IPC:%.2f:DIST:%.2f:LAT:%.2f" %
(DIRECTORY, compname, ipc, ga_params[0], ga_params[1]),
ga_params[0], ga_params[1]
)
print_info("Solution generated")
return True
def generate(test_definition, outputfile, target, **kwargs):
"""
Microbenchmark generation policy
:param test_definition: Test definition object
:type test_definition: :class:`MicroprobeTestDefinition`
:param outputfile: Outputfile name
:type outputfile: :class:`str`
:param target: Target definition object
:type target: :class:`Target`
"""
variables = test_definition.variables
print_info("Interpreting assembly...")
sequence = interpret_asm(test_definition.code, target,
[var.name for var in variables])
if len(sequence) < 1:
raise MicroprobeMPTFormatError(
"No instructions found in the 'instructions' entry of the MPT"
" file. Check the input file.")
registers = test_definition.registers
raw = test_definition.raw
for instruction_def in sequence:
if instruction_def.address is not None:
print_warning("Instruction address not needed in '%s'" %
instruction_def.asm)
if test_definition.default_data_address is not None:
print_warning("Default data address not needed")
if test_definition.default_code_address is not None:
print_warning("Default code address not needed")
wrapper_name = "CWrapper"
if kwargs.get("endless", False):
print_warning("Using endless C wrapper")
wrapper_name = "CInfGen"
print_info("Creating benchmark synthesizer...")
try:
cwrapper = microprobe.code.get_wrapper(wrapper_name)
except MicroprobeValueError as exc:
raise MicroprobeException(
"Wrapper '%s' not available. Check if you have the wrappers "
"of the target installed or set up an appropriate "
"MICROPROBEWRAPPERS environment variable. Original error was: %s" %
(wrapper_name, str(exc)))
wrapper_kwargs = {}
wrapper = cwrapper(**wrapper_kwargs)
synth = microprobe.code.Synthesizer(target, wrapper, extra_raw=raw)
if len(registers) >= 0:
print_info("Add register initialization pass")
synth.add_pass(
microprobe.passes.initialization.InitializeRegistersPass(
registers, skip_unknown=True, warn_unknown=True))
synth.add_pass(
microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence)))
synth.add_pass(microprobe.passes.variable.DeclareVariablesPass(variables))
synth.add_pass(
microprobe.passes.instruction.ReproduceSequencePass(sequence))
if kwargs.get("fix_memory_registers", False):
kwargs["fix_memory_references"] = True
if kwargs.get("fix_memory_references", False):
synth.add_pass(
microprobe.passes.memory.FixMemoryReferencesPass(
reset_registers=kwargs.get("fix_memory_registers", False)))
if kwargs.get("fix_memory_registers", False):
print_info("Fix memory registers: On")
synth.add_pass(microprobe.passes.register.NoHazardsAllocationPass())
if kwargs.get("fix_branch_next", False):
print_info("Force branch to next: On")
synth.add_pass(
microprobe.passes.address.UpdateInstructionAddressesPass())
synth.add_pass(microprobe.passes.branch.BranchNextPass(force=True))
if kwargs.get("fix_indirect_branches", False):
print_info("Fix indirect branches: On")
synth.add_pass(
microprobe.passes.address.UpdateInstructionAddressesPass())
synth.add_pass(microprobe.passes.branch.FixIndirectBranchPass())
print_info("Synthesizing...")
bench = synth.synthesize()
# Save the microbenchmark
synth.save(outputfile, bench=bench)
return
def __call__(self, building_block, target):
"""
:param building_block:
:param target:
"""
if not self._skip_unknown:
for register_name in self._reg_dict:
if register_name not in list(target.registers.keys()):
raise MicroprobeCodeGenerationError(
"Unknown register name: '%s'. Unable to set it" %
register_name)
if self._warn_unknown:
for register_name in self._reg_dict:
if register_name not in list(target.registers.keys()):
print_warning(
"Unknown register name: '%s'. Unable to set it" %
register_name)
for reg in target.registers.values():
value = None
elemsize = None
if reg.name in self._reg_dict:
value = self._reg_dict[reg.name]
self._reg_dict.pop(reg.name)
if (reg in building_block.context.reserved_registers
and not self._force_reserved):
LOG.debug("Skip reserved - %s", reg)
continue
elif reg in target.control_registers:
LOG.debug("Skip control - %s", reg)
continue
if value is None:
if reg.used_for_vector_arithmetic:
if self._vect_value is not None:
value = self._vect_value
elemsize = self._vect_elemsize
else:
LOG.debug("Skip no vector default value provided - %s",
reg)
continue
elif reg.used_for_float_arithmetic:
if self._fp_value is not None:
value = self._fp_value
else:
LOG.debug("Skip no float default value provided - %s",
reg)
continue
else:
if self._value is not None:
value = self._value
else:
LOG.debug("Skip no default value provided - %s", reg)
continue
while callable(value):
value = value()
if reg.used_for_float_arithmetic:
value = ieee_float_to_int64(float(value))
elif reg.used_for_vector_arithmetic:
if isinstance(value, float):
if elemsize != 64:
raise MicroprobeCodeGenerationError(
"Unable to initialize '%s' to '%s'. Only 64bit"
" vector element initialization is supported" %
(reg.name, (value, elemsize)))
value = ieee_float_to_int64(float(value))
value = "%d_%d" % (value, elemsize)
else:
value = "%d_%d" % (value, elemsize)
LOG.debug("Set '%s' to '0x%x'", reg, value)
if (target.wrapper.direct_initialization_support
and not self._force_code):
try:
target.wrapper.register_direct_init(reg, value)
except MicroprobeCodeGenerationError:
building_block.add_init(
target.set_register(reg, value,
building_block.context))
except MicroprobeDuplicatedValueError:
LOG.debug("Skip already set - %s", reg)
else:
building_block.add_init(
target.set_register(reg, value, building_block.context))
building_block.context.set_register_value(reg, value)
def dump_objdump(target, arguments):
"""
:param target:
:type target:
:param arguments:
:type arguments:
"""
ifile = open(arguments['input_dma_file'], 'r')
dataw = arguments['width_bytes']
inputlines = ifile.readlines()
ifile.close()
progress = Progress(len(inputlines), msg="Lines parsed:")
lines_dict = {}
for idx, line in enumerate(inputlines):
splitline = line.upper().split(" ")
if len(splitline) != 3:
raise MicroprobeDMAFormatError("Unable to parse line %d: %s" %
(idx, line))
if (splitline[0] != "D" or len(splitline[1]) != 16
or len(splitline[1]) != 16):
raise MicroprobeDMAFormatError("Unable to parse line %d: %s" %
(idx, line))
key = int(splitline[1], base=16)
if key in lines_dict:
print_warning("Address (%s) in line %d overwrites previous entry" %
(splitline[1], idx))
lines_dict[key] = splitline[2][:-1]
progress()
current_key = None
progress = Progress(len(list(lines_dict.keys())),
msg="Detecting segments:")
for key in sorted(lines_dict):
progress()
if current_key is None:
current_key = key
continue
current_address = current_key + (len(lines_dict[current_key]) // 2)
if current_address == key:
lines_dict[current_key] += lines_dict[key]
lines_dict.pop(key)
else:
current_key = key
instrs = []
progress = Progress(len(list(lines_dict.keys())),
msg="Interpreting segments:")
for key in sorted(lines_dict):
progress()
current_instrs = interpret_bin(lines_dict[key],
target,
safe=not arguments['strict'])
current_instrs[0].address = Address(base_address='code',
displacement=key)
instrs += current_instrs
maxlen = max([ins.format.length
for ins in target.instructions.values()] + [dataw]) * 2
maxlen += maxlen // 2
counter = 0
label_dict = RejectingDict()
instr_dict = RejectingDict()
range_num = 1
progress = Progress(len(instrs), msg="Computing labels:")
for instr_def in instrs:
progress()
if instr_def.address is not None:
counter = instr_def.address.displacement
if instr_def.instruction_type is None:
for idx in range(0, len(instr_def.asm), dataw * 2):
label = None
if instr_def.address is not None and idx == 0:
label = ".range_%x" % range_num
label_dict[counter] = label
range_num += 1
elif counter in label_dict:
label = label_dict[counter]
idx2 = min(idx + (dataw * 2), len(instr_def.asm))
masm = instr_def.asm[idx:idx2].lower()
binary = " ".join([
str(masm)[i:i + 2] for i in range(0, len(masm), 2)
]).lower()
instr_dict[counter] = [binary, "0x" + masm, label, None, None]
counter += (idx2 - idx) // 2
continue
instr = instruction_from_definition(instr_def, fix_relative=False)
asm = instr.assembly().lower()
relative = None
absolute = None
if instr.branch:
for memoperand in instr.memory_operands():
if not memoperand.descriptor.is_branch_target:
continue
for operand in memoperand.operands:
if operand.type.address_relative:
relative = operand.value
if operand.type.address_absolute:
absolute = operand.value
masm = instr_def.asm[2:]
if len(masm) % 2 != 0:
masm = "0" + masm
binary = " ".join([str(masm)[i:i + 2] for i in range(0, len(masm), 2)])
label = None
if instr_def.address is not None:
if counter not in label_dict:
label = ".range_%x" % range_num
label_dict[counter] = label
range_num += 1
else:
label = label_dict[counter]
elif counter in label_dict:
label = label_dict[counter]
rtarget = None
atarget = None
if relative is not None or absolute is not None:
if relative is not None:
assert absolute is None
if isinstance(relative, int):
target_addr = counter + relative
rtarget = relative
elif isinstance(relative, Address):
target_addr = counter + relative.displacement
rtarget = relative.displacement
else:
raise NotImplementedError
if absolute is not None:
assert relative is None
if isinstance(absolute, int):
target_addr = absolute
atarget = absolute
elif isinstance(absolute, Address):
target_addr = absolute.displacement
atarget = absolute.displacement
else:
raise NotImplementedError
if target_addr not in label_dict:
label_dict[target_addr] = "branch_%x" % target_addr
if target_addr in instr_dict:
instr_dict[target_addr][2] = label_dict[target_addr]
instr_dict[counter] = [binary, asm, label, rtarget, atarget]
counter = counter + (len(masm) // 2)
print("")
print("%s:\tfile format raw %s" %
(os.path.basename(arguments['input_dma_file']), target.isa.name))
print("")
print("")
print("Disassembly of section .code:")
print("")
str_format = "%8s:\t%-" + str(maxlen) + "s\t%s"
for counter in sorted(instr_dict.keys()):
binary, asm, label, rtarget, atarget = instr_dict[counter]
if label is not None:
print("%016x <%s>:" % (counter, label))
cformat = str_format
if rtarget is not None:
cformat = str_format + "\t <%s>" % (label_dict[counter + rtarget])
elif atarget is not None:
cformat = str_format + "\t <%s>" % (label_dict[atarget])
print(cformat % (hex(counter)[2:], binary, asm))
def _process_preload_data(data, args, offset=0):
data_size = args['preload_data'] / args['smt_mode']
code_size = args['preload_code'] / args['smt_mode']
only_loads = args.get('preload_data_only_loads', False)
only_stores = args.get('preload_data_only_stores', False)
if only_loads and only_stores:
print_warning(
"Both --preload-data-only-loads and --preload-data-only-stores"
" specified. No data will be pre-load!",
)
if only_loads:
data = [
access for access in data if access.data_type == "I" or
(
access.data_type ==
"D" and
access.access_type == "R"
)
]
if only_stores:
data = [
access for access in data if access.data_type == "I" or
(
access.data_type ==
"D" and
access.access_type ==
"R"
)
]
fdata = []
if len(args['preload_data_range']) > 0:
for access in data:
if access.data_type == "I":
fdata.append(access)
continue
for start, end in args['preload_data_range']:
if access.address >= start and (
access.address + access.length - 1) < end:
fdata.append(access)
break
data = fdata
addresses_read = []
ccode = 0
cdata = 0
for access in data:
if access.data_type == "D" and cdata >= data_size:
continue
if access.data_type == "I" and ccode >= code_size:
continue
for address in range(access.address, access.address + access.length):
if address in addresses_read:
continue
addresses_read.append(address)
if access.data_type == "D":
cdata += 1
else:
ccode += 1
for idx in range(0, len(addresses_read)):
addresses_read[idx] = addresses_read[idx] + offset
return set(addresses_read)
def _shift_and_fix_code(
target, code, offset, addresses, reset_steps, registers,
):
"""Shift code and fix reset code."""
scode = []
for instruction in code:
instruction = instruction.copy()
# Fix and shift decorators (not need to modify code)
for key, values in instruction.decorators.items():
if not isinstance(values, list):
values = [values]
if key in ['MA', 'BT']:
for idx in range(0, len(values)):
if addresses[0] <= values[idx] <= addresses[1]:
values[idx] = values[idx] + offset
scode.append(instruction)
cidx = 0
context = Context()
context.set_symbolic(False)
for reset_step in reset_steps:
rins = reset_step[0]
cins = scode[cidx:cidx+len(rins)]
rreg = reset_step[1]
try:
rval = [reg for reg in registers if reg.name == rreg.name][0].value
except IndexError:
# This was a support register to compute an address,
# it should be fixed
rval = 0
for rin, cin in zip(rins, cins):
if rin.name != cin.instruction_type.name:
print_error("Unable to fix the reset code")
exit(1)
if len(reset_step) == 3:
rins, reset_register, value = reset_step
if not isinstance(value, six.integer_types):
# All addresses should be offset
value += offset
nins = target.set_register(
reset_register,
value.displacement,
context,
opt=False,
)
print_info(
"Fixing reset code for reg: %s. "
"New value: 0x%016X" %
(rreg.name, value.displacement),
)
else:
if abs(value-rval) == offset:
# This has been shifted, force offset
value += offset
print_info(
"Fixing reset code for reg: %s. "
"New value: 0x%016X" %
(rreg.name, value),
)
nins = target.set_register(
reset_register, value, context, opt=False,
)
context.set_register_value(reset_register, value)
elif len(reset_step) == 4:
rins, reset_register, address_obj, length = reset_step
# All addresses should be offsetted
address_obj += offset
nins = target.store_integer(
reset_register, address_obj, length * 8, context,
)
print_info(
"Fixing reset code for reg: %s. "
"New value: 0x%016X" %
(rreg.name, address_obj.displacement),
)
else:
raise NotImplementedError(
"Unable to shift and fix code"
)
if len(rins) != len(nins):
print_error("Original resetting code:")
for ins in rins:
print_error(ins.assembly())
print_error("New resetting code:")
for ins in nins:
print_error(ins.assembly())
print_error("New resetting code differs from original in length")
exit(1)
for ins in nins:
if len(reset_step) == 3:
if not isinstance(value, six.integer_types):
value = value.displacement
ins.add_comment(
"Reset code. Setting %s to 0X%016X" %
(reset_register.name, value),
)
else:
ins.add_comment(
"Reset code. Setting mem content in 0X%016X" %
(address_obj.displacement),
)
for idx, (nin, cin) in enumerate(zip(nins, cins)):
if nin.name != cin.instruction_type.name:
print_warning("New code differs from original in opcodes")
scode[cidx+idx] = instruction_to_definition(nin)
scode[cidx+idx].comments = scode[cidx+idx].comments[1:]
cidx += len(rins)
return scode
def _compute_offset(smt_shift, target, test_def):
"""Compute required offset bettween threads to avoid conflicts."""
addresses = []
instructions = interpret_asm(
test_def.code, target, [var.name for var in test_def.variables],
)
instructions = [
instruction_from_definition(instr) for instr in instructions
]
address = test_def.default_code_address
for instr in instructions:
if instr.address is not None:
if instr.address.base_address == "code":
address = test_def.default_code_address + \
instr.address.displacement
instr.set_address(address)
else:
address = address + instr.architecture_type.format.length
addresses.append(address)
for var in test_def.variables:
addresses.append(var.address)
if var.var_type.upper() in ["CHAR", "UINT8_T"]:
addresses.extend(
range(var.address, var.address + var.num_elements),
)
else:
raise NotImplementedError(
"Unable to compute touched addresses for "
"type '%s'" % var.var_type
)
if test_def.roi_memory_access_trace:
addresses = []
for access in test_def.roi_memory_access_trace:
addresses.extend(
range(access.address, access.address + access.length),
)
offset = ((max(addresses) / (4 * 1024)) + 1) * (4 * 1024)
offset = int(offset)
max_range = offset
min_range = (min(addresses) / (4 * 1024)) * (4 * 1024)
print_info("Computed offset needed: %d bytes" % offset)
print_info(
"Computed offset needed: %d megabytes" %
(offset / (1024 * 1024)),
)
if smt_shift != -1:
if offset > smt_shift:
print_warning(
"Offset forced to be %d bytes. There is overlapping accross "
"threads" % smt_shift,
)
else:
print_info(
"Offset forced to be %d bytes." % smt_shift,
)
return smt_shift, (min_range, max_range, set(addresses))
return offset, (min_range, max_range, set(addresses))