def self_assembly_function(self): """ self_assembly_function """ target = self.target instr = target.instructions[self.instr_name] repetition = 0 while repetition < REPETITIONS: instruction = microprobe.code.ins.Instruction() instruction.set_arch_type(instr) print(instruction) for operand in instruction.operands(): operand.set_value(operand.type.random_value()) print(operand) assembly = instruction.assembly() print("Assembly: %s" % assembly) instr_def = interpret_asm([assembly], target, [])[0] print("%s == %s ?" % (instr, instr_def.instruction_type)) for trial in range(0, TRIALS): print("Trial: %s" % trial) try: if instr == instr_def.instruction_type: break except NotImplementedError: if trial == TRIALS - 1: self.assertEqual(instr, instr_def.instruction_type) for orig_operand, new_operand in zip( instruction.operands(), instr_def.operands ): print("%s == %s ?" % (orig_operand.value, new_operand)) self.assertEqual(orig_operand.value, new_operand) repetition += 1 self.assertEqual(repetition, REPETITIONS)
def __call__(self, building_block, target): """ :param building_block: :param target: """ instructions_def = interpret_asm(self._asm, target, building_block.labels) instructions = [] for definition in instructions_def: instruction = microprobe.code.ins.Instruction() instruction_set_def_properties(instruction, definition, building_block=building_block, target=target, allowed_registers=self._aregs) instructions.append(instruction) if self._index < 0: building_block.add_instructions( instructions, after=building_block.cfg.bbls[-1].instrs[-1]) elif self._index == 0: building_block.add_instructions( instructions, before=building_block.cfg.bbls[0].instrs[0]) else: cindex = 0 ains = None instr = None for bbl in building_block.cfg.bbls: for instr in bbl.instrs: cindex = cindex + 1 if instr is None: raise MicroprobeCodeGenerationError( "Empty basic block found") if cindex == self._index: ains = instr building_block.add_instructions(instructions, after=ains) return building_block.add_instructions( instructions, after=building_block.cfg.bbls[-1].instrs[-1])
def __call__(self, building_block, target): """ :param building_block: :param target: """ instructions_def = interpret_asm(self._asm, target, building_block.labels) instructions = [] for definition in instructions_def: instruction = microprobe.code.ins.Instruction() instruction_set_def_properties(instruction, definition, building_block=building_block, target=target, allowed_registers=self._aregs) instructions.append(instruction) building_block.add_init(instructions)
def generate(test_definition, outputfile, target, **kwargs): """ Microbenchmark generation policy :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param outputfile: Outputfile name :type outputfile: :class:`str` :param target: Target definition object :type target: :class:`Target` """ variables = test_definition.variables print_info("Interpreting assembly...") sequence = interpret_asm( test_definition.code, target, [var.name for var in variables] ) if len(sequence) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file." ) max_ins = test_definition.instruction_count if kwargs["max_trace_size"] != _DEFAULT_MAX_INS or max_ins is None: max_ins = kwargs["max_trace_size"] # registers = test_definition.registers # raw = test_definition.raw # if test_definition.default_data_address is None: # print_error("Default data address is needed") # exit(-1) # if test_definition.default_code_address is None: # print_error("Default code address is needed") # exit(-1) wrapper_name = "QTrace" print_info("Creating benchmark synthesizer...") try: cwrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers " "of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error was: %s" % (wrapper_name, str(exc)) ) start_addr = [ reg.value for reg in test_definition.registers if reg.name in ["PC", "IAR"] ] if start_addr: start_addr = start_addr[0] else: start_addr = test_definition.default_code_address if start_addr != test_definition.default_code_address: print_error( "Default code address does not match register state " "PC/IAR value. Check MPT consistency." ) exit(-1) wrapper_kwargs = {} wrapper_kwargs["init_code_address"] = start_addr wrapper_kwargs["init_data_address"] = test_definition.default_data_address wrapper = cwrapper(**wrapper_kwargs) synth = microprobe.code.TraceSynthesizer( target, wrapper, show_trace=kwargs.get( "show_trace", False), maxins=max_ins, start_addr=start_addr, no_scratch=True ) # if len(registers) >= 0: # synth.add_pass( # microprobe.passes.initialization.InitializeRegistersPass( # registers, skip_unknown=True, warn_unknown=True # ) # ) synth.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence)) ) synth.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence) ) synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass() ) synth.add_pass( microprobe.passes.branch.NormalizeBranchTargetsPass() ) synth.add_pass( microprobe.passes.memory.InitializeMemoryDecorator( default=kwargs.get("default_memory_access_pattern", None) ) ) synth.add_pass( microprobe.passes.branch.InitializeBranchDecorator( default=kwargs.get("default_branch_pattern", None), indirect=kwargs.get("default_branch_indirect_target_pattern", None) ) ) synth.add_pass( microprobe.passes.symbol.ResolveSymbolicReferencesPass() ) print_info("Synthesizing...") bench = synth.synthesize() # Save the microbenchmark synth.save(outputfile, bench=bench) return
def generate(test_definition, output_file, target, **kwargs): """ Microbenchmark generation policy. :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param output_file: Output file name :type output_file: :class:`str` :param target: Target definition object :type target: :class:`Target` """ end_address_orig = None overhead = 0 if len(test_definition.dat_mappings) > 0: # # Assuming MPT generated from MAMBO full system # dat = target.get_dat(dat_map=test_definition.dat_mappings) dat.control['DAT'] = True for instr in test_definition.code: instr.address = dat.translate(instr.address, rev=True) # Address order might have changed after mapping test_definition.set_instruction_definitions( sorted(test_definition.code, key=lambda x: x.address) ) # remove not translated addresses (needed?) # variables = [var for var in test_definition.variables # if var.address != dat.translate(var.address, rev=True)] # test_definition.set_variables_definition(variables) for var in test_definition.variables: var.address = dat.translate(var.address, rev=True) # Address order might have changed after mapping test_definition.set_variables_definition( sorted(test_definition.variables, key=lambda x: x.address) ) if test_definition.default_code_address != 0: test_definition.default_code_address = dat.translate( test_definition.default_code_address, rev=True ) if test_definition.default_data_address != 0: test_definition.default_data_address = dat.translate( test_definition.default_data_address, rev=True ) for access in test_definition.roi_memory_access_trace: access.address = dat.translate( access.address, rev=True ) if 'raw_bin' in kwargs: print_info("Interpreting RAW dump...") sequence = [] raw_dict = {} current_address = 0 # Assume state file provides the initial code address displ = test_definition.default_code_address test_definition.set_default_code_address(0) for entry in test_definition.code: if (not entry.assembly.upper().startswith("0X") and not entry.assembly.upper().startswith("0B")): raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "assembly (%s)" % entry.assembly ) if entry.label is not None: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "labels (%s)" % entry.label ) if entry.decorators not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "decorators (%s)" % entry.decorators ) if entry.comments not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "comments (%s)" % entry.comments ) if entry.address is not None: current_address = entry.address + displ if current_address not in raw_dict: raw_dict[current_address] = "" # Assume that raw dump use a 4 bytes hex dump if len(entry.assembly) != 10: raise MicroprobeMPTFormatError( "This is not a RAW 4-byte dump as it contains " "lines with other formats (%s)" % entry.assembly ) raw_dict[current_address] += entry.assembly[2:] if len(raw_dict) > 1: address_ant = sorted(raw_dict.keys())[0] len_ant = len(raw_dict[address_ant])//2 # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 for address in sorted(raw_dict.keys())[1:]: if address_ant + len_ant == address: raw_dict[address_ant] += raw_dict[address] len_ant = len(raw_dict[address_ant])//2 raw_dict.pop(address) else: len_ant = len(raw_dict[address])//2 address_ant = address # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 sequence = [] for address in sorted(raw_dict.keys()): # Endianess will be big endian, because we are concatenating # full words resulting in the higher bits being encoded first code = interpret_bin( raw_dict[address], target, safe=True, little_endian=False, word_length=4 ) for instr in code: instr.address = address instr = instruction_from_definition(instr) address = address + instr.architecture_type.format.length instr = instruction_to_asm_definition(instr) sequence.append(instr) test_definition.set_instruction_definitions(sequence) reset_steps = [] if 'no_wrap_test' not in kwargs: if test_definition.default_code_address != 0: print_error("Default code address should be zero") exit(-1) print_info("Wrapping function...") start_symbol = "START_TEST" init_address = test_definition.default_code_address for register in test_definition.registers: if register.name == "PC": init_address = register.value if register.name == "PSW_ADDR": init_address = register.value displacements = [] for elem in test_definition.code: if elem.address is not None: if len(displacements) == 0: displacements.append( (elem.address, 4*1024, elem.address - init_address) ) else: displacements.append( (elem.address, elem.address - displacements[-1][0], elem.address - init_address) ) # Get ranges with enough space to put init code # Assuming 4K space is enough displacements = [ displ for displ in displacements if displ[1] >= 4*1024 and displ[2] <= 0 ] if len(displacements) == 0: print_error( "Unable to find space for the initialization code. " "Check the mpt initial code address or state of PC " "for correctness." ) exit(-1) displ_fixed = False if kwargs['fix_start_address']: displacement = kwargs['fix_start_address'] print_info("Start point set to 0x%X" % displacement) displ_fixed = True elif 'fix_long_jump' in kwargs: displacement = sorted(displacements, key=lambda x: x[0])[0][0] if displacement > 2**32: displacement = 0x1000000 print_info("Start point set to 0x%X" % displacement) displ_fixed = True else: displacement = sorted(displacements, key=lambda x: x[2])[-1][0] start_symbol = None init_found = False for instr in test_definition.code: if instr.address == init_address: if instr.label is None: instr.label = "START_TEST" start_symbol = instr.label init_found = True break if not init_found: print_error( "Initial instruction address (%s) not found" % hex(init_address) ) exit(-1) if start_symbol is None: if test_definition.code[0].label is None: test_definition.code[0].label = "START_TEST" start_symbol = test_definition.code[0].label if displacement is None: displacement = 0 instructions = [] reset_steps = [] if 'wrap_endless' in kwargs and 'reset' in kwargs: target.scratch_var.set_address( Address( base_address=target.scratch_var.name ) ) new_ins, overhead, reset_steps = _compute_reset_code( target, test_definition, kwargs, ) instructions += new_ins if 'fix_long_jump' in kwargs: instructions += target.function_call( init_address, long_jump=True ) else: instructions += target.function_call( ("%s" % start_symbol).replace("+0x-", "-0x"), ) if 'wrap_endless' not in kwargs: instructions += [target.nop()] else: instructions += _compute_reset_jump(target, instructions) instructions_definitions = [] for instruction in instructions: instruction.set_label(None) if not displ_fixed: displacement = (displacement - instruction.architecture_type.format.length) current_instruction = MicroprobeAsmInstructionDefinition( instruction.assembly(), None, None, None, instruction.comments, ) instructions_definitions.append(current_instruction) instruction = target.nop() instruction.set_label(None) if not displ_fixed: displacement = (displacement - instruction.architecture_type.format.length) # To avoid overlaps if not displ_fixed: align = 0x100 displacement = ((displacement // align) + 0) * align instructions_definitions[0].address = displacement assert instructions_definitions[0].address is not None # instr = MicroprobeAsmInstructionDefinition( # instruction.assembly(), "ELF_ABI_EXIT", None, None, None) # end_address_orig = \ # (test_definition.default_code_address + displacement_end - # instruction.architecture_type.format.length) instructions_definitions[0].label = "mpt2elf_endless" test_definition.register_instruction_definitions( instructions_definitions, prepend=True, ) assert test_definition.code[0].address is not None if not displ_fixed: test_definition.set_default_code_address( test_definition.default_code_address + displacement, ) for elem in test_definition.code: if elem.address is not None: elem.address = elem.address - displacement else: test_definition.set_default_code_address( displacement ) for elem in test_definition.code: if elem.address is not None: elem.address = elem.address - displacement variables = test_definition.variables variables = [var for var in test_definition.variables if var.address is None or var.address >= 0x00100000] test_definition.set_variables_definition(variables) print_info("Interpreting asm ...") sequence_orig = interpret_asm( test_definition.code, target, [var.name for var in variables] + [target.scratch_var.name], show_progress=True, ) if len(sequence_orig) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file.", ) raw = test_definition.raw raw['FILE_FOOTER'] = "# mp_mpt2elf: Wrapping overhead: %03.2f %%" \ % overhead # end_address = end_address_orig ckwargs = { # 'end_address': end_address, # 'reset': False, # 'endless': 'endless' in kwargs } wrapper_name = "AsmLd" if test_definition.default_data_address is not None: ckwargs['init_data_address'] = \ test_definition.default_data_address if test_definition.default_code_address is not None: ckwargs['init_code_address'] = \ test_definition.default_code_address try: code_wrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers" " of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error " "was: %s" % (wrapper_name, str(exc)), ) wrapper = code_wrapper(**ckwargs) print_info("Setup synthesizer ...") synthesizer = microprobe.code.Synthesizer( target, wrapper, no_scratch=False, extra_raw=raw, ) variables = test_definition.variables registers = test_definition.registers sequence = sequence_orig if len(registers) >= 0: cr_reg = [ register for register in registers if register.name == "CR" ] registers = [ register for register in registers if register.name != "CR" ] if cr_reg: value = cr_reg[0].value for idx in range(0, 8): cr = MicroprobeTestRegisterDefinition( "CR%d" % idx, (value >> (28 - (idx * 4))) & 0xF, ) registers.append(cr) synthesizer.add_pass( microprobe.passes.initialization.InitializeRegistersPass( registers, skip_unknown=True, warn_unknown=True, skip_control=True, force_reserved=True ), ) synthesizer.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass( len(sequence), ), ) synthesizer.add_pass( microprobe.passes.variable.DeclareVariablesPass( variables, ), ) synthesizer.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence), ) if target.name.startswith("power"): fix_branches = [instr.name for instr in target.instructions.values() if instr.branch_conditional] if 'raw_bin' in kwargs: # We do not know what is code and what is data, so we safely # disable the asm generation and keep the values for orig in [21, 17, 19]: synthesizer.add_pass( microprobe.passes.instruction.DisableAsmByOpcodePass( fix_branches, 0, ifval=orig ) ) else: # We know what is code and what is data, so we can safely # fix the branch instructions for new, orig in [(20, 21), (16, 17), (18, 19)]: synthesizer.add_pass( SetInstructionOperandsByOpcodePass( fix_branches, 0, new, force=True, ifval=orig ) ) if kwargs.get("fix_memory_registers", False): kwargs["fix_memory_references"] = True if kwargs.get("fix_memory_references", False): print_info("Fix memory references: On") synthesizer.add_pass( microprobe.passes.memory.FixMemoryReferencesPass( reset_registers=kwargs.get("fix_memory_registers", False), ), ) synthesizer.add_pass( microprobe.passes.register.FixRegistersPass( forbid_writes=['GPR3'], ), ) if kwargs.get("fix_memory_registers", False): print_info("Fix memory registers: On") synthesizer.add_pass( microprobe.passes.register.NoHazardsAllocationPass(), ) if kwargs.get("fix_branch_next", False): print_info("Force branch to next: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( force="fix_flatten_code" in kwargs, noinit=True ) ) synthesizer.add_pass( microprobe.passes.branch.BranchNextPass(force=True), ) if kwargs.get("fix_indirect_branches", False): print_info("Fix indirect branches: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( noinit=True ), ) synthesizer.add_pass( microprobe.passes.branch.FixIndirectBranchPass(), ) if displ_fixed: synthesizer.add_pass( microprobe.passes.address.SetInitAddressPass(displacement) ) synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( noinit=True, init_from_first=not displ_fixed, ), ) synthesizer.add_pass( microprobe.passes.variable.UpdateVariableAddressesPass( ), ) synthesizer.add_pass( microprobe.passes.symbol.ResolveSymbolicReferencesPass( onlyraw=True ), ) print_info("Start synthesizer ...") bench = synthesizer.synthesize() # Save the microbenchmark synthesizer.save(output_file, bench=bench) print_info("'%s' generated!" % output_file) _compile(output_file, target, **kwargs) return
def _compute_reset_code(target, test_def, args): instructions = interpret_asm( test_def.code, target, [var.name for var in test_def.variables], show_progress=True, ) # TODO: This can be done in parallel or look for speed up the process instructions = [ instruction_from_definition(instr) for instr in instructions ] instruction_dict = {} address = test_def.default_code_address progress = Progress( len(test_def.roi_memory_access_trace), msg="Building instruction dictionary", ) for instr in instructions: progress() if instr.address is not None: if instr.address.base_address == "code": address = test_def.default_code_address + \ instr.address.displacement instr.set_address(address) else: address = address + instr.architecture_type.format.length instr.set_address(address) instruction_dict[instr.address] = instr free_regs = [] written_after_read_regs = [] read_regs = [] level = 0 dynamic_count = 0 progress = Progress( len(test_def.roi_memory_access_trace), msg="Evaluating register usage", ) reset_regs = set() for access in test_def.roi_memory_access_trace: progress() if access.data_type == "D": continue dynamic_count += 1 try: instr = instruction_dict[access.address] uses = instruction_dict[access.address].uses() sets = instruction_dict[access.address].sets() except KeyError: print_error( "Access to from instruction at address " "0x%016X registered but such instruction is not" " present in the definition." % access.address, ) exit(1) # Calls if instr.mnemonic == "BL": level += 1 elif instr.mnemonic == "BCL": level += 1 elif instr.mnemonic == "BCCTRL": if instr.operands()[2].value in [0, 3]: level += 1 # Returns if instr.mnemonic == "BCLR": if (((instr.operands()[0].value & 0b10100) == 20) and (instr.operands()[2].value == 0)): level -= 1 # TODO: this should include Z and RISCV instructions for call # and return, but currently we do not have memory access traces # for such platforms for reg in uses: if reg not in read_regs: read_regs.append(reg) for reg in sets: if reg in free_regs: continue elif reg not in read_regs: free_regs.append(reg) elif reg not in written_after_read_regs: written_after_read_regs.append(reg) reset_regs = set(read_regs).intersection( set(written_after_read_regs), ) reset_regs = sorted(reset_regs) assert len(free_regs) == len(set(free_regs)) assert len(set(free_regs).intersection(set(reset_regs))) == 0 if len(test_def.roi_memory_access_trace) == 0: # We do not have memory access trace, assume calling conventions reset_regs = target.volatile_registers reset_regs = [ reg for reg in reset_regs if reg in target.volatile_registers] if len(reset_regs) == 0 and len(test_def.roi_memory_access_trace) == 0: print_info( "No memory access trace found. Resetting volatile registers." ) reset_regs = target.volatile_registers unused_regs = sorted( (reg for reg in target.registers.values() if reg not in read_regs), ) # # Make sure scratch registers are reset last # for reg in target.scratch_registers: if reg in reset_regs: reset_regs.remove(reg) reset_regs.append(reg) free_regs = unused_regs + free_regs # Know which ones are not used (or written) and which ones are used # Use them as base / temporal registers for addresses # Check addresses conflict_addresses = {} new_ins = [] progress = Progress( len(test_def.roi_memory_access_trace), msg="Evaluating memory usage", ) for access in test_def.roi_memory_access_trace: progress() if access.data_type == "I": continue val = conflict_addresses.get( access.address, [access.length, access.access_type], ) if access.access_type not in val[1]: val[1] += access.access_type val[0] = max(val[0], access.length) conflict_addresses[access.address] = val fix_addresses = [] for address in conflict_addresses: value = conflict_addresses[address] if value[1] == "RW": wvalue = None for var in test_def.variables: if var.var_type.upper() in ["CHAR", "UINT8_T"]: elem_size = 1 else: raise NotImplementedError end_address = var.address + var.num_elements * elem_size if var.address <= address <= end_address: offset = int((address - var.address) / elem_size) svalue = var.init_value[ offset:offset + int(value[0] / elem_size) ] svalue = "".join(["%02X" % tval for tval in svalue]) wvalue = int(svalue, 16) break if wvalue is None: print_error( "Unable to restore original value for address 0x%X" % address, ) exit(1) if value[0] <= 8: fix_addresses.append((address, value[0], wvalue)) else: for selem in range(0, value[0]//8): sfmt = "%%0%dX" % (2*value[0]) nvalue = sfmt % wvalue nvalue = int(nvalue[selem*16:(selem+1)*16], 16) fix_addresses.append( (address + selem * 8, 8, nvalue) ) reset_steps = [] context = Context() context.set_symbolic(True) if len(fix_addresses) > 0: # TODO: This can be optimized. Reduce the number of instructions to # be added by sorting the reset code (shared values or similar # addresses) # TODO: This can be optimized for use vector registers when # needed # print_info("Adding instructions to reset memory state") reset_register = [ reg for reg in free_regs if reg.type.used_for_address_arithmetic and reg.name != "GPR0" ][0] for address, length, value in fix_addresses: address_obj = Address(base_address="data", displacement=address) new_instructions = target.set_register( reset_register, value, context, opt=False, ) for ins in new_instructions: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (reset_register.name, value), ) reset_steps.append([new_instructions[:], reset_register, value]) context.set_register_value(reset_register, value) try: store_ins = target.store_integer( reset_register, address_obj, length * 8, context, ) new_instructions += store_ins reset_steps.append( [store_ins, reset_register, address_obj, length], ) except MicroprobeCodeGenerationError: areg = [ reg for reg in free_regs if reg.type.used_for_address_arithmetic and reg.name != "GPR0" ][1] set_ins = target.set_register( areg, address, context, opt=False, ) new_instructions += set_ins reset_steps.append([set_ins, areg, address_obj]) context.set_register_value(areg, address_obj) store_ins = target.store_integer( reset_register, address_obj, length * 8, context, ) new_instructions += store_ins reset_steps.append( [store_ins, reset_register, address_obj, length], ) for ins in set_ins: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (areg.name, address), ) for ins in store_ins: ins.add_comment( "Reset code. Setting mem content in 0X%016X" % (address), ) new_ins.extend(new_instructions) # Reset contents of used registers for reset_register in reset_regs: try: value = [ reg for reg in test_def.registers if reg.name == reset_register.name ][0].value except IndexError: continue new_instructions = target.set_register( reset_register, value, context, opt=False, ) reset_steps.append([new_instructions, reset_register, value]) context.set_register_value(reset_register, value) for ins in new_instructions: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (reset_register.name, value), ) new_ins.extend(new_instructions) try: overhead = (((len(new_ins) * 1.0) / dynamic_count) * 100) except ZeroDivisionError: print_warning("Unable to compute overhead. Zero dynamic instruction " "count") overhead = 0 print_info( "%03.2f%% overhead added by resetting code" % overhead, ) if overhead > args['wrap_endless_threshold']: print_error( "Instructions added: %d" % len(new_ins), ) print_error( "Total instructions: %d" % dynamic_count, ) print_error( "Reset code above --wrap-endless-threshold. Stopping generation.", ) exit(1) return new_ins, overhead, reset_steps
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import from __future__ import print_function import sys from microprobe.target import import_definition from microprobe.utils.bin import interpret_bin from microprobe.code.ins import instruction_from_definition from microprobe.utils.asm import interpret_asm target = import_definition(sys.argv[1]) print(sys.argv[2:]) for elem in sys.argv[2:]: instr_def = interpret_bin(elem, target)[0] instr = instruction_from_definition(instr_def) codification = int(instr.binary(), 2) assembly = instr.assembly() instr_def2 = interpret_asm(assembly, target, [])[0] print(hex(codification)) instr_def3 = interpret_bin(hex(codification)[2:], target)[0] instr2 = instruction_from_definition(instr_def2) instr3 = instruction_from_definition(instr_def3) assert instr.assembly() == instr2.assembly() assert instr2.assembly() == instr3.assembly() assert instr.binary() == instr2.binary() assert instr2.binary() == instr3.binary() print(instr3.assembly())
def interpret_objdump( objdump_output, target, strict=False, sections=None, start_address=-1, end_address=float('+inf') ): """ Returns a :class:`~.MicroprobeTestDefinition` object that results from interpreting the objdump output. The *target* object is used to validate the existence of the instruction and operands. :param objdump_output: Assembly to interpret :type objdump_output: Objdump textual output :param target: Target definition :type target: :class:`~.Target` object :param strict: If set, fail if an opcode can not be interpreted. (Default: False) :type strict: :class:`~.bool` :param sections: List of section names to parse :type sections: :class:`~.list` of :class:`~.str` :param start_address: Start address to interpret :type start_address: ::class:`~.int` :param end_address: End address to interpret :type end_address: ::class:`~.int` :return: An object representing the microprobe test :rtype: :class:`~.list` of :class:`~.MicroprobeTestDefinition` :raise microprobe.exceptions.MicroprobeObjdumpError: if something is wrong during the interpretation of the objdump """ if not strict: MICROPROBE_RC['safe_bin'] = True if isinstance(objdump_output, str): objdump_output = objdump_output.replace('\r', '\n') objdump_output = objdump_output.split('\n') filtered_objdump_output = _objdump_cleanup( objdump_output, sections, start_address, end_address ) code_labels = _find_code_labels(filtered_objdump_output) var_labels = _find_var_labels(filtered_objdump_output, code_labels) labels = code_labels + var_labels label_pattern = _generate_label_pattern(labels) binary_format = _binary_reformat(filtered_objdump_output) asm_format = _asm_reformat(filtered_objdump_output) assert len(binary_format) == len(asm_format) instr_defs = [] current_labels = var_labels[:] progress = Progress(len(binary_format), msg="Lines parsed:") for binary, asm in zip(binary_format, asm_format): instr_def = None if not label_pattern.search(asm): try: instr_def = interpret_asm(binary, target, current_labels) except MicroprobeBinaryError: if strict: raise MicroprobeObjdumpError( "Unable to interpret binary '%s' (asm:' %s')" % (binary, asm) ) else: LOG.warning("Skiping binary '%s' (asm:' %s')", binary, asm) instr_def = None else: try: instr_def = interpret_asm( asm, target, current_labels, log=False ) except MicroprobeAsmError: instr_def = interpret_asm(binary, target, current_labels) if instr_def is not None: fixed_instr_def = _fix_instr_definition(instr_def[0], asm, target) instr_defs.append(fixed_instr_def) if fixed_instr_def.label is not None: current_labels.append(fixed_instr_def.label) progress() variable_defs = [] required_defs = [] for var_label in var_labels: var_def = _interpret_variable(var_label, objdump_output) if var_def is not None: variable_defs.append(var_def) else: LOG.warning( "Variable label: '%s' referenced but not found " "in the dump" ) required_defs.append(_default_variable(var_label)) return variable_defs, required_defs, instr_defs
def dump_mpt(input_file_fd, target, arguments): """ :param input_file_fd: :type input_file_fd: :param target: :type target: :param arguments: :type arguments: """ try: contents = input_file_fd.read() except KeyboardInterrupt: print_info("No input data provided. Exiting...") exit(1) print_info("Parsing input file...") skip = True data_mode = False reg_comment = re.compile("^ [0-9][0-9][0-9][0-9][0-9][0-9]: ") reg_label = re.compile(":.*EQU \\*") reg_data = re.compile(" +DC +X'") reg_entry = re.compile(" +DS +") instruction_definitions = [] data_definitions = [] clabel = None hlabel = None paddress = None for line in contents.split("\n"): if " PROC " in line: skip = False if skip: continue if line.startswith("1PP "): continue if line.replace(" ", "") == "": continue if " CONSTANT AREA " in line: data_mode = True continue if "***" in line: continue if data_mode is True and "|" not in line: break if line.startswith("0"): line = line.replace("0", " ", 1) sline = [elem for elem in line.split("|")[0].split(" ") if elem != ""] if data_mode: if line.startswith("0 0"): line = line.replace("0 0", " 0", 1) address = int(sline[0], 16) if len(data_definitions) > 0 and \ (address == data_definitions[-1][0] + len(data_definitions[-1][1]) // 2): data_definitions[-1][1] += "".join(sline[2:]) else: data_definitions.append( [address, "".join(sline[2:]), "constant_area"]) continue if reg_comment.search(line): continue if reg_label.search(line): clabel = hlabel + "_" + sline[2].replace(":", "") continue if sline[2] == "PROC": hlabel = sline[3] continue if sline[2] == "RET": continue if reg_entry.search(line): continue if reg_data.search(line): address = int(sline[0], 16) if len(data_definitions) > 0 and \ (address == data_definitions[-1][0] + len(data_definitions[-1][1]) // 2): data_definitions[-1][1] += "".join(sline[1:3]) data_definitions[-1][2] += "".join(sline[4:]) else: data_definitions.append( [address, "".join(sline[1:3]), " ".join(sline[4:])]) continue idx = 1 value = "" while len(sline[idx]) == 4: value += sline[idx] idx += 1 address = int(sline[0], 16) if address - (len(value) // 2) == paddress: address = None comment = " ".join(sline[idx + 1:]) asm = "0x%s" % value asm_def = MicroprobeAsmInstructionDefinition(asm, clabel, address, None, comment) instruction_definitions.append(asm_def) clabel = None paddress = int(sline[0], 16) instr_defs = interpret_asm(instruction_definitions, target, [], show_progress=True) var_defs = [] for idx, data in enumerate(data_definitions): address = data[0] var_name = "data_%010d" % idx len_init_value = len(data[1]) // 2 init_value = [ int(data[1][2 * idx:2 * (idx + 1)], 16) for idx in range(0, len_init_value) ] var_defs.append( MicroprobeTestVariableDefinition(var_name, "char", len_init_value, address, None, init_value)) print_info("Input file parsed") print_info("%d instructions processed from the input file" % len(instr_defs)) if var_defs != []: print_info("Variables referenced and detected in the dump: %s" % ','.join([var.name for var in var_defs])) print_info("Generating the MPT contents...") mpt_config = mpt_configuration_factory() if 'default_code_address' in arguments: mpt_config.set_default_code_address(arguments['default_code_address']) else: mpt_config.set_default_code_address(instr_defs[0].address.displacement) if 'default_data_address' in arguments: mpt_config.set_default_data_address(arguments['default_data_address']) else: mpt_config.set_default_data_address(0) for var in var_defs: mpt_config.register_variable_definition(var) mpt_config.register_instruction_definitions(instr_defs) print_info("Dumping MPT to '%s'" % arguments['output_mpt_file']) mpt_parser = mpt_parser_factory() mpt_parser.dump_mpt_config(mpt_config, arguments['output_mpt_file'])
def generate(test_definition, outputfile, target, **kwargs): """ Microbenchmark generation policy :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param outputfile: Outputfile name :type outputfile: :class:`str` :param target: Target definition object :type target: :class:`Target` """ variables = test_definition.variables if 'raw_bin' in kwargs: print_info("Interpreting RAW dump...") sequence = [] raw_dict = {} current_address = 0 for entry in test_definition.code: if (not entry.assembly.upper().startswith("0X") and not entry.assembly.upper().startswith("0B")): raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "assembly (%s)" % entry.assembly) if entry.label is not None: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "labels (%s)" % entry.label) if entry.decorators not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "decorators (%s)" % entry.decorators) if entry.comments not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "comments (%s)" % entry.comments) if entry.address is not None: current_address = entry.address if current_address not in raw_dict: raw_dict[current_address] = "" # Assume that raw dump use a 4 bytes hex dump if len(entry.assembly) != 10: raise MicroprobeMPTFormatError( "This is not a RAW 4-byte dump as it contains " "lines with other formats (%s)" % entry.assembly) raw_dict[current_address] += entry.assembly[2:] if len(raw_dict) > 1: address_ant = sorted(raw_dict.keys())[0] len_ant = len(raw_dict[address_ant]) // 2 # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 for address in sorted(raw_dict.keys())[1:]: if address_ant + len_ant == address: raw_dict[address_ant] += raw_dict[address] len_ant = len(raw_dict[address_ant]) // 2 raw_dict.pop(address) else: len_ant = len(raw_dict[address]) // 2 address_ant = address # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 sequence = [] for address in sorted(raw_dict.keys()): # Endianess will be big endian, because we are concatenating # full words resulting in the higher bits being encoded first code = interpret_bin(raw_dict[address], target, safe=True, little_endian=False, word_length=4) code[0].address = address sequence.extend(code) else: print_info("Interpreting assembly...") sequence = interpret_asm(test_definition.code, target, [var.name for var in variables]) if len(sequence) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file.") registers = test_definition.registers raw = test_definition.raw for instruction_def in sequence: if instruction_def.address is not None: print_warning("Instruction address not needed in '%s'" % instruction_def.asm) if test_definition.default_data_address is not None: print_warning("Default data address not needed") if test_definition.default_code_address is not None: print_warning("Default code address not needed") wrapper_name = "CWrapper" if kwargs.get("endless", False): print_warning("Using endless C wrapper") wrapper_name = "CInfGen" print_info("Creating benchmark synthesizer...") try: cwrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers " "of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error was: %s" % (wrapper_name, str(exc))) wrapper_kwargs = {} wrapper = cwrapper(**wrapper_kwargs) synth = microprobe.code.Synthesizer(target, wrapper, extra_raw=raw) if len(registers) >= 0: print_info("Add register initialization pass") synth.add_pass( microprobe.passes.initialization.InitializeRegistersPass( registers, skip_unknown=True, warn_unknown=True)) synth.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence))) synth.add_pass(microprobe.passes.variable.DeclareVariablesPass(variables)) synth.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence)) if kwargs.get("fix_indirect_branches", False): print_info("Fix indirect branches: On") synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass()) synth.add_pass(microprobe.passes.branch.FixIndirectBranchPass()) if kwargs.get("fix_branch_next", False): print_info("Force branch to next: On") synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass()) synth.add_pass(microprobe.passes.branch.BranchNextPass(force=True)) if kwargs.get("fix_memory_registers", False): kwargs["fix_memory_references"] = True if kwargs.get("fix_memory_references", False): synth.add_pass( microprobe.passes.memory.FixMemoryReferencesPass( reset_registers=kwargs.get("fix_memory_registers", False))) if kwargs.get("fix_memory_registers", False): print_info("Fix memory registers: On") synth.add_pass(microprobe.passes.register.NoHazardsAllocationPass()) print_info("Synthesizing...") bench = synth.synthesize() # Save the microbenchmark synth.save(outputfile, bench=bench) return
def generate(test_definition, outputfile, target, **kwargs): """ Microbenchmark generation policy :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param outputfile: Outputfile name :type outputfile: :class:`str` :param target: Target definition object :type target: :class:`Target` """ variables = test_definition.variables print_info("Interpreting assembly...") sequence = interpret_asm(test_definition.code, target, [var.name for var in variables]) if len(sequence) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file.") registers = test_definition.registers raw = test_definition.raw for instruction_def in sequence: if instruction_def.address is not None: print_warning("Instruction address not needed in '%s'" % instruction_def.asm) if test_definition.default_data_address is not None: print_warning("Default data address not needed") if test_definition.default_code_address is not None: print_warning("Default code address not needed") wrapper_name = "CWrapper" if kwargs.get("endless", False): print_warning("Using endless C wrapper") wrapper_name = "CInfGen" print_info("Creating benchmark synthesizer...") try: cwrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers " "of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error was: %s" % (wrapper_name, str(exc))) wrapper_kwargs = {} wrapper = cwrapper(**wrapper_kwargs) synth = microprobe.code.Synthesizer(target, wrapper, extra_raw=raw) if len(registers) >= 0: print_info("Add register initialization pass") synth.add_pass( microprobe.passes.initialization.InitializeRegistersPass( registers, skip_unknown=True, warn_unknown=True)) synth.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence))) synth.add_pass(microprobe.passes.variable.DeclareVariablesPass(variables)) synth.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence)) if kwargs.get("fix_memory_registers", False): kwargs["fix_memory_references"] = True if kwargs.get("fix_memory_references", False): synth.add_pass( microprobe.passes.memory.FixMemoryReferencesPass( reset_registers=kwargs.get("fix_memory_registers", False))) if kwargs.get("fix_memory_registers", False): print_info("Fix memory registers: On") synth.add_pass(microprobe.passes.register.NoHazardsAllocationPass()) if kwargs.get("fix_branch_next", False): print_info("Force branch to next: On") synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass()) synth.add_pass(microprobe.passes.branch.BranchNextPass(force=True)) if kwargs.get("fix_indirect_branches", False): print_info("Fix indirect branches: On") synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass()) synth.add_pass(microprobe.passes.branch.FixIndirectBranchPass()) print_info("Synthesizing...") bench = synth.synthesize() # Save the microbenchmark synth.save(outputfile, bench=bench) return
def generate(test_definition, output_file, target, **kwargs): """ Microbenchmark generation policy. :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param output_file: Output file name :type output_file: :class:`str` :param target: Target definition object :type target: :class:`Target` """ end_address_orig = None overhead = 0 reset_steps = [] if 'no_wrap_test' not in kwargs and False: start_symbol = "START_TEST" displacement = None for elem in test_definition.code: if elem.address is not None: if displacement is None: displacement = elem.address displacement = min(displacement, elem.address) if test_definition.code[0].label is not None: start_symbol = test_definition.code[0].label else: test_definition.code[0].label = "START_TEST" if displacement is None: displacement = 0 displacement_end = displacement instructions = [] reset_steps = [] if 'wrap_endless' in kwargs: new_ins, overhead, reset_steps = _compute_reset_code( target, test_definition, kwargs, ) instructions += new_ins instructions += target.function_call( ("%s+0x%x" % (start_symbol, (-1) * displacement)).replace("+0x-", "-0x"), ) if 'wrap_endless' not in kwargs: instructions += [target.nop()] else: instructions += _compute_reset_jump(target, len(instructions)) instructions_definitions = [] for instruction in instructions: instruction.set_label(None) displacement = (displacement - instruction.architecture_type.format.length) current_instruction = MicroprobeAsmInstructionDefinition( instruction.assembly(), None, None, None, instruction.comments, ) instructions_definitions.append(current_instruction) instruction = target.nop() instruction.set_label(None) displacement = (displacement - instruction.architecture_type.format.length) end_address_orig = \ (test_definition.default_code_address + displacement_end - instruction.architecture_type.format.length) test_definition.register_instruction_definitions( instructions_definitions, prepend=True, ) test_definition.set_default_code_address( test_definition.default_code_address + displacement, ) for elem in test_definition.code: if elem.address is not None: elem.address = elem.address - displacement variables = test_definition.variables print_info("Interpreting asm ...") sequence_orig = interpret_asm( test_definition.code, target, [var.name for var in variables], show_progress=True, ) if len(sequence_orig) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file.", ) raw = test_definition.raw shift_offset, addresses = _compute_offset( 0, target, test_definition, ) thread_id = 1 shift_value = shift_offset * (thread_id - 1) end_address = end_address_orig if end_address: end_address = end_address_orig + shift_value ckwargs = { 'reset': False, 'endless': 'endless' in kwargs } # if test_definition.default_data_address is not None: # ckwargs['init_data_address'] = \ # test_definition.default_data_address + shift_value # if test_definition.default_code_address is not None: # ckwargs['init_code_address'] = \ # test_definition.default_code_address + shift_value wrapper_name = "Cronus" try: code_wrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers" " of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error " "was: %s" % (wrapper_name, str(exc)), ) wrapper = code_wrapper(os.path.basename(output_file), **ckwargs) print_info("Setup synthesizer ...") synthesizer = microprobe.code.Synthesizer( target, wrapper, no_scratch=True, extra_raw=raw, ) print_info("Processing thread-%d" % thread_id) synthesizer.set_current_thread(thread_id) variables = test_definition.variables registers = test_definition.registers sequence = sequence_orig cr_reg = [ register for register in registers if register.name == "CR" ] registers = [ register for register in registers if register.name != "CR" ] if cr_reg: value = cr_reg[0].value for idx in range(0, 8): cr = MicroprobeTestRegisterDefinition( "CR%d" % idx, (value >> (28 - (idx * 4))) & 0xF, ) registers.append(cr) synthesizer.add_pass( microprobe.passes.initialization.InitializeRegistersPass( registers, skip_unknown=True, warn_unknown=True, force_reserved=False, skip_control=True ), ) synthesizer.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass( len(sequence), ), ) synthesizer.add_pass( microprobe.passes.variable.DeclareVariablesPass( variables, ), ) synthesizer.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence), ) if kwargs.get("fix_memory_registers", False): kwargs["fix_memory_references"] = True if kwargs.get("fix_memory_references", False): print_info("Fix memory references: On") synthesizer.add_pass( microprobe.passes.memory.FixMemoryReferencesPass( reset_registers=kwargs.get("fix_memory_registers", False), ), ) synthesizer.add_pass( microprobe.passes.register.FixRegistersPass( forbid_writes=['GPR3'], ), ) if kwargs.get("fix_memory_registers", False): print_info("Fix memory registers: On") synthesizer.add_pass( microprobe.passes.register.NoHazardsAllocationPass(), ) if kwargs.get("fix_branch_next", False): print_info("Force branch to next: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( force="fix_flatten_code" in kwargs ) ) synthesizer.add_pass( microprobe.passes.branch.BranchNextPass(force=True), ) if kwargs.get("fix_indirect_branches", False): print_info("Fix indirect branches: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass(), ) synthesizer.add_pass( microprobe.passes.branch.FixIndirectBranchPass(), ) synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass(), ) synthesizer.add_pass( microprobe.passes.symbol.ResolveSymbolicReferencesPass(), ) print_info("Start synthesizer ...") bench = synthesizer.synthesize() # Save the microbenchmark synthesizer.save(output_file, bench=bench) return
def _compute_offset(smt_shift, target, test_def): """Compute required offset bettween threads to avoid conflicts.""" addresses = [] instructions = interpret_asm( test_def.code, target, [var.name for var in test_def.variables], ) instructions = [ instruction_from_definition(instr) for instr in instructions ] address = test_def.default_code_address for instr in instructions: if instr.address is not None: if instr.address.base_address == "code": address = test_def.default_code_address + \ instr.address.displacement instr.set_address(address) else: address = address + instr.architecture_type.format.length addresses.append(address) for var in test_def.variables: addresses.append(var.address) if var.var_type.upper() in ["CHAR", "UINT8_T"]: addresses.extend( range(var.address, var.address + var.num_elements), ) else: raise NotImplementedError( "Unable to compute touched addresses for " "type '%s'" % var.var_type ) if test_def.roi_memory_access_trace: addresses = [] for access in test_def.roi_memory_access_trace: addresses.extend( range(access.address, access.address + access.length), ) offset = ((max(addresses) / (4 * 1024)) + 1) * (4 * 1024) offset = int(offset) max_range = offset min_range = (min(addresses) / (4 * 1024)) * (4 * 1024) print_info("Computed offset needed: %d bytes" % offset) print_info( "Computed offset needed: %d megabytes" % (offset / (1024 * 1024)), ) if smt_shift != -1: if offset > smt_shift: print_warning( "Offset forced to be %d bytes. There is overlapping accross " "threads" % smt_shift, ) else: print_info( "Offset forced to be %d bytes." % smt_shift, ) return smt_shift, (min_range, max_range, set(addresses)) return offset, (min_range, max_range, set(addresses))