def _main(arguments): """Program main, after processing the command line arguments. :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ print_info("Arguments processed!") print_info("Importing target definition...") if 'raw_bin' in arguments: arguments['safe_bin'] = True if 'safe_bin' in arguments: microprobe.MICROPROBE_RC['safe_bin'] = True if 'fix_start_address' not in arguments: arguments['fix_start_address'] = None if "no_wrap_test" in arguments and "wrap_endless" in arguments: print_error( "--no-wrap-test specified and --wrap-endless specified. " "Incompatible options." ) target = import_definition(arguments.pop('target')) test_definition = arguments['mpt_definition'] output_file = arguments['elf_output_file'] print_info("Start generating '%s'" % output_file) generate( test_definition, output_file, target, **arguments )
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ target = import_definition(arguments['target']) if arguments['od_bin'] is None: print_error("Unable to find a 'od' utility. Edit your $PATH or use" " the --od-bin parameter") exit(-1) if "strict" not in arguments: arguments["strict"] = False dump_objdump(target, arguments)
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ if "dump_c2mpt_template" in arguments: dump_c2mpt_template(arguments) _exit(0) if "input_c_file" not in arguments: print_error("argument -i/--input-c-file is required") _exit(-1) print_info("Arguments processed!") print_info("Importing target definition...") target = import_definition(arguments['target']) c2mpt(target, arguments)
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ print_info("Arguments processed!") print_info("Importing target definition...") target = import_definition(arguments['target']) if arguments['od_bin'] is None: print_error("Unable to find a 'od' utility. Edit your $PATH or use" " the --od-bin parameter") exit(-1) if 'safe' not in arguments: arguments['safe'] = False dump_bin(target, arguments)
def dump_c2mpt_template(arguments): """ :param arguments: :type arguments: """ print_info("Creating a c2mpt C file template...") template_file = findfiles(MICROPROBE_RC['template_paths'], "c2mpt_template.c$")[0] if not os.path.isfile(template_file): print_error("Unable to find base template file: %s" % template_file) _exit(-1) output_file = arguments['output_mpt_file'].rstrip(".mpt") + ".c" print_info("Output file name: %s" % output_file) if os.path.isfile(output_file): print_error("Output file '%s' already exists" % output_file) _exit(-1) try: shutil.copy(template_file, output_file) except IOError: print_error("Something wron when copying '%s' to '%s'\nError: %s " % (template_file, output_file, IOError)) _exit(-1) print_info("Done! You can start editing %s " % output_file)
def _process_branch_pattern(patterns): rpattern = [] for pattern in patterns: if pattern.replace("0", "").replace("1", "") == "": rpattern.append(pattern) elif pattern.startswith("L"): pattern_lengths = int_range(1, 100)(pattern[1:]) for pattern_length in pattern_lengths: strfmt = "0%db" % pattern_length rpattern += [ format(elem, strfmt) for elem in range(0, 2**pattern_length) ] else: print_error("Wrong format for branch pattern '%s'." % pattern) exit(-1) rpattern = sorted( list(set(rpattern)), key=lambda x: len(x) # pylint: disable=unnecessary-lambda ) numbers = set([len(elem) for elem in rpattern]) cmul = 1 for num in numbers: cmul = cmul * num for idx, elem in enumerate(rpattern): rpattern[idx] = [elem, len(elem), elem * (cmul // len(elem))] rlist = [] llist = [] for elem in rpattern: if elem[2] in llist: continue rlist.append(elem[0]) llist.append(elem[2]) return rlist
def dump_objdump(target, arguments): """ :param target: :type target: :param arguments: :type arguments: """ cmd = "'%s' -Ax -tx1 -v '%s'" % (arguments['od_bin'], arguments['input_bin_file']) text_string = run_cmd_output(cmd) bintext = [] for line in text_string.split('\n'): if line == "": continue bintext.append("".join(line.split(' ')[1:])) instrs = interpret_bin("".join(bintext), target) print("") print("%s:\tfile format raw %s" % (os.path.basename(arguments['input_bin_file']), target.isa.name)) print("") print("") print("Disassembly of section .raw:") print("") maxlen = max(len(instr.asm[2:]) for instr in instrs) if maxlen % 2 != 0: maxlen = maxlen + 1 maxlen += maxlen // 2 counter = arguments['start_address'] label_dict = RejectingOrderedDict() instr_dict = RejectingOrderedDict() for instr_def in instrs: instr = instruction_from_definition(instr_def, fix_relative=False) asm = instr.assembly().lower() relative = None absolute = None if instr.branch: for memoperand in instr.memory_operands(): if not memoperand.descriptor.is_branch_target: continue for operand in memoperand.operands: if operand.type.address_relative: relative = operand.value if operand.type.address_absolute: absolute = operand.value masm = instr_def.asm[2:] if len(masm) % 2 != 0: masm = "0" + masm binary = " ".join([str(masm)[i:i + 2] for i in range(0, len(masm), 2)]) label = None if counter == 0: label = ".raw" label_dict[counter] = label elif counter in label_dict: label = label_dict[counter] rtarget = None atarget = None if relative is not None or absolute is not None: if relative is not None: assert absolute is None if isinstance(relative, six.integer_types): target_addr = counter + relative rtarget = relative elif isinstance(relative, Address): target_addr = counter + relative.displacement rtarget = relative.displacement else: raise NotImplementedError if absolute is not None: assert relative is None if isinstance(absolute, six.integer_types): target_addr = absolute atarget = absolute elif isinstance(absolute, Address): target_addr = absolute.displacement atarget = absolute.displacement else: raise NotImplementedError if target_addr not in label_dict: label_dict[target_addr] = "branch_%x" % target_addr if target_addr in instr_dict: instr_dict[target_addr][2] = label_dict[target_addr] instr_dict[counter] = [binary, asm, label, rtarget, atarget] counter = counter + len(masm) // 2 str_format = "%8s:\t%-" + str(maxlen) + "s\t%s" addresses = [] for counter, values in instr_dict.items(): binary, asm, label, rtarget, atarget = values if label is not None: print("%016x <%s>:" % (counter, label)) cformat = str_format if rtarget is not None: cformat = str_format + "\t <%s>" % (label_dict[counter + rtarget]) elif atarget is not None: cformat = str_format + "\t <%s>" % (label_dict[atarget]) print(cformat % (hex(counter)[2:], binary, asm)) addresses.append(counter) error = False for key in label_dict.keys(): if key not in addresses: print_warning("Target address '%s' not in the objdump" % hex(key)) error = True if error and arguments['strict']: print_error("Check target addresses of relative branches") exit(-1)
def dump_mpt(input_file, target, init_data, arguments): """ :param input_file: :type input_file: :param target: :type target: :param init_data: :type init_data: :param arguments: :type arguments: """ input_file_fd = io.open(input_file, 'r') contents = input_file_fd.read() if six.PY2: contents = contents.encode("ascii") elif six.PY3: pass print_info("Parsing input file...") var_defs, req_defs, instr_defs = \ interpret_objdump(contents, target, strict=arguments.get('strict', False), sections=["microprobe.text"]) print_info("Input file parsed") print_info("%d instructions processed from the input file" % len(instr_defs)) if var_defs != []: print_info("Variables referenced and detected in the dump: %s" % ','.join([var.name for var in var_defs])) if req_defs != []: print_warning( "Variables referenced and *NOT* detected in the dump: %s" % ','.join([var.name for var in req_defs])) print_warning("You might need to edit the generated MPT to fix the" " declaration of such variables") print_info("Generating the MPT contents...") mpt_config = mpt_configuration_factory() mpt_config.set_default_code_address(arguments['default_code_address']) mpt_config.set_default_data_address(arguments['default_data_address']) kwargs = {} if "stack_name" in arguments: kwargs["stack_name"] = arguments["stack_name"] if "stack_address" in arguments: kwargs["stack_address"] = Address( base_address="code", displacement=arguments["stack_address"]) variables, instructions = target.elf_abi(arguments["stack_size"], "c2mpt_function", **kwargs) for variable in variables: req_defs.append(variable_to_test_definition(variable)) address = instr_defs[0].address for instr in reversed(instructions): instr_defs = [instruction_to_definition(instr)] + instr_defs address -= instr.architecture_type.format.length if address.displacement < 0: print_error("Default code address is below zero after" " adding the initialization code.") print_error("Check/modify the objdump provided or do not use" " the elf_abi flag.") _exit(-1) if "end_branch_to_itself" in arguments: instr = target.branch_to_itself() else: instr = target.nop() instr.set_label("ELF_ABI_EXIT") instr_defs.append(instruction_to_definition(instr)) mpt_config.set_default_code_address(address.displacement) initialized_variables = {} mindisplacement = None for data in init_data.split('\n'): if data.strip() == "": continue if data.startswith("WARNING"): print_warning(data.split(":")[1].strip()) continue name = data.split("=")[0].strip() values = ast.literal_eval(data.split("=")[1].strip()) initialized_variables[name] = values if mindisplacement is None: mindisplacement = values[2] # TODO: this is unsafe. We should compute the real size maxdisplacement = values[2] + (values[1] * 8) else: mindisplacement = min(values[2], mindisplacement) maxdisplacement = max(values[2] + (values[1] * 8), maxdisplacement) if "host_displacement" in arguments: mindisplacement = arguments.get("host_displacement", mindisplacement) for name, values in initialized_variables.items(): values[2] = values[2] - mindisplacement + \ arguments['default_data_address'] if 'fix_displacement' in arguments: for name, values in initialized_variables.items(): if "*" in values[0] and isinstance(values[4], list): new_values = [] for value in values[4]: if value <= maxdisplacement and value >= mindisplacement: value = value - mindisplacement + \ arguments['default_data_address'] new_values.append(value) values[4] = new_values elif "*" in values[0] and isinstance(values[4], int): if (values[4] <= maxdisplacement and values[4] >= mindisplacement): values[4] = values[4] - mindisplacement + \ arguments['default_data_address'] elif values[0] == "uint8_t" and isinstance(values[4], list): if len(values[4]) > 8: new_values = "".join(["%02x" % elem for elem in values[4]]) # Enable this for testing switched endianness # new_values = [new_values[idx:idx + 2] # for idx in range(0, len(new_values), 2)] # new_values = new_values[::-1] # new_values = "".join(new_values) new_values = _fix_displacement(new_values, mindisplacement, maxdisplacement, arguments) values[4] = [ int(new_values[idx:idx + 2], 16) for idx in range(0, len(new_values), 2) ] for name, values in initialized_variables.items(): mpt_config.register_variable_definition( MicroprobeTestVariableDefinition(name, *values)) print_info("Init values for variable '%s' parsed" % name) for var in var_defs + req_defs: if var.name in list(initialized_variables.keys()): continue if var.name != arguments["stack_name"]: print_warning("Variable '%s' not registered in the C file " "using the macros provided!" % var.name) mpt_config.register_variable_definition(var) mpt_config.register_instruction_definitions(instr_defs) print_info("Dumping MPT to '%s'" % arguments['output_mpt_file']) mpt_parser = mpt_parser_factory() mpt_parser.dump_mpt_config(mpt_config, arguments['output_mpt_file'])
def _generic_policy_wrapper(all_arguments): instruction, outputdir, outputname, target, kwargs = all_arguments bname = instruction.name bname = bname + "#DD_%d" % kwargs['dependency_distance'] bname = bname + "#BS_%d" % kwargs['benchmark_size'] bname = bname + "#DI_%s" % kwargs['data_init'] if MICROPROBE_RC['debugwrapper']: policy = find_policy(target.name, 'debug') else: policy = find_policy(target.name, 'epi') extension = "" if target.name.endswith("z64_mesa_st") or target.name.endswith( "z64_mesa_smt2"): wrapper_name = "Avp" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset'], ) extension = "avp" elif target.name.endswith("ppc64_mesa"): wrapper_name = "Tst" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class( outputname.replace("%INSTR%", bname).replace("%EXT%", "tst"), reset=kwargs['reset'], ) extension = "tst" elif target.name.endswith("linux_gcc"): wrapper_name = "CInfGen" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset']) extension = "c" elif target.name.endswith("poe"): wrapper_name = "Poe" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset'], ) extension = "bin" elif target.name.endswith("cronus"): wrapper_name = "Cronus" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset'], ) extension = "bin" elif target.name.endswith("test_p"): wrapper_name = "RiscvTestsP" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset'], ) extension = "S" else: raise NotImplementedError("Unsupported configuration '%s'" % target.name) if MICROPROBE_RC['debugwrapper']: extension = "s" extra_arguments = {} extra_arguments['instruction'] = instruction extra_arguments['benchmark_size'] = kwargs['benchmark_size'] extra_arguments['dependency_distance'] = kwargs['dependency_distance'] extra_arguments['data_init'] = kwargs['data_init'] outputfile = os.path.join(outputdir, outputname) outputfile = outputfile.replace("%INSTR%", bname) outputfile = outputfile.replace("%EXT%", extension) if wrapper.outputname(outputfile) != outputfile: print_error("Fix outputname to have a proper extension. E.g. '%s'" % wrapper.outputname(outputfile)) exit(-1) if instruction.unsupported: print_info("%s not supported!" % instruction.name) return if kwargs['skip'] and os.path.isfile(outputfile): print_info("%s already generated!" % outputfile) return outputfile = new_file(wrapper.outputname(outputfile), internal=True) synth = policy.apply(target, wrapper, **extra_arguments) print_info("Generating %s..." % outputfile) if not kwargs["ignore_errors"]: bench = synth.synthesize() else: if os.path.exists("%s.fail" % outputfile): print_error("%s failed before. Skip." % outputfile) return try: bench = synth.synthesize() except (MicroprobeException, AssertionError) as exc: with open("%s.fail" % outputfile, 'a'): os.utime("%s.fail" % outputfile, None) print_error("%s" % exc) print_error("Generation failed and ignore error flag set") return synth.save(outputfile, bench=bench) print_info("%s generated!" % outputfile) return
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ print_info("Arguments processed!") print_info("Checking input arguments for consistency...") if ("epi_output_file" not in arguments and "epi_output_dir" not in arguments): print_error("No output provided") exit(-1) if ("epi_output_file" in arguments and "epi_output_dir" in arguments): print_error("--epi-output-file and --epi-output-dir options conflict") print_error("Use one of them") exit(-1) if ("instructions" not in arguments and "epi_output_dir" not in arguments): print_error("If --instructions is not provided, you need to provide") print_error("--epi-output-dir and not --epi-output-file") exit(-1) print_info("Importing target definition...") target = import_definition(arguments.pop('target')) policy = find_policy(target.name, 'epi') if policy is None: print_error("Target does not implement the default EPI policy") exit(-1) if 'instructions' not in arguments: arguments['instructions'] = list(target.isa.instructions.values()) outputdir = arguments['epi_output_dir'] outputname = "%INSTR%.%EXT%" else: arguments['instructions'] = parse_instruction_list( target, arguments['instructions']) if ("epi_output_file" in arguments and len(arguments['instructions']) != 1): print_error("More than one microbenchmark to generate.") print_error("Use --epi-output-dir parameter") exit(-1) if "epi_output_file" in arguments: outputdir = os.path.dirname(arguments['epi_output_file']) outputname = arguments['epi_output_file'] else: outputdir = arguments['epi_output_dir'] outputname = "%INSTR%.%EXT%" if 'reset' not in arguments: arguments['reset'] = False if 'skip' not in arguments: arguments['skip'] = False if 'ignore_errors' not in arguments: arguments['ignore_errors'] = False if 'parallel' not in arguments: print_info("Start sequential generation. Use parallel flag to speed") print_info("up the benchmark generation.") for instruction in arguments['instructions']: _generic_policy_wrapper( (instruction, outputdir, outputname, target, arguments)) else: print_info("Start parallel generation. Threads: %s" % mp.cpu_count()) pool = mp.Pool(processes=mp.cpu_count()) pool.map(_generic_policy_wrapper, [(instruction, outputdir, outputname, target, arguments) for instruction in arguments['instructions']])
def _generic_policy_wrapper(all_arguments): instructions, outputdir, outputname, target, kwargs = all_arguments extension = "" if target.name.endswith("linux_gcc"): wrapper_name = "CInfGen" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset']) extension = "c" elif target.name.endswith("cronus"): wrapper_name = "Cronus" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset']) extension = "bin" else: raise NotImplementedError("Unsupported configuration '%s'" % target.name) if MICROPROBE_RC['debugwrapper']: extension = "s" outputfile = os.path.join(outputdir, "%DIRTREE%", outputname) outputfile = outputfile.replace( "%DIRTREE%", os.path.join(*[instr.name for instr in instructions])) outputfile = outputfile.replace( "%INSTR%", "_".join(instr.name for instr in instructions)) outputfile = outputfile.replace("%EXT%", extension) if kwargs['skip'] and outputfile in _DIRCONTENTS: print_info("%s already exists!" % outputfile) return if kwargs['skip'] and os.path.isfile(outputfile): print_info("%s already exists!" % outputfile) return extra_arguments = {} extra_arguments['instructions'] = instructions extra_arguments['benchmark_size'] = kwargs['benchmark_size'] extra_arguments['dependency_distance'] = kwargs['dependency_distance'] extra_arguments['force_switch'] = kwargs['force_switch'] if wrapper.outputname(outputfile) != outputfile: print_error("Fix outputname to have a proper extension. E.g. '%s'" % wrapper.outputname(outputfile)) exit(-1) for instr in instructions: if instr.unsupported: print_info("%s not supported!" % instr.name) return policy = find_policy(target.name, 'seq') if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) outputfile = new_file(wrapper.outputname(outputfile), internal=True) print_info("Generating %s..." % outputfile) synth = policy.apply(target, wrapper, **extra_arguments) bench = synth.synthesize() synth.save(outputfile, bench=bench) print_info("%s generated!" % outputfile) return
def dump_mpt(input_file_fd, target, arguments): """ :param input_file_fd: :type input_file_fd: :param target: :type target: :param arguments: :type arguments: """ try: contents = input_file_fd.read() if six.PY3 and not isinstance(contents, str): contents = contents.decode() except KeyboardInterrupt: print_info("No input data provided. Exiting...") exit(1) print_info("Parsing input file...") print_info("Sections to parse: %s" % arguments['sections']) var_defs, req_defs, instr_defs = \ interpret_objdump(contents, target, strict=arguments.get('strict', False), sections=arguments['sections'], start_address=arguments['from_address'], end_address=arguments['to_address']) print_info("Input file parsed") print_info( "%d instructions processed from the input file" % len(instr_defs) ) if var_defs != []: print_info( "Variables referenced and detected in the dump: %s" % ','.join([var.name for var in var_defs]) ) if req_defs != []: print_warning( "Variables referenced and *NOT* detected in the dump: %s" % ','.join([var.name for var in req_defs]) ) print_warning( "You might need to edit the generated MPT to fix the" " declaration of such variables" ) print_info("Generating the MPT contents...") mpt_config = mpt_configuration_factory() if 'default_code_address' in arguments: mpt_config.set_default_code_address(arguments['default_code_address']) else: mpt_config.set_default_code_address(instr_defs[0].address.displacement) if 'default_data_address' in arguments: mpt_config.set_default_data_address(arguments['default_data_address']) else: mpt_config.set_default_data_address(0) if arguments.get('elf_abi', False): kwargs = {} if "stack_name" in arguments: kwargs["stack_name"] = arguments["stack_name"] if "stack_address" in arguments: kwargs["stack_address"] = Address( base_address="code", displacement=arguments["stack_address"] ) variables, instructions = target.elf_abi( arguments["stack_size"], arguments.get( "start_symbol", None ), **kwargs ) for variable in variables: req_defs.append(variable_to_test_definition(variable)) address = instr_defs[0].address for instr in reversed(instructions): instr_defs = [instruction_to_definition(instr)] + instr_defs address -= instr.architecture_type.format.length if address.displacement < 0: print_error( "Default code address is below zero after" " adding the initialization code." ) print_error( "Check/modify the objdump provided or do not use" " the elf_abi flag." ) exit(-1) mpt_config.set_default_code_address(address.displacement) instr = None if "end_branch_to_itself" in arguments: instr = target.branch_to_itself() elif arguments.get('elf_abi', False): instr = target.nop() if instr is not None: instr.set_label("ELF_ABI_EXIT") instr_defs.append(instruction_to_definition(instr)) for var in var_defs + req_defs: mpt_config.register_variable_definition(var) mpt_config.register_instruction_definitions(instr_defs) print_info("Dumping MPT to '%s'" % arguments['output_mpt_file']) mpt_parser = mpt_parser_factory() mpt_parser.dump_mpt_config(mpt_config, arguments['output_mpt_file'])
def _compute_reset_code(target, test_def, args): instructions = interpret_asm( test_def.code, target, [var.name for var in test_def.variables], show_progress=True, ) # TODO: This can be done in parallel or look for speed up the process instructions = [ instruction_from_definition(instr) for instr in instructions ] instruction_dict = {} address = test_def.default_code_address progress = Progress( len(test_def.roi_memory_access_trace), msg="Building instruction dictionary", ) for instr in instructions: progress() if instr.address is not None: if instr.address.base_address == "code": address = test_def.default_code_address + \ instr.address.displacement instr.set_address(address) else: address = address + instr.architecture_type.format.length instr.set_address(address) instruction_dict[instr.address] = instr free_regs = [] written_after_read_regs = [] read_regs = [] level = 0 dynamic_count = 0 progress = Progress( len(test_def.roi_memory_access_trace), msg="Evaluating register usage", ) reset_regs = set() for access in test_def.roi_memory_access_trace: progress() if access.data_type == "D": continue dynamic_count += 1 try: instr = instruction_dict[access.address] uses = instruction_dict[access.address].uses() sets = instruction_dict[access.address].sets() except KeyError: print_error( "Access to from instruction at address " "0x%016X registered but such instruction is not" " present in the definition." % access.address, ) exit(1) # Calls if instr.mnemonic == "BL": level += 1 elif instr.mnemonic == "BCL": level += 1 elif instr.mnemonic == "BCCTRL": if instr.operands()[2].value in [0, 3]: level += 1 # Returns if instr.mnemonic == "BCLR": if (((instr.operands()[0].value & 0b10100) == 20) and (instr.operands()[2].value == 0)): level -= 1 # TODO: this should include Z and RISCV instructions for call # and return, but currently we do not have memory access traces # for such platforms for reg in uses: if reg not in read_regs: read_regs.append(reg) for reg in sets: if reg in free_regs: continue elif reg not in read_regs: free_regs.append(reg) elif reg not in written_after_read_regs: written_after_read_regs.append(reg) reset_regs = set(read_regs).intersection( set(written_after_read_regs), ) reset_regs = sorted(reset_regs) assert len(free_regs) == len(set(free_regs)) assert len(set(free_regs).intersection(set(reset_regs))) == 0 if len(test_def.roi_memory_access_trace) == 0: # We do not have memory access trace, assume calling conventions reset_regs = target.volatile_registers reset_regs = [ reg for reg in reset_regs if reg in target.volatile_registers] if len(reset_regs) == 0 and len(test_def.roi_memory_access_trace) == 0: print_info( "No memory access trace found. Resetting volatile registers." ) reset_regs = target.volatile_registers unused_regs = sorted( (reg for reg in target.registers.values() if reg not in read_regs), ) # # Make sure scratch registers are reset last # for reg in target.scratch_registers: if reg in reset_regs: reset_regs.remove(reg) reset_regs.append(reg) free_regs = unused_regs + free_regs # Know which ones are not used (or written) and which ones are used # Use them as base / temporal registers for addresses # Check addresses conflict_addresses = {} new_ins = [] progress = Progress( len(test_def.roi_memory_access_trace), msg="Evaluating memory usage", ) for access in test_def.roi_memory_access_trace: progress() if access.data_type == "I": continue val = conflict_addresses.get( access.address, [access.length, access.access_type], ) if access.access_type not in val[1]: val[1] += access.access_type val[0] = max(val[0], access.length) conflict_addresses[access.address] = val fix_addresses = [] for address in conflict_addresses: value = conflict_addresses[address] if value[1] == "RW": wvalue = None for var in test_def.variables: if var.var_type.upper() in ["CHAR", "UINT8_T"]: elem_size = 1 else: raise NotImplementedError end_address = var.address + var.num_elements * elem_size if var.address <= address <= end_address: offset = int((address - var.address) / elem_size) svalue = var.init_value[ offset:offset + int(value[0] / elem_size) ] svalue = "".join(["%02X" % tval for tval in svalue]) wvalue = int(svalue, 16) break if wvalue is None: print_error( "Unable to restore original value for address 0x%X" % address, ) exit(1) if value[0] <= 8: fix_addresses.append((address, value[0], wvalue)) else: for selem in range(0, value[0]//8): sfmt = "%%0%dX" % (2*value[0]) nvalue = sfmt % wvalue nvalue = int(nvalue[selem*16:(selem+1)*16], 16) fix_addresses.append( (address + selem * 8, 8, nvalue) ) reset_steps = [] context = Context() context.set_symbolic(True) if len(fix_addresses) > 0: # TODO: This can be optimized. Reduce the number of instructions to # be added by sorting the reset code (shared values or similar # addresses) # TODO: This can be optimized for use vector registers when # needed # print_info("Adding instructions to reset memory state") reset_register = [ reg for reg in free_regs if reg.type.used_for_address_arithmetic and reg.name != "GPR0" ][0] for address, length, value in fix_addresses: address_obj = Address(base_address="data", displacement=address) new_instructions = target.set_register( reset_register, value, context, opt=False, ) for ins in new_instructions: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (reset_register.name, value), ) reset_steps.append([new_instructions[:], reset_register, value]) context.set_register_value(reset_register, value) try: store_ins = target.store_integer( reset_register, address_obj, length * 8, context, ) new_instructions += store_ins reset_steps.append( [store_ins, reset_register, address_obj, length], ) except MicroprobeCodeGenerationError: areg = [ reg for reg in free_regs if reg.type.used_for_address_arithmetic and reg.name != "GPR0" ][1] set_ins = target.set_register( areg, address, context, opt=False, ) new_instructions += set_ins reset_steps.append([set_ins, areg, address_obj]) context.set_register_value(areg, address_obj) store_ins = target.store_integer( reset_register, address_obj, length * 8, context, ) new_instructions += store_ins reset_steps.append( [store_ins, reset_register, address_obj, length], ) for ins in set_ins: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (areg.name, address), ) for ins in store_ins: ins.add_comment( "Reset code. Setting mem content in 0X%016X" % (address), ) new_ins.extend(new_instructions) # Reset contents of used registers for reset_register in reset_regs: try: value = [ reg for reg in test_def.registers if reg.name == reset_register.name ][0].value except IndexError: continue new_instructions = target.set_register( reset_register, value, context, opt=False, ) reset_steps.append([new_instructions, reset_register, value]) context.set_register_value(reset_register, value) for ins in new_instructions: ins.add_comment( "Reset code. Setting %s to 0X%016X" % (reset_register.name, value), ) new_ins.extend(new_instructions) try: overhead = (((len(new_ins) * 1.0) / dynamic_count) * 100) except ZeroDivisionError: print_warning("Unable to compute overhead. Zero dynamic instruction " "count") overhead = 0 print_info( "%03.2f%% overhead added by resetting code" % overhead, ) if overhead > args['wrap_endless_threshold']: print_error( "Instructions added: %d" % len(new_ins), ) print_error( "Total instructions: %d" % dynamic_count, ) print_error( "Reset code above --wrap-endless-threshold. Stopping generation.", ) exit(1) return new_ins, overhead, reset_steps
__copyright__ = "Copyright 2011-2021 IBM Corporation" __credits__ = [] __license__ = "IBM (c) 2011-2021 All rights reserved" __version__ = "0.5" __maintainer__ = "Ramon Bertran" __email__ = "*****@*****.**" __status__ = "Development" # "Prototype", "Development", or "Production" # Benchmark size BENCHMARK_SIZE = 20 # Get the target definition try: TARGET = import_definition("power_v206-power7-ppc64_linux_gcc") except MicroprobeTargetDefinitionError as exc: print_error("Unable to import target definition") print_error("Exception message: %s" % str(exc)) exit(-1) def main(): """Main function.""" component_list = ["FXU", "FXU-noLSU", "FXU-LSU", "VSU", "VSU-FXU"] ipcs = [float(x) / 10 for x in range(1, 41)] ipcs = ipcs[5:] + ipcs[:5] for name in component_list: for ipc in ipcs: generate_genetic(name, ipc)
from microprobe.utils.cmdline import print_error, print_info __author__ = "Ramon Bertran" __copyright__ = "Copyright 2011-2021 IBM Corporation" __credits__ = [] __license__ = "IBM (c) 2011-2021 All rights reserved" __version__ = "0.5" __maintainer__ = "Ramon Bertran" __email__ = "*****@*****.**" __status__ = "Development" # "Prototype", "Development", or "Production" # Get the target definition try: TARGET = import_definition("power_v206-power7-ppc64_linux_gcc") except MicroprobeTargetDefinitionError as exc: print_error("Unable to import target definition") print_error("Exception message: %s" % str(exc)) exit(-1) BASE_ELEMENT = [ element for element in TARGET.elements.values() if element.name == 'L1D' ][0] CACHE_HIERARCHY = TARGET.cache_hierarchy.get_data_hierarchy_from_element( BASE_ELEMENT) # Benchmark size BENCHMARK_SIZE = 8 * 1024 # Fill a list of the models to be generated MEMORY_MODELS = []
def main_setup(): """ Set up the command line interface (CLI) with the arguments required by this command line tool. """ args = sys.argv[1:] # Get the target definition try: target = import_definition("power_v206-power7-ppc64_linux_gcc") except MicroprobeTargetDefinitionError as exc: print_error("Unable to import target definition") print_error("Exception message: %s" % str(exc)) exit(-1) func_units = {} valid_units = [elem.name for elem in target.elements.values()] for instr in target.isa.instructions.values(): if instr.execution_units == "None": LOG.debug("Execution units for: '%s' not defined", instr.name) continue for unit in instr.execution_units: if unit not in valid_units: continue if unit not in func_units: func_units[unit] = [ elem for elem in target.elements.values() if elem.name == unit ][0] # Create the CLI interface object cmdline = microprobe.utils.cmdline.CLI("ISA power v206 profile example", config_options=False, target_options=False, debug_options=False) # Add the different parameters for this particular tool cmdline.add_option("functional_unit", "f", [func_units['ALU']], "Functional units to stress. Default: ALU", required=False, nargs="+", choices=func_units, opt_type=dict_key(func_units), metavar="FUNCTIONAL_UNIT_NAME") cmdline.add_option( "output_prefix", None, "POWER_V206_FU_STRESS", "Output prefix of the generated files. Default: POWER_V206_FU_STRESS", opt_type=str, required=False, metavar="PREFIX") cmdline.add_option("output_path", "O", "./", "Output path. Default: current path", opt_type=existing_dir, metavar="PATH") cmdline.add_option( "size", "S", 64, "Benchmark size (number of instructions in the endless loop). " "Default: 64 instructions", opt_type=int_type(1, 2**20), metavar="BENCHMARK_SIZE") cmdline.add_option("dependency_distance", "D", 1000, "Average dependency distance between the instructions. " "Default: 1000 (no dependencies)", opt_type=int_type(1, 1000), metavar="DEPENDECY_DISTANCE") cmdline.add_option("average_latency", "L", 2, "Average latency of the selected instructins. " "Default: 2 cycles", opt_type=float_type(1, 1000), metavar="AVERAGE_LATENCY") # Start the main print_info("Processing input arguments...") cmdline.main(args, _main)
def _generate_benchmark(target, output_prefix, args): """ Actual benchmark generation policy. This is the function that defines how the microbenchmark are going to be generated """ functional_units, size, latency, distance = args try: # Name of the output file func_unit_names = [unit.name for unit in functional_units] fname = "%s%s" % (output_prefix, "_".join(func_unit_names)) fname = "%s_LAT_%s" % (fname, latency) fname = "%s_DEP_%s" % (fname, distance) # Name of the fail output file (generated in case of exception) ffname = "%s.c.fail" % (fname) print_info("Generating %s ..." % (fname)) # Get the wrapper object. The wrapper object is in charge of # translating the internal representation of the microbenchmark # to the final output format. # # In this case, we obtain the 'CInfGen' wrapper, which embeds # the generated code within an infinite loop using C plus # in-line assembly statements. cwrapper = microprobe.code.get_wrapper("CInfGen") # Create the synthesizer object, which is in charge of driving the # generation of the microbenchmark, given a set of passes # (a.k.a. transformations) to apply to the an empty internal # representation of the microbenchmark synth = microprobe.code.Synthesizer(target, cwrapper(), value=0b01010101) # Add the transformation passes ####################################################################### # Pass 1: Init integer registers to a given value # ####################################################################### synth.add_pass( microprobe.passes.initialization.InitializeRegistersPass( value=_init_value())) ####################################################################### # Pass 2: Add a building block of size 'size' # ####################################################################### synth.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass(size)) ####################################################################### # Pass 3: Fill the building block with the instruction sequence # ####################################################################### synth.add_pass( microprobe.passes.instruction.SetInstructionTypeByElementPass( target, functional_units, {})) ####################################################################### # Pass 4: Compute addresses of instructions (this pass is needed to # # update the internal representation information so that in # # case addresses are required, they are up to date). # ####################################################################### synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass()) ####################################################################### # Pass 5: Set target of branches to be the next instruction in the # # instruction stream # ####################################################################### synth.add_pass(microprobe.passes.branch.BranchNextPass()) ####################################################################### # Pass 6: Set memory-related operands to access 16 storage locations # # in a round-robin fashion in stride 256 bytes. # # The pattern would be: 0, 256, 512, .... 3840, 0, 256, ... # ####################################################################### synth.add_pass(microprobe.passes.memory.SingleMemoryStreamPass( 16, 256)) ####################################################################### # Pass 7.A: Initialize the storage locations accessed by floating # # point instructions to have a valid floating point value # ####################################################################### synth.add_pass( microprobe.passes.float.InitializeMemoryFloatPass( value=1.000000000000001)) ####################################################################### # Pass 7.B: Initialize the storage locations accessed by decimal # # instructions to have a valid decimal value # ####################################################################### synth.add_pass( microprobe.passes.decimal.InitializeMemoryDecimalPass(value=1)) ####################################################################### # Pass 8: Set the remaining instructions operands (if not set) # # (Required to set remaining immediate operands) # ####################################################################### synth.add_pass( microprobe.passes.register.DefaultRegisterAllocationPass( dd=distance)) # Synthesize the microbenchmark.The synthesize applies the set of # transformation passes added before and returns object representing # the microbenchmark bench = synth.synthesize() # Save the microbenchmark to the file 'fname' synth.save(fname, bench=bench) print_info("%s generated!" % (fname)) # Remove fail file if exists if os.path.isfile(ffname): os.remove(ffname) except MicroprobeException: # In case of exception during the generation of the microbenchmark, # print the error, write the fail file and exit print_error(traceback.format_exc()) open(ffname, 'a').close() exit(-1)
def _generic_policy_wrapper(all_arguments): configuration, outputdir, outputname, target, kwargs = all_arguments instructions, mem_switch, data_switch, switch_branch, branch_pattern, \ replace_every, add_every, memory_streams, \ benchmark_size = configuration extrapath = [] extrapath.append("BS_%d" % benchmark_size) extrapath.append("MS_%d" % mem_switch) extrapath.append("DS_%d" % data_switch) extrapath.append("BP_%s" % branch_pattern) extrapath.append("SB_%d" % switch_branch) for repl in replace_every: extrapath.append("RE_%d_%s_%s" % (repl[2], repl[0].name, repl[1].name)) for addl in add_every: extrapath.append("AE_%d_%s" % (addl[1], "_".join( [elem.name for elem in addl[0]] ) ) ) for mems in memory_streams: extrapath.append("ME_%d_%d_%d_%d_%d" % mems) outputfile = os.path.join(outputdir, "%DIRTREE%", outputname) outputfile = outputfile.replace( "%DIRTREE%", os.path.join( *([instr.name for instr in instructions] + extrapath))) outputfile = outputfile.replace( "%BASENAME%", "_".join( instr.name for instr in instructions) + "#" + "#".join(extrapath)) extension = "" if target.name.endswith("linux_gcc"): wrapper_name = "CInfGen" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class( reset=kwargs['reset'] ) extension = "c" elif target.name.endswith("cronus"): wrapper_name = "Cronus" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class( reset=kwargs['reset'] ) extension = "bin" elif target.name.endswith("mesa"): wrapper_name = "Tst" extension = "tst" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class( os.path.basename(outputfile.replace("%EXT%", extension)), reset=kwargs['reset'] ) elif target.name.endswith("riscv64_test_p"): wrapper_name = "RiscvTestsP" extension = "S" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class( reset=kwargs['reset'], endless=True ) else: raise NotImplementedError( "Unsupported configuration '%s'" % target.name ) if MICROPROBE_RC['debugwrapper']: extension = "s" outputfile = outputfile.replace("%EXT%", extension) if kwargs['skip']: if os.path.exists(outputfile): print_info("%s already exists!" % outputfile) return if len(memory_streams) == 0: warnings.warn( "No memory streams provided " "using 1K stream stride 64 bytes" ) memory_streams = [(1, 1024, 1, 64, 1)] streamid = 0 new_memory_streams = [] for stream in memory_streams: for elem in range(0, stream[0]): new_memory_streams.append([streamid] + list(stream)[1:]) streamid += 1 extra_arguments = {} extra_arguments['instructions'] = instructions extra_arguments['benchmark_size'] = benchmark_size extra_arguments['dependency_distance'] = kwargs['dependency_distance'] extra_arguments['mem_switch'] = mem_switch extra_arguments['data_switch'] = data_switch extra_arguments['branch_pattern'] = branch_pattern extra_arguments['replace_every'] = replace_every extra_arguments['add_every'] = add_every extra_arguments['memory_streams'] = new_memory_streams extra_arguments['branch_switch'] = switch_branch if wrapper.outputname(outputfile) != outputfile: print_error( "Fix outputname to have a proper extension. E.g. '%s'" % wrapper.outputname(outputfile) ) exit(-1) for instr in instructions: if instr.unsupported: print_info("%s not supported!" % instr.name) return policy = find_policy(target.name, 'seqtune') if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) outputfile = new_file(wrapper.outputname(outputfile), internal=True) print_info("Generating %s..." % outputfile) synth = policy.apply(target, wrapper, **extra_arguments) if not kwargs["ignore_errors"]: bench = synth.synthesize() else: if os.path.exists("%s.fail" % outputfile): print_error("%s failed before. Not generating" % outputfile) return try: bench = synth.synthesize() except Exception as exc: with open("%s.fail" % outputfile, 'a'): os.utime("%s.fail" % outputfile, None) print_error("%s" % exc) print_error("Generation failed for current configuration.") print_error("Generating next configurations.") return synth.save(outputfile, bench=bench) print_info("%s generated!" % outputfile) return
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ print_info("Arguments processed!") print_info("Checking input arguments for consistency...") flags = ["data_switch", "memory_switch", "ignore_errors", "switch_branch"] for flag in flags: if flag not in arguments: arguments[flag] = False args = ["replace_every", "add_every", "branch_every", "branch_pattern", "memory_stream"] for arg in args: if arg not in arguments: arguments[arg] = [] if (len(arguments['branch_every']) > 0 and len(arguments['branch_pattern']) > 0): print_error("--branch-every and --branch-pattern flags are " "mutually exclusive. Use only on of them.") exit(-1) elif len(arguments['branch_pattern']) > 0: arguments['branch_pattern'] = _process_branch_pattern( arguments['branch_pattern'] ) elif len(arguments['branch_every']) > 0: arguments['branch_pattern'] = _process_branch_every( arguments['branch_every'] ) else: arguments['branch_pattern'] = ['0'] print_info("Importing target definition...") target = import_definition(arguments.pop('target')) policy = find_policy(target.name, 'seqtune') if policy is None: print_error("Target does not implement the default SEQTUNE policy") exit(-1) arguments['sequence'] = parse_instruction_list( target, arguments['sequence']) for elem in arguments["replace_every"]: elem[0] = parse_instruction_list(target, elem[0]) elem[1] = parse_instruction_list(target, elem[1]) for elem in arguments["add_every"]: elem[0] = parse_instruction_list(target, elem[0]) configurations = _generate_configurations(arguments) configurations = _filter_configurations(configurations, arguments) if 'count' in arguments: print_info("Total number of variations defined : %s" % len(list(configurations))) exit(0) outputdir = arguments['seq_output_dir'] outputname = "%BASENAME%.%EXT%" if 'reset' not in arguments: arguments['reset'] = False if 'skip' not in arguments: arguments['skip'] = False if 'parallel' not in arguments: print_info("Start sequential generation. Use parallel flag to speed") print_info("up the benchmark generation.") for params in configurations: _generic_policy_wrapper( (params, outputdir, outputname, target, arguments)) else: print_info("Start parallel generation. Threads: %s" % mp.cpu_count()) pool = mp.Pool(processes=mp.cpu_count()) pool.map(_generic_policy_wrapper, [(params, outputdir, outputname, target, arguments) for params in configurations] )
def _compile(filename, target, **kwargs): if "compiler" not in kwargs: print_info("Compiler not provided") print_info("To compiler the code, first extract the custom") print_info("ld script embedded in the assembly as comments to do") print_info("so execute:") print_info("grep 'MICROPROBE LD' %s | cut -d '@' -f 2" % filename) print_info("then compile using gcc and providing the ld script") print_info("using the -T option.") return print_info("Compiling %s ..." % filename) outputname = ".".join(filename.split(".")[:-1]+["elf"]) ldscriptname = ".".join(filename.split(".")[:-1]+["ldscript"]) fdout = open(ldscriptname, "w") fdin = open(filename, "r") for line in fdin.readlines(): if "MICROPROBE LD" in line: line = line.split("@")[1] fdout.write(line) fdout.close() fdin.close() try: baseldscriptname = findfiles( MICROPROBE_RC['template_paths'], "%s.ldscript" % target.name )[0] except IndexError: print_error("Unable to find template ld script: %s.ldscript" % target.name) exit(-1) cprog = kwargs["compiler"] cflags = " -o %s" % outputname cflags += " -T %s" % ldscriptname cflags += " -T %s " % baseldscriptname cflags += kwargs["compiler_flags"] # We only support BFD linker cflags += " -fuse-ld=bfd " cmd = "%s %s %s" % (cprog, cflags, filename) print_info("Executing compilation command") print_info("%s" % cmd) try: run_cmd(cmd) print_info("'%s' generated!" % outputname) except MicroprobeRunCmdError: cflags += " -static" cmd = "%s %s %s" % (cprog, cflags, filename) print_info("Executing compilation command (statically)") print_info("%s" % cmd) try: run_cmd(cmd) print_info("'%s' generated!" % outputname) except MicroprobeRunCmdError: print_info("'%s' not generated due compilation" " issues. Try manual compilation." % outputname)
def _generic_policy_wrapper(all_arguments): instructions, outputdir, outputname, target, kwargs = all_arguments outputfile = os.path.join(outputdir, "%DIRTREE%", outputname) outputfile = outputfile.replace( "%DIRTREE%", os.path.join(*[instr.name for instr in instructions])) if kwargs['shortnames']: outputfile = outputfile.replace( "%INSTR%", "mp_seq_%s" % hashlib.sha1("_".join( instr.name for instr in instructions).encode()).hexdigest()) else: outputfile = outputfile.replace( "%INSTR%", "_".join(instr.name for instr in instructions)) extension = "" if target.name.endswith("linux_gcc"): wrapper_name = "CInfGen" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset']) extension = "c" elif target.name.endswith("cronus"): wrapper_name = "Cronus" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(reset=kwargs['reset'], endless=kwargs['endless']) extension = "bin" elif target.name.endswith("mesa"): wrapper_name = "Tst" extension = "tst" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(os.path.basename( outputfile.replace("%EXT%", extension)), endless=kwargs['endless'], reset=kwargs['reset']) elif target.name.endswith("riscv64_test_p"): wrapper_name = "RiscvTestsP" extension = "S" wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(endless=kwargs['endless'], reset=kwargs['reset']) elif target.environment.default_wrapper: wrapper_name = target.environment.default_wrapper wrapper_class = _get_wrapper(wrapper_name) wrapper = wrapper_class(endless=kwargs['endless'], reset=kwargs['reset']) outputfile = outputfile.replace(".%EXT%", "") outputfile = wrapper.outputname(outputfile) else: raise NotImplementedError("Unsupported configuration '%s'" % target.name) if MICROPROBE_RC['debugwrapper']: extension = "s" outputfile = outputfile.replace("%EXT%", extension) if kwargs['skip'] and outputfile in _DIRCONTENTS: print_info("%s already exists!" % outputfile) return if (kwargs['skip'] and "%s.gz" % outputfile in _DIRCONTENTS and kwargs["compress"]): print_info("%s.gz already exists!" % outputfile) return if kwargs['skip'] and os.path.isfile(outputfile): print_info("%s already exists!" % outputfile) return if (kwargs['skip'] and os.path.isfile("%s.gz" % outputfile) and kwargs["compress"]): print_info("%s already exists!" % outputfile) return extra_arguments = {} extra_arguments['instructions'] = instructions extra_arguments['benchmark_size'] = kwargs['benchmark_size'] extra_arguments['dependency_distance'] = kwargs['dependency_distance'] extra_arguments['force_switch'] = kwargs['force_switch'] extra_arguments['endless'] = kwargs['endless'] if wrapper.outputname(outputfile) != outputfile: print_error( "Fix outputname '%s' to have a proper extension. E.g. '%s'" % (outputfile, wrapper.outputname(outputfile))) exit(-1) for instr in instructions: if instr.unsupported: print_info("%s not supported!" % instr.name) return policy = find_policy(target.name, 'seq') if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) outputfile = new_file(wrapper.outputname(outputfile), internal=True) print_info("Generating %s..." % outputfile) synth = policy.apply(target, wrapper, **extra_arguments) bench = synth.synthesize() synth.save(outputfile, bench=bench) print_info("%s generated!" % outputfile) if kwargs['compress']: move_file(outputfile, "%s.gz" % outputfile) print_info("%s compressed to %s.gz" % (outputfile, outputfile)) return
def generate(test_definition, output_file, target, **kwargs): """ Microbenchmark generation policy. :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param output_file: Output file name :type output_file: :class:`str` :param target: Target definition object :type target: :class:`Target` """ end_address_orig = None overhead = 0 if len(test_definition.dat_mappings) > 0: # # Assuming MPT generated from MAMBO full system # dat = target.get_dat(dat_map=test_definition.dat_mappings) dat.control['DAT'] = True for instr in test_definition.code: instr.address = dat.translate(instr.address, rev=True) # Address order might have changed after mapping test_definition.set_instruction_definitions( sorted(test_definition.code, key=lambda x: x.address) ) # remove not translated addresses (needed?) # variables = [var for var in test_definition.variables # if var.address != dat.translate(var.address, rev=True)] # test_definition.set_variables_definition(variables) for var in test_definition.variables: var.address = dat.translate(var.address, rev=True) # Address order might have changed after mapping test_definition.set_variables_definition( sorted(test_definition.variables, key=lambda x: x.address) ) if test_definition.default_code_address != 0: test_definition.default_code_address = dat.translate( test_definition.default_code_address, rev=True ) if test_definition.default_data_address != 0: test_definition.default_data_address = dat.translate( test_definition.default_data_address, rev=True ) for access in test_definition.roi_memory_access_trace: access.address = dat.translate( access.address, rev=True ) if 'raw_bin' in kwargs: print_info("Interpreting RAW dump...") sequence = [] raw_dict = {} current_address = 0 # Assume state file provides the initial code address displ = test_definition.default_code_address test_definition.set_default_code_address(0) for entry in test_definition.code: if (not entry.assembly.upper().startswith("0X") and not entry.assembly.upper().startswith("0B")): raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "assembly (%s)" % entry.assembly ) if entry.label is not None: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "labels (%s)" % entry.label ) if entry.decorators not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "decorators (%s)" % entry.decorators ) if entry.comments not in ['', ' ', None, []]: raise MicroprobeMPTFormatError( "This is not a RAW dump as it contains " "comments (%s)" % entry.comments ) if entry.address is not None: current_address = entry.address + displ if current_address not in raw_dict: raw_dict[current_address] = "" # Assume that raw dump use a 4 bytes hex dump if len(entry.assembly) != 10: raise MicroprobeMPTFormatError( "This is not a RAW 4-byte dump as it contains " "lines with other formats (%s)" % entry.assembly ) raw_dict[current_address] += entry.assembly[2:] if len(raw_dict) > 1: address_ant = sorted(raw_dict.keys())[0] len_ant = len(raw_dict[address_ant])//2 # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 for address in sorted(raw_dict.keys())[1:]: if address_ant + len_ant == address: raw_dict[address_ant] += raw_dict[address] len_ant = len(raw_dict[address_ant])//2 raw_dict.pop(address) else: len_ant = len(raw_dict[address])//2 address_ant = address # Assume that raw dump use a 4 bytes hex dump assert len_ant % 4 == 0 sequence = [] for address in sorted(raw_dict.keys()): # Endianess will be big endian, because we are concatenating # full words resulting in the higher bits being encoded first code = interpret_bin( raw_dict[address], target, safe=True, little_endian=False, word_length=4 ) for instr in code: instr.address = address instr = instruction_from_definition(instr) address = address + instr.architecture_type.format.length instr = instruction_to_asm_definition(instr) sequence.append(instr) test_definition.set_instruction_definitions(sequence) reset_steps = [] if 'no_wrap_test' not in kwargs: if test_definition.default_code_address != 0: print_error("Default code address should be zero") exit(-1) print_info("Wrapping function...") start_symbol = "START_TEST" init_address = test_definition.default_code_address for register in test_definition.registers: if register.name == "PC": init_address = register.value if register.name == "PSW_ADDR": init_address = register.value displacements = [] for elem in test_definition.code: if elem.address is not None: if len(displacements) == 0: displacements.append( (elem.address, 4*1024, elem.address - init_address) ) else: displacements.append( (elem.address, elem.address - displacements[-1][0], elem.address - init_address) ) # Get ranges with enough space to put init code # Assuming 4K space is enough displacements = [ displ for displ in displacements if displ[1] >= 4*1024 and displ[2] <= 0 ] if len(displacements) == 0: print_error( "Unable to find space for the initialization code. " "Check the mpt initial code address or state of PC " "for correctness." ) exit(-1) displ_fixed = False if kwargs['fix_start_address']: displacement = kwargs['fix_start_address'] print_info("Start point set to 0x%X" % displacement) displ_fixed = True elif 'fix_long_jump' in kwargs: displacement = sorted(displacements, key=lambda x: x[0])[0][0] if displacement > 2**32: displacement = 0x1000000 print_info("Start point set to 0x%X" % displacement) displ_fixed = True else: displacement = sorted(displacements, key=lambda x: x[2])[-1][0] start_symbol = None init_found = False for instr in test_definition.code: if instr.address == init_address: if instr.label is None: instr.label = "START_TEST" start_symbol = instr.label init_found = True break if not init_found: print_error( "Initial instruction address (%s) not found" % hex(init_address) ) exit(-1) if start_symbol is None: if test_definition.code[0].label is None: test_definition.code[0].label = "START_TEST" start_symbol = test_definition.code[0].label if displacement is None: displacement = 0 instructions = [] reset_steps = [] if 'wrap_endless' in kwargs and 'reset' in kwargs: target.scratch_var.set_address( Address( base_address=target.scratch_var.name ) ) new_ins, overhead, reset_steps = _compute_reset_code( target, test_definition, kwargs, ) instructions += new_ins if 'fix_long_jump' in kwargs: instructions += target.function_call( init_address, long_jump=True ) else: instructions += target.function_call( ("%s" % start_symbol).replace("+0x-", "-0x"), ) if 'wrap_endless' not in kwargs: instructions += [target.nop()] else: instructions += _compute_reset_jump(target, instructions) instructions_definitions = [] for instruction in instructions: instruction.set_label(None) if not displ_fixed: displacement = (displacement - instruction.architecture_type.format.length) current_instruction = MicroprobeAsmInstructionDefinition( instruction.assembly(), None, None, None, instruction.comments, ) instructions_definitions.append(current_instruction) instruction = target.nop() instruction.set_label(None) if not displ_fixed: displacement = (displacement - instruction.architecture_type.format.length) # To avoid overlaps if not displ_fixed: align = 0x100 displacement = ((displacement // align) + 0) * align instructions_definitions[0].address = displacement assert instructions_definitions[0].address is not None # instr = MicroprobeAsmInstructionDefinition( # instruction.assembly(), "ELF_ABI_EXIT", None, None, None) # end_address_orig = \ # (test_definition.default_code_address + displacement_end - # instruction.architecture_type.format.length) instructions_definitions[0].label = "mpt2elf_endless" test_definition.register_instruction_definitions( instructions_definitions, prepend=True, ) assert test_definition.code[0].address is not None if not displ_fixed: test_definition.set_default_code_address( test_definition.default_code_address + displacement, ) for elem in test_definition.code: if elem.address is not None: elem.address = elem.address - displacement else: test_definition.set_default_code_address( displacement ) for elem in test_definition.code: if elem.address is not None: elem.address = elem.address - displacement variables = test_definition.variables variables = [var for var in test_definition.variables if var.address is None or var.address >= 0x00100000] test_definition.set_variables_definition(variables) print_info("Interpreting asm ...") sequence_orig = interpret_asm( test_definition.code, target, [var.name for var in variables] + [target.scratch_var.name], show_progress=True, ) if len(sequence_orig) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file.", ) raw = test_definition.raw raw['FILE_FOOTER'] = "# mp_mpt2elf: Wrapping overhead: %03.2f %%" \ % overhead # end_address = end_address_orig ckwargs = { # 'end_address': end_address, # 'reset': False, # 'endless': 'endless' in kwargs } wrapper_name = "AsmLd" if test_definition.default_data_address is not None: ckwargs['init_data_address'] = \ test_definition.default_data_address if test_definition.default_code_address is not None: ckwargs['init_code_address'] = \ test_definition.default_code_address try: code_wrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers" " of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error " "was: %s" % (wrapper_name, str(exc)), ) wrapper = code_wrapper(**ckwargs) print_info("Setup synthesizer ...") synthesizer = microprobe.code.Synthesizer( target, wrapper, no_scratch=False, extra_raw=raw, ) variables = test_definition.variables registers = test_definition.registers sequence = sequence_orig if len(registers) >= 0: cr_reg = [ register for register in registers if register.name == "CR" ] registers = [ register for register in registers if register.name != "CR" ] if cr_reg: value = cr_reg[0].value for idx in range(0, 8): cr = MicroprobeTestRegisterDefinition( "CR%d" % idx, (value >> (28 - (idx * 4))) & 0xF, ) registers.append(cr) synthesizer.add_pass( microprobe.passes.initialization.InitializeRegistersPass( registers, skip_unknown=True, warn_unknown=True, skip_control=True, force_reserved=True ), ) synthesizer.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass( len(sequence), ), ) synthesizer.add_pass( microprobe.passes.variable.DeclareVariablesPass( variables, ), ) synthesizer.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence), ) if target.name.startswith("power"): fix_branches = [instr.name for instr in target.instructions.values() if instr.branch_conditional] if 'raw_bin' in kwargs: # We do not know what is code and what is data, so we safely # disable the asm generation and keep the values for orig in [21, 17, 19]: synthesizer.add_pass( microprobe.passes.instruction.DisableAsmByOpcodePass( fix_branches, 0, ifval=orig ) ) else: # We know what is code and what is data, so we can safely # fix the branch instructions for new, orig in [(20, 21), (16, 17), (18, 19)]: synthesizer.add_pass( SetInstructionOperandsByOpcodePass( fix_branches, 0, new, force=True, ifval=orig ) ) if kwargs.get("fix_memory_registers", False): kwargs["fix_memory_references"] = True if kwargs.get("fix_memory_references", False): print_info("Fix memory references: On") synthesizer.add_pass( microprobe.passes.memory.FixMemoryReferencesPass( reset_registers=kwargs.get("fix_memory_registers", False), ), ) synthesizer.add_pass( microprobe.passes.register.FixRegistersPass( forbid_writes=['GPR3'], ), ) if kwargs.get("fix_memory_registers", False): print_info("Fix memory registers: On") synthesizer.add_pass( microprobe.passes.register.NoHazardsAllocationPass(), ) if kwargs.get("fix_branch_next", False): print_info("Force branch to next: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( force="fix_flatten_code" in kwargs, noinit=True ) ) synthesizer.add_pass( microprobe.passes.branch.BranchNextPass(force=True), ) if kwargs.get("fix_indirect_branches", False): print_info("Fix indirect branches: On") synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( noinit=True ), ) synthesizer.add_pass( microprobe.passes.branch.FixIndirectBranchPass(), ) if displ_fixed: synthesizer.add_pass( microprobe.passes.address.SetInitAddressPass(displacement) ) synthesizer.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass( noinit=True, init_from_first=not displ_fixed, ), ) synthesizer.add_pass( microprobe.passes.variable.UpdateVariableAddressesPass( ), ) synthesizer.add_pass( microprobe.passes.symbol.ResolveSymbolicReferencesPass( onlyraw=True ), ) print_info("Start synthesizer ...") bench = synthesizer.synthesize() # Save the microbenchmark synthesizer.save(output_file, bench=bench) print_info("'%s' generated!" % output_file) _compile(output_file, target, **kwargs) return
def _main(arguments): """ Program main, after processing the command line arguments :param arguments: Dictionary with command line arguments and values :type arguments: :class:`dict` """ print_info("Arguments processed!") print_info("Checking input arguments for consistency...") slots = arguments['instruction_slots'] instruction_groups = list( iter_flatten(arguments.get('instruction_groups', []))) check_group_length_func = int_type(1, len(instruction_groups)) check_slots_length_func = int_type(0, slots) instruction_map = arguments.get('instruction_map', None) if instruction_map is None: instruction_map = ['-1'] * slots else: instruction_map = list(iter_flatten(instruction_map)) if len(instruction_map) != slots: print_error('Instruction map: %s' % instruction_map) print_error('Instruction map incorrect. Length should be: %s' % slots) exit(-1) new_map = [] for gmap in instruction_map: ngmap = [] if gmap == "-1": ngmap = list(range(1, len(instruction_groups) + 1)) new_map.append(ngmap) continue for cmap in gmap.split(','): try: ngmap.append(check_group_length_func(cmap)) except argparse.ArgumentTypeError as exc: print_error('Instruction map incorrect') print_error(exc) exit(-1) new_map.append(ngmap) instruction_map = new_map group_max = arguments.get('group_max', None) if group_max is None: group_max = ['-1'] * len(instruction_groups) else: group_max = list(iter_flatten(group_max)) if len(group_max) != len(instruction_groups): print_error('Group max: %s' % group_max) print_error('Group max incorrect. Length should be: %s' % len(instruction_groups)) exit(-1) new_map = [] for gmap in group_max: if gmap == "-1": new_map.append(slots) continue try: new_map.append(check_slots_length_func(gmap)) except argparse.ArgumentTypeError as exc: print_error('Group max incorrect') print_error(exc) exit(-1) group_max = new_map group_min = arguments.get('group_min', None) if group_min is None: group_min = ['-1'] * len(instruction_groups) else: group_min = list(iter_flatten(group_min)) if len(group_min) != len(instruction_groups): print_error('Group min: %s' % group_min) print_error('Group min incorrect. Length should be: %s' % len(instruction_groups)) exit(-1) new_map = [] for gmap in group_min: if gmap == "-1": new_map.append(0) continue try: new_map.append(check_slots_length_func(gmap)) except argparse.ArgumentTypeError as exc: print_error('Group min incorrect') print_error(exc) exit(-1) group_min = new_map print_info("Importing target definition...") target = import_definition(arguments.pop('target')) policy = find_policy(target.name, 'seq') if policy is None: print_error("Target does not implement the default SEQ policy") exit(-1) instruction_groups = [ parse_instruction_list(target, group) for group in instruction_groups ] base_seq = [] if 'base_seq' in arguments: base_seq = parse_instruction_list(target, arguments['base_seq']) sequences = [base_seq] if len(instruction_groups) > 0: sequences = _generate_sequences(slots, instruction_groups, instruction_map, group_max, group_min, base_seq) if 'count' in arguments: print_info("Total number of sequences defined : %s" % len(list(sequences))) exit(0) outputdir = arguments['seq_output_dir'] if _BALANCE_EXECUTION: global _DIRCONTENTS # pylint: disable=global-statement _DIRCONTENTS = set(findfiles([outputdir], "")) outputname = "%INSTR%.%EXT%" if 'reset' not in arguments: arguments['reset'] = False if 'skip' not in arguments: arguments['skip'] = False if 'force_switch' not in arguments: arguments['force_switch'] = False if 'parallel' not in arguments: print_info("Start sequential generation. Use parallel flag to speed") print_info("up the benchmark generation.") for sequence in sequences: _generic_policy_wrapper( (sequence, outputdir, outputname, target, arguments)) else: print_info("Start parallel generation. Threads: %s" % mp.cpu_count()) pool = mp.Pool(processes=mp.cpu_count()) pool.map(_generic_policy_wrapper, [(sequence, outputdir, outputname, target, arguments) for sequence in sequences])
def generate(test_definition, outputfile, target, **kwargs): """ Microbenchmark generation policy :param test_definition: Test definition object :type test_definition: :class:`MicroprobeTestDefinition` :param outputfile: Outputfile name :type outputfile: :class:`str` :param target: Target definition object :type target: :class:`Target` """ variables = test_definition.variables print_info("Interpreting assembly...") sequence = interpret_asm( test_definition.code, target, [var.name for var in variables] ) if len(sequence) < 1: raise MicroprobeMPTFormatError( "No instructions found in the 'instructions' entry of the MPT" " file. Check the input file." ) max_ins = test_definition.instruction_count if kwargs["max_trace_size"] != _DEFAULT_MAX_INS or max_ins is None: max_ins = kwargs["max_trace_size"] # registers = test_definition.registers # raw = test_definition.raw # if test_definition.default_data_address is None: # print_error("Default data address is needed") # exit(-1) # if test_definition.default_code_address is None: # print_error("Default code address is needed") # exit(-1) wrapper_name = "QTrace" print_info("Creating benchmark synthesizer...") try: cwrapper = microprobe.code.get_wrapper(wrapper_name) except MicroprobeValueError as exc: raise MicroprobeException( "Wrapper '%s' not available. Check if you have the wrappers " "of the target installed or set up an appropriate " "MICROPROBEWRAPPERS environment variable. Original error was: %s" % (wrapper_name, str(exc)) ) start_addr = [ reg.value for reg in test_definition.registers if reg.name in ["PC", "IAR"] ] if start_addr: start_addr = start_addr[0] else: start_addr = test_definition.default_code_address if start_addr != test_definition.default_code_address: print_error( "Default code address does not match register state " "PC/IAR value. Check MPT consistency." ) exit(-1) wrapper_kwargs = {} wrapper_kwargs["init_code_address"] = start_addr wrapper_kwargs["init_data_address"] = test_definition.default_data_address wrapper = cwrapper(**wrapper_kwargs) synth = microprobe.code.TraceSynthesizer( target, wrapper, show_trace=kwargs.get( "show_trace", False), maxins=max_ins, start_addr=start_addr, no_scratch=True ) # if len(registers) >= 0: # synth.add_pass( # microprobe.passes.initialization.InitializeRegistersPass( # registers, skip_unknown=True, warn_unknown=True # ) # ) synth.add_pass( microprobe.passes.structure.SimpleBuildingBlockPass(len(sequence)) ) synth.add_pass( microprobe.passes.instruction.ReproduceSequencePass(sequence) ) synth.add_pass( microprobe.passes.address.UpdateInstructionAddressesPass() ) synth.add_pass( microprobe.passes.branch.NormalizeBranchTargetsPass() ) synth.add_pass( microprobe.passes.memory.InitializeMemoryDecorator( default=kwargs.get("default_memory_access_pattern", None) ) ) synth.add_pass( microprobe.passes.branch.InitializeBranchDecorator( default=kwargs.get("default_branch_pattern", None), indirect=kwargs.get("default_branch_indirect_target_pattern", None) ) ) synth.add_pass( microprobe.passes.symbol.ResolveSymbolicReferencesPass() ) print_info("Synthesizing...") bench = synth.synthesize() # Save the microbenchmark synth.save(outputfile, bench=bench) return
def _shift_and_fix_code( target, code, offset, addresses, reset_steps, registers, ): """Shift code and fix reset code.""" scode = [] for instruction in code: instruction = instruction.copy() # Fix and shift decorators (not need to modify code) for key, values in instruction.decorators.items(): if not isinstance(values, list): values = [values] if key in ['MA', 'BT']: for idx in range(0, len(values)): if addresses[0] <= values[idx] <= addresses[1]: values[idx] = values[idx] + offset scode.append(instruction) cidx = 0 context = Context() context.set_symbolic(False) for reset_step in reset_steps: rins = reset_step[0] cins = scode[cidx:cidx+len(rins)] rreg = reset_step[1] try: rval = [reg for reg in registers if reg.name == rreg.name][0].value except IndexError: # This was a support register to compute an address, # it should be fixed rval = 0 for rin, cin in zip(rins, cins): if rin.name != cin.instruction_type.name: print_error("Unable to fix the reset code") exit(1) if len(reset_step) == 3: rins, reset_register, value = reset_step if not isinstance(value, six.integer_types): # All addresses should be offset value += offset nins = target.set_register( reset_register, value.displacement, context, opt=False, ) print_info( "Fixing reset code for reg: %s. " "New value: 0x%016X" % (rreg.name, value.displacement), ) else: if abs(value-rval) == offset: # This has been shifted, force offset value += offset print_info( "Fixing reset code for reg: %s. " "New value: 0x%016X" % (rreg.name, value), ) nins = target.set_register( reset_register, value, context, opt=False, ) context.set_register_value(reset_register, value) elif len(reset_step) == 4: rins, reset_register, address_obj, length = reset_step # All addresses should be offsetted address_obj += offset nins = target.store_integer( reset_register, address_obj, length * 8, context, ) print_info( "Fixing reset code for reg: %s. " "New value: 0x%016X" % (rreg.name, address_obj.displacement), ) else: raise NotImplementedError( "Unable to shift and fix code" ) if len(rins) != len(nins): print_error("Original resetting code:") for ins in rins: print_error(ins.assembly()) print_error("New resetting code:") for ins in nins: print_error(ins.assembly()) print_error("New resetting code differs from original in length") exit(1) for ins in nins: if len(reset_step) == 3: if not isinstance(value, six.integer_types): value = value.displacement ins.add_comment( "Reset code. Setting %s to 0X%016X" % (reset_register.name, value), ) else: ins.add_comment( "Reset code. Setting mem content in 0X%016X" % (address_obj.displacement), ) for idx, (nin, cin) in enumerate(zip(nins, cins)): if nin.name != cin.instruction_type.name: print_warning("New code differs from original in opcodes") scode[cidx+idx] = instruction_to_definition(nin) scode[cidx+idx].comments = scode[cidx+idx].comments[1:] cidx += len(rins) return scode