def _main(arguments):
"""
Program main, after processing the command line arguments
:param arguments: Dictionary with command line arguments and values
:type arguments: :class:`dict`
"""
print_info("Arguments processed!")
print_info("Checking input arguments for consistency...")
if ("epi_output_file" not in arguments
and "epi_output_dir" not in arguments):
print_error("No output provided")
exit(-1)
if ("epi_output_file" in arguments and "epi_output_dir" in arguments):
print_error("--epi-output-file and --epi-output-dir options conflict")
print_error("Use one of them")
exit(-1)
if ("instructions" not in arguments and "epi_output_dir" not in arguments):
print_error("If --instructions is not provided, you need to provide")
print_error("--epi-output-dir and not --epi-output-file")
exit(-1)
print_info("Importing target definition...")
target = import_definition(arguments.pop('target'))
policy = find_policy(target.name, 'epi')
if policy is None:
print_error("Target does not implement the default EPI policy")
exit(-1)
if 'instructions' not in arguments:
arguments['instructions'] = list(target.isa.instructions.values())
outputdir = arguments['epi_output_dir']
outputname = "%INSTR%.%EXT%"
else:
arguments['instructions'] = parse_instruction_list(
target, arguments['instructions'])
if ("epi_output_file" in arguments
and len(arguments['instructions']) != 1):
print_error("More than one microbenchmark to generate.")
print_error("Use --epi-output-dir parameter")
exit(-1)
if "epi_output_file" in arguments:
outputdir = os.path.dirname(arguments['epi_output_file'])
outputname = arguments['epi_output_file']
else:
outputdir = arguments['epi_output_dir']
outputname = "%INSTR%.%EXT%"
if 'reset' not in arguments:
arguments['reset'] = False
if 'skip' not in arguments:
arguments['skip'] = False
if 'ignore_errors' not in arguments:
arguments['ignore_errors'] = False
if 'parallel' not in arguments:
print_info("Start sequential generation. Use parallel flag to speed")
print_info("up the benchmark generation.")
for instruction in arguments['instructions']:
_generic_policy_wrapper(
(instruction, outputdir, outputname, target, arguments))
else:
print_info("Start parallel generation. Threads: %s" % mp.cpu_count())
pool = mp.Pool(processes=mp.cpu_count())
pool.map(_generic_policy_wrapper,
[(instruction, outputdir, outputname, target, arguments)
for instruction in arguments['instructions']])
def _generic_policy_wrapper(all_arguments):
instruction, outputdir, outputname, target, kwargs = all_arguments
bname = instruction.name
bname = bname + "#DD_%d" % kwargs['dependency_distance']
bname = bname + "#BS_%d" % kwargs['benchmark_size']
bname = bname + "#DI_%s" % kwargs['data_init']
if MICROPROBE_RC['debugwrapper']:
policy = find_policy(target.name, 'debug')
else:
policy = find_policy(target.name, 'epi')
extension = ""
if target.name.endswith("z64_mesa_st") or target.name.endswith(
"z64_mesa_smt2"):
wrapper_name = "Avp"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'], )
extension = "avp"
elif target.name.endswith("ppc64_mesa"):
wrapper_name = "Tst"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(
outputname.replace("%INSTR%", bname).replace("%EXT%", "tst"),
reset=kwargs['reset'],
)
extension = "tst"
elif target.name.endswith("linux_gcc"):
wrapper_name = "CInfGen"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'])
extension = "c"
elif target.name.endswith("poe"):
wrapper_name = "Poe"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'], )
extension = "bin"
elif target.name.endswith("cronus"):
wrapper_name = "Cronus"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'], )
extension = "bin"
elif target.name.endswith("test_p"):
wrapper_name = "RiscvTestsP"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'], )
extension = "S"
else:
raise NotImplementedError("Unsupported configuration '%s'" %
target.name)
if MICROPROBE_RC['debugwrapper']:
extension = "s"
extra_arguments = {}
extra_arguments['instruction'] = instruction
extra_arguments['benchmark_size'] = kwargs['benchmark_size']
extra_arguments['dependency_distance'] = kwargs['dependency_distance']
extra_arguments['data_init'] = kwargs['data_init']
outputfile = os.path.join(outputdir, outputname)
outputfile = outputfile.replace("%INSTR%", bname)
outputfile = outputfile.replace("%EXT%", extension)
if wrapper.outputname(outputfile) != outputfile:
print_error("Fix outputname to have a proper extension. E.g. '%s'" %
wrapper.outputname(outputfile))
exit(-1)
if instruction.unsupported:
print_info("%s not supported!" % instruction.name)
return
if kwargs['skip'] and os.path.isfile(outputfile):
print_info("%s already generated!" % outputfile)
return
outputfile = new_file(wrapper.outputname(outputfile), internal=True)
synth = policy.apply(target, wrapper, **extra_arguments)
print_info("Generating %s..." % outputfile)
if not kwargs["ignore_errors"]:
bench = synth.synthesize()
else:
if os.path.exists("%s.fail" % outputfile):
print_error("%s failed before. Skip." % outputfile)
return
try:
bench = synth.synthesize()
except (MicroprobeException, AssertionError) as exc:
with open("%s.fail" % outputfile, 'a'):
os.utime("%s.fail" % outputfile, None)
print_error("%s" % exc)
print_error("Generation failed and ignore error flag set")
return
synth.save(outputfile, bench=bench)
print_info("%s generated!" % outputfile)
return
def _generic_policy_wrapper(all_arguments):
instructions, outputdir, outputname, target, kwargs = all_arguments
extension = ""
if target.name.endswith("linux_gcc"):
wrapper_name = "CInfGen"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'])
extension = "c"
elif target.name.endswith("cronus"):
wrapper_name = "Cronus"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'])
extension = "bin"
else:
raise NotImplementedError("Unsupported configuration '%s'" %
target.name)
if MICROPROBE_RC['debugwrapper']:
extension = "s"
outputfile = os.path.join(outputdir, "%DIRTREE%", outputname)
outputfile = outputfile.replace(
"%DIRTREE%", os.path.join(*[instr.name for instr in instructions]))
outputfile = outputfile.replace(
"%INSTR%", "_".join(instr.name for instr in instructions))
outputfile = outputfile.replace("%EXT%", extension)
if kwargs['skip'] and outputfile in _DIRCONTENTS:
print_info("%s already exists!" % outputfile)
return
if kwargs['skip'] and os.path.isfile(outputfile):
print_info("%s already exists!" % outputfile)
return
extra_arguments = {}
extra_arguments['instructions'] = instructions
extra_arguments['benchmark_size'] = kwargs['benchmark_size']
extra_arguments['dependency_distance'] = kwargs['dependency_distance']
extra_arguments['force_switch'] = kwargs['force_switch']
if wrapper.outputname(outputfile) != outputfile:
print_error("Fix outputname to have a proper extension. E.g. '%s'" %
wrapper.outputname(outputfile))
exit(-1)
for instr in instructions:
if instr.unsupported:
print_info("%s not supported!" % instr.name)
return
policy = find_policy(target.name, 'seq')
if not os.path.exists(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
outputfile = new_file(wrapper.outputname(outputfile), internal=True)
print_info("Generating %s..." % outputfile)
synth = policy.apply(target, wrapper, **extra_arguments)
bench = synth.synthesize()
synth.save(outputfile, bench=bench)
print_info("%s generated!" % outputfile)
return
def _main(arguments):
"""
Program main, after processing the command line arguments
:param arguments: Dictionary with command line arguments and values
:type arguments: :class:`dict`
"""
print_info("Arguments processed!")
print_info("Checking input arguments for consistency...")
slots = arguments['instruction_slots']
instruction_groups = list(
iter_flatten(arguments.get('instruction_groups', [])))
check_group_length_func = int_type(1, len(instruction_groups))
check_slots_length_func = int_type(0, slots)
instruction_map = arguments.get('instruction_map', None)
if instruction_map is None:
instruction_map = ['-1'] * slots
else:
instruction_map = list(iter_flatten(instruction_map))
if len(instruction_map) != slots:
print_error('Instruction map: %s' % instruction_map)
print_error('Instruction map incorrect. Length should be: %s' % slots)
exit(-1)
new_map = []
for gmap in instruction_map:
ngmap = []
if gmap == "-1":
ngmap = list(range(1, len(instruction_groups) + 1))
new_map.append(ngmap)
continue
for cmap in gmap.split(','):
try:
ngmap.append(check_group_length_func(cmap))
except argparse.ArgumentTypeError as exc:
print_error('Instruction map incorrect')
print_error(exc)
exit(-1)
new_map.append(ngmap)
instruction_map = new_map
group_max = arguments.get('group_max', None)
if group_max is None:
group_max = ['-1'] * len(instruction_groups)
else:
group_max = list(iter_flatten(group_max))
if len(group_max) != len(instruction_groups):
print_error('Group max: %s' % group_max)
print_error('Group max incorrect. Length should be: %s' %
len(instruction_groups))
exit(-1)
new_map = []
for gmap in group_max:
if gmap == "-1":
new_map.append(slots)
continue
try:
new_map.append(check_slots_length_func(gmap))
except argparse.ArgumentTypeError as exc:
print_error('Group max incorrect')
print_error(exc)
exit(-1)
group_max = new_map
group_min = arguments.get('group_min', None)
if group_min is None:
group_min = ['-1'] * len(instruction_groups)
else:
group_min = list(iter_flatten(group_min))
if len(group_min) != len(instruction_groups):
print_error('Group min: %s' % group_min)
print_error('Group min incorrect. Length should be: %s' %
len(instruction_groups))
exit(-1)
new_map = []
for gmap in group_min:
if gmap == "-1":
new_map.append(0)
continue
try:
new_map.append(check_slots_length_func(gmap))
except argparse.ArgumentTypeError as exc:
print_error('Group min incorrect')
print_error(exc)
exit(-1)
group_min = new_map
print_info("Importing target definition...")
target = import_definition(arguments.pop('target'))
policy = find_policy(target.name, 'seq')
if policy is None:
print_error("Target does not implement the default SEQ policy")
exit(-1)
instruction_groups = [
parse_instruction_list(target, group) for group in instruction_groups
]
base_seq = []
if 'base_seq' in arguments:
base_seq = parse_instruction_list(target, arguments['base_seq'])
sequences = [base_seq]
if len(instruction_groups) > 0:
sequences = _generate_sequences(slots, instruction_groups,
instruction_map, group_max, group_min,
base_seq)
if 'count' in arguments:
print_info("Total number of sequences defined : %s" %
len(list(sequences)))
exit(0)
outputdir = arguments['seq_output_dir']
if _BALANCE_EXECUTION:
global _DIRCONTENTS # pylint: disable=global-statement
_DIRCONTENTS = set(findfiles([outputdir], ""))
outputname = "%INSTR%.%EXT%"
if 'reset' not in arguments:
arguments['reset'] = False
if 'skip' not in arguments:
arguments['skip'] = False
if 'force_switch' not in arguments:
arguments['force_switch'] = False
if 'parallel' not in arguments:
print_info("Start sequential generation. Use parallel flag to speed")
print_info("up the benchmark generation.")
for sequence in sequences:
_generic_policy_wrapper(
(sequence, outputdir, outputname, target, arguments))
else:
print_info("Start parallel generation. Threads: %s" % mp.cpu_count())
pool = mp.Pool(processes=mp.cpu_count())
pool.map(_generic_policy_wrapper,
[(sequence, outputdir, outputname, target, arguments)
for sequence in sequences])
def _generic_policy_wrapper(all_arguments):
instructions, outputdir, outputname, target, kwargs = all_arguments
outputfile = os.path.join(outputdir, "%DIRTREE%", outputname)
outputfile = outputfile.replace(
"%DIRTREE%", os.path.join(*[instr.name for instr in instructions]))
if kwargs['shortnames']:
outputfile = outputfile.replace(
"%INSTR%", "mp_seq_%s" % hashlib.sha1("_".join(
instr.name for instr in instructions).encode()).hexdigest())
else:
outputfile = outputfile.replace(
"%INSTR%", "_".join(instr.name for instr in instructions))
extension = ""
if target.name.endswith("linux_gcc"):
wrapper_name = "CInfGen"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'])
extension = "c"
elif target.name.endswith("cronus"):
wrapper_name = "Cronus"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(reset=kwargs['reset'],
endless=kwargs['endless'])
extension = "bin"
elif target.name.endswith("mesa"):
wrapper_name = "Tst"
extension = "tst"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(os.path.basename(
outputfile.replace("%EXT%", extension)),
endless=kwargs['endless'],
reset=kwargs['reset'])
elif target.name.endswith("riscv64_test_p"):
wrapper_name = "RiscvTestsP"
extension = "S"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(endless=kwargs['endless'],
reset=kwargs['reset'])
elif target.environment.default_wrapper:
wrapper_name = target.environment.default_wrapper
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(endless=kwargs['endless'],
reset=kwargs['reset'])
outputfile = outputfile.replace(".%EXT%", "")
outputfile = wrapper.outputname(outputfile)
else:
raise NotImplementedError("Unsupported configuration '%s'" %
target.name)
if MICROPROBE_RC['debugwrapper']:
extension = "s"
outputfile = outputfile.replace("%EXT%", extension)
if kwargs['skip'] and outputfile in _DIRCONTENTS:
print_info("%s already exists!" % outputfile)
return
if (kwargs['skip'] and "%s.gz" % outputfile in _DIRCONTENTS
and kwargs["compress"]):
print_info("%s.gz already exists!" % outputfile)
return
if kwargs['skip'] and os.path.isfile(outputfile):
print_info("%s already exists!" % outputfile)
return
if (kwargs['skip'] and os.path.isfile("%s.gz" % outputfile)
and kwargs["compress"]):
print_info("%s already exists!" % outputfile)
return
extra_arguments = {}
extra_arguments['instructions'] = instructions
extra_arguments['benchmark_size'] = kwargs['benchmark_size']
extra_arguments['dependency_distance'] = kwargs['dependency_distance']
extra_arguments['force_switch'] = kwargs['force_switch']
extra_arguments['endless'] = kwargs['endless']
if wrapper.outputname(outputfile) != outputfile:
print_error(
"Fix outputname '%s' to have a proper extension. E.g. '%s'" %
(outputfile, wrapper.outputname(outputfile)))
exit(-1)
for instr in instructions:
if instr.unsupported:
print_info("%s not supported!" % instr.name)
return
policy = find_policy(target.name, 'seq')
if not os.path.exists(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
outputfile = new_file(wrapper.outputname(outputfile), internal=True)
print_info("Generating %s..." % outputfile)
synth = policy.apply(target, wrapper, **extra_arguments)
bench = synth.synthesize()
synth.save(outputfile, bench=bench)
print_info("%s generated!" % outputfile)
if kwargs['compress']:
move_file(outputfile, "%s.gz" % outputfile)
print_info("%s compressed to %s.gz" % (outputfile, outputfile))
return
def _generic_policy_wrapper(all_arguments):
configuration, outputdir, outputname, target, kwargs = all_arguments
instructions, mem_switch, data_switch, switch_branch, branch_pattern, \
replace_every, add_every, memory_streams, \
benchmark_size = configuration
extrapath = []
extrapath.append("BS_%d" % benchmark_size)
extrapath.append("MS_%d" % mem_switch)
extrapath.append("DS_%d" % data_switch)
extrapath.append("BP_%s" % branch_pattern)
extrapath.append("SB_%d" % switch_branch)
for repl in replace_every:
extrapath.append("RE_%d_%s_%s" % (repl[2], repl[0].name, repl[1].name))
for addl in add_every:
extrapath.append("AE_%d_%s" % (addl[1],
"_".join(
[elem.name for elem in addl[0]]
)
)
)
for mems in memory_streams:
extrapath.append("ME_%d_%d_%d_%d_%d" % mems)
outputfile = os.path.join(outputdir, "%DIRTREE%", outputname)
outputfile = outputfile.replace(
"%DIRTREE%", os.path.join(
*([instr.name for instr in instructions] + extrapath)))
outputfile = outputfile.replace(
"%BASENAME%", "_".join(
instr.name for instr in instructions) + "#" + "#".join(extrapath))
extension = ""
if target.name.endswith("linux_gcc"):
wrapper_name = "CInfGen"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(
reset=kwargs['reset']
)
extension = "c"
elif target.name.endswith("cronus"):
wrapper_name = "Cronus"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(
reset=kwargs['reset']
)
extension = "bin"
elif target.name.endswith("mesa"):
wrapper_name = "Tst"
extension = "tst"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(
os.path.basename(outputfile.replace("%EXT%", extension)),
reset=kwargs['reset']
)
elif target.name.endswith("riscv64_test_p"):
wrapper_name = "RiscvTestsP"
extension = "S"
wrapper_class = _get_wrapper(wrapper_name)
wrapper = wrapper_class(
reset=kwargs['reset'],
endless=True
)
else:
raise NotImplementedError(
"Unsupported configuration '%s'" % target.name
)
if MICROPROBE_RC['debugwrapper']:
extension = "s"
outputfile = outputfile.replace("%EXT%", extension)
if kwargs['skip']:
if os.path.exists(outputfile):
print_info("%s already exists!" % outputfile)
return
if len(memory_streams) == 0:
warnings.warn(
"No memory streams provided "
"using 1K stream stride 64 bytes"
)
memory_streams = [(1, 1024, 1, 64, 1)]
streamid = 0
new_memory_streams = []
for stream in memory_streams:
for elem in range(0, stream[0]):
new_memory_streams.append([streamid] + list(stream)[1:])
streamid += 1
extra_arguments = {}
extra_arguments['instructions'] = instructions
extra_arguments['benchmark_size'] = benchmark_size
extra_arguments['dependency_distance'] = kwargs['dependency_distance']
extra_arguments['mem_switch'] = mem_switch
extra_arguments['data_switch'] = data_switch
extra_arguments['branch_pattern'] = branch_pattern
extra_arguments['replace_every'] = replace_every
extra_arguments['add_every'] = add_every
extra_arguments['memory_streams'] = new_memory_streams
extra_arguments['branch_switch'] = switch_branch
if wrapper.outputname(outputfile) != outputfile:
print_error(
"Fix outputname to have a proper extension. E.g. '%s'" %
wrapper.outputname(outputfile)
)
exit(-1)
for instr in instructions:
if instr.unsupported:
print_info("%s not supported!" % instr.name)
return
policy = find_policy(target.name, 'seqtune')
if not os.path.exists(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
outputfile = new_file(wrapper.outputname(outputfile), internal=True)
print_info("Generating %s..." % outputfile)
synth = policy.apply(target, wrapper, **extra_arguments)
if not kwargs["ignore_errors"]:
bench = synth.synthesize()
else:
if os.path.exists("%s.fail" % outputfile):
print_error("%s failed before. Not generating" % outputfile)
return
try:
bench = synth.synthesize()
except Exception as exc:
with open("%s.fail" % outputfile, 'a'):
os.utime("%s.fail" % outputfile, None)
print_error("%s" % exc)
print_error("Generation failed for current configuration.")
print_error("Generating next configurations.")
return
synth.save(outputfile, bench=bench)
print_info("%s generated!" % outputfile)
return
def _main(arguments):
"""
Program main, after processing the command line arguments
:param arguments: Dictionary with command line arguments and values
:type arguments: :class:`dict`
"""
print_info("Arguments processed!")
print_info("Checking input arguments for consistency...")
flags = ["data_switch", "memory_switch",
"ignore_errors", "switch_branch"]
for flag in flags:
if flag not in arguments:
arguments[flag] = False
args = ["replace_every", "add_every", "branch_every", "branch_pattern",
"memory_stream"]
for arg in args:
if arg not in arguments:
arguments[arg] = []
if (len(arguments['branch_every']) >
0 and len(arguments['branch_pattern']) > 0):
print_error("--branch-every and --branch-pattern flags are "
"mutually exclusive. Use only on of them.")
exit(-1)
elif len(arguments['branch_pattern']) > 0:
arguments['branch_pattern'] = _process_branch_pattern(
arguments['branch_pattern']
)
elif len(arguments['branch_every']) > 0:
arguments['branch_pattern'] = _process_branch_every(
arguments['branch_every']
)
else:
arguments['branch_pattern'] = ['0']
print_info("Importing target definition...")
target = import_definition(arguments.pop('target'))
policy = find_policy(target.name, 'seqtune')
if policy is None:
print_error("Target does not implement the default SEQTUNE policy")
exit(-1)
arguments['sequence'] = parse_instruction_list(
target,
arguments['sequence'])
for elem in arguments["replace_every"]:
elem[0] = parse_instruction_list(target, elem[0])
elem[1] = parse_instruction_list(target, elem[1])
for elem in arguments["add_every"]:
elem[0] = parse_instruction_list(target, elem[0])
configurations = _generate_configurations(arguments)
configurations = _filter_configurations(configurations, arguments)
if 'count' in arguments:
print_info("Total number of variations defined : %s" %
len(list(configurations)))
exit(0)
outputdir = arguments['seq_output_dir']
outputname = "%BASENAME%.%EXT%"
if 'reset' not in arguments:
arguments['reset'] = False
if 'skip' not in arguments:
arguments['skip'] = False
if 'parallel' not in arguments:
print_info("Start sequential generation. Use parallel flag to speed")
print_info("up the benchmark generation.")
for params in configurations:
_generic_policy_wrapper(
(params,
outputdir,
outputname,
target,
arguments))
else:
print_info("Start parallel generation. Threads: %s" % mp.cpu_count())
pool = mp.Pool(processes=mp.cpu_count())
pool.map(_generic_policy_wrapper,
[(params, outputdir, outputname, target,
arguments)
for params in configurations]
)