def run(args):
if not cere_configure.init():
return False
args.regions_file = os.path.realpath(args.regions_file)
if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
logger.error("File not found: {0}".format(cere_configure.cere_config["regions_infos"]))
logger.error("Did you run cere regions?")
return False
instru_cmd = ""
#Check for instrumentation
if args.instrumentation:
instru_cmd = "--instrument --wrapper={0}".format(var.RDTSC_WRAPPER)
logger.info("Compiling hybrid binary with instrumentation")
#Remove objects file
try:
os.remove(os.path.realpath("__cere__objects"))
except OSError as e:
if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
raise #re-raise exception if a different error occured
#Compile hybrid binary
try:
logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"original --hybrid {2} --hybrid-regions={3} \
--regions-infos={4} --cere-objects={5} --extraction-lvl={6}\"".format(cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], instru_cmd, args.regions_file,\
cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), args.extraction_lvl), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.critical(str(err))
logger.critical(err.output)
return False
logger.info("Hybrid compiled successfully")
return True
def run_instrument(args_region=None,
args_regions_file=None,
args_plugin_instr=var.RDTSC_WRAPPER,
args_invocation=0,
args_norun=False,
args_force=False):
if not cere_configure.init():
return False
if utils.is_invalid(args_region) and not args_force:
logger.error(
"{0} is invalid. Skipping instrumentation".format(args_region))
return False
if args_regions_file:
region_input = "--regions-file={0}".format(args_regions_file)
elif args_region:
region_input = "--region={0}".format(args_region)
else:
logger.error(
"No region specified, use at least one of the following: --region, --regions-file"
)
return False
if args_invocation != 0:
logger.info(
"Compiling instrumentation mode for {0} invocation {1}".format(
region_input, args_invocation))
mode = " --invocation={0}".format(args_invocation)
else:
logger.info(
"Compiling instrumentation mode for {0}".format(region_input))
mode = ""
try:
logger.debug(
subprocess.check_output(
"{0} MODE=\"original {1} --instrument {2} --wrapper={3}\" -B".
format(cere_configure.cere_config["build_cmd"], region_input,
mode, args_plugin_instr),
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Compiling instrumentation failed")
return False
if not args_norun:
logger.info("Running instrumentation for {0}".format(region_input))
try:
logger.info(
subprocess.check_output(cere_configure.cere_config["run_cmd"],
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Instrumentation failed")
return False
return True
def run(args):
if not cere_configure.init():
return False
invocation_file = os.path.join(var.CERE_TRACES_PATH,
args.region + ".invocations")
if os.path.isfile(invocation_file) and not args.force:
logger.info(
"Invocations already selected for {0}. If you want to reselect use -f"
.format(args.region, invocation_file))
return True
if not utils.trace_exists(args.region):
logger.error("No trace for {0}.\n\
Please first run cere trace --region={0}".format(
args.region))
return False
bin_file = os.path.join(var.CERE_TRACES_PATH, args.region + ".bin")
csv_file = os.path.join(var.CERE_TRACES_PATH, args.region + ".csv")
clusterize_invocations(args.region, csv_file, bin_file)
try:
shutil.move(
"{0}.invocations".format(args.region),
"{0}/{1}.invocations".format(var.CERE_TRACES_PATH, args.region))
except IOError as err:
logger.error(str(err))
return False
return True
def run(args):
if not cere_configure.init():
return False
profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
regions_file = cere_configure.cere_config["regions_infos"]
new_regions_file = "tmp.csv"
build_cmd = cere_configure.cere_config["build_cmd"]
run_cmd = cere_configure.cere_config["run_cmd"]
clean_cmd = cere_configure.cere_config["clean_cmd"]
logger.info("Removing previous regions list")
if os.path.isfile(regions_file):
os.remove(regions_file)
#Build again the application to be sure we give the right binary to pprof
try:
logger.info(subprocess.check_output("{0} && {1} CERE_MODE=\"original --instrument --instrument-app --regions-infos={2}\"".format(clean_cmd, build_cmd, regions_file), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
return False
if not os.path.isfile(regions_file):
logger.critical("Failed to generate region list.")
return False
logger.info("Regions list dumped in {0} file".format(regions_file))
if not args.static:
if not os.path.isfile(profile_file):
logger.critical('No profiling file. Please first run cere profile or run cere regions --static')
return False
binary = which(run_cmd)
if not binary:
logger.critical("Cannot find the binary. Please provide binary name through cere configure --binary")
return False
add_header(regions_file, new_regions_file)
#regular expression to parse the gperf tool output
regex_list = [r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,']
cmd = subprocess.Popen("{0} -dot {1} {2}".format(var.PPROF, binary, profile_file), shell=True, stdout=subprocess.PIPE)
for line in cmd.stdout:
matchObj, step = parse_line(regex_list, line)
if step < 2 :
add_coverage(regions_file, new_regions_file, matchObj)
else:
continue
try:
shutil.move(new_regions_file, regions_file)
except IOError as err:
logger.critical(str(err))
logger.error(err.output)
return False
return True
def run(args):
if not cere_configure.init():
return False
if utils.is_invalid(args.region) and not args.force:
logger.warning("{0} is invalid. Skipping replay".format(args.region))
return False
invocations = find_invocations(args.invocation, args.region)
if not invocations:
return False
if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
logger.warning("You are not using the default library. Computing predicted time\n\
may not work if the replay output is not the same")
for invocation, part in invocations.iteritems():
if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force:
logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation))
continue
if not utils.dump_exist(args.region, invocation):
logger.error("Memory dump is missing for {0} invocation {1}.\n\
Run cere capture --region={0} [--invocation={1}]".format(args.region, invocation))
return False
if args.noinstrumentation:
instru_cmd = ""
logger.info("Compiling replay mode for region {0} invocation {1} without instrumentation".format(args.region, invocation))
else:
instru_cmd = "--instrument"
logger.info("Compiling replay mode for region {0} invocation {1} with instrumentation".format(args.region, invocation))
try:
logger.debug(subprocess.check_output("{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EREPLAY)
return False
if not args.norun:
logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation))
try:
logger.debug(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EREPLAY)
return False
#Save replay measures files
if PREDICTION_MODE:
try:
shutil.move("{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation))
except IOError as err:
logger.error(str(err))
logger.error(" No results file. Maybe replay failed".format(invocation))
return False
if PREDICTION_MODE:
predicted_cycles = compute_predicted_time(args.region, invocations)
logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
return True
def run(args):
if not cere_configure.init():
return False
if not os.path.isfile("regions.csv"):
logger.critical("Regions.csv file missing. Please run cere regions")
return False
region = args.region
filename = ""
functionname=""
#Find the file where the region is
with open("regions.csv") as regions_list:
reader = csv.DictReader(regions_list)
for row in reader:
if region in row["Region Name"]:
filename = row["File Name"]
functionname = row["Function Name"]
break
if args.path:
filename = filename.rsplit('/', 1)
filename = args.path+"/"+filename[1]
filename = filename.replace(os.path.splitext(filename)[1], ".o")
logger.debug("The file is {0} and the function is {1}".format(filename, functionname))
#Now let's compile it in the orginal application
if not run_shell_command("{0} MODE=original -B".format(cere_configure.cere_config["build_cmd"])):
return False
original_lines = get_nlines(filename, functionname)
if not original_lines:
return False
#backup the original assembly file
if not run_shell_command("cp cere_tmp cere_original"):
return False
#Compile replay mode
#we accept that compilation fails because the dump does not have to be present.
try:
logger.debug(subprocess.check_output("{0} MODE=\"replay --region={1}\" -B".format(cere_configure.cere_config["build_cmd"], args.region), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.warning("If the dump is not present, skip these warnings")
replay_lines = get_nlines(filename, "run__cere__"+region)
if not replay_lines:
return False
#backup the replay assembly file
if not run_shell_command("mv cere_tmp cere_replay"):
return False
err = compute_error(original_lines, replay_lines)
if err <= args.max_error:
logger.info("Assembly matching: Original lines = {0} && replay lines = {1} (error = {2})".format(original_lines, replay_lines, err))
else:
logger.info("Assembly not matching: Original lines = {0} && replay lines = {1} (error = {2})".format(original_lines, replay_lines, err))
if args.diff_asm:
subprocess.call("vimdiff cere_original cere_replay", shell=True)
return True
def run(args):
if not cere_configure.init():
return False
if not (args.region or args.regions_file):
logger.error(
"No region specified, use at least one of the following: --region, --regions-file"
)
return False
if (args.regions_file and args.region):
logger.error("--region and --regions-file are exclusive")
return False
if args.region:
regions = [args.region]
elif args.regions_file:
if not os.path.isfile(args.regions_file):
logger.critical("\"{0}\" No such file.".format(args.regions_file))
return False
with open(args.regions_file, 'r') as regions_file:
regions = [region.strip() for region in regions_file]
err = False
allRegions = []
#For each region
for r in regions:
allRegions.append(Region(r))
#first we need the trace
res = allRegions[-1].measure_trace()
if not res: err = True
#Compute the coverage of this region
allRegions[-1].compute_coverage()
#We can clusterize invocations in performance classes
res = cere_selectinv.run(allRegions[-1])
if not res: err = True
trace_filename = create_region_to_trace_file(regions)
run_io_checker(trace_filename)
for idx, region in enumerate(allRegions):
#Replay representative invocations
res = allRegions[idx].replay_invocations(args.force)
if not res: err = True
#Compute error between invivo and in vitro
allRegions[idx].check_region_matching()
dump_results(allRegions, args.max_error)
graph = load_graph()
if graph != None:
update_nodes(graph, args.max_error)
return not err
def run(args):
if not cere_configure.init():
return False
if not args.app and not args.regions:
args.app = True
args.regions = True
res_1=True
res_2=True
if args.regions:
res_1 = instrument_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], cere_configure.cere_config["clean_cmd"], args.force)
if args.app:
res_2 = measure_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], cere_configure.cere_config["clean_cmd"], args.force)
return (res_1 and res_2)
def run(args):
if not cere_configure.init():
return False
if not args.app and not args.regions:
args.app = True
args.regions = True
res_1=True
res_2=True
if args.regions:
res_1 = instrument_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], args.force)
if args.app:
res_2 = measure_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], args.force)
return (res_1 and res_2)
def run(args):
if not cere_configure.init():
return False
if not check_dependancies(args):
return False
if not check_arguments(args):
return False
#Find matching codelets
if not update(args):
return False
#Select matching codelets with best coverage
if not solve_with_best_granularity(args):
return False
return True
def run(args):
if not cere_configure.init():
return False
if not (args.region or args.regions_file):
logger.error("No region specified, use at least one of the following: --region, --regions-file")
return False
if (args.regions_file and args.region):
logger.error("--region and --regions-file are exclusive")
return False
if args.region:
regions = [args.region]
elif args.regions_file:
if not os.path.isfile(args.regions_file):
logger.critical("\"{0}\" No such file.".format(args.regions_file))
return False
with open(args.regions_file, 'r') as regions_file:
regions = [region.strip() for region in regions_file]
err=False
allRegions = []
#For each region
for r in regions:
allRegions.append(Region(r))
#first we need the trace
res = allRegions[-1].measure_trace()
if not res: err=True
#Compute the coverage of this region
allRegions[-1].compute_coverage()
#We can clusterize invocations in performance classes
res = cere_selectinv.run(allRegions[-1])
if not res: err=True
for idx, region in enumerate(allRegions):
#Replay representative invocations
res = allRegions[idx].replay_invocations(args.force)
if not res: err=True
#Compute error between invivo and in vitro
allRegions[idx].check_region_matching()
dump_results(allRegions, args.max_error)
graph = load_graph()
if graph != None:
update_nodes(graph, args.max_error)
return not err
def run(args):
logger.info("Start report")
if not cere_configure.init():
return False
graph = load_graph()
if graph == None:
logger.critical("Can't load graph. Did you run cere profile?")
return False
with context(args.path):
bench = os.getcwd().split("/")[-1]
REPORT = Report(bench, graph, args.mincycles)
REPORT.write_report()
logger.info("Report created")
return True
def run(args):
logger.info('Start report')
if not cere_configure.init():
return False
graph = load_graph()
if graph == None:
logger.critical("Can't load graph. Did you run cere profile?")
return False
with context(args.path):
bench = os.getcwd().split("/")[-1]
REPORT = Report(bench, graph, args.mincycles)
REPORT.write_report()
logger.info("Report created")
return True
def run(args):
if not cere_configure.init():
return False
if not check_arguments(args):
return False
# Find loops to trace
regions = find_regions_to_trace(args)
# When force is off, filter already measured regions
if not args.force:
regions = [r for r in regions if need_to_measure(r)]
result = True
# Are there any regions to trace ?
if len(regions) == 0:
logger.info("No regions to trace")
else:
result = launch_trace(args, regions)
# Move trace files to cere_measures
if not args.norun:
for region in regions:
try:
shutil.move("{0}.bin".format(region),
"{0}/{1}.bin".format(var.CERE_TRACES_PATH, region))
shutil.move("{0}.csv".format(region),
"{0}/{1}.csv".format(var.CERE_TRACES_PATH, region))
except IOError as err:
logger.error(str(err))
logger.error(
"Trace failed for region {0}: No output files, maybe the selected region does not exist."
.format(region))
utils.mark_invalid(region, cere_error.ETRACE)
result = False
# If read is used, read given invocation from trace file
if args.read and utils.trace_exists(args.region):
base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region)
trace = parse_trace_file(base_name + '.bin')
cycles = int(trace['cycles'][trace['invocations'] == args.read])
print(cycles)
return result
def run(args):
if not cere_configure.init():
return False
invocations = find_invocations(args.invocation, args.region)
if not invocations:
return False
for invocation in invocations:
if not args.force:
if utils.dump_exist(args.region, invocation):
logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation))
return True
if utils.is_invalid(args.region):
logger.warning("{0} is invalid. Skipping capture".format(args.region))
return False
else:
shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True)
logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation))
try:
logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"dump --region={2} --invocation={3}\"".format(cere_configure.cere_config["clean_cmd"],
cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EDUMP)
return False
if not args.norun:
logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation))
try:
logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
#even if the capture run fails, maybe the region is dumped.
logger.error(str(err))
logger.error(err.output)
if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)):
logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EDUMP)
return False
else:
logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation))
return True
def run(args):
if not cere_configure.init():
return False
if not check_arguments(args):
return False
# Find loops to trace
regions = find_regions_to_trace(args)
# When force is off, filter already measured regions
if not args.force:
regions = [r for r in regions if need_to_measure(r)]
result = True
# Are there any regions to trace ?
if len(regions) == 0:
logger.info("No regions to trace")
else:
result = launch_trace(args, regions)
# Move trace files to cere_measures
if not args.norun:
for region in regions:
try:
shutil.move("{0}.bin".format(region), var.CERE_TRACES_PATH)
shutil.move("{0}.csv".format(region), var.CERE_TRACES_PATH)
except IOError as err:
logger.error(str(err))
logger.error("Trace failed for region {0}: No output files, maybe the selected region does not exist.".format(region))
utils.mark_invalid(region, cere_error.ETRACE)
result = False
# If read is used, read given invocation from trace file
if args.read and utils.trace_exists(args.region):
base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region)
trace = parse_trace_file(base_name + '.bin')
cycles = int(trace['cycles'][trace['invocations'] == args.read])
print(cycles)
return result
def run(args):
if not cere_configure.init():
return False
args.regions_file = os.path.realpath(args.regions_file)
if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
logger.error("No such file: {0}".format(cere_configure.cere_config["regions_infos"]))
logger.error("Did you run cere regions?")
return False
region_input=""
if args.region:
region_input="--region={0}".format(args.region)
try:
logger.debug(subprocess.check_output("{0} MODE=\"original --hybrid --instrument {5} --wrapper={1} --hybrid-regions={2} \
--regions-infos={3} --cere-objects={4}\"".format(cere_configure.cere_config["build_cmd"], var.RDTSC_WRAPPER, args.regions_file, \
cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), region_input), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.critical(str(err))
logger.critical(err.output)
return False
return True
def run_instrument(args_region=None, args_regions_file=None, args_plugin_instr=var.RDTSC_WRAPPER, args_invocation=0, args_norun=False, args_force=False):
if not cere_configure.init():
return False
if utils.is_invalid(args_region) and not args_force:
logger.error("{0} is invalid. Skipping instrumentation".format(args_region))
return False
if args_regions_file:
region_input = "--regions-file={0}".format(args_regions_file)
elif args_region:
region_input = "--region={0}".format(args_region)
else:
logger.error("No region specified, use at least one of the following: --region, --regions-file")
return False
if args_invocation != 0 :
logger.info("Compiling instrumentation mode for {0} invocation {1}".format(region_input, args_invocation))
mode = " --invocation={0}".format(args_invocation)
else:
logger.info("Compiling instrumentation mode for {0}".format(region_input))
mode = ""
try:
logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"original {2} --instrument {3} --wrapper={4}\"".format(cere_configure.cere_config["clean_cmd"],
cere_configure.cere_config["build_cmd"], region_input, mode, args_plugin_instr), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Compiling instrumentation failed")
return False
if not args_norun:
logger.info("Running instrumentation for {0}".format(region_input))
try:
logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Instrumentation failed")
return False
return True
def run_io_checker(args_region=None, args_regions_file=None, args_invocation=1, args_force=False):
if not cere_configure.init():
return False
if not check_arguments(args_region, args_regions_file, args_invocation, args_force):
return False
if args_regions_file:
# Make regions_file path absolute
args_regions_file = os.path.abspath(args_regions_file)
regions_to_trace = find_regions_to_trace(args_region, args_regions_file, args_invocation, args_force)
if len(regions_to_trace) == 0:
logger.info("No regions to trace")
else:
if not launch_io_trace(args_force, regions_to_trace):
return False
parse_io_trace(regions_to_trace)
return True
def run(args):
if not cere_configure.init():
return False
args.regions_file = os.path.realpath(args.regions_file)
if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
logger.error("No such file: {0}".format(
cere_configure.cere_config["regions_infos"]))
logger.error("Did you run cere regions?")
return False
region_input = ""
if args.region:
region_input = "--region={0}".format(args.region)
try:
logger.debug(subprocess.check_output("{0} MODE=\"original --hybrid --instrument {5} --wrapper={1} --hybrid-regions={2} \
--regions-infos={3} --cere-objects={4}\"" .format(cere_configure.cere_config["build_cmd"], var.RDTSC_WRAPPER, args.regions_file, \
cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), region_input), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.critical(str(err))
logger.critical(err.output)
return False
return True
def run(args):
if not cere_configure.init():
return False
invocation_file = os.path.join(var.CERE_TRACES_PATH, args.region+".invocations")
if os.path.isfile(invocation_file) and not args.force:
logger.info("Invocations already selected for {0}. If you want to reselect use -f".format(args.region, invocation_file))
return True
if not utils.trace_exists(args.region):
logger.error("No trace for {0}.\n\
Please first run cere trace --region={0}".format(args.region))
return False
bin_file = os.path.join(var.CERE_TRACES_PATH, args.region+".bin")
csv_file = os.path.join(var.CERE_TRACES_PATH, args.region+".csv")
clusterize_invocations(args.region, csv_file, bin_file)
try:
shutil.move("{0}.invocations".format(args.region), "{0}/{1}.invocations".format(var.CERE_TRACES_PATH, args.region))
except IOError as err:
logger.error(str(err))
return False
return True
#!/usr/bin/env python
import sys
import networkx as nx
sys.path.insert(0, "../../src/cere/")
from create_graph import remove_cycles, fix_self_coverage
import cere_configure
from graph_utils import plot, save_graph
if __name__ == "__main__":
cere_configure.init()
digraph = nx.DiGraph()
nodes = [1,2,3,4,5,6,7]
samples = 100
digraph.add_nodes_from(nodes)
for i in nodes:
digraph.node[i]['_name'] = "Node_" + str(i)
digraph.node[i]['_matching'] = True
digraph.node[i]['_valid'] = True
digraph.node[i]['_to_test'] = False
digraph.node[i]['_small'] = False
digraph.node[i]['_tested'] = False
#Assign coverage and self_coverage
digraph.node[1]['_self_coverage'] = 10.
digraph.node[1]['_coverage'] = 60.
digraph.node[2]['_self_coverage'] = 5.
digraph.node[2]['_coverage'] = 40.
digraph.node[3]['_self_coverage'] = 30.
digraph.node[3]['_coverage'] = 13.
digraph.node[4]['_self_coverage'] = 10.
digraph.node[4]['_coverage'] = 0.
def run(args):
if not cere_configure.init():
return False
profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
regions_file = cere_configure.cere_config["regions_infos"]
new_regions_file = "tmp.csv"
logger.info("Removing previous regions list")
if os.path.isfile(regions_file):
os.remove(regions_file)
if args.dynamic: args.static = True
if not args.static and not args.dynamic:
args.static = True
args.dynamic = True
if(args.static):
mydump = Dump()
res = cere_capture.run(mydump)
logger.info("Regions list dumped in {0} file".format(regions_file))
if not res:
logger.warning("Regions listing encountered and error: list may be incomplete or empty")
if(args.dynamic):
if not os.path.isfile(profile_file):
logger.critical('No profiling file. Please run cere profile')
return False
if not os.path.isfile(regions_file):
logger.critical('Regions file does not exists. Please re-run cere regions with static enabled.')
return False
build_cmd = cere_configure.cere_config["build_cmd"]
run_cmd = cere_configure.cere_config["run_cmd"]
#Build again the application to be sure we give the right binary to pprof
try:
logger.debug(subprocess.check_output("{0} MODE=\"original --instrument --instrument-app\" -B".format(build_cmd), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
return False
binary = which(run_cmd)
if not binary:
logger.critical("Cannot find the binary. Please provide binary name through cere configure --binary")
return False
add_header(regions_file, new_regions_file)
#regular expression to parse the gperf tool output
regex_list = [r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,']
cmd = subprocess.Popen("pprof -dot {0} {1}".format(binary, profile_file), shell=True, stdout=subprocess.PIPE)
for line in cmd.stdout:
matchObj, step = parse_line(regex_list, line)
if step < 2 :
add_coverage(regions_file, new_regions_file, matchObj)
else:
continue
try:
shutil.move(new_regions_file, regions_file)
except IOError as err:
logger.critical(str(err))
logger.error(err.output)
return False
return True
def run(args):
if not cere_configure.init():
return False
#Need to save this because args.force will be changed by the replay pass
force_flags = args.force
#Array to store best flags for every regions
best_flags = []
graph = graph_utils.load_graph()
if graph == None:
logger.error("CERE graph is missing. Please run cere profile")
return False
#Load previous best results
best_history = {}
if os.path.isfile("{0}/regions_flags.csv".format(var.CERE_FLAGS_PATH)) and not force_flags:
flags_file = read_csv(open("{0}/regions_flags.csv".format(var.CERE_FLAGS_PATH), 'rb'))
for row in flags_file:
best_history[row["Region"]] = {"best_id":row["Id"], "best_cycles":row["Cycles"], "original_cycles":row["Original_cycles"]}
#Use llc as backend
os.environ["CERE_LLC"]="llc"
#Read flags file
flags = read_csv(args.FLAGSFILE)
#Run the requested region
if args.region:
found = False
#Load region information
for n, d in graph.nodes(data=True):
if d['_name'] == args.region:
found = True
break
if not found:
logger.error("{0} does not exist. Valid region name can be found in {1}`".format(args.region, cere_configure.cere_config["regions_infos"]))
return False
if not d['_tested']:
logger.error("{0} not tested. Please run `cere check-matching --region={0}`".format(args.region))
return False
if not d['_matching']:
logger.warning("{0} replay is marked as not matching. Results might not be accurate.".format(args.region))
#Load any previous measures for this region
flags_history = get_region_history(args.region, force_flags)
#Replay region with flags to test
best_id, best_cycles = run_replay(flags, args, flags_history)
best_flags.append(get_best_flags(args.region, best_history, d['_invivo'], best_id, best_cycles))
#or run selected regions from ilp selector
elif not args.region:
graph = graph_utils.load_graph()
if graph == None or graph.graph['selector'] != "ILP":
logger.error("Selected regions missing. First run cere select-ilp or choose a region manually with cere flag --region.")
return False
#Run selected regions
for n, d in graph.nodes(data=True):
if d["_selected"]:
args.region=d['_name']
#Load any previous measures for this region
flags_history = get_region_history(args.region, force_flags)
#Replay region with flags to test
best_id, best_cycles = run_replay(flags, args, flags_history)
best_flags.append(get_best_flags(args.region, best_history, d['_invivo'], best_id, best_cycles))
compute_theorical_speedup(best_flags, graph)
dump_results(best_flags, args.FLAGSFILE, flags)
clean_environ()
return True
def run(args):
if not cere_configure.init():
return False
if not os.path.isfile("regions.csv"):
logger.critical("Regions.csv file missing. Please run cere regions")
return False
region = args.region
filename = ""
functionname = ""
#Find the file where the region is
with open("regions.csv") as regions_list:
reader = csv.DictReader(regions_list)
for row in reader:
if region in row["Region Name"]:
filename = row["File Name"]
functionname = row["Function Name"]
break
if args.path:
filename = filename.rsplit('/', 1)
filename = args.path + "/" + filename[1]
filename = filename.replace(os.path.splitext(filename)[1], ".o")
logger.debug("The file is {0} and the function is {1}".format(
filename, functionname))
#Now let's compile it in the orginal application
if not run_shell_command("{0} MODE=original -B".format(
cere_configure.cere_config["build_cmd"])):
return False
original_lines = get_nlines(filename, functionname)
if not original_lines:
return False
#backup the original assembly file
if not run_shell_command("cp cere_tmp cere_original"):
return False
#Compile replay mode
#we accept that compilation fails because the dump does not have to be present.
try:
logger.debug(
subprocess.check_output(
"{0} MODE=\"replay --region={1}\" -B".format(
cere_configure.cere_config["build_cmd"], args.region),
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.warning("If the dump is not present, skip these warnings")
replay_lines = get_nlines(filename, "run__cere__" + region)
if not replay_lines:
return False
#backup the replay assembly file
if not run_shell_command("mv cere_tmp cere_replay"):
return False
err = compute_error(original_lines, replay_lines)
if err <= args.max_error:
logger.info(
"Assembly matching: Original lines = {0} && replay lines = {1} (error = {2})"
.format(original_lines, replay_lines, err))
else:
logger.info(
"Assembly not matching: Original lines = {0} && replay lines = {1} (error = {2})"
.format(original_lines, replay_lines, err))
if args.diff_asm:
subprocess.call("vimdiff cere_original cere_replay", shell=True)
return True
def run(args):
if not cere_configure.init():
return False
profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
regions_file = cere_configure.cere_config["regions_infos"]
new_regions_file = "tmp.csv"
logger.info("Removing previous regions list")
if os.path.isfile(regions_file):
os.remove(regions_file)
if args.dynamic: args.static = True
if not args.static and not args.dynamic:
args.static = True
args.dynamic = True
if (args.static):
mydump = Dump()
res = cere_capture.run(mydump)
logger.info("Regions list dumped in {0} file".format(regions_file))
if not res:
logger.warning(
"Regions listing encountered and error: list may be incomplete or empty"
)
if (args.dynamic):
if not os.path.isfile(profile_file):
logger.critical('No profiling file. Please run cere profile')
return False
if not os.path.isfile(regions_file):
logger.critical(
'Regions file does not exists. Please re-run cere regions with static enabled.'
)
return False
build_cmd = cere_configure.cere_config["build_cmd"]
run_cmd = cere_configure.cere_config["run_cmd"]
#Build again the application to be sure we give the right binary to pprof
try:
logger.debug(
subprocess.check_output(
"{0} MODE=\"original --instrument --instrument-app\" -B".
format(build_cmd),
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
return False
binary = which(run_cmd)
if not binary:
logger.critical(
"Cannot find the binary. Please provide binary name through cere configure --binary"
)
return False
add_header(regions_file, new_regions_file)
#regular expression to parse the gperf tool output
regex_list = [
r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,'
]
cmd = subprocess.Popen("pprof -dot {0} {1}".format(
binary, profile_file),
shell=True,
stdout=subprocess.PIPE)
for line in cmd.stdout:
matchObj, step = parse_line(regex_list, line)
if step < 2:
add_coverage(regions_file, new_regions_file, matchObj)
else:
continue
try:
shutil.move(new_regions_file, regions_file)
except IOError as err:
logger.critical(str(err))
logger.error(err.output)
return False
return True
def run(args):
if not cere_configure.init():
return False
if(args.region):
invocations = find_invocations(args.invocation, args.region)
if not invocations:
return False
for invocation in invocations:
if not args.force:
if utils.dump_exist(args.region, invocation):
logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation))
return True
if not args.no_io_trace:
#Check IOs
if not cere_check_io.run_io_checker(args.region, None, invocation, args.force):
utils.mark_invalid(args.region, "Failed to run IO checker")
if utils.is_invalid(args.region):
logger.warning("{0} is invalid. Skipping capture".format(args.region))
return False
else:
shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True)
logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation))
try:
logger.debug(subprocess.check_output("{0} MODE=\"dump --region={1} --invocation={2}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EDUMP)
return False
if not args.norun:
logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation))
try:
logger.info(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
#even if the capture run fails, maybe the region is dumped.
logger.error(str(err))
logger.error(err.output)
if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)):
logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EDUMP)
return False
else:
logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation))
#Global dump
else:
logger.info("Compiling capture mode for all regions")
try:
logger.debug(subprocess.check_output("{0} MODE=\"dump\" -B".format(cere_configure.cere_config["build_cmd"]), stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.info("Compiling capture mode for all regions failed")
return False
if not args.norun:
logger.info("Capturing all regions")
try:
logger.debug(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Capturing all regions failed")
return False
else:
logger.info("All regions succesfully captured")
return True
def run(args):
if not cere_configure.init():
return False
if utils.is_invalid(args.region) and not args.force:
logger.warning("{0} is invalid. Skipping replay".format(args.region))
return False
invocations = find_invocations(args.invocation, args.region)
if not invocations:
return False
if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
logger.warning(
"You are not using the default library. Computing predicted time\n\
may not work if the replay output is not the same")
for invocation, part in invocations.iteritems():
if os.path.isfile(
"{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region,
invocation)) and not args.force:
logger.warning(
"Replay already measured for {0} invocation {1}.".format(
args.region, invocation))
continue
if not utils.dump_exist(args.region, invocation):
logger.error("Memory dump is missing for {0} invocation {1}.\n\
Run cere capture --region={0} [--invocation={1}]".format(
args.region, invocation))
return False
if args.noinstrumentation:
instru_cmd = ""
logger.info(
"Compiling replay mode for region {0} invocation {1} without instrumentation"
.format(args.region, invocation))
else:
instru_cmd = "--instrument"
logger.info(
"Compiling replay mode for region {0} invocation {1} with instrumentation"
.format(args.region, invocation))
try:
logger.debug(
subprocess.check_output(
"{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B"
.format(cere_configure.cere_config["build_cmd"],
args.region, invocation, instru_cmd,
args.plugin_instr, args.invitro_callcount),
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error(
"Compiling replay mode for region {0} invocation {1} Failed".
format(args.region, invocation))
utils.mark_invalid(args.region, cere_error.EREPLAY)
return False
if not args.norun:
logger.info("Replaying invocation {1} for region {0}".format(
args.region, invocation))
try:
logger.debug(
subprocess.check_output(
cere_configure.cere_config["run_cmd"],
stderr=subprocess.STDOUT,
shell=True))
except subprocess.CalledProcessError as err:
logger.error(str(err))
logger.error(err.output)
logger.error("Replay failed for {0} invocation {1}".format(
args.region, invocation))
utils.mark_invalid(args.region, cere_error.EREPLAY)
return False
#Save replay measures files
if PREDICTION_MODE:
try:
shutil.move(
"{0}.csv".format(args.region),
"{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH,
args.region, invocation))
except IOError as err:
logger.error(str(err))
logger.error(
" No results file. Maybe replay failed".format(
invocation))
return False
if PREDICTION_MODE:
predicted_cycles = compute_predicted_time(args.region, invocations)
logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
return True
#!/usr/bin/env python
import sys
import networkx as nx
sys.path.insert(0, "../../src/cere/")
from create_graph import remove_cycles, fix_self_coverage
import cere_configure
from graph_utils import plot, save_graph
if __name__ == "__main__":
cere_configure.init()
digraph = nx.DiGraph()
nodes = [1, 2, 3, 4, 5, 6, 7]
samples = 100
digraph.add_nodes_from(nodes)
for i in nodes:
digraph.node[i]["_name"] = "Node_" + str(i)
digraph.node[i]["_matching"] = True
digraph.node[i]["_valid"] = True
digraph.node[i]["_to_test"] = False
digraph.node[i]["_small"] = False
digraph.node[i]["_tested"] = False
# Assign coverage and self_coverage
digraph.node[1]["_self_coverage"] = 10.0
digraph.node[1]["_coverage"] = 60.0
digraph.node[2]["_self_coverage"] = 5.0
digraph.node[2]["_coverage"] = 40.0
digraph.node[3]["_self_coverage"] = 30.0
digraph.node[3]["_coverage"] = 13.0
digraph.node[4]["_self_coverage"] = 10.0
