def update_nodes(graph, max_allowed_error): lines = read_csv("{0}/matching_error.csv".format(var.CERE_REPLAY_PATH)) for line in lines: #for region_name, error in matching.iteritems(): #find the node in the graph for n,d in graph.nodes(data=True): if line["Codelet Name"] == d['_name']: d['_invivo'] = float(line["Invivo"]) d['_invitro'] = float(line["Invitro"]) d['_tested'] = True if float(line["Error"]) <= max_allowed_error: d['_matching'] = True else: d['_valid'] = False d['_error'] = float(line["Error"]) if utils.is_invalid(d['_name']): d['_error_message'] = utils.get_error_message(d['_name']) invocations = read_csv("{0}/invocations_error.csv".format(var.CERE_REPLAY_PATH)) del d['_invocations'][:] for inv in invocations: if inv["Codelet Name"] == d['_name']: d['_invocations'].append({"Cluster":inv["Cluster"], "Invocation":inv["Invocation"], "Part":round(float(inv["Part"]), 2), "Invivo (cycles)":"{:e}".format(float(inv["Invivo"])), "Invitro (cycles)":"{:e}".format(float(inv["Invitro"])), "Error (%)":round(float(inv["Error"]), 2)}) d['_tested'] = True d['_to_test'] = False save_graph(graph) plot(graph)
def check_arguments(args): """ Check arguments """ if not (args.region or args.regions_file): logger.error( "No region specified, use at least one of the following: --region, --regions-file" ) return False if (args.regions_file and args.region): logger.error("--region and --regions-file are exclusive") return False if (args.read and not args.region): logger.error("--read can only be used with --region") return False if args.regions_file: if not os.path.isfile(args.regions_file): logger.error("No such file: {0}".format(args.regions_file)) return False else: # Make regions_file path absolute args.regions_file = os.path.abspath(args.regions_file) if args.region and utils.is_invalid(args.region): logger.warning("{0} is invalid. Skipping trace".format(args.region)) return False return True
def need_to_measure(region): if utils.is_invalid(region): return False if utils.trace_exists(region): return False return True
def run(args): if not cere_configure.init(): return False invocations = find_invocations(args.invocation, args.region) if not invocations: return False for invocation in invocations: if not args.force: if utils.dump_exist(args.region, invocation): logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation)) return True if utils.is_invalid(args.region): logger.warning("{0} is invalid. Skipping capture".format(args.region)) return False else: shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True) logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation)) try: logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"dump --region={2} --invocation={3}\"".format(cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EDUMP) return False if not args.norun: logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation)) try: logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: #even if the capture run fails, maybe the region is dumped. logger.error(str(err)) logger.error(err.output) if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)): logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EDUMP) return False else: logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation)) return True
def run_instrument(args_region=None, args_regions_file=None, args_plugin_instr=var.RDTSC_WRAPPER, args_invocation=0, args_norun=False, args_force=False): if not cere_configure.init(): return False if utils.is_invalid(args_region) and not args_force: logger.error("{0} is invalid. Skipping instrumentation".format(args_region)) return False if args_regions_file: region_input = "--regions-file={0}".format(args_regions_file) elif args_region: region_input = "--region={0}".format(args_region) else: logger.error("No region specified, use at least one of the following: --region, --regions-file") return False if args_invocation != 0 : logger.info("Compiling instrumentation mode for {0} invocation {1}".format(region_input, args_invocation)) mode = " --invocation={0}".format(args_invocation) else: logger.info("Compiling instrumentation mode for {0}".format(region_input)) mode = "" try: logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"original {2} --instrument {3} --wrapper={4}\"".format(cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], region_input, mode, args_plugin_instr), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling instrumentation failed") return False if not args_norun: logger.info("Running instrumentation for {0}".format(region_input)) try: logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Instrumentation failed") return False return True
def update_nodes(graph, max_allowed_error): lines = read_csv("{0}/matching_error.csv".format(var.CERE_REPLAY_PATH)) for line in lines: #for region_name, error in matching.iteritems(): #find the node in the graph for n, d in graph.nodes(data=True): if line["Codelet Name"] == d['_name']: d['_invivo'] = float(line["Invivo"]) d['_invitro'] = float(line["Invitro"]) d['_tested'] = True if float(line["Error"]) <= max_allowed_error: d['_matching'] = True else: d['_valid'] = False d['_error'] = float(line["Error"]) if utils.is_invalid(d['_name']): d['_error_message'] = utils.get_error_message(d['_name']) invocations = read_csv("{0}/invocations_error.csv".format( var.CERE_REPLAY_PATH)) del d['_invocations'][:] for inv in invocations: if inv["Codelet Name"] == d['_name']: d['_invocations'].append({ "Cluster": inv["Cluster"], "Invocation": inv["Invocation"], "Part": round(float(inv["Part"]), 2), "Invivo (cycles)": "{:e}".format(float(inv["Invivo"])), "Invitro (cycles)": "{:e}".format(float(inv["Invitro"])), "Error (%)": round(float(inv["Error"]), 2) }) d['_tested'] = True d['_to_test'] = False save_graph(graph) plot(graph)
def run_instrument(args_region=None, args_regions_file=None, args_plugin_instr=var.RDTSC_WRAPPER, args_invocation=0, args_norun=False, args_force=False): if not cere_configure.init(): return False if utils.is_invalid(args_region) and not args_force: logger.error( "{0} is invalid. Skipping instrumentation".format(args_region)) return False if args_regions_file: region_input = "--regions-file={0}".format(args_regions_file) elif args_region: region_input = "--region={0}".format(args_region) else: logger.error( "No region specified, use at least one of the following: --region, --regions-file" ) return False if args_invocation != 0: logger.info( "Compiling instrumentation mode for {0} invocation {1}".format( region_input, args_invocation)) mode = " --invocation={0}".format(args_invocation) else: logger.info( "Compiling instrumentation mode for {0}".format(region_input)) mode = "" try: logger.debug( subprocess.check_output( "{0} && {1} CERE_MODE=\"original {2} --instrument {3} --wrapper={4}\"" .format(cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], region_input, mode, args_plugin_instr), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling instrumentation failed") return False if not args_norun: logger.info("Running instrumentation for {0}".format(region_input)) try: logger.info( subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Instrumentation failed") return False return True
def run(args): if not cere_configure.init(): return False if utils.is_invalid(args.region) and not args.force: logger.warning("{0} is invalid. Skipping replay".format(args.region)) return False invocations = find_invocations(args.invocation, args.region) if not invocations: return False if PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER: logger.warning( "You are not using the default library. Computing predicted time\n\ may not work if the replay output is not the same" ) for invocation, part in invocations.iteritems(): if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force: logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation)) continue if not utils.dump_exist(args.region, invocation): logger.error( "Memory dump is missing for {0} invocation {1}.\n\ Run cere capture --region={0} [--invocation={1}]".format( args.region, invocation ) ) return False if args.noinstrumentation: instru_cmd = "" logger.info( "Compiling replay mode for region {0} invocation {1} without instrumentation".format( args.region, invocation ) ) else: instru_cmd = "--instrument" logger.info( "Compiling replay mode for region {0} invocation {1} with instrumentation".format( args.region, invocation ) ) if args.static: static_cmd = "--static" logger.info("Static mode enabled".format(args.region, invocation)) else: static_cmd = "" try: logger.debug( subprocess.check_output( '{0} && {1} CERE_REPLAY_REPETITIONS={6} CERE_MODE="replay --region={2} --invocation={3} {4} --wrapper={5} {7}"'.format( cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount, static_cmd, ), stderr=subprocess.STDOUT, shell=True, ) ) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False if not args.norun: logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation)) try: logger.debug( subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True) ) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False # Save replay measures files if PREDICTION_MODE: try: shutil.move( "{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation), ) except IOError as err: logger.error(str(err)) logger.error(" No results file. Maybe replay failed".format(invocation)) return False if PREDICTION_MODE: predicted_cycles = compute_predicted_time(args.region, invocations) dump_result(args.region, predicted_cycles) logger.info(" Overall predicted cycles = {0}".format(predicted_cycles)) return True
def run(args): if not cere_configure.init(): return False if utils.is_invalid(args.region) and not args.force: logger.warning("{0} is invalid. Skipping replay".format(args.region)) return False invocations = find_invocations(args.invocation, args.region) if not invocations: return False if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER): logger.warning("You are not using the default library. Computing predicted time\n\ may not work if the replay output is not the same") for invocation, part in invocations.iteritems(): if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force: logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation)) continue if not utils.dump_exist(args.region, invocation): logger.error("Memory dump is missing for {0} invocation {1}.\n\ Run cere capture --region={0} [--invocation={1}]".format(args.region, invocation)) return False if args.noinstrumentation: instru_cmd = "" logger.info("Compiling replay mode for region {0} invocation {1} without instrumentation".format(args.region, invocation)) else: instru_cmd = "--instrument" logger.info("Compiling replay mode for region {0} invocation {1} with instrumentation".format(args.region, invocation)) if args.static: static_cmd = "--static" logger.info("Static mode enabled".format(args.region, invocation)) else: static_cmd = "" try: logger.debug( subprocess.check_output("{0} && {1} CERE_REPLAY_REPETITIONS={6} CERE_MODE=\"replay --region={2} --invocation={3} {4} --wrapper={5} {7}\"".format( cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount, static_cmd), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False if not args.norun: logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation)) try: logger.debug(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False #Save replay measures files if PREDICTION_MODE: try: shutil.move("{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) except IOError as err: logger.error(str(err)) logger.error(" No results file. Maybe replay failed".format(invocation)) return False if PREDICTION_MODE: predicted_cycles = compute_predicted_time(args.region, invocations) dump_result(args.region, predicted_cycles) logger.info(" Overall predicted cycles = {0}".format(predicted_cycles)) return True