def run(args): if not cere_configure.init(): return False if utils.is_invalid(args.region) and not args.force: logger.warning("{0} is invalid. Skipping replay".format(args.region)) return False invocations = find_invocations(args.invocation, args.region) if not invocations: return False if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER): logger.warning("You are not using the default library. Computing predicted time\n\ may not work if the replay output is not the same") for invocation, part in invocations.iteritems(): if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force: logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation)) continue if not utils.dump_exist(args.region, invocation): logger.error("Memory dump is missing for {0} invocation {1}.\n\ Run cere capture --region={0} [--invocation={1}]".format(args.region, invocation)) return False if args.noinstrumentation: instru_cmd = "" logger.info("Compiling replay mode for region {0} invocation {1} without instrumentation".format(args.region, invocation)) else: instru_cmd = "--instrument" logger.info("Compiling replay mode for region {0} invocation {1} with instrumentation".format(args.region, invocation)) try: logger.debug(subprocess.check_output("{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False if not args.norun: logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation)) try: logger.debug(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False #Save replay measures files if PREDICTION_MODE: try: shutil.move("{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) except IOError as err: logger.error(str(err)) logger.error(" No results file. Maybe replay failed".format(invocation)) return False if PREDICTION_MODE: predicted_cycles = compute_predicted_time(args.region, invocations) logger.info(" Overall predicted cycles = {0}".format(predicted_cycles)) return True
def parse_io_trace(regions_to_trace): missing_trace = [] file_stack = [] region_stack = [] for region, invocations in regions_to_trace.iteritems(): for invocation in invocations: trace_dir = os.path.join(var.CERE_IO_TRACES_PATH, region, str(invocation), region) # If strace has not generated a trace file it means: # there is no IO or the trace is included in another trace if not os.path.isfile(trace_dir): missing_trace.append(trace_dir) continue region_stack.append(region) regex = r"(.*)\(([0-9]*)\,\s\"(.*)\"" # Parse the IO trace, if a read or write is detected, the region is invalid. with open(trace_dir, "r") as io_trace: content = io_trace.readlines() for line in content: matchObj = re.match(regex, line) if not matchObj: continue op_type = matchObj.group(1) fd = int(matchObj.group(2)) text = matchObj.group(3) if "CERE_" in text: nested_region = text.split()[1] nested_invocation = text.split()[2].replace("\\n", "") # Starting a nested region if "CERE_START" in text: # create the region trace trace_file = os.path.join( var.CERE_IO_TRACES_PATH, nested_region, nested_invocation, nested_region ) nested_file = open(trace_file, "a") # remove it from the missing trace if trace_file in missing_trace: missing_trace.remove(trace_file) file_stack.append(nested_file) region_stack.append(nested_region) elif "CERE_STOP" in text: # Close the nested region trace file_stack.pop().close() region_stack.pop() else: for f in file_stack: f.write(line) if (op_type in forbidden_ios) and (fd not in tolerated_fd): for r in region_stack: utils.mark_invalid(r, cere_error.EIO) region_stack.pop() for r in region_stack: utils.mark_invalid(r, "Failed to run IO checker")
def run(args): if not cere_configure.init(): return False if not check_arguments(args): return False # Find loops to trace regions = find_regions_to_trace(args) # When force is off, filter already measured regions if not args.force: regions = [r for r in regions if need_to_measure(r)] result = True # Are there any regions to trace ? if len(regions) == 0: logger.info("No regions to trace") else: result = launch_trace(args, regions) # Move trace files to cere_measures if not args.norun: for region in regions: try: shutil.move("{0}.bin".format(region), var.CERE_TRACES_PATH) shutil.move("{0}.csv".format(region), var.CERE_TRACES_PATH) except IOError as err: logger.error(str(err)) logger.error( "Trace failed for region {0}: No output files, maybe the selected region does not exist." .format(region)) utils.mark_invalid(region, cere_error.ETRACE) result = False # If read is used, read given invocation from trace file if args.read and utils.trace_exists(args.region): base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region) trace = parse_trace_file(base_name + '.bin') cycles = int(trace['cycles'][trace['invocations'] == args.read]) print(cycles) return result
def run(args): if not cere_configure.init(): return False if not check_arguments(args): return False # Find loops to trace regions = find_regions_to_trace(args) # When force is off, filter already measured regions if not args.force: regions = [r for r in regions if need_to_measure(r)] result = True # Are there any regions to trace ? if len(regions) == 0: logger.info("No regions to trace") else: result = launch_trace(args, regions) # Move trace files to cere_measures if not args.norun: for region in regions: try: shutil.move("{0}.bin".format(region), var.CERE_TRACES_PATH) shutil.move("{0}.csv".format(region), var.CERE_TRACES_PATH) except IOError as err: logger.error(str(err)) logger.error("Trace failed for region {0}: No output files, maybe the selected region does not exist.".format(region)) utils.mark_invalid(region, cere_error.ETRACE) result = False # If read is used, read given invocation from trace file if args.read and utils.trace_exists(args.region): base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region) trace = parse_trace_file(base_name + '.bin') cycles = int(trace['cycles'][trace['invocations'] == args.read]) print(cycles) return result
def run(args): if not cere_configure.init(): return False if utils.is_invalid(args.region) and not args.force: logger.warning("{0} is invalid. Skipping replay".format(args.region)) return False invocations = find_invocations(args.invocation, args.region) if not invocations: return False if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER): logger.warning( "You are not using the default library. Computing predicted time\n\ may not work if the replay output is not the same") for invocation, part in invocations.iteritems(): if os.path.isfile( "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force: logger.warning( "Replay already measured for {0} invocation {1}.".format( args.region, invocation)) continue if not utils.dump_exist(args.region, invocation): logger.error("Memory dump is missing for {0} invocation {1}.\n\ Run cere capture --region={0} [--invocation={1}]".format( args.region, invocation)) return False if args.noinstrumentation: instru_cmd = "" logger.info( "Compiling replay mode for region {0} invocation {1} without instrumentation" .format(args.region, invocation)) else: instru_cmd = "--instrument" logger.info( "Compiling replay mode for region {0} invocation {1} with instrumentation" .format(args.region, invocation)) try: logger.debug( subprocess.check_output( "{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B" .format(cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error( "Compiling replay mode for region {0} invocation {1} Failed". format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False if not args.norun: logger.info("Replaying invocation {1} for region {0}".format( args.region, invocation)) try: logger.debug( subprocess.check_output( cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Replay failed for {0} invocation {1}".format( args.region, invocation)) utils.mark_invalid(args.region, cere_error.EREPLAY) return False #Save replay measures files if PREDICTION_MODE: try: shutil.move( "{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) except IOError as err: logger.error(str(err)) logger.error( " No results file. Maybe replay failed".format( invocation)) return False if PREDICTION_MODE: predicted_cycles = compute_predicted_time(args.region, invocations) logger.info(" Overall predicted cycles = {0}".format(predicted_cycles)) return True
def run(args): if not cere_configure.init(): return False if(args.region): invocations = find_invocations(args.invocation, args.region) if not invocations: return False for invocation in invocations: if not args.force: if utils.dump_exist(args.region, invocation): logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation)) return True if not args.no_io_trace: #Check IOs if not cere_check_io.run_io_checker(args.region, None, invocation, args.force): utils.mark_invalid(args.region, "Failed to run IO checker") if utils.is_invalid(args.region): logger.warning("{0} is invalid. Skipping capture".format(args.region)) return False else: shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True) logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation)) try: logger.debug(subprocess.check_output("{0} MODE=\"dump --region={1} --invocation={2}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EDUMP) return False if not args.norun: logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation)) try: logger.info(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: #even if the capture run fails, maybe the region is dumped. logger.error(str(err)) logger.error(err.output) if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)): logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation)) utils.mark_invalid(args.region, cere_error.EDUMP) return False else: logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation)) #Global dump else: logger.info("Compiling capture mode for all regions") try: logger.debug(subprocess.check_output("{0} MODE=\"dump\" -B".format(cere_configure.cere_config["build_cmd"]), stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.info("Compiling capture mode for all regions failed") return False if not args.norun: logger.info("Capturing all regions") try: logger.debug(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True)) except subprocess.CalledProcessError as err: logger.error(str(err)) logger.error(err.output) logger.error("Capturing all regions failed") return False else: logger.info("All regions succesfully captured") return True