Пример #1
0
def run(args):
  if not cere_configure.init():
    return False

  args.regions_file = os.path.realpath(args.regions_file)

  if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
    logger.error("File not found: {0}".format(cere_configure.cere_config["regions_infos"]))
    logger.error("Did you run cere regions?")
    return False

  instru_cmd = ""
  #Check for instrumentation
  if args.instrumentation:
    instru_cmd = "--instrument --wrapper={0}".format(var.RDTSC_WRAPPER)
    logger.info("Compiling hybrid binary with instrumentation")

  #Remove objects file
  try:
    os.remove(os.path.realpath("__cere__objects"))
  except OSError as e:
    if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory
      raise #re-raise exception if a different error occured

  #Compile hybrid binary
  try:
    logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"original --hybrid {2} --hybrid-regions={3} \
    --regions-infos={4} --cere-objects={5} --extraction-lvl={6}\"".format(cere_configure.cere_config["clean_cmd"], cere_configure.cere_config["build_cmd"], instru_cmd, args.regions_file,\
    cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), args.extraction_lvl), stderr=subprocess.STDOUT, shell=True))
  except subprocess.CalledProcessError as err:
    logger.critical(str(err))
    logger.critical(err.output)
    return False
  logger.info("Hybrid compiled successfully")
  return True
Пример #2
0
def run_instrument(args_region=None,
                   args_regions_file=None,
                   args_plugin_instr=var.RDTSC_WRAPPER,
                   args_invocation=0,
                   args_norun=False,
                   args_force=False):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args_region) and not args_force:
        logger.error(
            "{0} is invalid. Skipping instrumentation".format(args_region))
        return False

    if args_regions_file:
        region_input = "--regions-file={0}".format(args_regions_file)
    elif args_region:
        region_input = "--region={0}".format(args_region)
    else:
        logger.error(
            "No region specified, use at least one of the following: --region, --regions-file"
        )
        return False

    if args_invocation != 0:
        logger.info(
            "Compiling instrumentation mode for {0} invocation {1}".format(
                region_input, args_invocation))
        mode = " --invocation={0}".format(args_invocation)
    else:
        logger.info(
            "Compiling instrumentation mode for {0}".format(region_input))
        mode = ""

    try:
        logger.debug(
            subprocess.check_output(
                "{0} MODE=\"original {1} --instrument {2} --wrapper={3}\" -B".
                format(cere_configure.cere_config["build_cmd"], region_input,
                       mode, args_plugin_instr),
                stderr=subprocess.STDOUT,
                shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling instrumentation failed")
        return False

    if not args_norun:
        logger.info("Running instrumentation for {0}".format(region_input))
        try:
            logger.info(
                subprocess.check_output(cere_configure.cere_config["run_cmd"],
                                        stderr=subprocess.STDOUT,
                                        shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error("Instrumentation failed")
            return False
    return True
Пример #3
0
def run(args):
    if not cere_configure.init():
        return False

    invocation_file = os.path.join(var.CERE_TRACES_PATH,
                                   args.region + ".invocations")

    if os.path.isfile(invocation_file) and not args.force:
        logger.info(
            "Invocations already selected for {0}. If you want to reselect use -f"
            .format(args.region, invocation_file))
        return True
    if not utils.trace_exists(args.region):
        logger.error("No trace for {0}.\n\
                     Please first run cere trace --region={0}".format(
            args.region))
        return False
    bin_file = os.path.join(var.CERE_TRACES_PATH, args.region + ".bin")
    csv_file = os.path.join(var.CERE_TRACES_PATH, args.region + ".csv")

    clusterize_invocations(args.region, csv_file, bin_file)
    try:
        shutil.move(
            "{0}.invocations".format(args.region),
            "{0}/{1}.invocations".format(var.CERE_TRACES_PATH, args.region))
    except IOError as err:
        logger.error(str(err))
        return False
    return True
Пример #4
0
def run(args):
  if not cere_configure.init():
      return False
  profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
  regions_file = cere_configure.cere_config["regions_infos"]
  new_regions_file = "tmp.csv"
  build_cmd = cere_configure.cere_config["build_cmd"]
  run_cmd = cere_configure.cere_config["run_cmd"]
  clean_cmd = cere_configure.cere_config["clean_cmd"]

  logger.info("Removing previous regions list")
  if os.path.isfile(regions_file):
      os.remove(regions_file)

  #Build again the application to be sure we give the right binary to pprof
  try:
      logger.info(subprocess.check_output("{0} && {1} CERE_MODE=\"original --instrument --instrument-app --regions-infos={2}\"".format(clean_cmd, build_cmd, regions_file), stderr=subprocess.STDOUT, shell=True))
  except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      return False
  if not os.path.isfile(regions_file):
    logger.critical("Failed to generate region list.")
    return False

  logger.info("Regions list dumped in {0} file".format(regions_file))

  if not args.static:
    if not os.path.isfile(profile_file):
      logger.critical('No profiling file. Please first run cere profile or run cere regions --static')
      return False

    binary = which(run_cmd)
    if not binary:
      logger.critical("Cannot find the binary. Please provide binary name through cere configure --binary")
      return False

    add_header(regions_file, new_regions_file)

    #regular expression to parse the gperf tool output
    regex_list = [r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
              r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
              r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,']

    cmd = subprocess.Popen("{0} -dot {1} {2}".format(var.PPROF, binary, profile_file), shell=True, stdout=subprocess.PIPE)

    for line in cmd.stdout:
      matchObj, step = parse_line(regex_list, line)
      if step < 2 :
        add_coverage(regions_file, new_regions_file, matchObj)
      else:
        continue
    try:
      shutil.move(new_regions_file, regions_file)
    except IOError as err:
      logger.critical(str(err))
      logger.error(err.output)
      return False
  return True
Пример #5
0
def run(args):
  if not cere_configure.init():
    return False
  if utils.is_invalid(args.region) and not args.force:
    logger.warning("{0} is invalid. Skipping replay".format(args.region))
    return False

  invocations = find_invocations(args.invocation, args.region)
  if not invocations:
    return False
  if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
    logger.warning("You are not using the default library. Computing predicted time\n\
                    may not work if the replay output is not the same")
  for invocation, part in invocations.iteritems():
    if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force:
      logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation))
      continue
    if not utils.dump_exist(args.region, invocation):
      logger.error("Memory dump is missing for {0} invocation {1}.\n\
                    Run cere capture --region={0} [--invocation={1}]".format(args.region, invocation))
      return False
    if args.noinstrumentation:
      instru_cmd = ""
      logger.info("Compiling replay mode for region {0} invocation {1} without instrumentation".format(args.region, invocation))
    else:
      instru_cmd = "--instrument"
      logger.info("Compiling replay mode for region {0} invocation {1} with instrumentation".format(args.region, invocation))
    try:
      logger.debug(subprocess.check_output("{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation))
      utils.mark_invalid(args.region, cere_error.EREPLAY)
      return False
    if not args.norun:
      logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation))
      try:
        logger.debug(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation))
        utils.mark_invalid(args.region, cere_error.EREPLAY)
        return False
      #Save replay measures files
      if PREDICTION_MODE:
        try:
          shutil.move("{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation))
        except IOError as err:
          logger.error(str(err))
          logger.error("  No results file. Maybe replay failed".format(invocation))
          return False
  if PREDICTION_MODE:
    predicted_cycles = compute_predicted_time(args.region, invocations)
    logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
  return True
Пример #6
0
def run(args):
    if not cere_configure.init():
        return False
    if not os.path.isfile("regions.csv"):
        logger.critical("Regions.csv file missing. Please run cere regions")
        return False
    region = args.region
    filename = ""
    functionname=""
    #Find the file where the region is
    with open("regions.csv") as regions_list:
        reader = csv.DictReader(regions_list)
        for row in reader:
            if region in row["Region Name"]:
                filename = row["File Name"]
                functionname = row["Function Name"]
                break
    if args.path:
        filename = filename.rsplit('/', 1)
        filename = args.path+"/"+filename[1]
    filename = filename.replace(os.path.splitext(filename)[1], ".o")
    logger.debug("The file is {0} and the function is {1}".format(filename, functionname))

    #Now let's compile it in the orginal application
    if not run_shell_command("{0} MODE=original -B".format(cere_configure.cere_config["build_cmd"])):
        return False
    original_lines = get_nlines(filename, functionname)
    if not original_lines:
        return False
    #backup the original assembly file
    if not run_shell_command("cp cere_tmp cere_original"):
        return False

    #Compile replay mode
    #we accept that compilation fails because the dump does not have to be present.
    try:
        logger.debug(subprocess.check_output("{0} MODE=\"replay --region={1}\" -B".format(cere_configure.cere_config["build_cmd"], args.region), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.warning("If the dump is not present, skip these warnings")
    replay_lines = get_nlines(filename, "run__cere__"+region)
    if not replay_lines:
        return False
    #backup the replay assembly file
    if not run_shell_command("mv cere_tmp cere_replay"):
        return False

    err = compute_error(original_lines, replay_lines)
    if err <= args.max_error:
        logger.info("Assembly matching: Original lines = {0} && replay lines = {1} (error = {2})".format(original_lines, replay_lines, err))
    else:
        logger.info("Assembly not matching: Original lines = {0} && replay lines = {1} (error = {2})".format(original_lines, replay_lines, err))
    if args.diff_asm:
        subprocess.call("vimdiff cere_original cere_replay", shell=True)
    return True
Пример #7
0
def run(args):
    if not cere_configure.init():
        return False

    if not (args.region or args.regions_file):
        logger.error(
            "No region specified, use at least one of the following: --region, --regions-file"
        )
        return False

    if (args.regions_file and args.region):
        logger.error("--region and --regions-file are exclusive")
        return False

    if args.region:
        regions = [args.region]
    elif args.regions_file:
        if not os.path.isfile(args.regions_file):
            logger.critical("\"{0}\" No such file.".format(args.regions_file))
            return False
        with open(args.regions_file, 'r') as regions_file:
            regions = [region.strip() for region in regions_file]

    err = False
    allRegions = []
    #For each region
    for r in regions:
        allRegions.append(Region(r))
        #first we need the trace
        res = allRegions[-1].measure_trace()
        if not res: err = True

        #Compute the coverage of this region
        allRegions[-1].compute_coverage()
        #We can clusterize invocations in performance classes
        res = cere_selectinv.run(allRegions[-1])
        if not res: err = True

    trace_filename = create_region_to_trace_file(regions)
    run_io_checker(trace_filename)

    for idx, region in enumerate(allRegions):
        #Replay representative invocations
        res = allRegions[idx].replay_invocations(args.force)
        if not res: err = True

        #Compute error between invivo and in vitro
        allRegions[idx].check_region_matching()

    dump_results(allRegions, args.max_error)
    graph = load_graph()
    if graph != None:
        update_nodes(graph, args.max_error)
    return not err
Пример #8
0
def run(args):
    if not cere_configure.init():
        return False

    if not args.app and not args.regions:
        args.app = True
        args.regions = True
    res_1=True
    res_2=True
    if args.regions:
        res_1 = instrument_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], cere_configure.cere_config["clean_cmd"], args.force)
    if args.app:
        res_2 = measure_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], cere_configure.cere_config["clean_cmd"], args.force)
    return (res_1 and res_2)
Пример #9
0
def run(args):
    if not cere_configure.init():
        return False

    if not args.app and not args.regions:
        args.app = True
        args.regions = True
    res_1=True
    res_2=True
    if args.regions:
        res_1 = instrument_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], args.force)
    if args.app:
        res_2 = measure_application(cere_configure.cere_config["run_cmd"], cere_configure.cere_config["build_cmd"], args.force)
    return (res_1 and res_2)
Пример #10
0
def run(args):
    if not cere_configure.init():
        return False
    if not check_dependancies(args):
        return False
    if not check_arguments(args):
        return False
    #Find matching codelets
    if not update(args):
        return False
    #Select matching codelets with best coverage
    if not solve_with_best_granularity(args):
        return False
    return True
Пример #11
0
def run(args):
    if not cere_configure.init():
        return False

    if not (args.region or args.regions_file):
        logger.error("No region specified, use at least one of the following: --region, --regions-file")
        return False

    if (args.regions_file and args.region):
        logger.error("--region and --regions-file are exclusive")
        return False

    if args.region:
      regions = [args.region]
    elif args.regions_file:
      if not os.path.isfile(args.regions_file):
        logger.critical("\"{0}\" No such file.".format(args.regions_file))
        return False
      with open(args.regions_file, 'r') as regions_file:
        regions = [region.strip() for region in regions_file]

    err=False
    allRegions = []
    #For each region
    for r in regions:
        allRegions.append(Region(r))
        #first we need the trace
        res = allRegions[-1].measure_trace()
        if not res: err=True

        #Compute the coverage of this region
        allRegions[-1].compute_coverage()
        #We can clusterize invocations in performance classes
        res = cere_selectinv.run(allRegions[-1])
        if not res: err=True

    for idx, region in enumerate(allRegions):
        #Replay representative invocations
        res = allRegions[idx].replay_invocations(args.force)
        if not res: err=True

        #Compute error between invivo and in vitro
        allRegions[idx].check_region_matching()

    dump_results(allRegions, args.max_error)
    graph = load_graph()
    if graph != None:
        update_nodes(graph, args.max_error)
    return not err
Пример #12
0
def run(args):
    logger.info("Start report")
    if not cere_configure.init():
        return False
    graph = load_graph()
    if graph == None:
        logger.critical("Can't load graph. Did you run cere profile?")
        return False

    with context(args.path):
        bench = os.getcwd().split("/")[-1]
        REPORT = Report(bench, graph, args.mincycles)
        REPORT.write_report()
    logger.info("Report created")
    return True
Пример #13
0
def run(args):
    logger.info('Start report')
    if not cere_configure.init():
        return False
    graph = load_graph()
    if graph == None:
        logger.critical("Can't load graph. Did you run cere profile?")
        return False

    with context(args.path):
        bench = os.getcwd().split("/")[-1]
        REPORT = Report(bench, graph, args.mincycles)
        REPORT.write_report()
    logger.info("Report created")
    return True
Пример #14
0
def run(args):
    if not cere_configure.init():
        return False

    if not check_arguments(args):
        return False

    # Find loops to trace
    regions = find_regions_to_trace(args)

    # When force is off, filter already measured regions
    if not args.force:
        regions = [r for r in regions if need_to_measure(r)]

    result = True

    # Are there any regions to trace ?
    if len(regions) == 0:
        logger.info("No regions to trace")
    else:
        result = launch_trace(args, regions)

    # Move trace files to cere_measures
    if not args.norun:
        for region in regions:
            try:
                shutil.move("{0}.bin".format(region),
                            "{0}/{1}.bin".format(var.CERE_TRACES_PATH, region))
                shutil.move("{0}.csv".format(region),
                            "{0}/{1}.csv".format(var.CERE_TRACES_PATH, region))
            except IOError as err:
                logger.error(str(err))
                logger.error(
                    "Trace failed for region {0}: No output files, maybe the selected region does not exist."
                    .format(region))
                utils.mark_invalid(region, cere_error.ETRACE)
                result = False

    # If read is used, read given invocation from trace file
    if args.read and utils.trace_exists(args.region):
        base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region)
        trace = parse_trace_file(base_name + '.bin')
        cycles = int(trace['cycles'][trace['invocations'] == args.read])
        print(cycles)

    return result
Пример #15
0
def run(args):
  if not cere_configure.init():
    return False

  invocations = find_invocations(args.invocation, args.region)
  if not invocations:
    return False
  for invocation in invocations:
    if not args.force:
      if utils.dump_exist(args.region, invocation):
        logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation))
        return True
      if utils.is_invalid(args.region):
        logger.warning("{0} is invalid. Skipping capture".format(args.region))
        return False
    else:
      shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True)

    logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation))
    try:
      logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"dump --region={2} --invocation={3}\"".format(cere_configure.cere_config["clean_cmd"],
                                          cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation))
      utils.mark_invalid(args.region, cere_error.EDUMP)
      return False
    if not args.norun:
      logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation))
      try:
        logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        #even if the capture run fails, maybe the region is dumped.
        logger.error(str(err))
        logger.error(err.output)
      if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)):
        logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation))
        utils.mark_invalid(args.region, cere_error.EDUMP)
        return False
      else:
        logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation))
  return True
Пример #16
0
def run(args):
    if not cere_configure.init():
        return False

    if not check_arguments(args):
        return False

    # Find loops to trace
    regions = find_regions_to_trace(args)

    # When force is off, filter already measured regions
    if not args.force:
        regions = [r for r in regions if need_to_measure(r)]

    result = True

    # Are there any regions to trace ?
    if len(regions) == 0:
        logger.info("No regions to trace")
    else:
        result = launch_trace(args, regions)

    # Move trace files to cere_measures
    if not args.norun:
        for region in regions:
            try:
                shutil.move("{0}.bin".format(region), var.CERE_TRACES_PATH)
                shutil.move("{0}.csv".format(region), var.CERE_TRACES_PATH)
            except IOError as err:
                logger.error(str(err))
                logger.error("Trace failed for region {0}: No output files, maybe the selected region does not exist.".format(region))
                utils.mark_invalid(region, cere_error.ETRACE)
                result = False

    # If read is used, read given invocation from trace file
    if args.read and utils.trace_exists(args.region):
        base_name = "{0}/{1}".format(var.CERE_TRACES_PATH, args.region)
        trace = parse_trace_file(base_name + '.bin')
        cycles = int(trace['cycles'][trace['invocations'] == args.read])
        print(cycles)

    return result
Пример #17
0
def run(args):
  if not cere_configure.init():
    return False
  args.regions_file = os.path.realpath(args.regions_file)
  if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
    logger.error("No such file: {0}".format(cere_configure.cere_config["regions_infos"]))
    logger.error("Did you run cere regions?")
    return False
  region_input=""
  if args.region:
    region_input="--region={0}".format(args.region)
  try:
    logger.debug(subprocess.check_output("{0} MODE=\"original --hybrid --instrument {5} --wrapper={1} --hybrid-regions={2} \
    --regions-infos={3} --cere-objects={4}\"".format(cere_configure.cere_config["build_cmd"], var.RDTSC_WRAPPER, args.regions_file, \
    cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), region_input), stderr=subprocess.STDOUT, shell=True))
  except subprocess.CalledProcessError as err:
    logger.critical(str(err))
    logger.critical(err.output)
    return False
  return True
Пример #18
0
def run_instrument(args_region=None, args_regions_file=None, args_plugin_instr=var.RDTSC_WRAPPER, args_invocation=0, args_norun=False, args_force=False):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args_region) and not args_force:
        logger.error("{0} is invalid. Skipping instrumentation".format(args_region))
        return False

    if args_regions_file:
        region_input = "--regions-file={0}".format(args_regions_file)
    elif args_region:
        region_input = "--region={0}".format(args_region)
    else:
        logger.error("No region specified, use at least one of the following: --region, --regions-file")
        return False

    if args_invocation != 0 :
        logger.info("Compiling instrumentation mode for {0} invocation {1}".format(region_input, args_invocation))
        mode = " --invocation={0}".format(args_invocation)
    else:
        logger.info("Compiling instrumentation mode for {0}".format(region_input))
        mode = ""

    try:
        logger.debug(subprocess.check_output("{0} && {1} CERE_MODE=\"original {2} --instrument {3} --wrapper={4}\"".format(cere_configure.cere_config["clean_cmd"],
                                              cere_configure.cere_config["build_cmd"], region_input, mode, args_plugin_instr), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling instrumentation failed")
        return False

    if not args_norun:
        logger.info("Running instrumentation for {0}".format(region_input))
        try:
            logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error("Instrumentation failed")
            return False
    return True
Пример #19
0
def run_io_checker(args_region=None, args_regions_file=None, args_invocation=1, args_force=False):
    if not cere_configure.init():
        return False

    if not check_arguments(args_region, args_regions_file, args_invocation, args_force):
        return False

    if args_regions_file:
        # Make regions_file path absolute
        args_regions_file = os.path.abspath(args_regions_file)

    regions_to_trace = find_regions_to_trace(args_region, args_regions_file, args_invocation, args_force)
    if len(regions_to_trace) == 0:
        logger.info("No regions to trace")
    else:
        if not launch_io_trace(args_force, regions_to_trace):
            return False

    parse_io_trace(regions_to_trace)

    return True
Пример #20
0
def run(args):
    if not cere_configure.init():
        return False
    args.regions_file = os.path.realpath(args.regions_file)
    if not os.path.isfile(cere_configure.cere_config["regions_infos"]):
        logger.error("No such file: {0}".format(
            cere_configure.cere_config["regions_infos"]))
        logger.error("Did you run cere regions?")
        return False
    region_input = ""
    if args.region:
        region_input = "--region={0}".format(args.region)
    try:
        logger.debug(subprocess.check_output("{0} MODE=\"original --hybrid --instrument {5} --wrapper={1} --hybrid-regions={2} \
    --regions-infos={3} --cere-objects={4}\""                                                 .format(cere_configure.cere_config["build_cmd"], var.RDTSC_WRAPPER, args.regions_file, \
        cere_configure.cere_config["regions_infos"], os.path.realpath("__cere__objects"), region_input), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
        logger.critical(str(err))
        logger.critical(err.output)
        return False
    return True
Пример #21
0
def run(args):
    if not cere_configure.init():
        return False

    invocation_file = os.path.join(var.CERE_TRACES_PATH, args.region+".invocations")

    if os.path.isfile(invocation_file) and not args.force:
        logger.info("Invocations already selected for {0}. If you want to reselect use -f".format(args.region, invocation_file))
        return True
    if not utils.trace_exists(args.region):
        logger.error("No trace for {0}.\n\
                     Please first run cere trace --region={0}".format(args.region))
        return False
    bin_file = os.path.join(var.CERE_TRACES_PATH, args.region+".bin")
    csv_file = os.path.join(var.CERE_TRACES_PATH, args.region+".csv")

    clusterize_invocations(args.region, csv_file, bin_file)
    try:
        shutil.move("{0}.invocations".format(args.region), "{0}/{1}.invocations".format(var.CERE_TRACES_PATH, args.region))
    except IOError as err:
        logger.error(str(err))
        return False
    return True
Пример #22
0
#!/usr/bin/env python
import sys
import networkx as nx
sys.path.insert(0, "../../src/cere/")
from create_graph import remove_cycles, fix_self_coverage
import cere_configure
from graph_utils import plot, save_graph

if __name__ == "__main__":
    cere_configure.init()
    digraph = nx.DiGraph()
    nodes = [1,2,3,4,5,6,7]
    samples = 100
    digraph.add_nodes_from(nodes)
    for i in nodes:
        digraph.node[i]['_name'] = "Node_" + str(i)
        digraph.node[i]['_matching'] = True
        digraph.node[i]['_valid'] = True
        digraph.node[i]['_to_test'] = False
        digraph.node[i]['_small'] = False
        digraph.node[i]['_tested'] = False

    #Assign coverage and self_coverage
    digraph.node[1]['_self_coverage'] = 10.
    digraph.node[1]['_coverage'] = 60.
    digraph.node[2]['_self_coverage'] = 5.
    digraph.node[2]['_coverage'] = 40.
    digraph.node[3]['_self_coverage'] = 30.
    digraph.node[3]['_coverage'] = 13.
    digraph.node[4]['_self_coverage'] = 10.
    digraph.node[4]['_coverage'] = 0.
Пример #23
0
def run(args):
    if not cere_configure.init():
        return False
    profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
    regions_file = cere_configure.cere_config["regions_infos"]
    new_regions_file = "tmp.csv"

    logger.info("Removing previous regions list")
    if os.path.isfile(regions_file):
        os.remove(regions_file)

    if args.dynamic: args.static = True
    if not args.static and not args.dynamic:
        args.static = True
        args.dynamic = True

    if(args.static):
        mydump = Dump()
        res = cere_capture.run(mydump)
        logger.info("Regions list dumped in {0} file".format(regions_file))
        if not res:
            logger.warning("Regions listing encountered and error: list may be incomplete or empty")
    if(args.dynamic):
        if not os.path.isfile(profile_file):
            logger.critical('No profiling file. Please run cere profile')
            return False
        if not os.path.isfile(regions_file):
            logger.critical('Regions file does not exists. Please re-run cere regions with static enabled.')
            return False

        build_cmd = cere_configure.cere_config["build_cmd"]
        run_cmd = cere_configure.cere_config["run_cmd"]

        #Build again the application to be sure we give the right binary to pprof
        try:
            logger.debug(subprocess.check_output("{0} MODE=\"original --instrument --instrument-app\" -B".format(build_cmd), stderr=subprocess.STDOUT, shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            return False

        binary = which(run_cmd)
        if not binary:
            logger.critical("Cannot find the binary. Please provide binary name through cere configure --binary")
            return False

        add_header(regions_file, new_regions_file)

        #regular expression to parse the gperf tool output
        regex_list = [r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
                  r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
                  r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,']

        cmd = subprocess.Popen("pprof -dot {0} {1}".format(binary, profile_file), shell=True, stdout=subprocess.PIPE)

        for line in cmd.stdout:
            matchObj, step = parse_line(regex_list, line)
            if step < 2 :
                add_coverage(regions_file, new_regions_file, matchObj)
            else:
                continue
        try:
            shutil.move(new_regions_file, regions_file)
        except IOError as err:
            logger.critical(str(err))
            logger.error(err.output)
            return False
    return True
Пример #24
0
def run(args):
  if not cere_configure.init():
    return False
  #Need to save this because args.force will be changed by the replay pass
  force_flags = args.force
  #Array to store best flags for every regions
  best_flags = []
  
  graph = graph_utils.load_graph()
  if graph == None:
    logger.error("CERE graph is missing. Please run cere profile")
    return False

  #Load previous best results
  best_history = {}
  if os.path.isfile("{0}/regions_flags.csv".format(var.CERE_FLAGS_PATH)) and not force_flags:
    flags_file = read_csv(open("{0}/regions_flags.csv".format(var.CERE_FLAGS_PATH), 'rb'))
    for row in flags_file:
      best_history[row["Region"]] = {"best_id":row["Id"], "best_cycles":row["Cycles"], "original_cycles":row["Original_cycles"]}

  #Use llc as backend
  os.environ["CERE_LLC"]="llc"
  #Read flags file
  flags = read_csv(args.FLAGSFILE)

  #Run the requested region
  if args.region:
    found = False
    #Load region information
    for n, d in graph.nodes(data=True):
      if d['_name'] == args.region:
        found = True
        break
    if not found:
      logger.error("{0} does not exist. Valid region name can be found in {1}`".format(args.region, cere_configure.cere_config["regions_infos"]))
      return False
    if not d['_tested']:
      logger.error("{0} not tested. Please run `cere check-matching --region={0}`".format(args.region))
      return False
    if not d['_matching']:
      logger.warning("{0} replay is marked as not matching. Results might not be accurate.".format(args.region))

    #Load any previous measures for this region
    flags_history = get_region_history(args.region, force_flags)

    #Replay region with flags to test
    best_id, best_cycles = run_replay(flags, args, flags_history)

    best_flags.append(get_best_flags(args.region, best_history, d['_invivo'], best_id, best_cycles))

  #or run selected regions from ilp selector
  elif not args.region:
    graph = graph_utils.load_graph()
    if graph == None or graph.graph['selector'] != "ILP":
      logger.error("Selected regions missing. First run cere select-ilp or choose a region manually with cere flag --region.")
      return False
    #Run selected regions
    for n, d in graph.nodes(data=True):
      if d["_selected"]:
        args.region=d['_name']

        #Load any previous measures for this region
        flags_history = get_region_history(args.region, force_flags)

        #Replay region with flags to test
        best_id, best_cycles = run_replay(flags, args, flags_history)

        best_flags.append(get_best_flags(args.region, best_history, d['_invivo'], best_id, best_cycles))

  compute_theorical_speedup(best_flags, graph)
  dump_results(best_flags, args.FLAGSFILE, flags)
  clean_environ()
  return True
Пример #25
0
def run(args):
    if not cere_configure.init():
        return False
    if not os.path.isfile("regions.csv"):
        logger.critical("Regions.csv file missing. Please run cere regions")
        return False
    region = args.region
    filename = ""
    functionname = ""
    #Find the file where the region is
    with open("regions.csv") as regions_list:
        reader = csv.DictReader(regions_list)
        for row in reader:
            if region in row["Region Name"]:
                filename = row["File Name"]
                functionname = row["Function Name"]
                break
    if args.path:
        filename = filename.rsplit('/', 1)
        filename = args.path + "/" + filename[1]
    filename = filename.replace(os.path.splitext(filename)[1], ".o")
    logger.debug("The file is {0} and the function is {1}".format(
        filename, functionname))

    #Now let's compile it in the orginal application
    if not run_shell_command("{0} MODE=original -B".format(
            cere_configure.cere_config["build_cmd"])):
        return False
    original_lines = get_nlines(filename, functionname)
    if not original_lines:
        return False
    #backup the original assembly file
    if not run_shell_command("cp cere_tmp cere_original"):
        return False

    #Compile replay mode
    #we accept that compilation fails because the dump does not have to be present.
    try:
        logger.debug(
            subprocess.check_output(
                "{0} MODE=\"replay --region={1}\" -B".format(
                    cere_configure.cere_config["build_cmd"], args.region),
                stderr=subprocess.STDOUT,
                shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.warning("If the dump is not present, skip these warnings")
    replay_lines = get_nlines(filename, "run__cere__" + region)
    if not replay_lines:
        return False
    #backup the replay assembly file
    if not run_shell_command("mv cere_tmp cere_replay"):
        return False

    err = compute_error(original_lines, replay_lines)
    if err <= args.max_error:
        logger.info(
            "Assembly matching: Original lines = {0} && replay lines = {1} (error = {2})"
            .format(original_lines, replay_lines, err))
    else:
        logger.info(
            "Assembly not matching: Original lines = {0} && replay lines = {1} (error = {2})"
            .format(original_lines, replay_lines, err))
    if args.diff_asm:
        subprocess.call("vimdiff cere_original cere_replay", shell=True)
    return True
Пример #26
0
def run(args):
    if not cere_configure.init():
        return False
    profile_file = "{0}/app.prof".format(var.CERE_PROFILE_PATH)
    regions_file = cere_configure.cere_config["regions_infos"]
    new_regions_file = "tmp.csv"

    logger.info("Removing previous regions list")
    if os.path.isfile(regions_file):
        os.remove(regions_file)

    if args.dynamic: args.static = True
    if not args.static and not args.dynamic:
        args.static = True
        args.dynamic = True

    if (args.static):
        mydump = Dump()
        res = cere_capture.run(mydump)
        logger.info("Regions list dumped in {0} file".format(regions_file))
        if not res:
            logger.warning(
                "Regions listing encountered and error: list may be incomplete or empty"
            )
    if (args.dynamic):
        if not os.path.isfile(profile_file):
            logger.critical('No profiling file. Please run cere profile')
            return False
        if not os.path.isfile(regions_file):
            logger.critical(
                'Regions file does not exists. Please re-run cere regions with static enabled.'
            )
            return False

        build_cmd = cere_configure.cere_config["build_cmd"]
        run_cmd = cere_configure.cere_config["run_cmd"]

        #Build again the application to be sure we give the right binary to pprof
        try:
            logger.debug(
                subprocess.check_output(
                    "{0} MODE=\"original --instrument --instrument-app\" -B".
                    format(build_cmd),
                    stderr=subprocess.STDOUT,
                    shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            return False

        binary = which(run_cmd)
        if not binary:
            logger.critical(
                "Cannot find the binary. Please provide binary name through cere configure --binary"
            )
            return False

        add_header(regions_file, new_regions_file)

        #regular expression to parse the gperf tool output
        regex_list = [
            r'(N.*)\s\[label\=\"(.*?)\\n([0-9]*)\s\((.*)\%\)\\rof\s(.*)\s\((.*)\%\)\\r',
            r'(N.*)\s\[label\=\"(.*)\\n([0-9]*)\s\((.*)\%\)\\r',
            r'(N.*)\s\-\>\s(N.*)\s\[label\=([0-9]*)\,'
        ]

        cmd = subprocess.Popen("pprof -dot {0} {1}".format(
            binary, profile_file),
                               shell=True,
                               stdout=subprocess.PIPE)

        for line in cmd.stdout:
            matchObj, step = parse_line(regex_list, line)
            if step < 2:
                add_coverage(regions_file, new_regions_file, matchObj)
            else:
                continue
        try:
            shutil.move(new_regions_file, regions_file)
        except IOError as err:
            logger.critical(str(err))
            logger.error(err.output)
            return False
    return True
Пример #27
0
def run(args):
  if not cere_configure.init():
    return False

  if(args.region):
    invocations = find_invocations(args.invocation, args.region)
    if not invocations:
      return False
    for invocation in invocations:
      if not args.force:
        if utils.dump_exist(args.region, invocation):
          logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation))
          return True

        if not args.no_io_trace:
          #Check IOs
          if not cere_check_io.run_io_checker(args.region, None, invocation, args.force):
            utils.mark_invalid(args.region, "Failed to run IO checker")

        if utils.is_invalid(args.region):
          logger.warning("{0} is invalid. Skipping capture".format(args.region))
          return False
      else:
        shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True)

      logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation))
      try:
        logger.debug(subprocess.check_output("{0} MODE=\"dump --region={1} --invocation={2}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation))
        utils.mark_invalid(args.region, cere_error.EDUMP)
        return False
      if not args.norun:
        logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation))
        try:
          logger.info(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
        except subprocess.CalledProcessError as err:
          #even if the capture run fails, maybe the region is dumped.
          logger.error(str(err))
          logger.error(err.output)
        if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)):
          logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation))
          utils.mark_invalid(args.region, cere_error.EDUMP)
          return False
        else:
          logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation))
  #Global dump
  else:
    logger.info("Compiling capture mode for all regions")
    try:
      logger.debug(subprocess.check_output("{0} MODE=\"dump\" -B".format(cere_configure.cere_config["build_cmd"]), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      logger.info("Compiling capture mode for all regions failed")
      return False
    if not args.norun:
      logger.info("Capturing all regions")
      try:
        logger.debug(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Capturing all regions failed")
        return False
      else:
        logger.info("All regions succesfully captured")
  return True
Пример #28
0
def run(args):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args.region) and not args.force:
        logger.warning("{0} is invalid. Skipping replay".format(args.region))
        return False

    invocations = find_invocations(args.invocation, args.region)
    if not invocations:
        return False
    if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
        logger.warning(
            "You are not using the default library. Computing predicted time\n\
                    may not work if the replay output is not the same")
    for invocation, part in invocations.iteritems():
        if os.path.isfile(
                "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region,
                                         invocation)) and not args.force:
            logger.warning(
                "Replay already measured for {0} invocation {1}.".format(
                    args.region, invocation))
            continue
        if not utils.dump_exist(args.region, invocation):
            logger.error("Memory dump is missing for {0} invocation {1}.\n\
                    Run cere capture --region={0} [--invocation={1}]".format(
                args.region, invocation))
            return False
        if args.noinstrumentation:
            instru_cmd = ""
            logger.info(
                "Compiling replay mode for region {0} invocation {1} without instrumentation"
                .format(args.region, invocation))
        else:
            instru_cmd = "--instrument"
            logger.info(
                "Compiling replay mode for region {0} invocation {1} with instrumentation"
                .format(args.region, invocation))
        try:
            logger.debug(
                subprocess.check_output(
                    "{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B"
                    .format(cere_configure.cere_config["build_cmd"],
                            args.region, invocation, instru_cmd,
                            args.plugin_instr, args.invitro_callcount),
                    stderr=subprocess.STDOUT,
                    shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error(
                "Compiling replay mode for region {0} invocation {1} Failed".
                format(args.region, invocation))
            utils.mark_invalid(args.region, cere_error.EREPLAY)
            return False
        if not args.norun:
            logger.info("Replaying invocation {1} for region {0}".format(
                args.region, invocation))
            try:
                logger.debug(
                    subprocess.check_output(
                        cere_configure.cere_config["run_cmd"],
                        stderr=subprocess.STDOUT,
                        shell=True))
            except subprocess.CalledProcessError as err:
                logger.error(str(err))
                logger.error(err.output)
                logger.error("Replay failed for {0} invocation {1}".format(
                    args.region, invocation))
                utils.mark_invalid(args.region, cere_error.EREPLAY)
                return False
            #Save replay measures files
            if PREDICTION_MODE:
                try:
                    shutil.move(
                        "{0}.csv".format(args.region),
                        "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH,
                                                 args.region, invocation))
                except IOError as err:
                    logger.error(str(err))
                    logger.error(
                        "  No results file. Maybe replay failed".format(
                            invocation))
                    return False
    if PREDICTION_MODE:
        predicted_cycles = compute_predicted_time(args.region, invocations)
        logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
    return True
Пример #29
0
#!/usr/bin/env python
import sys
import networkx as nx

sys.path.insert(0, "../../src/cere/")
from create_graph import remove_cycles, fix_self_coverage
import cere_configure
from graph_utils import plot, save_graph

if __name__ == "__main__":
    cere_configure.init()
    digraph = nx.DiGraph()
    nodes = [1, 2, 3, 4, 5, 6, 7]
    samples = 100
    digraph.add_nodes_from(nodes)
    for i in nodes:
        digraph.node[i]["_name"] = "Node_" + str(i)
        digraph.node[i]["_matching"] = True
        digraph.node[i]["_valid"] = True
        digraph.node[i]["_to_test"] = False
        digraph.node[i]["_small"] = False
        digraph.node[i]["_tested"] = False

    # Assign coverage and self_coverage
    digraph.node[1]["_self_coverage"] = 10.0
    digraph.node[1]["_coverage"] = 60.0
    digraph.node[2]["_self_coverage"] = 5.0
    digraph.node[2]["_coverage"] = 40.0
    digraph.node[3]["_self_coverage"] = 30.0
    digraph.node[3]["_coverage"] = 13.0
    digraph.node[4]["_self_coverage"] = 10.0