Beispiel #1
0
def check_arguments(args):
    """
    Check arguments
    """

    if not (args.region or args.regions_file):
        logger.error("No region specified, use at least one of the following: --region, --regions-file")
        return False

    if (args.regions_file and args.region):
        logger.error("--region and --regions-file are exclusive")
        return False

    if (args.read and not args.region):
        logger.error("--read can only be used with --region")
        return False

    if args.regions_file:
        if not os.path.isfile(args.regions_file):
            logger.error("No such file: {0}".format(args.regions_file))
            return False
        else:
            # Make regions_file path absolute
            args.regions_file = os.path.abspath(args.regions_file)

    if args.region and utils.is_invalid(args.region):
        logger.warning("{0} is invalid. Skipping trace".format(args.region))
        return False

    return True
Beispiel #2
0
def update_nodes(graph, lines, max_allowed_error):
    for line in lines:
        #for region_name, error in matching.iteritems():
        #find the node in the graph
        for n,d in graph.nodes(data=True):
            if line["Codelet Name"] == d['_name'] and not d['_tested']:
                d['_invivo'] = float(line["Invivo"])
                d['_invitro'] = float(line["Invitro"])
                d['_tested'] = True
                if float(line["Error"]) <= max_allowed_error:
                    d['_matching'] = True
                else:
                    d['_valid'] = False
                d['_error'] = float(line["Error"])
                if utils.is_invalid(d['_name']):
                    d['_error_message'] = utils.get_error_message(d['_name'])
                invocations = read_csv("{0}/invocations_error.csv".format(var.CERE_REPLAY_PATH))
                for inv in invocations:
                    if inv["Codelet Name"] == d['_name']:
                        d['_invocations'].append({"Cluster":inv["Cluster"], "Invocation":inv["Invocation"],
                          "Part":round(float(inv["Part"]), 2), "Invivo (cycles)":"{:e}".format(float(inv["Invivo"])),
                          "Invitro (cycles)":"{:e}".format(float(inv["Invitro"])), "Error (%)":round(float(inv["Error"]), 2)})
                d['_tested'] = True
                d['_to_test'] = False
    return graph
Beispiel #3
0
def run_instrument(args_region=None,
                   args_regions_file=None,
                   args_plugin_instr=var.RDTSC_WRAPPER,
                   args_invocation=0,
                   args_norun=False,
                   args_force=False):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args_region) and not args_force:
        logger.error(
            "{0} is invalid. Skipping instrumentation".format(args_region))
        return False

    if args_regions_file:
        region_input = "--regions-file={0}".format(args_regions_file)
    elif args_region:
        region_input = "--region={0}".format(args_region)
    else:
        logger.error(
            "No region specified, use at least one of the following: --region, --regions-file"
        )
        return False

    if args_invocation != 0:
        logger.info(
            "Compiling instrumentation mode for {0} invocation {1}".format(
                region_input, args_invocation))
        mode = " --invocation={0}".format(args_invocation)
    else:
        logger.info(
            "Compiling instrumentation mode for {0}".format(region_input))
        mode = ""

    try:
        logger.debug(
            subprocess.check_output(
                "{0} MODE=\"original {1} --instrument {2} --wrapper={3}\" -B".
                format(cere_configure.cere_config["build_cmd"], region_input,
                       mode, args_plugin_instr),
                stderr=subprocess.STDOUT,
                shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling instrumentation failed")
        return False

    if not args_norun:
        logger.info("Running instrumentation for {0}".format(region_input))
        try:
            logger.info(
                subprocess.check_output(cere_configure.cere_config["run_cmd"],
                                        stderr=subprocess.STDOUT,
                                        shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error("Instrumentation failed")
            return False
    return True
Beispiel #4
0
def check_arguments(args):
    """
    Check arguments
    """

    if not (args.region or args.regions_file):
        logger.error(
            "No region specified, use at least one of the following: --region, --regions-file"
        )
        return False

    if (args.regions_file and args.region):
        logger.error("--region and --regions-file are exclusive")
        return False

    if (args.read and not args.region):
        logger.error("--read can only be used with --region")
        return False

    if args.regions_file:
        if not os.path.isfile(args.regions_file):
            logger.error("No such file: {0}".format(args.regions_file))
            return False
        else:
            # Make regions_file path absolute
            args.regions_file = os.path.abspath(args.regions_file)

    if args.region and utils.is_invalid(args.region):
        logger.warning("{0} is invalid. Skipping trace".format(args.region))
        return False

    return True
Beispiel #5
0
def need_to_measure(region):
    if utils.is_invalid(region):
        return False

    if utils.trace_exists(region):
        return False

    return True
Beispiel #6
0
def need_to_measure(region):
    if utils.is_invalid(region):
        return False

    if utils.trace_exists(region):
        return False

    return True
Beispiel #7
0
def run(args):
  if not cere_configure.init():
    return False
  if utils.is_invalid(args.region) and not args.force:
    logger.warning("{0} is invalid. Skipping replay".format(args.region))
    return False

  invocations = find_invocations(args.invocation, args.region)
  if not invocations:
    return False
  if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
    logger.warning("You are not using the default library. Computing predicted time\n\
                    may not work if the replay output is not the same")
  for invocation, part in invocations.iteritems():
    if os.path.isfile("{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation)) and not args.force:
      logger.warning("Replay already measured for {0} invocation {1}.".format(args.region, invocation))
      continue
    if not utils.dump_exist(args.region, invocation):
      logger.error("Memory dump is missing for {0} invocation {1}.\n\
                    Run cere capture --region={0} [--invocation={1}]".format(args.region, invocation))
      return False
    if args.noinstrumentation:
      instru_cmd = ""
      logger.info("Compiling replay mode for region {0} invocation {1} without instrumentation".format(args.region, invocation))
    else:
      instru_cmd = "--instrument"
      logger.info("Compiling replay mode for region {0} invocation {1} with instrumentation".format(args.region, invocation))
    try:
      logger.debug(subprocess.check_output("{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation, instru_cmd, args.plugin_instr, args.invitro_callcount), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      logger.error("Compiling replay mode for region {0} invocation {1} Failed".format(args.region, invocation))
      utils.mark_invalid(args.region, cere_error.EREPLAY)
      return False
    if not args.norun:
      logger.info("Replaying invocation {1} for region {0}".format(args.region, invocation))
      try:
        logger.debug(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Replay failed for {0} invocation {1}".format(args.region, invocation))
        utils.mark_invalid(args.region, cere_error.EREPLAY)
        return False
      #Save replay measures files
      if PREDICTION_MODE:
        try:
          shutil.move("{0}.csv".format(args.region), "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region, invocation))
        except IOError as err:
          logger.error(str(err))
          logger.error("  No results file. Maybe replay failed".format(invocation))
          return False
  if PREDICTION_MODE:
    predicted_cycles = compute_predicted_time(args.region, invocations)
    logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
  return True
Beispiel #8
0
def run_instrument(args_region=None, args_regions_file=None, args_plugin_instr=var.RDTSC_WRAPPER, args_invocation=0, args_norun=False, args_force=False):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args_region) and not args_force:
        logger.error("{0} is invalid. Skipping instrumentation".format(args_region))
        return False

    if args_regions_file:
        region_input = "--regions-file={0}".format(args_regions_file)
    elif args_region:
        region_input = "--region={0}".format(args_region)
    else:
        logger.error("No region specified, use at least one of the following: --region, --regions-file")
        return False

    if args_invocation != 0 :
        logger.info("Compiling instrumentation mode for {0} invocation {1}".format(region_input, args_invocation))
        mode = " --invocation={0}".format(args_invocation)
    else:
        logger.info("Compiling instrumentation mode for {0}".format(region_input))
        mode = ""

    try:
        logger.debug(subprocess.check_output("{0} MODE=\"original {1} --instrument {2} --wrapper={3}\" -B".format(cere_configure.cere_config["build_cmd"], region_input, mode, args_plugin_instr), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling instrumentation failed")
        return False

    if not args_norun:
        logger.info("Running instrumentation for {0}".format(region_input))
        try:
            logger.info(subprocess.check_output(cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error("Instrumentation failed")
            return False
    return True
Beispiel #9
0
def update_nodes(graph, max_allowed_error):
    lines = read_csv("{0}/matching_error.csv".format(var.CERE_REPLAY_PATH))
    for line in lines:
        #for region_name, error in matching.iteritems():
        #find the node in the graph
        for n, d in graph.nodes(data=True):
            if line["Codelet Name"] == d['_name']:
                d['_invivo'] = float(line["Invivo"])
                d['_invitro'] = float(line["Invitro"])
                d['_tested'] = True
                if float(line["Error"]) <= max_allowed_error:
                    d['_matching'] = True
                else:
                    d['_valid'] = False
                d['_error'] = float(line["Error"])
                if utils.is_invalid(d['_name']):
                    d['_error_message'] = utils.get_error_message(d['_name'])
                invocations = read_csv("{0}/invocations_error.csv".format(
                    var.CERE_REPLAY_PATH))
                del d['_invocations'][:]
                for inv in invocations:
                    if inv["Codelet Name"] == d['_name']:
                        d['_invocations'].append({
                            "Cluster":
                            inv["Cluster"],
                            "Invocation":
                            inv["Invocation"],
                            "Part":
                            round(float(inv["Part"]), 2),
                            "Invivo (cycles)":
                            "{:e}".format(float(inv["Invivo"])),
                            "Invitro (cycles)":
                            "{:e}".format(float(inv["Invitro"])),
                            "Error (%)":
                            round(float(inv["Error"]), 2)
                        })
                d['_tested'] = True
                d['_to_test'] = False
    save_graph(graph)
    plot(graph)
Beispiel #10
0
def run(args):
    if not cere_configure.init():
        return False
    if utils.is_invalid(args.region) and not args.force:
        logger.warning("{0} is invalid. Skipping replay".format(args.region))
        return False

    invocations = find_invocations(args.invocation, args.region)
    if not invocations:
        return False
    if (PREDICTION_MODE and args.plugin_instr != var.RDTSC_WRAPPER):
        logger.warning(
            "You are not using the default library. Computing predicted time\n\
                    may not work if the replay output is not the same")
    for invocation, part in invocations.iteritems():
        if os.path.isfile(
                "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH, args.region,
                                         invocation)) and not args.force:
            logger.warning(
                "Replay already measured for {0} invocation {1}.".format(
                    args.region, invocation))
            continue
        if not utils.dump_exist(args.region, invocation):
            logger.error("Memory dump is missing for {0} invocation {1}.\n\
                    Run cere capture --region={0} [--invocation={1}]".format(
                args.region, invocation))
            return False
        if args.noinstrumentation:
            instru_cmd = ""
            logger.info(
                "Compiling replay mode for region {0} invocation {1} without instrumentation"
                .format(args.region, invocation))
        else:
            instru_cmd = "--instrument"
            logger.info(
                "Compiling replay mode for region {0} invocation {1} with instrumentation"
                .format(args.region, invocation))
        try:
            logger.debug(
                subprocess.check_output(
                    "{0} INVITRO_CALL_COUNT={5} MODE=\"replay --region={1} --invocation={2} {3} --wrapper={4}\" -B"
                    .format(cere_configure.cere_config["build_cmd"],
                            args.region, invocation, instru_cmd,
                            args.plugin_instr, args.invitro_callcount),
                    stderr=subprocess.STDOUT,
                    shell=True))
        except subprocess.CalledProcessError as err:
            logger.error(str(err))
            logger.error(err.output)
            logger.error(
                "Compiling replay mode for region {0} invocation {1} Failed".
                format(args.region, invocation))
            utils.mark_invalid(args.region, cere_error.EREPLAY)
            return False
        if not args.norun:
            logger.info("Replaying invocation {1} for region {0}".format(
                args.region, invocation))
            try:
                logger.debug(
                    subprocess.check_output(
                        cere_configure.cere_config["run_cmd"],
                        stderr=subprocess.STDOUT,
                        shell=True))
            except subprocess.CalledProcessError as err:
                logger.error(str(err))
                logger.error(err.output)
                logger.error("Replay failed for {0} invocation {1}".format(
                    args.region, invocation))
                utils.mark_invalid(args.region, cere_error.EREPLAY)
                return False
            #Save replay measures files
            if PREDICTION_MODE:
                try:
                    shutil.move(
                        "{0}.csv".format(args.region),
                        "{0}/{1}_{2}.csv".format(var.CERE_REPLAY_PATH,
                                                 args.region, invocation))
                except IOError as err:
                    logger.error(str(err))
                    logger.error(
                        "  No results file. Maybe replay failed".format(
                            invocation))
                    return False
    if PREDICTION_MODE:
        predicted_cycles = compute_predicted_time(args.region, invocations)
        logger.info(" Overall predicted cycles = {0}".format(predicted_cycles))
    return True
Beispiel #11
0
def run(args):
  if not cere_configure.init():
    return False

  if(args.region):
    invocations = find_invocations(args.invocation, args.region)
    if not invocations:
      return False
    for invocation in invocations:
      if not args.force:
        if utils.dump_exist(args.region, invocation):
          logger.info("Dump already exists for region {0} invocation {1}".format(args.region, invocation))
          return True

        if not args.no_io_trace:
          #Check IOs
          if not cere_check_io.run_io_checker(args.region, None, invocation, args.force):
            utils.mark_invalid(args.region, "Failed to run IO checker")

        if utils.is_invalid(args.region):
          logger.warning("{0} is invalid. Skipping capture".format(args.region))
          return False
      else:
        shutil.rmtree(os.path.join(var.CERE_DUMPS_PATH, args.region, str(invocation)), ignore_errors=True)

      logger.info("Compiling capture mode for region {0} invocation {1}".format(args.region, invocation))
      try:
        logger.debug(subprocess.check_output("{0} MODE=\"dump --region={1} --invocation={2}\" -B".format(cere_configure.cere_config["build_cmd"], args.region, invocation), stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Compiling capture mode for region {0} invocation {1} failed".format(args.region, invocation))
        utils.mark_invalid(args.region, cere_error.EDUMP)
        return False
      if not args.norun:
        logger.info("Capturing invocation {1} for region {0}".format(args.region, invocation))
        try:
          logger.info(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
        except subprocess.CalledProcessError as err:
          #even if the capture run fails, maybe the region is dumped.
          logger.error(str(err))
          logger.error(err.output)
        if not os.path.isdir("{0}/{1}/{2}".format(var.CERE_DUMPS_PATH, args.region, invocation)):
          logger.error("Capture failed for region {0} invocation {1}".format(args.region, invocation))
          utils.mark_invalid(args.region, cere_error.EDUMP)
          return False
        else:
          logger.info("Invocation {1} succesfully captured for region {0} ".format(args.region, invocation))
  #Global dump
  else:
    logger.info("Compiling capture mode for all regions")
    try:
      logger.debug(subprocess.check_output("{0} MODE=\"dump\" -B".format(cere_configure.cere_config["build_cmd"]), stderr=subprocess.STDOUT, shell=True))
    except subprocess.CalledProcessError as err:
      logger.error(str(err))
      logger.error(err.output)
      logger.info("Compiling capture mode for all regions failed")
      return False
    if not args.norun:
      logger.info("Capturing all regions")
      try:
        logger.debug(subprocess.check_output("LD_BIND_NOW=1 " + cere_configure.cere_config["run_cmd"], stderr=subprocess.STDOUT, shell=True))
      except subprocess.CalledProcessError as err:
        logger.error(str(err))
        logger.error(err.output)
        logger.error("Capturing all regions failed")
        return False
      else:
        logger.info("All regions succesfully captured")
  return True