Ejemplo n.º 1
0
 def test_all(self, p_arity, p_type, config):
     conf = Confiture("config/templates/test.yaml")
     self.__pgm = conf.check_and_get("test/config/" + config)
     for pgm, data in self.__pgm.items():
         # Step One: execute program with arguments
         cmd = "{}/{}".format(data["bin"], pgm)
         self.out("Launching {0} inference on {1}".format(p_arity, cmd))
         p_arity.launch(cmd,
                        data["args"].split(" ") + [" > /dev/null"],
                        verbose=False)
         self.out("Launching {0} inference on {1}".format(p_type, cmd))
         p_type.launch(cmd,
                       data["args"].split(" ") + [" > /dev/null"],
                       verbose=False)
         # Step Two: parse source
         # Create a parser object
         src_data = Data(self.__config["data-path"], pgm)
         if src_data.parse(cmd,
                           self.__config["lib-path"],
                           data["src"],
                           force=False,
                           verbose=False):
             src_data.dump()
         else:
             src_data.load(verbose=False)
         # Finally, compare source with infered results
         self.__accuracy["arity"].append(
             p_arity.get_analysis(pgm, src_data).accuracy(get=True,
                                                          verbose=False))
         self.__accuracy["type"].append(
             p_type.get_analysis(pgm, src_data).accuracy(get=True,
                                                         verbose=False))
     self.display()
Ejemplo n.º 2
0
    def run(self, s, *args, **kwargs):
        try:
            pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
        except ValueError as e:
            raise e
        except KeyError:
            self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
            return
        except TypeError:
            self.stderr('Wrong argument(s) detected')
            return

        # Check CLANG configuration
        config = Confiture("config/templates/clang.yaml").check_and_get(
            "config/config.yaml")
        try:
            data = Data(config["clang"]["data-path"], pgm)
            data.load()
        except IOError:
            data = None
        if data is None:
            self.stderr(
                "error: you must parse source code of \"{0}\" first (use parsedata)"
                .format(pgm))
            return
        pintool.get_analysis(pgm, data).accuracy()
Ejemplo n.º 3
0
    def do_mismatch(self, s):
        """
            Displays all mismatch for a given program,
            by comparison with binary and source code.

        """
        try:
            pgm, pintool = self.__get_pgm_and_inf(s)
        except ValueError:
            return
        except KeyError:
            #TODO explicit message (w/ pintool and binary details)
            self.stderr("Pintool error")

        # Check CLANG configuration
        conf = Confiture("config/templates/clang.yaml")
        conf.check("config/config.yaml")
        try:
            data = Data(self.config["clang"]["data-path"], pgm)
            data.load()
        except IOError:
            data = None
        if data is None:
            self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
            return

        pintool.get_analysis(pgm, data).mismatch()
Ejemplo n.º 4
0
    def do_accuracy(self, s):
        """
            Analyse the results of inference for a given program,
            by comparison with binary and source code.

        """
        try:
            pgm, pintool = self.__get_pgm_and_inf(s)
        except ValueError as e:
            raise e
        except KeyError:
            self.stderr("Pintool error")
            return
        # Check CLANG configuration
        conf = Confiture("config/templates/clang.yaml")
        conf.check("config/config.yaml")
        try:
            data = Data(self.config["clang"]["data-path"], pgm)
            data.load()
        except IOError:
            data = None
        if data is None:
            self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
            return
        pintool.get_analysis(pgm, data).accuracy()
Ejemplo n.º 5
0
    def __run_arity(self):
        res = AccuracyRes()
        ignored = 0
        FNULL = open(os.devnull, "w")
        prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
        for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
            if prev_res.contains(pgm):
                continue
            if param["args"] == "":
                ignored += 1
                continue
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch without PIN
            start = datetime.now()
            call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
            stop = datetime.now()
            print stop - start
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch empty
            self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch program arity
            self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            # display arity accuracy
            try:
                if param["data"].endswith("/"):
                    data = Data(param["data"], pgm)
                else:
                    datadir = param["data"][:param["data"].rfind("/") + 1]
                    pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
                    data = Data(datadir, pgmname)
                data.load()
            except IOError as e:
                self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
                continue
            # Get time of execution with PIN and no instrumentation
            with open(self.__empty.get_logfile(pgm, prev=False)) as f:
                empty_time = float(f.read())
            ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False), data)
            res.add(ar.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "arity.res", empty_time=empty_time, no_pin_time=stop - start), pgm=pgm, verbose=True)

        print res
        print "IGNORED: {}".format(ignored)
Ejemplo n.º 6
0
    def __run_type(self):
        res = AccuracyRes()
        ignored = 0
        FNULL = open(os.devnull, "w")
        prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
        # used to get times of execution with no instrumentation
        arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
        for pgm, param in OrderedDict(
                sorted(self.__conf.items(), key=lambda a: a[0])).items():
            if prev_res.contains(pgm):
                continue
            # launch program type
            if param["args"] == "":
                ignored += 1
                continue
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            self.__type.launch(
                param["bin"],
                [param["args"], "1>/dev/null"])  #, verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            try:
                if param["data"].endswith("/"):
                    data = Data(param["data"], pgm)
                else:
                    datadir = param["data"][:param["data"].rfind("/") + 1]
                    pgmname = param["data"][param["data"].rfind("/") +
                                            1:param["data"].rfind(".")]
                    data = Data(datadir, pgmname)
                data.load()
            except IOError:
                self.stderr(
                    "error: you must parse source code of \"{0}\" first (use parsedata)"
                    .format(pgm))
                continue
            # Get times of execution with no instrumentation
            empty_time = arity_res.get_one(pgm).empty_time
            nopin_time = arity_res.get_one(pgm).nopin_time
            ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False),
                              data)
            res.add(ty.accuracy(get=True,
                                verbose=False,
                                log=self.__resdir + "/" + "type.res",
                                empty_time=empty_time,
                                nopin_time=nopin_time),
                    pgm=pgm)

        print res
        print "IGNORED: {}".format(ignored)
Ejemplo n.º 7
0
 def run(self, s, *args, **kwargs):
     try:
         pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
     except ValueError as e:
         raise e
     except KeyError:
          #TODO explicit message (w/ pintool and binary details)
         self.stderr("Pintool error")
         return
     # Check CLANG configuration
     config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
     try:
         data = Data(config["clang"]["data-path"], pgm)
         data.load()
     except IOError:
         data = None
     if data is None:
         self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
         return
     pintool.get_analysis(pgm, data).mismatch()
Ejemplo n.º 8
0
 def test_all(self, p_arity, p_type, config):
     conf = Confiture("config/templates/test.yaml")
     self.__pgm = conf.check_and_get("test/config/" + config)
     for pgm, data in self.__pgm.items():
         # Step One: execute program with arguments
         cmd = "{}/{}".format(data["bin"], pgm)
         self.out("Launching {0} inference on {1}".format(p_arity, cmd))
         p_arity.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
         self.out("Launching {0} inference on {1}".format(p_type, cmd))
         p_type.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
         # Step Two: parse source
         # Create a parser object
         src_data = Data(self.__config["data-path"], pgm)
         if src_data.parse(cmd, self.__config["lib-path"], data["src"], force=False, verbose=False):
             src_data.dump()
         else:
             src_data.load(verbose=False)
         # Finally, compare source with infered results
         self.__accuracy["arity"].append(p_arity.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
         self.__accuracy["type"].append(p_type.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
     self.display()
Ejemplo n.º 9
0
    def __run_one(self, params):
        res = AccuracyRes()
        FNULL = open(os.devnull, "w")
        for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
            if self.__res.contains(pgm, params):
                continue
            if self.__prev_pintool is not None and pgm not in self.__prev_treated:
                if "pre" in param.keys():
                    call(param["pre"], stdout=FNULL, shell=True)
                self.__prev_pintool.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
                if "post" in param.keys():
                    call(param["post"], stdout=FNULL, shell=True)
                self.__prev_treated.append(pgm)
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch program arity
            if param["args"] == "":
                continue
            self.__pintool.launch(param["bin"], [param["args"], "1>/dev/null"], params, verbose=True)
            # display arity accuracy
            try:
                if param["data"].endswith("/"):
                    data = Data(param["data"], pgm)
                else:
                    datadir = param["data"][:param["data"].rfind("/") + 1]
                    pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
                    data = Data(datadir, pgmname)
                data.load()
            except IOError:
                self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
                continue
            if str(self.__pintool) == "arity":
                an = ArityAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
            else:
                an = TypeAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
            res.add(an.accuracy(get=True, verbose=False, log=self.__resdir + "/" + str(self.__testname) + "_" + str(self.__pintool) + ".res"), pgm=pgm, verbose=True)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)

        print res
Ejemplo n.º 10
0
    def __run_type(self):
        res = AccuracyRes()
        ignored = 0
        FNULL = open(os.devnull, "w")
        prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
        # used to get times of execution with no instrumentation
        arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
        for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
            if prev_res.contains(pgm):
                continue
            # launch program type
            if param["args"] == "":
                ignored += 1
                continue
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            self.__type.launch(param["bin"], [param["args"], "1>/dev/null"])#, verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            try:
                if param["data"].endswith("/"):
                    data = Data(param["data"], pgm)
                else:
                    datadir = param["data"][:param["data"].rfind("/") + 1]
                    pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
                    data = Data(datadir, pgmname)
                data.load()
            except IOError:
                self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
                continue
            # Get times of execution with no instrumentation
            empty_time = arity_res.get_one(pgm).empty_time
            nopin_time = arity_res.get_one(pgm).nopin_time
            ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False), data)
            res.add(ty.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "type.res", empty_time=empty_time, nopin_time=nopin_time), pgm=pgm)

        print res
        print "IGNORED: {}".format(ignored)
Ejemplo n.º 11
0
    def run(self, s, *args, **kwargs):
        try:
            pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
        except ValueError as e:
            raise e
        except KeyError:
            self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
            return
        except TypeError:
            self.stderr('Wrong argument(s) detected')
            return

        # Check CLANG configuration
        config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
        try:
            data = Data(config["clang"]["data-path"], pgm)
            data.load()
        except IOError:
            data = None
        if data is None:
            self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
            return
        pintool.get_analysis(pgm, data).accuracy()
Ejemplo n.º 12
0
    def __run_arity(self):
        res = AccuracyRes()
        ignored = 0
        FNULL = open(os.devnull, "w")
        prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
        for pgm, param in OrderedDict(
                sorted(self.__conf.items(), key=lambda a: a[0])).items():
            if prev_res.contains(pgm):
                continue
            if param["args"] == "":
                ignored += 1
                continue
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch without PIN
            start = datetime.now()
            call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
            stop = datetime.now()
            print stop - start
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch empty
            self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"],
                                verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            if "pre" in param.keys():
                call(param["pre"], stdout=FNULL, shell=True)
            # launch program arity
            self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"],
                                verbose=False)
            if "post" in param.keys():
                call(param["post"], stdout=FNULL, shell=True)
            # display arity accuracy
            try:
                if param["data"].endswith("/"):
                    data = Data(param["data"], pgm)
                else:
                    datadir = param["data"][:param["data"].rfind("/") + 1]
                    pgmname = param["data"][param["data"].rfind("/") +
                                            1:param["data"].rfind(".")]
                    data = Data(datadir, pgmname)
                data.load()
            except IOError as e:
                self.stderr(
                    "error: you must parse source code of \"{0}\" first (use parsedata)"
                    .format(pgm))
                continue
            # Get time of execution with PIN and no instrumentation
            with open(self.__empty.get_logfile(pgm, prev=False)) as f:
                empty_time = float(f.read())
            ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False),
                               data)
            res.add(ar.accuracy(get=True,
                                verbose=False,
                                log=self.__resdir + "/" + "arity.res",
                                empty_time=empty_time,
                                no_pin_time=stop - start),
                    pgm=pgm,
                    verbose=True)

        print res
        print "IGNORED: {}".format(ignored)