def do_mismatch(self, s):
"""
Displays all mismatch for a given program,
by comparison with binary and source code.
"""
try:
pgm, pintool = self.__get_pgm_and_inf(s)
except ValueError:
return
except KeyError:
#TODO explicit message (w/ pintool and binary details)
self.stderr("Pintool error")
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
try:
data = Data(self.config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).mismatch()
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
return
except TypeError:
self.stderr('Wrong argument(s) detected')
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get(
"config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def do_accuracy(self, s):
"""
Analyse the results of inference for a given program,
by comparison with binary and source code.
"""
try:
pgm, pintool = self.__get_pgm_and_inf(s)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool error")
return
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
try:
data = Data(self.config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def __run_arity(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch without PIN
start = datetime.now()
call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
stop = datetime.now()
print stop - start
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch empty
self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError as e:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
# Get time of execution with PIN and no instrumentation
with open(self.__empty.get_logfile(pgm, prev=False)) as f:
empty_time = float(f.read())
ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False), data)
res.add(ar.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "arity.res", empty_time=empty_time, no_pin_time=stop - start), pgm=pgm, verbose=True)
print res
print "IGNORED: {}".format(ignored)
def test_all(self, p_arity, p_type, config):
conf = Confiture("config/templates/test.yaml")
self.__pgm = conf.check_and_get("test/config/" + config)
for pgm, data in self.__pgm.items():
# Step One: execute program with arguments
cmd = "{}/{}".format(data["bin"], pgm)
self.out("Launching {0} inference on {1}".format(p_arity, cmd))
p_arity.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
self.out("Launching {0} inference on {1}".format(p_type, cmd))
p_type.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
# Step Two: parse source
# Create a parser object
src_data = Data(self.__config["data-path"], pgm)
if src_data.parse(cmd,
self.__config["lib-path"],
data["src"],
force=False,
verbose=False):
src_data.dump()
else:
src_data.load(verbose=False)
# Finally, compare source with infered results
self.__accuracy["arity"].append(
p_arity.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.__accuracy["type"].append(
p_type.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.display()
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
#TODO explicit message (w/ pintool and binary details)
self.stderr("Pintool error")
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).mismatch()
def do_parsedata(self, s):
"""
Parse source code to test inference results
"""
# TODO check number of args
# TODO completion on args
split = s.split(" ")
if len(split) == 1:
binary, srcdir = split[0], None
else:
binary, srcdir = split
pgm = os.path.basename(binary)
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
# Create a parser object
data = Data(self.config["clang"]["data-path"], pgm)
data.parse(binary, self.config["clang"]["lib-path"], srcdir)
data.dump()
def __run_type(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
# used to get times of execution with no instrumentation
arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm):
continue
# launch program type
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__type.launch(param["bin"], [param["args"], "1>/dev/null"])#, verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
# Get times of execution with no instrumentation
empty_time = arity_res.get_one(pgm).empty_time
nopin_time = arity_res.get_one(pgm).nopin_time
ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False), data)
res.add(ty.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "type.res", empty_time=empty_time, nopin_time=nopin_time), pgm=pgm)
print res
print "IGNORED: {}".format(ignored)
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
return
except TypeError:
self.stderr('Wrong argument(s) detected')
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def __run_type(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
# used to get times of execution with no instrumentation
arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(
sorted(self.__conf.items(), key=lambda a: a[0])).items():
if prev_res.contains(pgm):
continue
# launch program type
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__type.launch(
param["bin"],
[param["args"], "1>/dev/null"]) #, verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/") +
1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
continue
# Get times of execution with no instrumentation
empty_time = arity_res.get_one(pgm).empty_time
nopin_time = arity_res.get_one(pgm).nopin_time
ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False),
data)
res.add(ty.accuracy(get=True,
verbose=False,
log=self.__resdir + "/" + "type.res",
empty_time=empty_time,
nopin_time=nopin_time),
pgm=pgm)
print res
print "IGNORED: {}".format(ignored)
def run(self, s, *args, **kwargs):
# TODO check number of args
# TODO completion on args
split = s.split(" ")
if len(split) == 1:
binary, srcdir = split[0], None
else:
binary, srcdir = split
pgm = os.path.basename(binary)
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
# Create a parser object
data = Data(config["clang"]["data-path"], pgm)
data.parse(binary, config["clang"]["lib-path"], srcdir)
data.dump()
def test_all(self, p_arity, p_type, config):
conf = Confiture("config/templates/test.yaml")
self.__pgm = conf.check_and_get("test/config/" + config)
for pgm, data in self.__pgm.items():
# Step One: execute program with arguments
cmd = "{}/{}".format(data["bin"], pgm)
self.out("Launching {0} inference on {1}".format(p_arity, cmd))
p_arity.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
self.out("Launching {0} inference on {1}".format(p_type, cmd))
p_type.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
# Step Two: parse source
# Create a parser object
src_data = Data(self.__config["data-path"], pgm)
if src_data.parse(cmd, self.__config["lib-path"], data["src"], force=False, verbose=False):
src_data.dump()
else:
src_data.load(verbose=False)
# Finally, compare source with infered results
self.__accuracy["arity"].append(p_arity.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
self.__accuracy["type"].append(p_type.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
self.display()
def __run_one(self, params):
res = AccuracyRes()
FNULL = open(os.devnull, "w")
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if self.__res.contains(pgm, params):
continue
if self.__prev_pintool is not None and pgm not in self.__prev_treated:
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__prev_pintool.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
self.__prev_treated.append(pgm)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
if param["args"] == "":
continue
self.__pintool.launch(param["bin"], [param["args"], "1>/dev/null"], params, verbose=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
if str(self.__pintool) == "arity":
an = ArityAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
else:
an = TypeAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
res.add(an.accuracy(get=True, verbose=False, log=self.__resdir + "/" + str(self.__testname) + "_" + str(self.__pintool) + ".res"), pgm=pgm, verbose=True)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
print res
def __run_arity(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(
sorted(self.__conf.items(), key=lambda a: a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch without PIN
start = datetime.now()
call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
stop = datetime.now()
print stop - start
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch empty
self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"],
verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"],
verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/") +
1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError as e:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
continue
# Get time of execution with PIN and no instrumentation
with open(self.__empty.get_logfile(pgm, prev=False)) as f:
empty_time = float(f.read())
ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False),
data)
res.add(ar.accuracy(get=True,
verbose=False,
log=self.__resdir + "/" + "arity.res",
empty_time=empty_time,
no_pin_time=stop - start),
pgm=pgm,
verbose=True)
print res
print "IGNORED: {}".format(ignored)