def __init__(self, test_conf, pintool, logdir, resdir, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__couple = pintool
self.__logdir = logdir
super(TestCouple, self).__init__(*args, **kwargs)
def test_all(self, p_arity, p_type, config):
conf = Confiture("config/templates/test.yaml")
self.__pgm = conf.check_and_get("test/config/" + config)
for pgm, data in self.__pgm.items():
# Step One: execute program with arguments
cmd = "{}/{}".format(data["bin"], pgm)
self.out("Launching {0} inference on {1}".format(p_arity, cmd))
p_arity.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
self.out("Launching {0} inference on {1}".format(p_type, cmd))
p_type.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
# Step Two: parse source
# Create a parser object
src_data = Data(self.__config["data-path"], pgm)
if src_data.parse(cmd,
self.__config["lib-path"],
data["src"],
force=False,
verbose=False):
src_data.dump()
else:
src_data.load(verbose=False)
# Finally, compare source with infered results
self.__accuracy["arity"].append(
p_arity.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.__accuracy["type"].append(
p_type.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.display()
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
return
except TypeError:
self.stderr('Wrong argument(s) detected')
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get(
"config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def __init__(self, test_conf, empty, arity, typ, logdir, resdir, *args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Inlcude sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__empty = empty
self.__arity = arity
self.__type = typ
self.__logdir = logdir
super(TestAccuracy, self).__init__(*args, **kwargs)
def __init__(self, test_conf, pintool, resdir, *args, **kwargs):
self.__resdir = resdir
self.__testname = os.path.splitext(os.path.basename(test_conf))[0]
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__pintool = pintool
if "prev_pintool" in kwargs.keys():
self.__prev_pintool = kwargs["prev_pintool"]
else:
self.__prev_pintool = None
self.__prev_treated = list()
if str(pintool) == "arity":
self.__res = ArityChart(self.__resdir + "/{}_arity.res".format(self.__testname), self.__conf)
else:
self.__res = TypeChart(self.__resdir + "/{}_type.res".format(self.__testname), self.__conf)
super(TestParameter, self).__init__()
def launch_analysis(args, analysis=None):
if analysis is None:
analysis = args.analysis
# Read configuration file
tpl = Confiture("config/template.yaml")
config = tpl.check_and_get("config/config.yaml")
# Check if temporary repo exists
if not os.path.exists("tmp"):
os.makedirs("tmp")
# Extract dwarf info
SymExtractor().extract(args.pgm, "tmp/dwarf.txt")
# Create Pintool object
pintool = Pintool(
config["pin"]["path"],
config["pin"]["bin"],
config["pin"]["compile-flags"],
config["pintool"]["src"],
config["pintool"]["obj"],
)
# Log file
if args.log:
logidir = args.log
else:
logdir = default_log_dir(args.pgm)
# Compile pintool
Log.stdout("compiling pintool")
if not pintool.compile(analysis if analysis else args.analysis):
Log.stdout("compilation exited with non-zero status")
exit()
if "oracle" in analysis:
infile = "tmp/dwarf.txt"
else:
infile = None
# Launch pintool
Log.stdout("launching analysis on {}".format(args.pgm))
pintool.launch(args.pgm, args.args, infile, logdir,
"-l true" if not args.ignore_libs else None)
# Get results
for an in analysis:
Log.stdout("extracting results")
res = Result(default_log_path(logdir, args.pgm, an))
# Print the results
Log.stdout("results of inference:")
res.display()
def __init__(self,
test_conf,
pintools,
logdir,
resdir,
alt_prev=False,
*args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__memalloc = pintools["memalloc"]
self.__type = pintools["type"]
self.__logdir = logdir
self.__alt_prev = alt_prev
super(TestAlloc, self).__init__(*args, **kwargs)
def run(self, s, *args, **kwargs):
# TODO check number of args
# TODO completion on args
split = s.split(" ")
if len(split) == 1:
binary, srcdir = split[0], None
else:
binary, srcdir = split
pgm = os.path.basename(binary)
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
# Create a parser object
data = Data(config["clang"]["data-path"], pgm)
data.parse(binary, config["clang"]["lib-path"], srcdir)
data.dump()
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
#TODO explicit message (w/ pintool and binary details)
self.stderr("Pintool error")
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).mismatch()
def pintool_analysis(analysis, args):
# Read configuration file
tpl = Confiture("config/template.yaml")
config = tpl.check_and_get("config/config.yaml")
# Check if temporary repo exists
if not os.path.exists("tmp"):
os.makedirs("tmp")
# Create Pintool object
call = Pintool(
config["pin"]["path"],
config["pin"]["bin"],
config["pin"]["compile-flags"],
config["analysis"][analysis]["src"],
config["analysis"][analysis]["obj"],
)
# Log file
if args.log:
log = args.log
else:
log = default_log_path(args.pgm, analysis)
# Compile pintool
Log.stdout("compiling pintool")
if not call.compile():
Log.stdout("compilation exited with non-zero status")
exit()
# Launch pintool
Log.stdout("launching analysis on {}".format(args.pgm))
call.launch(args.pgm, args.args, None, log)
# Get results
Log.stdout("extracting results")
res = Result(log, analysis)
# Print the results
Log.stdout("results of inference:")
res.display()
user = UserSection()
class VirtualHostSection(Section):
enable_ssl = Value(Boolean(), default=False)
path = PathSection()
_meta = {'repeat': many, 'unique': True}
class MyWebserverConfiguration(Section):
daemon = Value(Boolean(), default=False)
pidfile = Value(String(), default=None)
interface = Value(String(), default='127.0.0.1:80')
interface_ssl = Value(String(), default='127.0.0.1:443')
host = VirtualHostSection()
parsed_conf = Confiture(conf, schema=MyWebserverConfiguration()).parse()
types = [
'confiture.schema.types.Number', # validates using nubmers.Number ABC, casts using float() - always
'confiture.schema.types.Integer', # validate by casting, min/max, cast using int()
'confiture.schema.types.Float', # validation by casting only, NO min/max, as per Number
'confiture.schema.types.Boolean', # validation via 'value is not True and value is not False', cast with bool()
'confiture.schema.types.String', # validates by encoding (thus unicode input, binary output), doesn't cast
'confiture.schema.types.Regex', # custom error message, re.match to validate, no cast
'confiture.schema.types.NamedRegex', # returns .groupdict() instead of match.string
'confiture.schema.types.RegexPattern', # a literal regex pattern, validates by compiling
'confiture.schema.types.IPAddress', # uses ipaddr, no cast
'confiture.schema.types.IPNetwork', # uses ipaddr, no cast
'confiture.schema.types.Url', # validation via urlparse.urlparse()
def test_empty():
confiture = Confiture('tests/yaml/template/empty.yaml')
confiture.check('tests/yaml/config/empty_valid.yaml')
confiture.check('tests/yaml/config/simple_valid.yaml')
def test_missing_nested_field():
confiture = Confiture('tests/yaml/template/nested.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/simple_fail.yaml')
def test_nested():
confiture = Confiture('tests/yaml/template/nested.yaml')
confiture.check('tests/yaml/config/nested_valid.yaml')
def test_travis_configuration():
confiture = Confiture('tests/yaml/template/travis.yaml')
confiture.check('.travis.yml')
def test_template_not_found():
with pytest.raises(ConfigFileError):
Confiture('tests/yaml/template/404.yaml')
def test_simple():
confiture = Confiture('tests/yaml/template/simple.yaml')
confiture.check('tests/yaml/config/simple_valid.yaml')
from confiture import Confiture, ConfigFileError
print("[*] loading template")
confiture = Confiture("examples/templates/confiture.yaml")
print("[*] checking required files for blueberry")
try:
confiture.check("examples/config/blueberry_ok.yaml")
print("[*] blueberry file is correct")
except ConfigFileError as e:
print(e.message)
print("[*] checking required files for banana")
try:
confiture.check("examples/config/banana_ko.yaml")
print("[*] banana file is correct")
except ConfigFileError as e:
print(e.message)
def test_config_not_found():
confiture = Confiture('tests/yaml/template/simple.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/404.yaml')
def __init__(self, config_path="config/config.yaml"):
conf = Confiture("config/templates/general.yaml")
# Parse configuration file and get result
self.__config = conf.check_and_get(config_path)
# Set the log directory
self.__logdir = self.__config["log"]["path"]
# Available pintools
self.__pintools = dict()
for pintool in self.__config["pintool"]:
# Check the needed two arguments
req = ["src", "obj"]
for r in req:
if r not in self.__config["pintool"][pintool].keys():
raise PintoolFileNotFound("{0} file not found for pintool {1}".format(r, pintool))
src = self.__config["pintool"][pintool]["src"]
obj = self.__config["pintool"][pintool]["obj"]
# Check potential extra argument
if "prev_step" in self.__config["pintool"][pintool].keys():
prev_step = self.__config["pintool"][pintool]["prev_step"]
else:
prev_step = None
if "alt_prev_step" in self.__config["pintool"][pintool].keys():
alt_prev_step = self.__config["pintool"][pintool]["alt_prev_step"]
else:
alt_prev_step = None
# Create pintool object
pintool_obj = Pintool(
name=pintool,
src_path=src,
obj_path=obj,
pinconf=self.__config["pin"],
log_dir=self.__logdir,
prev_step=prev_step,
alt_prev_step=alt_prev_step
)
self.__pintools[pintool] = pintool_obj
# # Create a test object
# # Testing options
# kwargs = dict()
# # kwargs["log"] = self.out
# kwargs["clang"] = self.config["clang"]
# self.test = ScatTest(**kwargs)
# Enable commands
self.__cmds = dict()
self.__cmds["checkconfig"] = CheckConfigCmd(
logdir=self.__logdir,
pinpath=self.__config["pin"]["bin"],
)
self.__cmds["make"] = MakeCmd(
pintools=self.__pintools,
)
self.__cmds["display"] = DisplayCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["parsedata"] = ParseDataCmd()
self.__cmds["accuracy"] = AccuracyCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["mismatch"] = MismatchCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["launch"] = LaunchCmd(
pintools=self.__pintools,
)
self.__cmds["couple"] = CoupleCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["memcomb"] = MemCombCmd(
pintools=self.__pintools,
logdir=self.__logdir
)
if "param" in self.__config["test"].keys():
self.__cmds["test"] = TestCmd(
test_conf=self.__config["test"]["desc"],
param=self.__config["test"]["param"],
pintools=self.__pintools,
logdir=self.__logdir,
resdir=self.__config["test"]["res"],
)
self.__cmds["chart"] = ChartCmd(
resdir=self.__config["test"]["res"],
conf=self.__config["test"]["param"],
)
# Link methods to scat shell
for cmd, obj in self.__cmds.items():
setattr(self.__class__, "do_" + cmd, obj.run)
setattr(self.__class__, "help_" + cmd, obj.help)
setattr(self.__class__, "complete_" + cmd, obj.complete)
# Init shell
Cmd.__init__(self)
# Read history from file
try:
with open(".history", "r+") as history_file:
line = history_file.readline()
while line != '':
self.history.append(line[:-1])
line = history_file.readline()
except IOError:
pass
self.FSCORE = c["output"]["fscore"]
self.IGNORE = c["ignore"]
if "sorted" in c["output"].keys():
self.SORTED = c["output"]["sorted"]
else:
self.SORTED = False
if "variance" in c["output"].keys():
self.VARIANCE = c["output"]["variance"]
else:
self.VARIANCE = False
if len(sys.argv) < 2:
print "Usage: {} CONFIG".format(sys.argv[0])
exit()
c = Confiture("test/tab/template.yaml").check_and_get(sys.argv[1])
CONFIG = Config(c)
class Result(object):
def __init__(self, pgm):
self.pgm = pgm
self.res = dict()
for opti in CONFIG.OPTI_LVL:
self.res[opti] = dict()
for an in CONFIG.ALS: # ["jcall", "jmp", "iCi"]:
self.res[opti][an] = dict()
for v in ["miss", "extra", "u_miss", "u_extra", "total"]:
self.res[opti][an][v] = 0
self.res[opti][an]["time"] = list()
def run(self, s, *args, **kwargs):
split = s.split(" ")
# Watch for a particular test description file
if "-t" in split:
self.__testconf = Confiture(
"config/templates/empty.yaml").check_and_get(
split.pop(split.index("-t") + 1))
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
inf, param = split[:2]
if len(split) > 2:
pgm = split[2]
else:
pgm = "test"
if inf == "arity" or inf == "type":
if param != "accuracy" and param != "scalability" and param != "overhead":
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
inp = param in [
"min_calls", "param_threshold", "min_vals", "max_vals",
"addr_threshold"
]
outp = param in [
"min_calls", "ret_threshold", "min_vals", "max_vals",
"addr_threshold"
]
if inf == "arity":
chart = ArityChart(self.__get_res(inf, param, pgm),
self.__testconf)
else:
chart = TypeChart(self.__get_res(inf, param, pgm),
self.__testconf)
if param == "accuracy":
chart.draw_accuracy(chart.get_accuracy(), "accuracy")
elif param == "variability":
chart.draw_var(chart.get_var(pgm, defaults),
"{}_var".format(pgm))
elif param == "scalability":
chart.draw_scalability(chart.get_accuracy(), "scalability")
elif param == "overhead":
chart.draw_overhead(chart.get_overhead(self.__testconf))
else:
chart.draw(chart.get(param, defaults, inp=inp, outp=outp),
pgm + "_" + param)
elif inf == "couple":
if param == "general":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_accuracy(chart.get_accuracy(), "general")
else:
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
if param == "variability":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_var(chart.get_var(pgm, defaults), "var")
else:
chart = CoupleChart("{}/{}".format(self.__resdir, param))
chart.draw(chart.get(param, defaults), param)
elif inf == "alloc":
# compute oracle
oracle = dict()
if self.__testconf is not None:
# Inlcude sub configuration files
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
for pgm, vals in self.__testconf.items():
if "oracle" in vals.keys():
oracle[pgm] = dict(vals["oracle"])
for k, v in oracle[pgm].items():
if v is not None:
oracle[pgm][k] = v.split(" ")
else:
oracle[pgm][v] = list()
if param == "couple" or param == "type":
chart = AllocChart(
oracle,
"{}/alloc_{}_general.res".format(self.__resdir, param))
chart.table()
elif param == "compare":
chart = AllocChart(
oracle,
"{}/alloc_couple_general.res".format(self.__resdir))
chart.table_cmp(
AllocChart(
oracle,
"{}/alloc_type_general.res".format(self.__resdir)))
return
elif param == "consistency":
chart = AllocChart(
oracle,
"{}/alloc_couple_consistency.res".format(self.__resdir))
chart.draw_consistency()
