def test_all(self, p_arity, p_type, config):
conf = Confiture("config/templates/test.yaml")
self.__pgm = conf.check_and_get("test/config/" + config)
for pgm, data in self.__pgm.items():
# Step One: execute program with arguments
cmd = "{}/{}".format(data["bin"], pgm)
self.out("Launching {0} inference on {1}".format(p_arity, cmd))
p_arity.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
self.out("Launching {0} inference on {1}".format(p_type, cmd))
p_type.launch(cmd,
data["args"].split(" ") + [" > /dev/null"],
verbose=False)
# Step Two: parse source
# Create a parser object
src_data = Data(self.__config["data-path"], pgm)
if src_data.parse(cmd,
self.__config["lib-path"],
data["src"],
force=False,
verbose=False):
src_data.dump()
else:
src_data.load(verbose=False)
# Finally, compare source with infered results
self.__accuracy["arity"].append(
p_arity.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.__accuracy["type"].append(
p_type.get_analysis(pgm, src_data).accuracy(get=True,
verbose=False))
self.display()
def do_accuracy(self, s):
"""
Analyse the results of inference for a given program,
by comparison with binary and source code.
"""
try:
pgm, pintool = self.__get_pgm_and_inf(s)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool error")
return
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
try:
data = Data(self.config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def do_mismatch(self, s):
"""
Displays all mismatch for a given program,
by comparison with binary and source code.
"""
try:
pgm, pintool = self.__get_pgm_and_inf(s)
except ValueError:
return
except KeyError:
#TODO explicit message (w/ pintool and binary details)
self.stderr("Pintool error")
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
try:
data = Data(self.config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).mismatch()
def parse_string(self, string):
if not string.endswith('\n'):
string += '\n'
try:
dc = Confiture(string)
return Conf(dc.parse())
except ParsingError as e:
raise ValueError('At {0.position}, Cannot parse string, {0}'.format(e))
def __init__(self, test_conf, pintool, logdir, resdir, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__couple = pintool
self.__logdir = logdir
super(TestCouple, self).__init__(*args, **kwargs)
def launch_analysis(args, analysis=None):
if analysis is None:
analysis = args.analysis
# Read configuration file
tpl = Confiture("config/template.yaml")
config = tpl.check_and_get("config/config.yaml")
# Check if temporary repo exists
if not os.path.exists("tmp"):
os.makedirs("tmp")
# Extract dwarf info
SymExtractor().extract(args.pgm, "tmp/dwarf.txt")
# Create Pintool object
pintool = Pintool(
config["pin"]["path"],
config["pin"]["bin"],
config["pin"]["compile-flags"],
config["pintool"]["src"],
config["pintool"]["obj"],
)
# Log file
if args.log:
logidir = args.log
else:
logdir = default_log_dir(args.pgm)
# Compile pintool
Log.stdout("compiling pintool")
if not pintool.compile(analysis if analysis else args.analysis):
Log.stdout("compilation exited with non-zero status")
exit()
if "oracle" in analysis:
infile = "tmp/dwarf.txt"
else:
infile = None
# Launch pintool
Log.stdout("launching analysis on {}".format(args.pgm))
pintool.launch(args.pgm, args.args, infile, logdir,
"-l true" if not args.ignore_libs else None)
# Get results
for an in analysis:
Log.stdout("extracting results")
res = Result(default_log_path(logdir, args.pgm, an))
# Print the results
Log.stdout("results of inference:")
res.display()
def do_parsedata(self, s):
"""
Parse source code to test inference results
"""
# TODO check number of args
# TODO completion on args
pgm, srcdir = s.split(" ")
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
# Create a parser object
parser = SourceParser(self.config["clang"]["lib-path"], pgm, self.config["clang"]["data-path"], srcdir)
parser.parse()
def __init__(self, config_path="config/config.yaml"):
self.config_path = config_path
conf = Confiture("config/templates/general.yaml")
# Parse configuration file and get result
self.config = conf.check_and_get(config_path)
# Before we checked the config, it is considered KO
self.config_ok = False
# Set the log directory
self.log_dir = self.config["log"]["path"]
# Available pintools
self.__pintools = dict()
for pintool in self.config["pintool"]:
# Check the needed two arguments
req = ["src", "obj"]
for r in req:
if r not in self.config["pintool"][pintool].keys():
#TODO
raise Exception
src = self.config["pintool"][pintool]["src"]
obj = self.config["pintool"][pintool]["obj"]
# Check potential extra argument
if "prev_step" in self.config["pintool"][pintool].keys():
prev_step = self.config["pintool"][pintool]["prev_step"]
else:
prev_step = None
# Create pintool object
pintool_obj = Pintool(
name=pintool,
src_path=src,
obj_path=obj,
pinconf=self.config["pin"],
stdout=self.out,
stderr=self.out,
log_dir=self.log_dir,
prev_step=prev_step,
)
self.__pintools[pintool] = pintool_obj
# Create a test object
# Testing options
kwargs = dict()
kwargs["log"] = self.out
if "test" in self.config.keys() and "proto" in self.config["test"]:
kwargs["proto"] = self.config["test"]["proto"]
self.test = ScatTest(**kwargs)
# Init shell
Cmd.__init__(self, completekey='tab')
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
self.stderr("Pintool \"{}\" not found".format(s.split()[1]))
return
except TypeError:
self.stderr('Wrong argument(s) detected')
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get(
"config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
return
pintool.get_analysis(pgm, data).accuracy()
def __init__(self, test_conf, empty, arity, typ, logdir, resdir, *args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Inlcude sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__empty = empty
self.__arity = arity
self.__type = typ
self.__logdir = logdir
super(TestAccuracy, self).__init__(*args, **kwargs)
def __init__(self, config_path="config/config.yaml"):
self.config_path = config_path
conf = Confiture("config/templates/general.yaml")
# Parse configuration file and get result
self.config = conf.check_and_get(config_path)
# Before we checked the config, it is considered KO
self.config_ok = False
# Set the log directory
self.log_dir = self.config["log"]["path"]
# Create a result object
self.res = Result(self.log_dir)
# Create a pin object with pin executable path
# Get CLI options for pin
if "cli-options" in self.config["pin"].keys():
cli_options = self.config["pin"]["cli-options"]
else:
cli_options = ""
# Get function identification method
if "function-mode" in self.config.keys():
fn_mode = self.config["function-mode"]
else:
# For now, default is identifying functions by name
fn_mode = "name"
# Get information from config file for all needed pintools
kwargs = dict()
for inf_code, inf_name in INF_ALL:
if inf_code == INF_BASE:
continue
kwargs["{0}_src".format(inf_name)] = self.config["pin"]["pintool-src"][inf_name]
kwargs["{0}_obj".format(inf_name)] = self.config["pin"]["pintool-obj"][inf_name]
# Other Pin configuration options
kwargs["pinpath"] = self.config["pin"]["path"]
kwargs["options"] = cli_options
kwargs["log"] = self.out
kwargs["fn_mode"] = fn_mode
kwargs["pinbin"] = self.config["pin"]["bin"]
kwargs["respath"] = self.config["res"]["path"]
# Init Pin
self.__pin = Pin(**kwargs)
# Init shell
Cmd.__init__(self, completekey='tab')
def test_all(self, p_arity, p_type, config):
conf = Confiture("config/templates/test.yaml")
self.__pgm = conf.check_and_get("test/config/" + config)
for pgm, data in self.__pgm.items():
# Step One: execute program with arguments
cmd = "{}/{}".format(data["bin"], pgm)
self.out("Launching {0} inference on {1}".format(p_arity, cmd))
p_arity.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
self.out("Launching {0} inference on {1}".format(p_type, cmd))
p_type.launch(cmd, data["args"].split(" ") + [" > /dev/null"], verbose=False)
# Step Two: parse source
# Create a parser object
src_data = Data(self.__config["data-path"], pgm)
if src_data.parse(cmd, self.__config["lib-path"], data["src"], force=False, verbose=False):
src_data.dump()
else:
src_data.load(verbose=False)
# Finally, compare source with infered results
self.__accuracy["arity"].append(p_arity.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
self.__accuracy["type"].append(p_type.get_analysis(pgm, src_data).accuracy(get=True, verbose=False))
self.display()
def __init__(self, test_conf, pintool, resdir, *args, **kwargs):
self.__resdir = resdir
self.__testname = os.path.splitext(os.path.basename(test_conf))[0]
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__pintool = pintool
if "prev_pintool" in kwargs.keys():
self.__prev_pintool = kwargs["prev_pintool"]
else:
self.__prev_pintool = None
self.__prev_treated = list()
if str(pintool) == "arity":
self.__res = ArityChart(self.__resdir + "/{}_arity.res".format(self.__testname), self.__conf)
else:
self.__res = TypeChart(self.__resdir + "/{}_type.res".format(self.__testname), self.__conf)
super(TestParameter, self).__init__()
def do_parsedata(self, s):
"""
Parse source code to test inference results
"""
# TODO check number of args
# TODO completion on args
split = s.split(" ")
if len(split) == 1:
binary, srcdir = split[0], None
else:
binary, srcdir = split
pgm = os.path.basename(binary)
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
# Create a parser object
data = Data(self.config["clang"]["data-path"], pgm)
data.parse(binary, self.config["clang"]["lib-path"], srcdir)
data.dump()
def pintool_analysis(analysis, args):
# Read configuration file
tpl = Confiture("config/template.yaml")
config = tpl.check_and_get("config/config.yaml")
# Check if temporary repo exists
if not os.path.exists("tmp"):
os.makedirs("tmp")
# Create Pintool object
call = Pintool(
config["pin"]["path"],
config["pin"]["bin"],
config["pin"]["compile-flags"],
config["analysis"][analysis]["src"],
config["analysis"][analysis]["obj"],
)
# Log file
if args.log:
log = args.log
else:
log = default_log_path(args.pgm, analysis)
# Compile pintool
Log.stdout("compiling pintool")
if not call.compile():
Log.stdout("compilation exited with non-zero status")
exit()
# Launch pintool
Log.stdout("launching analysis on {}".format(args.pgm))
call.launch(args.pgm, args.args, None, log)
# Get results
Log.stdout("extracting results")
res = Result(log, analysis)
# Print the results
Log.stdout("results of inference:")
res.display()
def get_configuration():
"""
Get configuration from quart_config.init and CLI arguments.
:return: confiture object
"""
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--config_path', default='etc/quart_config.conf')
schema = Config()
schema.populate_argparse(arg_parser)
args = arg_parser.parse_args()
config = Confiture.from_filename(args.config_path, schema=schema).\
parse()
return config
def get_configuration():
"""
Get configuration from quart_config.init and CLI arguments.
:return: confiture object
"""
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--config_path', default='etc/quart_config.conf')
schema = Config()
schema.populate_argparse(arg_parser)
args = arg_parser.parse_args()
config = Confiture.from_filename(args.config_path, schema=schema).\
parse()
return config
def __init__(self, test_conf, empty, arity, typ, logdir, resdir, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Inlcude sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__empty = empty
self.__arity = arity
self.__type = typ
self.__logdir = logdir
super(TestAccuracy, self).__init__(*args, **kwargs)
def __init__(self,
test_conf,
pintools,
logdir,
resdir,
alt_prev=False,
*args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__memalloc = pintools["memalloc"]
self.__type = pintools["type"]
self.__logdir = logdir
self.__alt_prev = alt_prev
super(TestAlloc, self).__init__(*args, **kwargs)
def do_mismatch(self, s):
"""
Displays all mismatch for a given program,
by comparison with source code.
"""
try:
pgm, inf = self.__get_pgm_and_inf(s)
except ValueError:
return
inputfile = self.__get_inputfile(inf, pgm)
# Check CLANG configuration
conf = Confiture("config/templates/clang.yaml")
conf.check("config/config.yaml")
try:
data = SourceParser(self.config["clang"]["lib-path"], pgm, self.config["clang"]["data-path"]).load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
self.res.mismatch(pgm, inf, inputfile, data)
def __init__(self, test_conf, pintools, logdir, resdir,
alt_prev=False, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__memalloc = pintools["memalloc"]
self.__type = pintools["type"]
self.__logdir = logdir
self.__alt_prev = alt_prev
super(TestAlloc, self).__init__(*args, **kwargs)
def run(self, s, *args, **kwargs):
# TODO check number of args
# TODO completion on args
split = s.split(" ")
if len(split) == 1:
binary, srcdir = split[0], None
else:
binary, srcdir = split
pgm = os.path.basename(binary)
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
# Create a parser object
data = Data(config["clang"]["data-path"], pgm)
data.parse(binary, config["clang"]["lib-path"], srcdir)
data.dump()
class TestCouple(object):
def __init__(self, test_conf, pintool, logdir, resdir, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__couple = pintool
self.__logdir = logdir
super(TestCouple, self).__init__(*args, **kwargs)
def run(self, params=None, logname="couple_general.res"):
FNULL = open(os.devnull, "w")
prev_res = CoupleChart(self.__resdir + "/" + logname)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm, params):
continue
if param["args"] == "":
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
if params is not None and "rho" in params.keys():
min_rho = params.pop("rho")
else:
min_rho = 0.8
if params is not None and "min_vals" in params.keys():
min_vals = params.pop("min_vals")
else:
min_vals = 50
# launch program couple
self.__couple.launch(param["bin"], [param["args"], "1>/dev/null"], params=params)#, verbose=False)
logfile = self.__couple.get_logfile(pgm, prev=False)
# launch offline computation of couples
res = Couple(logfile, pgm, verbose=False).run(get=True, log=self.__resdir + "/" + logname, min_rho=min_rho, min_vals=min_vals)
print "{} | functions: {} - #f: {} - #g: {} - #couples: {}".format(pgm, *res)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
return
def run_params(self, params, name):
param = filter(lambda a: isinstance(a[1], list), [(k, v) for k, v in params.items()])[0][0]
vals = params.pop(param)
for val in vals:
p = dict()
p[param] = val
p.update(params)
self.run(p, name)
def run(self, s, *args, **kwargs):
try:
pgm, pintool = get_pgm_and_inf(s, self.__pintools, self.__logdir)
except ValueError as e:
raise e
except KeyError:
#TODO explicit message (w/ pintool and binary details)
self.stderr("Pintool error")
return
# Check CLANG configuration
config = Confiture("config/templates/clang.yaml").check_and_get("config/config.yaml")
try:
data = Data(config["clang"]["data-path"], pgm)
data.load()
except IOError:
data = None
if data is None:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
return
pintool.get_analysis(pgm, data).mismatch()
class ChartCmd(ICommand):
"""
usage: chart <analysis> <parameter> [program]
Non-optional arguments:
analysis: the analysis for which you want to get results (either arity or type)
parameter: the parameter to vary (e.g. min_calls)
Optional arguments:
program: program you want to analyse (by default, it gathers all available results)
Note that if param = "accuracy", it outputs data for every program with default parameter values.
"""
def __init__(self, resdir, conf, *args, **kwargs):
self.__resdir = resdir
self.__conf = conf
self.__testconf = None
super(ChartCmd, self).__init__(*args, **kwargs)
def __get_res(self, inf="", param="", pgm=""):
if inf == "arity" or inf == "type":
return "{}/{}.res".format(self.__resdir, inf)
elif inf == "couple":
return "{}/{}_{}.res".format(self.__resdir, inf, "general")
else:
return "{}/{}_{}.res".format(self.__resdir, pgm, inf)
def run(self, s, *args, **kwargs):
split = s.split(" ")
# Watch for a particular test description file
if "-t" in split:
self.__testconf = Confiture(
"config/templates/empty.yaml").check_and_get(
split.pop(split.index("-t") + 1))
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
inf, param = split[:2]
if len(split) > 2:
pgm = split[2]
else:
pgm = "test"
if inf == "arity" or inf == "type":
if param != "accuracy" and param != "scalability" and param != "overhead":
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
inp = param in [
"min_calls", "param_threshold", "min_vals", "max_vals",
"addr_threshold"
]
outp = param in [
"min_calls", "ret_threshold", "min_vals", "max_vals",
"addr_threshold"
]
if inf == "arity":
chart = ArityChart(self.__get_res(inf, param, pgm),
self.__testconf)
else:
chart = TypeChart(self.__get_res(inf, param, pgm),
self.__testconf)
if param == "accuracy":
chart.draw_accuracy(chart.get_accuracy(), "accuracy")
elif param == "variability":
chart.draw_var(chart.get_var(pgm, defaults),
"{}_var".format(pgm))
elif param == "scalability":
chart.draw_scalability(chart.get_accuracy(), "scalability")
elif param == "overhead":
chart.draw_overhead(chart.get_overhead(self.__testconf))
else:
chart.draw(chart.get(param, defaults, inp=inp, outp=outp),
pgm + "_" + param)
elif inf == "couple":
if param == "general":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_accuracy(chart.get_accuracy(), "general")
else:
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
if param == "variability":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_var(chart.get_var(pgm, defaults), "var")
else:
chart = CoupleChart("{}/{}".format(self.__resdir, param))
chart.draw(chart.get(param, defaults), param)
elif inf == "alloc":
# compute oracle
oracle = dict()
if self.__testconf is not None:
# Inlcude sub configuration files
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
for pgm, vals in self.__testconf.items():
if "oracle" in vals.keys():
oracle[pgm] = dict(vals["oracle"])
for k, v in oracle[pgm].items():
if v is not None:
oracle[pgm][k] = v.split(" ")
else:
oracle[pgm][v] = list()
if param == "couple" or param == "type":
chart = AllocChart(
oracle,
"{}/alloc_{}_general.res".format(self.__resdir, param))
chart.table()
elif param == "compare":
chart = AllocChart(
oracle,
"{}/alloc_couple_general.res".format(self.__resdir))
chart.table_cmp(
AllocChart(
oracle,
"{}/alloc_type_general.res".format(self.__resdir)))
return
elif param == "consistency":
chart = AllocChart(
oracle,
"{}/alloc_couple_consistency.res".format(self.__resdir))
chart.draw_consistency()
class TestAccuracy(Std):
def __init__(self, test_conf, empty, arity, typ, logdir, resdir, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Inlcude sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__empty = empty
self.__arity = arity
self.__type = typ
self.__logdir = logdir
super(TestAccuracy, self).__init__(*args, **kwargs)
def __run_arity(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch without PIN
start = datetime.now()
call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
stop = datetime.now()
print stop - start
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch empty
self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError as e:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
# Get time of execution with PIN and no instrumentation
with open(self.__empty.get_logfile(pgm, prev=False)) as f:
empty_time = float(f.read())
ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False), data)
res.add(ar.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "arity.res", empty_time=empty_time, no_pin_time=stop - start), pgm=pgm, verbose=True)
print res
print "IGNORED: {}".format(ignored)
def __run_type(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
# used to get times of execution with no instrumentation
arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm):
continue
# launch program type
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__type.launch(param["bin"], [param["args"], "1>/dev/null"])#, verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
# Get times of execution with no instrumentation
empty_time = arity_res.get_one(pgm).empty_time
nopin_time = arity_res.get_one(pgm).nopin_time
ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False), data)
res.add(ty.accuracy(get=True, verbose=False, log=self.__resdir + "/" + "type.res", empty_time=empty_time, nopin_time=nopin_time), pgm=pgm)
print res
print "IGNORED: {}".format(ignored)
def run(self, subcommand=None):
if subcommand is None or subcommand == "arity":
self.__run_arity()
if subcommand is None or subcommand == "type":
self.__run_type()
def test_empty():
confiture = Confiture("tests/yaml/template/empty.yaml")
confiture.check("tests/yaml/config/empty_valid.yaml")
confiture.check("tests/yaml/config/simple_valid.yaml")
def test_missing_nested_field():
confiture = Confiture('tests/yaml/template/nested.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/simple_fail.yaml')
class TestAlloc(object):
def __init__(self, test_conf, pintools, logdir, resdir,
alt_prev=False, *args, **kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__memalloc = pintools["memalloc"]
self.__type = pintools["type"]
self.__logdir = logdir
self.__alt_prev = alt_prev
super(TestAlloc, self).__init__(*args, **kwargs)
def run(self, params=None, logname="alloc_general.res", consistency=False):
FNULL = open(os.devnull, "w")
prev_res = AllocChart(None, self.__resdir + "/" + logname)
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch memalloc
self.__memalloc.launch(param["bin"], [param["args"], "1>/dev/null"], params=params, alt_prev=self.__alt_prev, verbose=False)
logfile = self.__memalloc.get_logfile(pgm, prev=False)
typelogfile = self.__type.get_logfile(pgm, prev=False)
if self.__alt_prev:
couplereslogfile = self.__memalloc.get_logfile(pgm, alt_prev=True)
else:
couplereslogfile = None
if "libraries" in param.keys():
libraries = param["libraries"]
else:
libraries = False
# launch offline computation of memcomb
res = MemComb(logfile, typelogfile, pgm, coupleres_log_file=couplereslogfile, verbose=False).run(get=True, libraries=libraries, log=self.__resdir + "/" + logname, test_consistency=consistency)
if res is not None:
if res[0] is not None and res[0] != "None":
alloc = res[0].split(":")
alloc[1] = hex(int(alloc[1]))
else:
alloc = "None"
if res[1] is not None and res[1] != "None":
free = res[1].split(":")
free[1] = hex(int(free[1]))
else:
free = "None"
print "{}: ALLOC|FREE {}|{} - ERRORS {}|{} - CALLS {}|{} - ONLINE {}|{} - OFFLINE {}".format(
pgm,
":".join(alloc),
":".join(free),
res[2][0][0],
res[2][0][1],
res[2][1][0],
res[2][1][1],
res[3][0],
res[3][1],
res[4],
)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
return
class TestAlloc(object):
def __init__(self,
test_conf,
pintools,
logdir,
resdir,
alt_prev=False,
*args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__memalloc = pintools["memalloc"]
self.__type = pintools["type"]
self.__logdir = logdir
self.__alt_prev = alt_prev
super(TestAlloc, self).__init__(*args, **kwargs)
def run(self, params=None, logname="alloc_general.res", consistency=False):
FNULL = open(os.devnull, "w")
prev_res = AllocChart(None, self.__resdir + "/" + logname)
for pgm, param in OrderedDict(
sorted(self.__conf.items(), key=lambda a: a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch memalloc
self.__memalloc.launch(param["bin"],
[param["args"], "1>/dev/null"],
params=params,
alt_prev=self.__alt_prev,
verbose=False)
logfile = self.__memalloc.get_logfile(pgm, prev=False)
typelogfile = self.__type.get_logfile(pgm, prev=False)
if self.__alt_prev:
couplereslogfile = self.__memalloc.get_logfile(pgm,
alt_prev=True)
else:
couplereslogfile = None
if "libraries" in param.keys():
libraries = param["libraries"]
else:
libraries = False
# launch offline computation of memcomb
res = MemComb(logfile,
typelogfile,
pgm,
coupleres_log_file=couplereslogfile,
verbose=False).run(get=True,
libraries=libraries,
log=self.__resdir + "/" + logname,
test_consistency=consistency)
if res is not None:
if res[0] is not None and res[0] != "None":
alloc = res[0].split(":")
alloc[1] = hex(int(alloc[1]))
else:
alloc = "None"
if res[1] is not None and res[1] != "None":
free = res[1].split(":")
free[1] = hex(int(free[1]))
else:
free = "None"
print "{}: ALLOC|FREE {}|{} - ERRORS {}|{} - CALLS {}|{} - ONLINE {}|{} - OFFLINE {}".format(
pgm,
":".join(alloc),
":".join(free),
res[2][0][0],
res[2][0][1],
res[2][1][0],
res[2][1][1],
res[3][0],
res[3][1],
res[4],
)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
return
def test_empty():
confiture = Confiture('tests/yaml/template/empty.yaml')
confiture.check('tests/yaml/config/empty_valid.yaml')
confiture.check('tests/yaml/config/simple_valid.yaml')
def test_config_not_found():
confiture = Confiture('tests/yaml/template/simple.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/404.yaml')
class TestAccuracy(Std):
def __init__(self, test_conf, empty, arity, typ, logdir, resdir, *args,
**kwargs):
self.__resdir = resdir
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(
test_conf)
# Inlcude sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__empty = empty
self.__arity = arity
self.__type = typ
self.__logdir = logdir
super(TestAccuracy, self).__init__(*args, **kwargs)
def __run_arity(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(
sorted(self.__conf.items(), key=lambda a: a[0])).items():
if prev_res.contains(pgm):
continue
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch without PIN
start = datetime.now()
call(param["bin"] + " " + param["args"], stdout=FNULL, shell=True)
stop = datetime.now()
print stop - start
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch empty
self.__empty.launch(param["bin"], [param["args"], "1>/dev/null"],
verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
self.__arity.launch(param["bin"], [param["args"], "1>/dev/null"],
verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/") +
1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError as e:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
continue
# Get time of execution with PIN and no instrumentation
with open(self.__empty.get_logfile(pgm, prev=False)) as f:
empty_time = float(f.read())
ar = ArityAnalysis(pgm, self.__arity.get_logfile(pgm, prev=False),
data)
res.add(ar.accuracy(get=True,
verbose=False,
log=self.__resdir + "/" + "arity.res",
empty_time=empty_time,
no_pin_time=stop - start),
pgm=pgm,
verbose=True)
print res
print "IGNORED: {}".format(ignored)
def __run_type(self):
res = AccuracyRes()
ignored = 0
FNULL = open(os.devnull, "w")
prev_res = TypeChart(self.__resdir + "/type.res", self.__conf)
# used to get times of execution with no instrumentation
arity_res = ArityChart(self.__resdir + "/arity.res", self.__conf)
for pgm, param in OrderedDict(
sorted(self.__conf.items(), key=lambda a: a[0])).items():
if prev_res.contains(pgm):
continue
# launch program type
if param["args"] == "":
ignored += 1
continue
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__type.launch(
param["bin"],
[param["args"], "1>/dev/null"]) #, verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/") +
1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr(
"error: you must parse source code of \"{0}\" first (use parsedata)"
.format(pgm))
continue
# Get times of execution with no instrumentation
empty_time = arity_res.get_one(pgm).empty_time
nopin_time = arity_res.get_one(pgm).nopin_time
ty = TypeAnalysis(pgm, self.__type.get_logfile(pgm, prev=False),
data)
res.add(ty.accuracy(get=True,
verbose=False,
log=self.__resdir + "/" + "type.res",
empty_time=empty_time,
nopin_time=nopin_time),
pgm=pgm)
print res
print "IGNORED: {}".format(ignored)
def run(self, subcommand=None):
if subcommand is None or subcommand == "arity":
self.__run_arity()
if subcommand is None or subcommand == "type":
self.__run_type()
def test_missing_nested_field():
confiture = Confiture('tests/yaml/template/nested.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/simple_fail.yaml')
def test_nested():
confiture = Confiture("tests/yaml/template/nested.yaml")
confiture.check("tests/yaml/config/nested_valid.yaml")
def test_template_not_found():
with pytest.raises(ConfigFileError):
Confiture('tests/yaml/template/404.yaml')
def __init__(self, config_path="config/config.yaml"):
conf = Confiture("config/templates/general.yaml")
# Parse configuration file and get result
self.__config = conf.check_and_get(config_path)
# Set the log directory
self.__logdir = self.__config["log"]["path"]
# Available pintools
self.__pintools = dict()
for pintool in self.__config["pintool"]:
# Check the needed two arguments
req = ["src", "obj"]
for r in req:
if r not in self.__config["pintool"][pintool].keys():
raise PintoolFileNotFound("{0} file not found for pintool {1}".format(r, pintool))
src = self.__config["pintool"][pintool]["src"]
obj = self.__config["pintool"][pintool]["obj"]
# Check potential extra argument
if "prev_step" in self.__config["pintool"][pintool].keys():
prev_step = self.__config["pintool"][pintool]["prev_step"]
else:
prev_step = None
if "alt_prev_step" in self.__config["pintool"][pintool].keys():
alt_prev_step = self.__config["pintool"][pintool]["alt_prev_step"]
else:
alt_prev_step = None
# Create pintool object
pintool_obj = Pintool(
name=pintool,
src_path=src,
obj_path=obj,
pinconf=self.__config["pin"],
log_dir=self.__logdir,
prev_step=prev_step,
alt_prev_step=alt_prev_step
)
self.__pintools[pintool] = pintool_obj
# # Create a test object
# # Testing options
# kwargs = dict()
# # kwargs["log"] = self.out
# kwargs["clang"] = self.config["clang"]
# self.test = ScatTest(**kwargs)
# Enable commands
self.__cmds = dict()
self.__cmds["checkconfig"] = CheckConfigCmd(
logdir=self.__logdir,
pinpath=self.__config["pin"]["bin"],
)
self.__cmds["make"] = MakeCmd(
pintools=self.__pintools,
)
self.__cmds["display"] = DisplayCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["parsedata"] = ParseDataCmd()
self.__cmds["accuracy"] = AccuracyCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["mismatch"] = MismatchCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["launch"] = LaunchCmd(
pintools=self.__pintools,
)
self.__cmds["couple"] = CoupleCmd(
pintools=self.__pintools,
logdir=self.__logdir,
)
self.__cmds["memcomb"] = MemCombCmd(
pintools=self.__pintools,
logdir=self.__logdir
)
if "param" in self.__config["test"].keys():
self.__cmds["test"] = TestCmd(
test_conf=self.__config["test"]["desc"],
param=self.__config["test"]["param"],
pintools=self.__pintools,
logdir=self.__logdir,
resdir=self.__config["test"]["res"],
)
self.__cmds["chart"] = ChartCmd(
resdir=self.__config["test"]["res"],
conf=self.__config["test"]["param"],
)
# Link methods to scat shell
for cmd, obj in self.__cmds.items():
setattr(self.__class__, "do_" + cmd, obj.run)
setattr(self.__class__, "help_" + cmd, obj.help)
setattr(self.__class__, "complete_" + cmd, obj.complete)
# Init shell
Cmd.__init__(self)
# Read history from file
try:
with open(".history", "r+") as history_file:
line = history_file.readline()
while line != '':
self.history.append(line[:-1])
line = history_file.readline()
except IOError:
pass
def test_nested():
confiture = Confiture('tests/yaml/template/nested.yaml')
confiture.check('tests/yaml/config/nested_valid.yaml')
def test_travis_configuration():
confiture = Confiture('tests/yaml/template/travis.yaml')
confiture.check('.travis.yml')
user = UserSection()
class VirtualHostSection(Section):
enable_ssl = Value(Boolean(), default=False)
path = PathSection()
_meta = {'repeat': many, 'unique': True}
class MyWebserverConfiguration(Section):
daemon = Value(Boolean(), default=False)
pidfile = Value(String(), default=None)
interface = Value(String(), default='127.0.0.1:80')
interface_ssl = Value(String(), default='127.0.0.1:443')
host = VirtualHostSection()
parsed_conf = Confiture(conf, schema=MyWebserverConfiguration()).parse()
types = [
'confiture.schema.types.Number', # validates using nubmers.Number ABC, casts using float() - always
'confiture.schema.types.Integer', # validate by casting, min/max, cast using int()
'confiture.schema.types.Float', # validation by casting only, NO min/max, as per Number
'confiture.schema.types.Boolean', # validation via 'value is not True and value is not False', cast with bool()
'confiture.schema.types.String', # validates by encoding (thus unicode input, binary output), doesn't cast
'confiture.schema.types.Regex', # custom error message, re.match to validate, no cast
'confiture.schema.types.NamedRegex', # returns .groupdict() instead of match.string
'confiture.schema.types.RegexPattern', # a literal regex pattern, validates by compiling
'confiture.schema.types.IPAddress', # uses ipaddr, no cast
'confiture.schema.types.IPNetwork', # uses ipaddr, no cast
'confiture.schema.types.Url', # validation via urlparse.urlparse()
def parse_config(filename):
conf = Confiture.from_filename(filename, schema=RootMartyConfig())
return conf.parse()
def run(self, s, *args, **kwargs):
split = s.split(" ")
# Watch for a particular test description file
if "-t" in split:
self.__testconf = Confiture(
"config/templates/empty.yaml").check_and_get(
split.pop(split.index("-t") + 1))
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
inf, param = split[:2]
if len(split) > 2:
pgm = split[2]
else:
pgm = "test"
if inf == "arity" or inf == "type":
if param != "accuracy" and param != "scalability" and param != "overhead":
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
inp = param in [
"min_calls", "param_threshold", "min_vals", "max_vals",
"addr_threshold"
]
outp = param in [
"min_calls", "ret_threshold", "min_vals", "max_vals",
"addr_threshold"
]
if inf == "arity":
chart = ArityChart(self.__get_res(inf, param, pgm),
self.__testconf)
else:
chart = TypeChart(self.__get_res(inf, param, pgm),
self.__testconf)
if param == "accuracy":
chart.draw_accuracy(chart.get_accuracy(), "accuracy")
elif param == "variability":
chart.draw_var(chart.get_var(pgm, defaults),
"{}_var".format(pgm))
elif param == "scalability":
chart.draw_scalability(chart.get_accuracy(), "scalability")
elif param == "overhead":
chart.draw_overhead(chart.get_overhead(self.__testconf))
else:
chart.draw(chart.get(param, defaults, inp=inp, outp=outp),
pgm + "_" + param)
elif inf == "couple":
if param == "general":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_accuracy(chart.get_accuracy(), "general")
else:
defaults = dict()
for k, v in self.__conf[inf][param].items():
if k not in ["min", "max", "step"]:
defaults[k] = v
if param == "variability":
chart = CoupleChart(self.__get_res(inf, param, pgm))
chart.draw_var(chart.get_var(pgm, defaults), "var")
else:
chart = CoupleChart("{}/{}".format(self.__resdir, param))
chart.draw(chart.get(param, defaults), param)
elif inf == "alloc":
# compute oracle
oracle = dict()
if self.__testconf is not None:
# Inlcude sub configuration files
for k, v in self.__testconf.items():
if "config" in v.keys():
subconf = Confiture(
"config/templates/empty.yaml").check_and_get(
v["config"])
self.__testconf.pop(k)
self.__testconf.update(subconf)
for pgm, vals in self.__testconf.items():
if "oracle" in vals.keys():
oracle[pgm] = dict(vals["oracle"])
for k, v in oracle[pgm].items():
if v is not None:
oracle[pgm][k] = v.split(" ")
else:
oracle[pgm][v] = list()
if param == "couple" or param == "type":
chart = AllocChart(
oracle,
"{}/alloc_{}_general.res".format(self.__resdir, param))
chart.table()
elif param == "compare":
chart = AllocChart(
oracle,
"{}/alloc_couple_general.res".format(self.__resdir))
chart.table_cmp(
AllocChart(
oracle,
"{}/alloc_type_general.res".format(self.__resdir)))
return
elif param == "consistency":
chart = AllocChart(
oracle,
"{}/alloc_couple_consistency.res".format(self.__resdir))
chart.draw_consistency()
Integer(),
default=514,
argparse_help='Port number where syslogs are sent.',
argparse_names=['-p', '--port']
)
facility = Value(
String(),
default='user',
argparse_help='Facility code for the syslog.',
argparse_names=['-f', '--facility']
)
schema = Config()
config = Confiture.from_filename(
'test_ressources/tests_jira_quart_config.conf',
schema=schema
).parse()
jira = JIRA(
options={'server': config.subsection('jira').get('url')},
basic_auth=(
str(config.subsection('jira').get('user')),
str(config.subsection('jira').get('password'))
)
)
#Test find_the_open_issue
#Get different issue types from Jira (needs regular updates
# for test to work)
canceled_issue = jira.issue('VUMA-511')
awaiting_validation = jira.issue('VUMA-512')
def parse_config(filename):
conf = Confiture.from_filename(filename, schema=RootMartyConfig())
return conf.parse()
plugin 'ugly' {
common_value = 456
ugly_value = no
}
'''
#
# This is where the magic happen:
#
if __name__ == '__main__':
my_plugins = (BeautifulPlugin, UglyPlugin)
enabled_plugins = []
# Parse the global configuration:
config = Confiture(TEST_CONFIG, schema=MainConfigurationSchema())
try:
pconfig = config.parse()
except (ValidationError, ParsingError) as err:
if err.position is not None:
print str(err.position)
print err
sys.exit(1)
else:
print 'Main configuration:'
pprint(pconfig.to_dict())
# Enable each used plugins:
for plugin_conf in pconfig.subsections('plugin'):
# Search the plugin:
for plugin in my_plugins:
from confiture import Confiture, ConfigFileError
print("[*] loading template")
confiture = Confiture("examples/templates/confiture.yaml")
print("[*] checking required files for blueberry")
try:
confiture.check("examples/config/blueberry_ok.yaml")
print("[*] blueberry file is correct")
except ConfigFileError as e:
print(e.message)
print("[*] checking required files for banana")
try:
confiture.check("examples/config/banana_ko.yaml")
print("[*] banana file is correct")
except ConfigFileError as e:
print(e.message)
class TestParameter(object):
def __init__(self, test_conf, pintool, resdir, *args, **kwargs):
self.__resdir = resdir
self.__testname = os.path.splitext(os.path.basename(test_conf))[0]
self.__conf = Confiture("config/templates/empty.yaml").check_and_get(test_conf)
# Include sub configuration files
for k, v in self.__conf.items():
if "config" in v.keys():
subconf = Confiture("config/templates/empty.yaml").check_and_get(v["config"])
self.__conf.pop(k)
self.__conf.update(subconf)
self.__pintool = pintool
if "prev_pintool" in kwargs.keys():
self.__prev_pintool = kwargs["prev_pintool"]
else:
self.__prev_pintool = None
self.__prev_treated = list()
if str(pintool) == "arity":
self.__res = ArityChart(self.__resdir + "/{}_arity.res".format(self.__testname), self.__conf)
else:
self.__res = TypeChart(self.__resdir + "/{}_type.res".format(self.__testname), self.__conf)
super(TestParameter, self).__init__()
def __run_one(self, params):
res = AccuracyRes()
FNULL = open(os.devnull, "w")
for pgm, param in OrderedDict(sorted(self.__conf.items(), key=lambda a:a[0])).items():
if self.__res.contains(pgm, params):
continue
if self.__prev_pintool is not None and pgm not in self.__prev_treated:
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
self.__prev_pintool.launch(param["bin"], [param["args"], "1>/dev/null"], verbose=False)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
self.__prev_treated.append(pgm)
if "pre" in param.keys():
call(param["pre"], stdout=FNULL, shell=True)
# launch program arity
if param["args"] == "":
continue
self.__pintool.launch(param["bin"], [param["args"], "1>/dev/null"], params, verbose=True)
# display arity accuracy
try:
if param["data"].endswith("/"):
data = Data(param["data"], pgm)
else:
datadir = param["data"][:param["data"].rfind("/") + 1]
pgmname = param["data"][param["data"].rfind("/")+1:param["data"].rfind(".")]
data = Data(datadir, pgmname)
data.load()
except IOError:
self.stderr("error: you must parse source code of \"{0}\" first (use parsedata)".format(pgm))
continue
if str(self.__pintool) == "arity":
an = ArityAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
else:
an = TypeAnalysis(pgm, self.__pintool.get_logfile(pgm, prev=False), data)
res.add(an.accuracy(get=True, verbose=False, log=self.__resdir + "/" + str(self.__testname) + "_" + str(self.__pintool) + ".res"), pgm=pgm, verbose=True)
if "post" in param.keys():
call(param["post"], stdout=FNULL, shell=True)
print res
def run(self, params):
param = filter(lambda a: isinstance(a[1], list), [(k, v) for k, v in params.items()])[0][0]
vals = params.pop(param)
for val in vals:
p = dict()
p[param] = val
p.update(params)
self.__run_one(p)
from confiture import Confiture, ConfigFileError
print("[*] loading template")
confiture = Confiture("examples/templates/confiture.yaml")
print("[*] checking required files for blueberry")
try:
confiture.check("examples/config/blueberry_ok.yaml")
print("[*] blueberry file is correct")
except ConfigFileError as e:
print(e.message)
print("[*] checking required files for banana")
try:
confiture.check("examples/config/banana_ko.yaml")
print("[*] banana file is correct")
except ConfigFileError as e:
print(e.message)
def test_config_not_found():
confiture = Confiture('tests/yaml/template/simple.yaml')
with pytest.raises(ConfigFileError):
confiture.check('tests/yaml/config/404.yaml')
ip = Value(String(),
default='localhost',
argparse_help='IP address where syslog are sent.',
argparse_names=['-ip'])
port = Value(Integer(),
default=514,
argparse_help='Port number where syslogs are sent.',
argparse_names=['-p', '--port'])
facility = Value(String(),
default='user',
argparse_help='Facility code for the syslog.',
argparse_names=['-f', '--facility'])
schema = Config()
config = Confiture.from_filename(
'test_ressources/tests_jira_quart_config.conf', schema=schema).parse()
jira = JIRA(options={'server': config.subsection('jira').get('url')},
basic_auth=(str(config.subsection('jira').get('user')),
str(config.subsection('jira').get('password'))))
#Test find_the_open_issue
#Get different issue types from Jira (needs regular updates
# for test to work)
canceled_issue = jira.issue('VUMA-511')
awaiting_validation = jira.issue('VUMA-512')
awaiting_fix = jira.issue('VUMA-513')
in_progress = jira.issue('VUMA-514')
more_info = jira.issue('VUMA-515')
done = jira.issue('VUMA-516')
def test_travis_configuration():
confiture = Confiture("tests/yaml/template/travis.yaml")
confiture.check(".travis.yml")
def test_simple():
confiture = Confiture('tests/yaml/template/simple.yaml')
confiture.check('tests/yaml/config/simple_valid.yaml')
def test_simple():
confiture = Confiture("tests/yaml/template/simple.yaml")
confiture.check("tests/yaml/config/simple_valid.yaml")
self.FSCORE = c["output"]["fscore"]
self.IGNORE = c["ignore"]
if "sorted" in c["output"].keys():
self.SORTED = c["output"]["sorted"]
else:
self.SORTED = False
if "variance" in c["output"].keys():
self.VARIANCE = c["output"]["variance"]
else:
self.VARIANCE = False
if len(sys.argv) < 2:
print "Usage: {} CONFIG".format(sys.argv[0])
exit()
c = Confiture("test/tab/template.yaml").check_and_get(sys.argv[1])
CONFIG = Config(c)
class Result(object):
def __init__(self, pgm):
self.pgm = pgm
self.res = dict()
for opti in CONFIG.OPTI_LVL:
self.res[opti] = dict()
for an in CONFIG.ALS: # ["jcall", "jmp", "iCi"]:
self.res[opti][an] = dict()
for v in ["miss", "extra", "u_miss", "u_extra", "total"]:
self.res[opti][an][v] = 0
self.res[opti][an]["time"] = list()
