def __init__(
self,
m_file_path,
benchmark_os_path,
parallel,
shared_engine=None #TAG:MSH
):
self.parallel = parallel
import matlab.engine as matlab_engine
import matlab as matlab
# Don't seem to nee matlab.engine other than to start the matlab
# session.
# self.matlab_engine = matlab_engine
self.matlab = matlab
#TAG:MSH
if shared_engine is None:
print 'starting matlab...'
self.eng = matlab_engine.start_matlab()
print 'done'
else:
print 'attempting to connect to an existing matlab session'
self.eng = matlab_engine.connect_matlab(shared_engine)
self.m_file = m_file_path
m_file_name_split = self.m_file.split('.')
if m_file_name_split[1].strip() != 'm':
raise err.Fatal('internal error!')
self.m_fun_str = m_file_name_split[0].strip()
# self.sim_fun = self.eng.simulate_m_file(1)
# add paths
# comm.call_function([], 'addpath', [SQ + benchmark_os_path + SQ])
self.eng.addpath(benchmark_os_path)
#TAG:CLSS
# TODO: Remove this hack. Added for backwards compatibility
# detect if the simulator file is a function or a class
# source: http://blogs.mathworks.com/loren/2013/08/26/what-kind-of-matlab-file-is-this/
# classy = 8 if class else is 0 (function or a script)
classy = self.eng.exist(self.m_fun_str, 'class')
if classy == 0.0:
print 'maltab file is a function'
self.sim_is_class = False
elif classy == 8.0:
print 'maltab file is a class'
self.sim_is_class = True
self.sim_obj = self.eng.init_plant(self.m_fun_str)
else:
raise err.Fatal(
'''Supplied matlab simulator is neither a class or a function:
possible floating point error?. exist() returned: {}'''.format(
classy))
def x_array(self, val):
if type(val) != np.ndarray:
raise err.Fatal('x_array is not a numpy array: type = {}'.format(
type(val)))
if val.ndim != 1:
raise err.Fatal(
'x_array: more than one x vector provided?: x = {}, ndim = {}'.
format(val, val.ndim))
self._x_array = val
def sanity_check(self):
for (i, j) in zip(self.l, self.h):
if i > j:
# # ##!!##logger.debug('IntervalCons sanity check failure!: l > h')
# # ##!!##logger.debug('l = {}, h = {}'.format(self.l, self.h))
raise err.Fatal('malformed interval!')
if len(self.l) != len(self.h):
raise err.Fatal('dimension mismatch between bounds!')
def input_array(self, val):
if type(val) != np.ndarray:
raise err.Fatal(
'input_array is not a numpy array: type = {}'.format(
type(val)))
if val.ndim != 1:
raise err.Fatal(
'input_array: more than one state vector provided?: ci = {}, ndim = {}'
.format(val, val.ndim))
self._input_array = val
def simulator_factory(
config_dict,
benchmark_os_path,
property_checker,
plt,
plant_pvt_init_data,
parallel=False,
test_params=None,
sim_args=None
):
# ##!!##logger.debug('requested simulator creation')
sim_type = config_dict['plant_description']
if sim_type == 'matlab':
logger.info('creating matlab simulator')
# get the m_file's path
m_file_path = config_dict['plant_path']
abs_m_file_path = fp.construct_path(m_file_path, benchmark_os_path)
if fp.validate_file_names([abs_m_file_path]):
# return MatlabSim(m_file_path, benchmark_os_path, parallel)
return MEngPy(m_file_path, benchmark_os_path, parallel, sim_args)
else:
raise err.FileNotFound('file does not exist: ' + m_file_path)
elif sim_type == 'simulink':
return SimulinkSim()
elif sim_type == 'python':
logger.info('creating Native Python simulator')
# get the file path
python_file_path = config_dict['plant_path']
#(module_name, file_ext) = python_file_path.split('.')
(module_name, file_ext) = fp.split_filename_ext(python_file_path)
if file_ext != 'py':
raise err.Fatal('Python file extension py expected, found: {}'.format(file_ext))
module_path = fp.construct_path(python_file_path, benchmark_os_path)
if fp.validate_file_names([module_path]):
return NativeSim(module_name, module_path,
property_checker, plt, plant_pvt_init_data, parallel)
else:
raise err.FileNotFound('file does not exist: ' + python_file_path)
elif sim_type == 'test':
return TestSim(test_params)
else:
raise err.Fatal('unknown sim type : {}'.format(sim_type))
def check(self, *args):
if self.CE_gen is None:
self.CE_gen = self.get_CE_gen()
self.compute_next_trace()
return InvarStatus.Safe if self.pwa_trace is None else InvarStatus.Unsafe
else:
raise err.Fatal('check should be called only once!')
def parse_config(self, config_dict):
# ##!!##logger.debug('parsing abstraction parameters')
if config_dict['type'] == 'string':
try:
grid_eps_str = config_dict['grid_eps']
# remove braces
grid_eps_str = grid_eps_str[1:-1]
self.eps = np.array(
[float(eps) for eps in grid_eps_str.split(',')])
pi_grid_eps_str = config_dict['pi_grid_eps']
# remove braces
pi_grid_eps_str = pi_grid_eps_str[1:-1]
#self.pi_eps = np.array([float(pi_eps) for pi_eps in pi_grid_eps_str.split(',')])
self.refinement_factor = float(
config_dict['refinement_factor'])
self.num_samples = int(config_dict['num_samples'])
self.delta_t = float(config_dict['delta_t'])
self.N = int(np.ceil(self.T / self.delta_t))
# Make the accessed data as None, so presence of spurious data can be detected in a
# sanity check
config_dict['grid_eps'] = None
config_dict['pi_grid_eps'] = None
config_dict['refinement_factor'] = None
config_dict['num_samples'] = None
config_dict['delta_t'] = None
except KeyError, key:
raise err.Fatal(
'expected abstraction parameter undefined: {}'.format(key))
def __init__(self, s_):
s = s_[0]
# remove everything after comma, i.e., the concrete values
# Assume the string to be multiline (re.M) and do it for every line
s = re.sub(r',.*$', r'', s, flags=re.M)
# remove return value
s = re.sub(r'return value.*$', r'', s, flags=re.M)
# change assignment '=' to equality constraint '=='
s = s.replace('=', '==')
# cleanup
s = s.strip()
# and all constraints now
s = s.replace('\n', ' and ')
s = remove_typecasts(s)
self.py_str = s
#############
# TODO: below is not being used. ast conversion is done at the end
# But its good for incremental error check, so let it stay for the time
# being!
# ###########
try:
self.py_ast = ast.parse(s)
except:
print s_[0], '\n########', s
raise err.Fatal('parsing failure')
return
def __init__(self, s_):
SimplifiedPathPred.ctr += 1
s = s_[0]
s = s.replace('AND', 'and')
s = s.replace('\n', ' ')
# prevent matching of '>=' and '=<'...
# usual replace fails: s = s.replace('=', '==')
# (?<!...): negative look behind for '>'
# (?!...): negative look forward for '<'
s = re.sub(r'(?<!>)=(?!<)', r'==', s)
#print s
s = s.replace('<>', '!=')
#print s
s = s.replace(r'=<', r'<=')
s = remove_typecasts(s)
#print s
self.py_str = s
#############
# TODO: below is not being used. ast conversion is done at the end
# But its good for incremental error check, so let it stay for the time
# being!
# ###########
# TODO: remove try block for efficiency
# current purpose is to improve error reporting
try:
self.py_ast = ast.parse(s)
except:
print s_[0], '\n########', s
raise err.Fatal('python ast parsing failure')
#self.z3_cons = py2z3.translate(py_ast)
return
def pyval2z3val(val):
if type(val) is float:
return z3.RealVal(val)
elif type(val) is int:
return z3.IntVal(val)
else:
raise err.Fatal('unhandled python val type!')
def class_factory(graph_lib):
"""class_factory
Parameters
----------
graph_lib : string for the graph lib
Returns
-------
corresponding class for the specified graph lib
Notes
------
"""
if graph_lib == 'nx':
from .graphNX import GraphNX
return GraphNX
elif graph_lib == 'nxlm':
from .graphNXlowmem import GraphNXLM
return GraphNXLM
elif graph_lib == 'gt':
from .graphGT import GraphGT
return GraphGT
elif graph_lib == 'g':
from .graph_generic import Graph
return Graph
else:
raise err.Fatal(
'unknown graph library requested: {}'.format(graph_lib))
def simulate(sys, prop):
plt = globalopts.opts.plotting
#plot = globalopts.opts.plot
if not isinstance(globalopts.opts.property_checker,
properties.PropertyChecker):
raise err.Fatal('property checker must be enabled when '
'random testing!')
trace_list = RT.simulate(sys, prop)
#print(len(list(trace_list)))
# for trace in trace_list:
# print(trace)
#for trace in trace_list:
# fp.append_data('trace_log', str(trace))
if globalopts.opts.dump_trace:
dump_trace(trace_list)
if settings.paper_plot:
# because the plot is craeted inside the simulator, get
# the global handle
plt.acquire_global_fig()
plt.plot_rect(prop.init_cons.rect(), 'g')
plt.plot_rect(prop.final_cons.rect(), 'r')
plt.set_range((-2.5, 2.5), (-8, 8))
plt.plot_trace_list(trace_list)
# if settings.paper_plot:
# plt.plot_rect(prop.init_cons.rect(), 'g')
# plt.plot_rect(prop.final_cons.rect(), 'r')
# plt.set_range((-2, 2), (-7, 7))
plt.show()
def get_concrete_state_obj(t0, x0, d0, pvt0, s0, ci, pi, u):
if x0.ndim == 1:
concrete_states = st.StateArray(
t=np.array([t0]),
x=np.array([x0]),
d=np.array([d0]),
pvt=np.array([pvt0]),
u=np.array([u]),
s=np.array([s0]),
pi=np.array([pi]),
ci=np.array([ci]),
)
elif x0.ndim == 2:
concrete_states = st.StateArray(
t=t0,
x=x0,
d=d0,
pvt=pvt0,
u=u,
s=s0,
pi=pi,
ci=ci,
)
else:
raise err.Fatal('dimension must be 1 or 2...: {}!'.format(x0.ndim))
return concrete_states
def check(self, depth):
yices2_not_found = 'yices2: not found'
self.dump()
try:
sal_path_ = os.environ[SAL_PATH] + SAL_INF_BMC
except KeyError:
raise err.Fatal("SAL environment variable is not defined. It\n"
"should point to sal's top-level directory")
#raise KeyError
sal_path = fops.sanitize_path(sal_path_)
sal_cmd = sal_run_cmd(
sal_path,
depth,
self.sal_file,
self.prop_name,
)
try:
sal_op = U.strict_call_get_op(sal_cmd)
except U.CallError as e:
if yices2_not_found in e.message:
print('SAL can not find yices2. Trying with yices...')
opts = SalOpts()
opts.yices = 1
sal_cmd = sal_run_cmd(sal_path, depth, self.sal_file,
self.prop_name, opts)
sal_op = U.strict_call_get_op(sal_cmd)
else:
raise err.Fatal('unknown SAL error!')
print(sal_op)
self.trace = sal_op_parser.parse_trace(sal_op, self.vs)
if self.trace is None:
print('BMC failed to find a CE')
return InvarStatus.Unknown
else:
#self.trace.set_vars(self.vs)
print('#' * 40)
print('# Cleaned up trace')
print('#' * 40)
print(self.trace)
print('#' * 40)
return InvarStatus.Unsafe
def falsify(sys, prop, opts, current_abs, sampler):
# sys
controller_sim = sys.controller_sim
plant_sim = sys.plant_sim
# prop
init_cons_list = prop.init_cons_list
init_cons = prop.init_cons
final_cons = prop.final_cons
ci = prop.ci
pi = prop.pi
initial_discrete_state = prop.initial_discrete_state
initial_controller_state = prop.initial_controller_state
MAX_ITER = prop.MAX_ITER
#TODO: hack to make random_test sample ci_cells when doing
# ss-concrete. It is false if ss-symex (and anything else) is
# asked for, because then ci_seq consists if concrete values. Can
# also be activated for symex as an option, but to be done later.
sample_ci = opts.METHOD == 'concrete'
# options
plot = opts.plot
initial_discrete_state = tuple(initial_discrete_state)
initial_controller_state = np.array(initial_controller_state)
# make a copy of the original initial plant constraints
original_plant_cons_list = init_cons_list
pi_ref = wmanager.WMap(pi, sys.pi_grid_eps)
ci_ref = wmanager.WMap(ci, sys.ci_grid_eps) if sample_ci else None
# f1 = plt.figure()
##
## plt.grid(True)
##
## ax = f1.gca()
## eps = current_abs.plant_abs.eps
## #ax.set_xticks(np.arange(0, 2, eps[0]))
## #ax.set_yticks(np.arange(0, 20, eps[1]))
##
## f1.suptitle('abstraction')
if opts.refine == 'init':
refine_init(current_abs, init_cons_list, final_cons,
initial_discrete_state, initial_controller_state,
plant_sim, controller_sim, ci, pi, sampler, plot,
init_cons, original_plant_cons_list, MAX_ITER, sample_ci,
pi_ref, ci_ref, opts)
# seed 4567432
elif opts.refine == 'trace':
refine_trace(current_abs, init_cons_list, final_cons,
initial_discrete_state, initial_controller_state,
plant_sim, controller_sim, ci, pi, sampler, plot,
init_cons, original_plant_cons_list)
else:
raise err.Fatal('internal')
def __init__(self, l, h):
if type(h) != np.ndarray or type(l) != np.ndarray:
raise err.Fatal('interval constraints should be expressed as np.ndarray expected'
)
self.h = h
self.l = l
self.dim = len(self.l)
self.sanity_check()
def __init__(self, s_):
s = s_[0]
#s = s.replace('\n', ' ')
# prevent matching of '>=' and '=<'...
# usual replace fails: s = s.replace('=', '==')
# (?<!...): negative look behind for '>'
# (?!...): negative look forward for '<'
s = re.sub(r'(?<!>)=(?!<)', r'==', s)
#print s
s = s.replace('<>', '!=')
#print s
s = s.replace(r'=<', r'<=')
s = remove_typecasts(s)
#print s
stmt_list = s.strip().split('\n')
stmt_list = [stmt.split(' ') for stmt in stmt_list]
ordered_id_cons_gen = ((stmt[1], stmt[2]) for stmt in stmt_list)
#self.ordered_hashID_parsedID_cons_list \
# = [(hash(ID), self.cond_parser(ID), cons) for ID, cons in ordered_id_cons_gen]
self.ordered_parsedID_cons_list \
= [(self.cond_parser(ID), cons) for ID, cons in ordered_id_cons_gen]
#lst = []
#prev_pred_ID = []
#for ID, cons in ordered_id_cons_gen:
# pred_ID = self.cond_parser(ID)
#for i in ordered_parsedId_cons_list:
# print i
# debug prints...
#for i in stmt_list:
# print i[0], '\t', i[1], '\t', i[2]
# print self.cond_parser(i[1])
#self.py_str = cons
#############
# TODO: below is not being used. Ast conversion is done at the end
# But its good for incremental error check, so let it stay for the time
# being!
# ###########
# TODO: remove try block for efficiency
# current purpose is to improve error reporting
try:
# 'Ands' all constraints (even the ones which should be OR-ed) and
# checks if they can be parsed by the python ast parser. Just a
# sanity check!!
pred_stmt_gen = (('(' + stmt[2] + ')') for stmt in stmt_list)
cons = reduce(lambda x, y: '{} and {}'.format(x, y), pred_stmt_gen)
ast.parse(cons)
except:
print '='*100
print s_[0], '\n########', cons
raise err.Fatal('python ast parsing failure')
#self.z3_cons = py2z3.translate(py_ast)
return
def matlab_communicator_factory(communicator_type):
if communicator_type == 'pymatlab':
logger.info('requested pymatlab communicator object')
return PyMatlab()
elif communicator_type == 'matlab_engine':
logger.info('requested matlab engine communicator object')
return MatlabEngine()
else:
raise err.Fatal('Internal Error')
def get_matlab_command_str(command, arg):
single_quotes = '\''
if command == 'addpath':
s = 'addpath({0}{1}{0})'.format(single_quotes, arg)
elif command == '':
s = ''
else:
raise err.Fatal('Internal error!')
return s
def draw_2d(self):
pos_dict = {}
for n in self.G.nodes():
if len(n.plant_state.cell_id) != 2:
raise err.Fatal(
'only 2d abstractions can be drawn, with each node representing the coordinates (x,y)!. Was given {}-d'
.format(len(n.plant_state.cell_id)))
pos_dict[n] = n.plant_state.cell_id
self.G.draw(pos_dict)
def compute_concrete_plant_output(
A,
plant_sim,
states,
total_num_samples,
property_checker,
):
# ##!!##logger.debug(U.decorate('simulating plant'))
concrete_states = states
# concrete_states = st.StateArray(
# t_array,
# x_array, # cont_state_array
# d_array, # abs_state.discrete_state,
# p_array, # abs_state.pvt_state
# None, # don't need it
# u_array)
# simulate to get reached concrete states
# ##!!##logger.debug('input concrete states\n{}'.format(concrete_states))
# rchd_concrete_state_array = plant_sim.simulate(concrete_states,
# A.delta_t,
# property_checker=None)
rchd_concrete_state_array = plant_sim.simulate(concrete_states, A.delta_t,
property_checker, [False])
# ##!!##logger.debug('output concrete states\n{}'.format(rchd_concrete_state_array))
# ASSUMES:
# simulation was successful for each sample
# This implies
# - The plant_sim returned a valid concrete state for each supplied
# sample
# - \forall i. output_array[i] = SIM(input_array[i]) is valid
# and len(output_array) = len(input_arra)
# This need not be true always and we need to add plant_sim errors,
# like returning (inf, inf, inf,...) ?
# Some indication that simulation failed for the given state, without
# destroying the assumed direct correspondance between
# input array and output array
if rchd_concrete_state_array.n != concrete_states.n:
print rchd_concrete_state_array
print
print concrete_states
raise err.Fatal('Internal')
# ##!!##logger.debug(U.decorate('simulating plant done'))
return rchd_concrete_state_array
def z3val2pyval(val):
if val.is_real():
return z3real2pyfloat(val)
# TODO: what is z3.is_int_value() ?
# from the docs, seem more appropriate. But, an analogoue for reals does
# not exist. Confusing....
elif val.is_int():
return z3int2pyint(val)
else:
raise err.Fatal('unhandled z3 val type')
def scaleNround(self, CONVERSION_FACTOR):
raise err.Fatal('#$%^$&#%#&%$^$%^$^#!@$')
# do not do inplace conversion, instead return a copy!
# self.h = (self.h * CONVERSION_FACTOR).astype(int)
# self.l = (self.l * CONVERSION_FACTOR).astype(int)
h = np.round(self.h * CONVERSION_FACTOR).astype(int)
l = np.round(self.l * CONVERSION_FACTOR).astype(int)
return IntervalCons(l, h)
def serialize_array(x):
if x.ndim > 2:
raise err.Fatal('Interface can only be used for matrices, dim <= 2')
flat_x = x.flatten()
if x.ndim == 1:
s = (1, x.shape[0])
else:
s = x.shape
tmp_x = np.append(s, flat_x)
x_ser = array.array(tmp_x.dtype.char, tmp_x)
return x_ser
def get_abs_state_from_concrete_state(
self,
concrete_state,
hd=0,
p='',
cp='',
):
def split_concrete_controller_state(concrete_state):
nsi = self.num_dims.si
#nsf = self.num_dims.sf
si = concrete_state[0:nsi]
sf = concrete_state[nsi:]
return si, sf
if hd != 0:
raise err.Fatal('investigate')
id_str = str(self.gen_id())
si_str = SI + id_str
sf_str = SF + id_str
si__str = SI + id_str + p
sf__str = SF + id_str + p
x_str = X + id_str
u_str = U + id_str
ci_str = I + id_str
si = self.create_smt_var(IV + SI, si_str)
sf = self.create_smt_var(IV + SF, sf_str)
si_ = self.create_smt_var(RV + SI, si__str)
sf_ = self.create_smt_var(RV + SF, sf__str)
u = self.create_smt_var(RV + U, u_str)
x = self.create_smt_var(IV + X, x_str)
ci = self.create_smt_var(IV + I, ci_str)
#concrete_si_, concrete_sf_ = split_concrete_controller_state(concrete_state)
#C = self.solver.And(
# self.solver.equal(si_, concrete_si_),
# self.solver.equal(sf_, concrete_sf_))
concrete_si, concrete_sf = split_concrete_controller_state(
concrete_state)
C = self.solver.And(self.solver.equal(si, concrete_si),
self.solver.equal(sf, concrete_sf))
return ControllerSymbolicAbstractState(
C,
si,
sf,
x,
si_,
sf_,
u,
p,
cp,
ci,
hd,
)
def one_shot_sim(x, t0, tf, w, simstate=None):
if t0 != 0:
raise err.Fatal('t0 must be 0!')
else:
# TODO: construct numpy arrays in loadsystem
# This will require changing code into quite afew
# places...carefull!
s = np.array(prop.initial_controller_state)
d = np.array(prop.initial_discrete_state)
pvt = np.zeros(1)
ci_array = w
return system_sim(x, s, d, pvt, t0, tf, ci_array)
def graph_factory(graph_type):
if graph_type == 'nx':
global nx
import networkx as nx
return GraphNX()
elif graph_type == 'gt':
global gt
import graph_tool.all as gt
return GraphGT()
else:
raise err.Fatal(
'unknown graph library requested: {}'.format(graph_type))
def __init__(
self,
m_file_path,
benchmark_os_path,
parallel,
):
super(MatlabSim, self).__init__()
self.parallel = parallel
# parse file name
self.m_file = m_file_path
m_file_name_split = self.m_file.split('.')
if m_file_name_split[1].strip() != 'm':
raise err.Fatal('internal error!')
self.m_fun = m_file_name_split[0].strip()
# instantiate matlab
self.communicator = matlab_communicator_factory('pymatlab')
comm = self.communicator
logger.info('created matlab simulator from file: %s', self.m_file)
# add paths
comm.call_function([], 'addpath', [SQ + benchmark_os_path + SQ])
# for matlab, matlabpool needs to be invoked if parallelism is
# requested
if self.parallel:
comm.exec_command('matlabpool')
[
pool_size,
] = comm.call_function_retVal(['pool_size'], 'matlabpool',
['{0}size{0}'.format(SQ)])
logger.info('simulator is parallel with #threads: %s',
str(pool_size))
else:
logger.info('simulator is instantiated as single threaded')
# load simulator functions into workspace
comm.put_value('SIM_Str', self.m_fun)
comm.exec_command('sim_function = str2func(SIM_Str)')
comm.exec_command('simulate_system = simulate_m_file(1)')
comm.exec_command('simulate_system_par = simulate_m_file(2)')
comm.exec_command('simulate_entire_trajectories = simulate_m_file(3)')
comm.exec_command(
'simulate_entire_trajectories_cont = simulate_m_file(4)')
logger.info('loaded functions from simulate_m_file.m')
def create_smt_var(self, id_str, aux_str=''):
var_details = self.var_name_2_len_dict[id_str]
name_str = var_details[0] + aux_str
length = var_details[1]
# if array length is 0, do not create a z3 var
#if length == 0:
# return None
if var_details[2] is float:
return self.solver.RealVector(name_str, length)
elif var_details[2] is int:
return self.solver.IntVector(name_str, length)
else:
raise err.Fatal('unhandled type')
def equal(self, x, val):
cons_list = []
if isinstance(x, z3.ExprRef):
if x.sort() == z3.ArraySort(z3.BitVecSort(32), z3.BitVecSort(8)):
for i in range(len(val)):
c_equal = z3.Concat(x[INT_SIZE * i + 3], x[INT_SIZE * i + 2],
x[INT_SIZE * i + 1], x[INT_SIZE * i + 0]) \
== val[i]
cons_list.append(c_equal)
elif x.sort() == z3.RealSort():
c_equal = x == val
cons_list.append(c_equal)
else:
raise err.Fatal('unhandled sort x: {}'.format(x.sort()))
elif isinstance(x, list):
cons_list = map((lambda x, c: x == c), x, val)
else:
raise err.Fatal('unhandled type: {}'.format(type(x)))
# # loop sentinel
# if cons_list == []:
# cons_list = True
return z3.And(cons_list)