def run_drivetrain(theta, scale_init=1.0):
'generate a drivetrain benchmark instance, run it and return the runtime'
title = "Drivetrain (Theta={}, InitScale={})".format(theta, scale_init)
output_path = "hylaa_drivetrain_theta{}_initscale{}.py".format(theta, str(scale_init).replace(".", "_"))
gen_param = '-theta {} -init_scale {} -reverse_errors -switch_time 0.20001'.format(theta, scale_init)
e = hypy.Engine('hylaa')
e.set_generator('drivetrain', gen_param)
e.set_output(output_path)
e.add_pass("sub_constants", "")
e.add_pass("simplify", "-p")
print 'Running ' + title
res = e.run(print_stdout=False, parse_output=True)
print 'Finished ' + title
if res['code'] != hypy.Engine.SUCCESS:
raise RuntimeError('Error generating ' + title + ': ' + str(res['code']))
runtime = res['tool_time']
safe = res['output']['safe']
return runtime, safe
def plot(plot_param):
'''run a tool and make a plot'''
name = plot_param[0]
model = plot_param[1]
tool = plot_param[2]
tool_params = plot_param[3]
e = hypy.Engine()
e.set_save_terminal_output(True)
#e.set_save_model_path("out.model")
e.set_model(model)
e.set_tool(tool)
if tool_params is not None:
e.set_tool_params(tool_params)
e.set_print_terminal_output(True)
e.set_output_image(name + '.png')
print 'Running ' + name
code = e.run()
print 'Finished ' + name
if code != hypy.RUN_CODES.SUCCESS:
print '\n\n-------------------\nError in ' + name + ': ' + str(
code) + "; terminal output was:"
print e.get_terminal_output()
raise RuntimeError('Error in ' + name + ': ' + str(code))
def run_spaceex(model, interval_bounds):
"""
Run SpaceEx on the given model.
@param model: path to model
@param interval_bounds: True to compute interval state bounds, False to produce plot image
"""
hyst_options = '-output_vars *' if interval_bounds else '-output-format GEN'
# we need to explicitly specify the output variables because all except the first two are lost somewhere in Hyst
# (Design limitation in Hyst, not easy to fix? AutomatonSettings.plotVariableNames has fixed length 2, output_vars uses that by default)
e = hypy.Engine('spaceex', hyst_options)
e.set_input(model) # sets input model path
image_path = None if interval_bounds else (get_script_dir() +
"/output_spaceex/" +
os.path.basename(model)[:-4] +
"__spaceex_y_vs_tau.png")
res = e.run(parse_output=True, image_path=image_path)
#PrettyPrinter().pprint(res)
assert res['code'] == hypy.Engine.SUCCESS
if not res['output']['fixpoint']:
res_print = deepcopy(res)
res_print['tool_stdout'] = '<omitted>'
res_print['stdout'] = '<omitted>'
PrettyPrinter().pprint(res_print)
raise Exception(
"SpaceEx did not find fixpoint - not proven stable. This should not happen (except if you modified the model)"
)
if interval_bounds:
print('State limits:', res['output']['variables'])
else:
print("Plot saved to {}".format(image_path))
def run_flowstar(model, time, timestep):
"""
run flowstar with given model file, final time and timestep
"""
print("\ntimestep: {}, final time: {}".format(timestep, time))
e = hypy.Engine('flowstar', '-step {ts} -time {t}'.format(ts=timestep,
t=time))
e.set_input(model)
modelname = model.split("/")[-1].split(".")[0]
filename = get_script_dir(
) + '/output_flowstar/' + modelname + '__timestep_{}_time_{}'.format(
timestep, time)
e.set_output(filename + '.flowstar')
result = e.run(image_path=filename + '.png',
parse_output=True,
timeout=2 * 60 * 60)
print(result['code'])
print("writing outputs to {}".format(filename))
with open(filename + ".output.txt", "w") as f:
f.write(PrettyPrinter().pformat(result))
if time <= 10 and timestep >= 1e-3:
assert result['code'] == hypy.Engine.SUCCESS, 'hyst or flowstar failed'
print("Runtime: {} s".format(result['time']))
if result['code'] == hypy.Engine.SUCCESS:
bounds = gnuplot_oct_data_to_bounds(
result['output']['gnuplot_oct_data'], xlabel='tau', ylabel='y')
print("state bounds computed by flowstar:" + str(bounds))
if bounds['y'][1] > 10:
print("does not converge -- state bounds far too high")
elif time >= 20:
print(
"may be converging -- to be sure, check with greater final time"
)
def plot(plot_param):
'''run a tool and make a plot'''
name = plot_param[0]
model = plot_param[1]
tool_name = plot_param[2]
tool_param = plot_param[3]
pass_name = plot_param[4]
pass_param = plot_param[5]
e = hypy.Engine(tool_name, tool_param)
e.set_input(model)
if pass_name is not None:
e.add_pass(pass_name, pass_param)
print 'Running ' + name
code = e.run(image_path=name + ".png", print_stdout=False)['code']
print 'Finished ' + name
if code != hypy.Engine.SUCCESS:
print '\n\n-------------------\nError in ' + name + ': ' + str(code)
raise RuntimeError('Error in ' + name + ': ' + str(code))
def gen_nav(name, a_matrix, i_list, width, start_point, time, noise):
'generate a navigation benchmark instance and plot a simulation'
gen_param = '-matrix ' + str(a_matrix[0][0]) + ' ' + str(a_matrix[0][1]) + \
' ' + str(a_matrix[1][0]) + ' ' + str(a_matrix[1][1]) + ' -i_list '
for i in i_list:
gen_param += str(i) + ' '
(start_x, start_y) = start_point
gen_param += '-width ' + str(width) + ' -startx ' + str(start_x) + ' -starty ' + str(start_y) + \
' -noise ' + str(noise)
tool_param = "-corners True -legend False -rand 100 -time {} -title {}".format(
time, name)
e = hypy.Engine('pysim', tool_param)
e.set_generator('nav', gen_param)
e.set_output(name + '.py')
print('Running ' + name)
e.run(print_stdout=True, image_path=name + '.png')
print('Finished ' + name)
res = e.run()
if res['code'] != hypy.Engine.SUCCESS:
raise RuntimeError('Error in ' + name + ': ' + str(res['code']))
def run_without_pi():
'run without pi pass'
e = hypy.Engine('flowstar')
e.set_input('neuron.xml')
code = e.run(image_path='original.png')['code']
if code != hypy.Engine.SUCCESS:
raise RuntimeError('Error:' + str(code))
def run(unsafe_condition, order, step_size):
'''run flowstar with the given unsafe condition, order, and step size
returns a 2-tuple: (is_safe, runtime_seconds)
'''
# we will add an error condition, and then find parameters in Flow* that prove safety with the error condition
input_model_file = 'vanderpol.xml'
output_model_file = 'out_flowstar.model'
flowstar_hyst_param = '-orders {} -step {} -unsafe "{}"'.format(order, step_size, unsafe_condition)
e = hypy.Engine('flowstar', flowstar_hyst_param)
e.set_input(input_model_file)
print_stdout = False
image_path = None
#### enable these for debugging ####
#e.set_output('out.model') # output the generated Flow* model to this path
#e.set_verbose(True) # print hyst verbose output
#print_stdout=True # print hyst/tool output
#image_path='out.png' # output image path (requires GIMP is setup according to Hyst README)
print "{}".format(step_size),
result = e.run(parse_output=True, image_path=image_path, print_stdout=print_stdout)
# result is a dictionary object with the following keys:
# 'code' - exit code - engine.SUCCESS if successful, an engine.ERROR_* code otherwise
# 'hyst_time' - time in seconds for hyst to run, only returned if run_hyst == True
# 'tool_time' - time in seconds for tool(+image) to run, only returned if run_tool == True
# 'time' - total run time in seconds
# 'stdout' - the list of lines anything produced to stdout, only returned if save_stdout == True
# 'tool_stdout' - the list of lines the tool produced to stdout, only returned if save_stdout == True
# 'hypy_stdout' - the list of lines hypy produces to stdout, only returned if save_stdout == True
# 'output' - tool-specific processed output object, only returned if successful and parse_output == True
if result['code'] != hypy.Engine.SUCCESS:
raise RuntimeError('Hypy Error: {}'.format(result['code']))
runtime = result['tool_time']
output = result['output']
#The output object is an ordered dictionary, with:
# 'terminated' -> True/False <-- did errors occur during computation (was 'terminated' printed?)
# 'mode_times' -> [(mode1, time1), ...] <-- list of reach-tmes computed in each mode, in order
# 'result' -> 'UNKNOWN'/'SAFE'/None <-- if unsafe set is used and 'terminated' is false, this stores
# the text after "Result: " in stdout
# 'safe' -> True iff 'result' == 'SAFE'
# 'gnuplot_oct_data' -> the octogon data in the tool's gnuplot output
# 'reachable_taylor_models' -> a list of TaylorModel objects parsed from out.flow
is_safe = output['result'] == "SAFE"
print "({}) ".format(output['result']),
return is_safe, runtime
def run_with_pi():
'run with pi pass'
e = hypy.Engine('flowstar')
e.add_pass('pi_sim', '-times 2.0 1.0')
e.set_input('neuron.xml')
code = e.run(image_path='pi.png')['code']
if code != hypy.Engine.SUCCESS:
raise RuntimeError(str(code))
def test_simple(self):
'''run the simple example from the hyst readme'''
model = get_script_dir() + "/../../../examples/toy/toy.xml"
e = hypy.Engine('pysim')
e.set_input(model) # sets input model path
res = e.run()
self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def test_pysim(self):
'''run the simple example from the hyst readme'''
model = get_script_dir() + "/../../../examples/toy/toy.xml"
e = hypy.Engine('pysim')
e.set_input(model) # sets input model path
res = e.run(parse_output=True)
self.assertEqual(res['code'], hypy.Engine.SUCCESS)
np.testing.assert_allclose(res['output']['interval_bounds'], np.array([[0, 9],[0, 20], [0, 20]]))
def run_without_pi():
'run without pi pass'
e = hypy.Engine()
e.set_print_terminal_output(False)
e.set_model('neuron.xml')
e.set_tool('flowstar')
e.set_output_image('original.png')
code = e.run()
if code != hypy.RUN_CODES.SUCCESS:
raise RuntimeError('Error:' + str(code))
def run_single((index, total, path, tool_tuple, quit_flag)):
'''run a single model with a single tool.
file is the path to the model file
tool is an entry of the TOOLS list
'''
if not quit_flag.value == 0:
return None
rv = None
filename = os.path.split(path)[1]
model = os.path.splitext(filename)[0]
(tool_name, tool_param) = tool_tuple
base = RESULT_PATH + '/' + model + '_' + tool_name
e = hypy.Engine(tool_name, tool_param)
e.set_input(path)
e.set_output(base + hypy.TOOLS[tool_name].default_ext())
to = TIMEOUT
print "{}/{} Running {} with {} and timeout {}".format(
index, total, model, tool_name, to)
sys.stdout.flush()
result = e.run(run_tool=SHOULD_RUN_TOOLS,
timeout=to,
save_stdout=True,
image_path=base + ".png")
if result['code'] in [
Engine.ERROR_TOOL, Engine.ERROR_CONVERSION,
Engine.TIMEOUT_CONVERSION
]:
message = "Test failed for {}/{} model {} with {}: {}".format(
index, total, model, tool_name, result['code'])
log = "\n".join(result['stdout'])
rv = (message, log)
quit_flag.value = 1
print "{}/{} Finished {} with {}".format(index, total, model, tool_name)
return rv
def run_with_pi():
'run with pi pass'
hyst_params = ['-pass_pi_sim', '-times 2.0 1.0']
e = hypy.Engine()
e.set_print_terminal_output(False)
e.set_model('neuron.xml')
e.set_tool('flowstar')
e.set_tool_params(hyst_params)
e.set_output_image('pi.png')
code = e.run()
if code != hypy.RUN_CODES.SUCCESS:
raise RuntimeError('Error:' + str(code))
def run_single((index, total, path, tool, quit_flag)):
'''run a single model with a single tool.
file is the path to the model file
tool is an entry of the TOOLS list
'''
if not quit_flag.value == 0:
return None
rv = None
filename = os.path.split(path)[1]
model = os.path.splitext(filename)[0]
tool_name = tool['name']
param = tool.get('param')
index_str = str(index+1) + "/" + str(total)
print index_str + " Running " + model + " with " + tool_name
sys.stdout.flush()
e = hypy.Engine()
e.set_timeout(TIMEOUT)
e.set_save_terminal_output(True)
e.set_model(path)
e.set_tool(tool_name)
if not param is None:
e.set_tool_params(['-tp', param])
base = RESULT_PATH + '/' + model + '_' + tool_name
e.set_output_image(base + '.png')
e.set_save_model_path(base + hypy.TOOLS[tool_name].default_ext())
code = e.run(run_tool=SHOULD_RUN_TOOLS)
if code in hypy.get_error_run_codes():
message = "Test failed for " + index_str + " model " + model + " with " + tool_name + ": " + str(code)
log = e.get_terminal_output()
rv = (message, log)
quit_flag.value = 1
print index_str + " Finished " + model + " with " + tool_name
return rv
def test_build_nondeterministic(self):
'generate a model with nondeterministic dynamics'
m = BuildGenArgsBuilder(["t", "x"])
m.add_init_condition("on", "t == 0 && x == 0")
m.set_time_bound(1.0)
m.add_mode("on", "true", ["1", " x + [0.1, 0.2]"])
e = hypy.Engine('flowstar')
e.set_verbose(True)
e.set_generator('build', m.get_generator_param())
res = e.run(run_tool=False, print_stdout=True)
self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def run_single((index, total, path, tool_tuple, quit_flag)):
'''run a single model with a single tool.
file is the path to the model file
tool is an entry of the TOOLS list
'''
if not quit_flag.value == 0:
return None
rv = None
filename = os.path.split(path)[1]
model = os.path.splitext(filename)[0]
(tool_name, tool_param) = tool_tuple
index_str = str(index + 1) + "/" + str(total)
print index_str + " Running " + model + " with " + tool_name
sys.stdout.flush()
base = RESULT_PATH + '/' + model + '_' + tool_name
e = hypy.Engine(tool_name, tool_param)
e.set_input(path)
e.set_output(base + hypy.TOOLS[tool_name].default_ext())
result = e.run(run_tool=SHOULD_RUN_TOOLS,
timeout=TIMEOUT,
save_stdout=True,
image_path=base + ".png")
if result['code'] in [
Engine.ERROR_TOOL, Engine.ERROR_CONVERSION,
Engine.TIMEOUT_CONVERSION
]:
message = "Test failed for " + index_str + " model " + model + " with " + tool_name + ": " + str(
result['code'])
log = "\n".join(result['stdout'])
rv = (message, log)
quit_flag.value = 1
print index_str + " Finished " + model + " with " + tool_name
return rv
def test_build_model_run_hyst(self):
'generate a model and run it through hypy'
m = BuildGenArgsBuilder(["t", "x"])
m.add_init_condition("on", "t == 0 && 41 <= x <= 42")
m.add_error_condition("on", "x >= 80")
m.set_time_bound(4.5)
m.add_mode("on", "true", ["1.2 * x", "x * t"])
m.add_mode("off", "t <= 10", ["0", "0"])
m.add_transition("on", "off", "x <= t", ["t", "1"])
e = hypy.Engine('pysim')
e.set_generator('build', m.get_generator_param())
res = e.run(run_tool=False)
self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def test_plot_limits(self):
'''test plot creation with xlim and ylim arguments'''
model = get_script_dir() + "/../../../examples/toy/toy.xml"
e = hypy.Engine('pysim')
e.set_input(model) # sets input model path
tmpdir = None
try:
tmpdir = tempfile.mkdtemp()
image_path = tmpdir + "/myimage.png"
res = e.run(image_path=image_path, xlim=[1, 5], ylim=[3, 7])
self.assertEqual(res['code'], hypy.Engine.SUCCESS)
self.assertTrue(os.path.exists(image_path))
# Note: the resulting image contents are not tested
finally:
if tmpdir:
shutil.rmtree(tmpdir)
def test_simple(self):
'''run the simple example from the hyst readme'''
model = "/home/stan/repositories/hyst/examples/toy/toy.xml"
out_image = "toy_output.png"
tool = "pysim" # pysim is the built-in simulator; try flowstar or spaceex
e = hypy.Engine()
e.set_model(model) # sets input model path
e.set_tool(tool) # sets tool name to use
#e.set_print_terminal_output(True) # print output to terminal?
#e.set_save_model_path(converted_model_path) # save converted model?
#e.set_output_image(out_image) # sets output image path
#e.set_tool_params(["-tp", "jumps=2"]) # sets parameters for hyst conversion
code = e.run(make_image=False)
self.assertEqual(code, hypy.RUN_CODES.SUCCESS)
def plot(pass_params):
'make the plot'
pass_params += ' -noerror' # skip error modes since we're plotting
e = hypy.Engine('spaceex')
e.set_input('vanderpol_althoff.xml')
e.set_output('out_vanderpol_hybridized_plot.xml')
e.add_pass('hybridizemt', pass_params)
start = time.time()
print 'Running computation + plot... ',
code = e.run(image_path='out_vanderpol_hybridized_plot.png')['code']
dif = time.time() - start
print 'done. Finished in ' + str(dif) + ' seconds'
if code != hypy.Engine.SUCCESS:
raise RuntimeError('Error: ' + str(code))
def check_dcem(pass_params):
'''check for domain contraction error modes (dcem)'''
e = hypy.Engine('spaceex')
e.set_input('vanderpol_althoff.xml')
e.add_pass('hybridizemt', pass_params)
start = time.time()
print 'Running DCEM checking computation... ',
result = e.run(parse_output=True)
dif = time.time() - start
print 'done. Finished in ' + str(dif) + ' seconds'
if result['output']['safe'] != True:
raise RuntimeError("Domain contraction error modes (DCEMs) were reachable.")
else:
print "Success: DCEMs were not reachable."
if result['code'] != hypy.Engine.SUCCESS:
raise RuntimeError("Hyst resturn code was: " + str(result['code']))
def spaceex_single(abortmin, abortmax, agg="none", tol=0.1, directions='box', timeout=5):
'''
run spaceex with the given settings. runs only one time
Switching to passive occurs at time [abortmin, abortmax]
returns ('safe', runtime_sec), ('unsafe', runtime_sec)
'''
init = 'abortmin=={} & abortmax=={} & t==0 & vx==0 & vy==0 & '.format(abortmin, abortmax) + \
'-925<=x<=-875 & -425<=y<=-375 & Tmax==250 & loc()==P2"'
params = '-output-format none -aggregation {} -flowpipe_tol {} -directions {}'.format(
agg, tol, directions)
e = hypy.Engine('spaceex', params)
e.set_verbose(True)
e.set_input('abort.xml')
e.set_init(init)
e.set_output('out.xml')
print "Running Spaceex with r={} and params {}...".format(70-abortmin, params),
result = e.run(parse_output=True, print_stdout=False, timeout=2*timeout) # use 2*timeout as a hard timeout
code = result['code']
if code == hypy.Engine.TIMEOUT_TOOL:
rv = ['unsafe', None] # no time means it was a timeout
elif code != hypy.Engine.SUCCESS:
raise RuntimeError('Error: ' + str(code))
elif result['output']['safe']:
rv = ['safe', result['tool_time']]
else:
rv = ['unsafe', result['tool_time']]
print rv
return rv
def plot_vanderpol(hyst_params):
'''plot widths using vanderpol althoff
'''
name = 'vanderpol_althoff'
e = hypy.Engine()
e.set_tool_params(hyst_params)
e.set_save_model_path('out_vanderpol_hybridized_plot.xml')
e.set_output_image('out_vanderpol_hybridized_plot.png')
e.set_print_terminal_output(True)
e.set_model(name + '.xml')
e.set_tool('spaceex')
start = time.time()
print 'Running computation... ',
code = e.run(make_image=True)
dif = time.time() - start
print 'done. Finished in ' + str(dif) + ' seconds'
if code != hypy.RUN_CODES.SUCCESS:
raise RuntimeError('Error in ' + name + ': ' + str(code))
def gen_drivetrain_pysim(theta):
'generate a drivetrain benchmark instance and plot a simulation'
title = "Drivetrain (Theta={})".format(theta)
image_path = "pysim_drivetrain_theta{}.png".format(theta)
output_path = "pysim_drivetrain{}.py".format(theta)
gen_param = '-theta {} -init_points 10'.format(theta)
tool_param = "-title \"{}\" -xdim 0 -ydim 2".format(title)
e = hypy.Engine('pysim', tool_param)
e.set_generator('drivetrain', gen_param)
#e.set_output(output_path)
#e.set_verbose(True)
#e.add_pass("sub_constants", "")
#e.add_pass("simplify", "-p")
print 'Running ' + title
res = e.run(print_stdout=True, image_path=image_path)
print 'Finished ' + title
if res['code'] != hypy.Engine.SUCCESS:
raise RuntimeError('Error in ' + title + ': ' + str(res['code']))
def run_analysis(self, name, outdir="./output/"):
'''
Analyse system with SpaceEx (practical stability via fixpoint computation) and pysim (Simulation).
This also generates plots and saves the model files used for analysis.
@param name System name for plots and other output files
@param outdir Output directory for plots and other output files
'''
print("")
print("====================")
print("Analysing System: " + name)
sys.stdout.flush()
# NOTE: Hyst's SpaceEx-conversion doesn't like dashes in filenames. (Bugfix submitted: https://github.com/verivital/hyst/pull/43 )
model_file = system.to_spaceex_files(outdir + name)
system.global_time = True
model_file_global_time = system.to_spaceex_files(
outdir + name + "__reachability_with_time_")
system.spaceex_only_simulate = True
system.to_spaceex_files(outdir + name + "__simulation_with_time_")
system.spaceex_only_simulate = False
system.use_urgent_semantics_and_pseudorandom_sequence = True
model_file_pysim = system.to_spaceex_files(outdir + name +
"__for_pysim__.")
# Nominal case?
if system.is_nominal_timing():
system.results[
'stability_eigenvalues'] = system.nominal_case_stability()
else:
# not the nominal case, cannot test stability via eigenvalues of nominal case
system.results['stability_eigenvalues'] = 'N/A'
if not hypy_available:
logging.warning(
"hypy is unavailable. Not running SpaceEx and other tools.")
return
# PySim config
# number of random initial states
# (mostly for illustration: usually these are not near maximum x_p, and therefore the resulting trajectories are not useful to determine the interval bounds)
pysim_rand_points = 50
if '--fast' in sys.argv:
pysim_rand_points = 3
# additionally use initial states at corners?
# Disabled if the number of states is large
pysim_initstate_corners = True
if (system.n_p + system.n_d + system.m +
system.p) > 8 or '--fast' in sys.argv:
pysim_initstate_corners = False
pysim_options = '-star True -corners {corners} -rand {rand}'.format(
corners=pysim_initstate_corners,
rand=pysim_rand_points) + ' -xdim {x} -ydim 2'
# Note for xdim/ydim: order of states in SpaceEx file: 0 = tau, 1 = t, 2...2+n-1 = x_p_1...n, ..., random_state_1 ... _(m+p)
e = hypy.Engine('pysim', pysim_options.format(x=1))
e.set_input(model_file_pysim)
e.set_output(outdir + name + "_pysim.py")
res = e.run(parse_output=True,
image_path=model_file_pysim + "_pysim_plot_xp1_over_t.png")
system.results['pysim_hypy'] = res
assert res['code'] == hypy.Engine.SUCCESS, "running pysim failed"
#PrettyPrinter(depth=3).pprint(res)
pysim_state_bounds = res['output']['interval_bounds']
#print("PySim min/max states, including simulation-internal states (global time, random number)")
#print(pysim_state_bounds)
print(
"PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}, u_{1...m}, y_{1...p})"
)
pysim_state_bounds = pysim_state_bounds[
[0] +
range(2, 2 + system.n_p + system.n_d + system.m + system.p), :]
system.results['pysim_state_bounds'] = pysim_state_bounds
print(pysim_state_bounds)
# PySim: plot over tau
e = hypy.Engine('pysim', pysim_options.format(x=0))
e.set_input(model_file_pysim)
assert e.run(image_path=model_file_pysim +
"_pysim_plot_xp1_over_tau.png"
)['code'] == hypy.Engine.SUCCESS, "running pysim failed"
# SpaceEx: interval bounds
e = hypy.Engine('spaceex', '-output-format INTV -output_vars *')
e.set_input(model_file) # sets input model path
res = e.run(parse_output=True,
timeout=10 if "--fast" in sys.argv else 7200)
system.results['spaceex_hypy'] = res
print("Result: {} after {} s".format(res['code'], res['time']))
def spaceex_bounds_to_array(res):
variables = res['output']['variables']
return np.block([[minmax[0], minmax[1]]
for minmax in variables.itervalues()])
#PrettyPrinter().pprint(res)
if res['code'] == 'Timeout (Tool)':
self.results['stability_spaceex'] = 'timeout'
elif res['code'] != hypy.Engine.SUCCESS:
# Tool crashed
print("SpaceEx or Hyst crashed: {}".format(res['code']))
last_stdout_lines = res.get('tool_stdout', [])[-10:]
if 'Error detected in file bflib/sgf.c at line 99' in last_stdout_lines:
print("SpaceEx failed in GLPK library (bflib/sgf.c)")
self.results['stability_spaceex'] = 'crash (GLPK)'
elif any([
'Segmentation fault (core dumped)' in line
for line in last_stdout_lines
]):
print("SpaceEx died with Segmentation fault")
self.results['stability_spaceex'] = 'crash'
elif '[stderr] caused by: glpk 4.55returned status 1(GLP_UNDEF).' in last_stdout_lines:
print(
"SpaceEx failed with GLPK-related error message GLP_UNDEF")
self.results['stability_spaceex'] = 'error (GLPK)'
elif '[stderr] caused by: Support function evaluation requested for an unbounded set:' in last_stdout_lines:
print("SpaceEx failed: unbounded set")
self.results['stability_spaceex'] = 'unbounded'
else:
print("Unknown error")
self.results['stability_spaceex'] = 'unknown failure'
print("stdout was: ...")
print("\n".join(last_stdout_lines))
else:
#res_print=deepcopy(res)
#res_print['tool_stdout']='<omitted>'
#res_print['stdout']='<omitted>'
#PrettyPrinter().pprint(res_print)
found_fixpoint = res['output']['fixpoint']
fixpoint_warning = "(incomplete result because no fixpoint was found)" if not found_fixpoint else ""
print("SpaceEx min/max state bounds " + fixpoint_warning)
spaceex_state_bounds = spaceex_bounds_to_array(res)
print(spaceex_state_bounds)
print(
"K-factor (by which factor must box(simulation) be scaled to be a superset of box(analysis)? 1 = optimal, >1 = analaysis is probably pessimistic)"
)
def kFactor(analysis_bounds, simulation_bounds):
return np.max(
np.max(np.abs(analysis_bounds), axis=1) /
np.max(np.abs(simulation_bounds), axis=1))
k = kFactor(spaceex_state_bounds, pysim_state_bounds)
print("{} {}".format(k, fixpoint_warning))
if found_fixpoint:
# a fixpoint was found: System is practically stable (neglecting floating point inaccuracy)
# and the state is within the computed bounds.
self.results['stability_spaceex'] = 'stable'
print(
"Stable (SpaceEx found fixpoint -- the above results are strict bounds)"
)
self.results['k'] = k
self.results[
'spaceex_pessimistic_state_bounds'] = spaceex_state_bounds
else:
self.results['k_lower_bound'] = k
if k > 1000:
print(
"SpaceEx iteration is diverging (K>1000 without finding a fixpoint)"
)
self.results['stability_spaceex'] = 'diverging'
else:
print(
"Unknown (SpaceEx did not find fixpoint within given number of iterations -- the above results are no strict bounds!)"
)
self.results[
'stability_spaceex'] = 'unknown, max iterations reached'
# SpaceEx: plot over tau
# hypy doesn't support multiple output formats, so we need to rerun SpaceEx.
e = hypy.Engine('spaceex',
'-output-format GEN -output_vars tau,x_p_1')
e.set_input(model_file)
assert e.run(
image_path=model_file + "__spaceex_plot_xp1_over_tau.png"
)['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
# SpaceEx: plot over global time
e = hypy.Engine('spaceex',
'-output-format GEN -output_vars t,x_p_1')
e.set_input(model_file_global_time)
assert e.run(
image_path=model_file_global_time +
"__spaceex_plot_xp1_over_t.png"
)['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
def run_analysis(self, name, outdir, extra_plots=True, print_header=True):
'''
Analyse system with SpaceEx (practical stability via fixpoint computation) and pysim (Simulation).
This also generates plots and saves the model files used for analysis.
@param name Title (SpaceEx "system name") for plots and other output files
@param outdir Output directory for plots and other output files
WARNING: this will modify parameters in self.s (TODO fix that)
'''
if print_header:
print("")
print("====================")
print("Analysing System: " + name)
if self.s.continuize:
print("Continuization enabled: Assuming a fixed bound for delta_p, delta_c, which is determined in an outer loop.")
sys.stdout.flush()
model_file = self.to_spaceex_files(outdir + name)
# FIXME: this will modify self.s (which is okay for how this function currently used, but may be annoying in the future)
self.s.global_time = True
model_file_global_time = self.to_spaceex_files(outdir + name + "__reachability_with_time_")
self.s.spaceex_only_simulate=True
self.to_spaceex_files(outdir + name + "__simulation_with_time_")
self.s.spaceex_only_simulate=False
self.s.use_urgent_semantics_and_pseudorandom_sequence = True
model_file_pysim = self.to_spaceex_files(outdir + name + "__for_pysim__.")
self.s.global_time = False
# Nominal case?
if self.s.is_nominal_timing():
self.results['stability_eigenvalues'] = self.s.nominal_case_stability()
else:
# not the nominal case, cannot test stability via eigenvalues of nominal case
self.results['stability_eigenvalues'] = 'N/A'
# PySim config
# number of random initial states
# (mostly for illustration: usually these are not near maximum x_p, and therefore the resulting trajectories are not useful to determine the interval bounds)
pysim_rand_points=50
if '--fast' in sys.argv:
pysim_rand_points=3
# additionally use initial states at corners?
# Disabled if the number of states is large
pysim_initstate_corners = True
if (self.s.n_p + self.s.n_d + self.s.m + self.s.p) > 8 or '--fast' in sys.argv:
pysim_initstate_corners = False
pysim_options = '-legend False -title " " -star True -corners {corners} -rand {rand}'.format(corners=pysim_initstate_corners, rand=pysim_rand_points) + ' -xdim {x} -ydim {y}'
# order of states in SpaceEx file:
if not self.s.continuize:
#normally: 0 = tau, 1 = t, 2...2+n-1 = x_p_1...n, ..., random_state_1 ... _(m+p)
idx_t = 1
idx_xp1 = 2
else:
# continuized: xp x_c_tilde delta_p delta_c t
idx_xp1 = 0
idx_t = 2 * (self.s.n_p + self.s.n_d)
idx_xd1 = idx_xp1 + self.s.n_p
e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xp1))
e.set_input(model_file_pysim)
e.set_output(outdir + name + "_pysim.py")
xlim = [0, (self.s.spaceex_iterations_for_global_time or self.s.spaceex_iterations) * self.s.T]
ylim_xp = [(self.s.plot_ylim_xp[i] if self.s.plot_ylim_xp else None) for i in range(self.s.n_p)]
ylim_xd = [(self.s.plot_ylim_xd[i] if self.s.plot_ylim_xd else None) for i in range(self.s.n_d)]
res = e.run(parse_output=True, image_path = model_file_pysim + "_pysim_plot_xp1_over_t.png", xlim=xlim, ylim=ylim_xp[0])
self.results['pysim_hypy'] = res
assert res['code'] == hypy.Engine.SUCCESS, "running pysim failed:" + repr(res)
#PrettyPrinter(depth=3).pprint(res)
pysim_state_bounds = res['output']['interval_bounds']
#print("PySim min/max states, including simulation-internal states (global time, random number)")
#print(pysim_state_bounds)
if self.s.continuize:
assert self.s.immediate_ctrl
print("PySim min/max states (x_p_{1...n_p}, x_d_{1...n_d}, delta_p, delta_c, t")
elif self.s.immediate_ctrl:
print("PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}")
pysim_state_bounds = pysim_state_bounds[[0] + list(range(2, 2 + self.s.n_p + self.s.n_d)), :]
else:
print("PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}, u_{1...m}, y_{1...p})")
pysim_state_bounds = pysim_state_bounds[[0] + list(range(2, 2 + self.s.n_p + self.s.n_d + self.s.m + self.s.p)), :]
self.results['pysim_state_bounds'] = pysim_state_bounds
print(pysim_state_bounds)
if extra_plots and not self.s.continuize:
# PySim: plot over tau
e = hypy.Engine('pysim', pysim_options.format(x=0, y=2))
e.set_input(model_file_pysim)
assert e.run(image_path = model_file_pysim + "_pysim_plot_xp1_over_tau.png")['code'] == hypy.Engine.SUCCESS, "running pysim failed"
if extra_plots:
# Pysim: plot all states over t
# note that xp1 was already plotted earlier.
# xp2 ... xpN
for i in range(1, self.s.n_p):
e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xp1 + i))
e.set_input(model_file_pysim)
assert e.run(image_path = model_file_pysim + f"_pysim_plot_xp{i+1}_over_t.png", xlim=xlim, ylim=ylim_xp[i])['code'] == hypy.Engine.SUCCESS, "running pysim failed"
# xd1 ... xdN
for i in range(0, self.s.n_d):
e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xd1 + i))
e.set_input(model_file_pysim)
assert e.run(image_path = model_file_pysim + f"_pysim_plot_xd{i+1}_over_t.png", xlim=xlim, ylim=ylim_xd[i])['code'] == hypy.Engine.SUCCESS, "running pysim failed"
# SpaceEx: interval bounds
e = hypy.Engine('spaceex', '-output-format INTV -output_vars *')
e.set_input(model_file) # sets input model path
res = e.run(parse_output=True, timeout=10 if "--fast" in sys.argv else self.s.spaceex_timeout)
self.results['spaceex_hypy'] = res
print("Result: {} after {} s".format(res['code'], res['time']))
#PrettyPrinter().pprint(res)
if res['code'] == 'Timeout (Tool)':
self.results['stability_spaceex'] = 'timeout'
elif res['code'] != hypy.Engine.SUCCESS:
# Tool crashed
print("SpaceEx or Hyst crashed: {}".format(res['code']))
last_stdout_lines = res.get('tool_stdout', [])[-10:]
if 'Error detected in file bflib/sgf.c at line 99' in last_stdout_lines:
print("SpaceEx failed in GLPK library (bflib/sgf.c)")
self.results['stability_spaceex'] = 'crash (GLPK)'
elif any(['Segmentation fault (core dumped)' in line for line in last_stdout_lines]):
print("SpaceEx died with Segmentation fault")
self.results['stability_spaceex'] = 'crash'
elif '[stderr] caused by: glpk 4.55returned status 1(GLP_UNDEF).' in last_stdout_lines:
print("SpaceEx failed with GLPK-related error message GLP_UNDEF")
self.results['stability_spaceex'] = 'error (GLPK)'
elif '[stderr] caused by: Support function evaluation requested for an unbounded set:' in last_stdout_lines:
print("SpaceEx failed: unbounded set")
self.results['stability_spaceex'] = 'unbounded'
else:
print("Unknown error")
self.results['stability_spaceex'] = 'unknown failure'
print("stdout was: ...")
print("\n".join(last_stdout_lines))
else:
#res_print=deepcopy(res)
#res_print['tool_stdout']='<omitted>'
#res_print['stdout']='<omitted>'
#PrettyPrinter().pprint(res_print)
spaceex_state_bounds = spaceex_bounds_to_array(res['output']['variables'])
found_fixpoint = res['output']['fixpoint']
fixpoint_warning = "(incomplete result because no fixpoint was found)" if not found_fixpoint else ""
inf_warning = "\n (note that +inf/-inf is shown for bounded disturbance variables due to technical reasons)" if (self.s.continuize and np.any(np.isinf(spaceex_state_bounds))) else ""
print("SpaceEx min/max state bounds " + fixpoint_warning + inf_warning)
print(spaceex_state_bounds)
if self.s.continuize:
k = float('nan')
else:
print("K-factor (by which factor must box(simulation) be scaled to be a superset of box(analysis)? 1 = optimal, >1 = analaysis is probably pessimistic)")
def kFactor(analysis_bounds, simulation_bounds):
return np.max(np.max(np.abs(analysis_bounds),axis=1) / np.max(np.abs(simulation_bounds),axis=1))
k = kFactor(spaceex_state_bounds, pysim_state_bounds)
print("{} {}".format(k, fixpoint_warning))
if found_fixpoint:
# a fixpoint was found: System is practically stable (neglecting floating point inaccuracy)
# and the state is within the computed bounds.
self.results['stability_spaceex'] = 'stable'
print("Stable (SpaceEx found fixpoint -- the above results are strict bounds" + (" if the assumed disturbance interval is correct " if self.s.continuize else "") + ")")
self.results['k'] = k
self.results['spaceex_pessimistic_state_bounds'] = spaceex_state_bounds
else:
self.results['k_lower_bound'] = k
if k>1000:
print("SpaceEx iteration is diverging (K>1000 without finding a fixpoint)")
self.results['stability_spaceex'] = 'diverging'
else:
print("Unknown (SpaceEx did not find fixpoint within given number of iterations -- the above results are no strict bounds!)")
self.results['stability_spaceex'] = 'unknown, max iterations reached'
if extra_plots and not self.s.continuize:
# SpaceEx: plot over tau
# hypy doesn't support multiple output formats, so we need to rerun SpaceEx.
e = hypy.Engine('spaceex', '-output-format GEN -output_vars tau,x_p_1')
e.set_input(model_file)
assert e.run(image_path = model_file + "__spaceex_plot_xp1_over_tau.png")['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
if extra_plots:
# SpaceEx: plot over global time
e = hypy.Engine('spaceex', '-output-format GEN -output_vars t,x_p_1')
e.set_input(model_file_global_time)
assert e.run(image_path = model_file_global_time + "__spaceex_plot_xp1_over_t.png", xlim=xlim, ylim=ylim_xp[0])['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
def measure():
'''run the measurements'''
timeout_secs = 15
num_trials = 10
tools = []
labels = []
tool_params = []
input_xml = []
# tools.append('flowstar')
# labels.append('Flowstar')
# tool_params.append('-orders 1')
# input_xml.append('io.xml')
# tools.append('spaceex')
# labels.append('SpaceEx')
# tool_params.append('-scenario supp -skiptol -flowpipe_tol 0 -output-format TXT')
# input_xml.append('io.xml')
tools.append('hylaa')
labels.append('Hylaa')
tool_params.append('-settings settings.print_output=False')
input_xml.append('io.xml')
tools.append('hylaa')
labels.append('Warm')
tool_params.append('-settings settings.print_output=False ' +
'settings.opt_decompose_lp=False')
input_xml.append('io.xml')
tools.append('hylaa')
labels.append('Decomp')
tool_params.append('-settings settings.print_output=False ' +
'settings.opt_warm_start_lp=False')
input_xml.append('io.xml')
tools.append('hylaa')
labels.append('Basic')
tool_params.append(
'-settings settings.print_output=False ' +
'settings.opt_decompose_lp=False settings.opt_warm_start_lp=False')
input_xml.append('io.xml')
tools.append('hylaa')
labels.append('NoInput')
tool_params.append('-settings settings.print_output=False')
input_xml.append('ha.xml')
assert len(labels) == len(tools)
start = time.time()
for i in xrange(len(tools)):
tool = tools[i]
label = labels[i]
with open('out/result_{}.dat'.format(label), 'w') as f:
step_size = 0.2
while True:
timeout = False
total_secs = 0.0
measured_secs = []
for _ in xrange(num_trials):
params = "{} -step {:.9f}".format(tool_params[i],
step_size)
e = hypy.Engine(tool, params)
e.set_input(input_xml[i])
#e.set_output('out/temp.flowstar')
print "Running '{}' with step_size = {:.9f}...".format(
label, step_size)
#e.set_verbose(True)
res = e.run(print_stdout=True, run_tool=True)
if res['code'] == hypy.Engine.SUCCESS:
runtime = res['tool_time']
print "Completed after {:2f} seconds".format(runtime)
measured_secs.append(runtime)
total_secs += runtime
else:
raise RuntimeError("Running Tool Failed: {}".format(
res['code']))
measured_secs.sort()
#avg_runtime = (total_secs - measured_secs[0] - measured_secs[-1]) / (num_trials - 2)
#lower_bound = measured_secs[1]
#upper_bound = measured_secs[-2]
avg_runtime = total_secs / num_trials
lower_bound = measured_secs[0]
upper_bound = measured_secs[-1]
print "Average runtime: {:2f}, range = [{:2f}, {:2f}]".format(
avg_runtime, lower_bound, upper_bound)
max_time = 6.28
num_steps = int(max_time / step_size)
f.write('{} {} {} {}\n'.format(num_steps, avg_runtime,
lower_bound, upper_bound))
if avg_runtime > timeout_secs:
break
step_size /= 1.3
dif = time.time() - start
print "Measurement Completed in {:2f} seconds".format(dif)
def run_single(xxx_todo_changeme):
'''run a single model with a single tool.
file is the path to the model file
tool is an entry of the TOOLS list
'''
(index, total, path, tool_tuple, quit_flag) = xxx_todo_changeme
if not quit_flag.value == 0:
return None
rv = None
filename = os.path.split(path)[1]
model = os.path.splitext(filename)[0]
(tool_name, tool_param) = tool_tuple
base = RESULT_PATH + '/' + model + '_' + tool_name
e = hypy.Engine(tool_name, tool_param)
e.set_input(path)
e.set_output(base + hypy.TOOLS[tool_name].default_ext())
current_timeout = TIMEOUT
allowed_results = [Engine.SUCCESS]
NONLINEAR_MODELS = ['biology7d', 'biology9d', 'brusselator', 'coupled_vanderpol', 'vanderpol', 'lorenz', 'stable_3d', 'neuron']
# Ignore rules: Which errors are okay? Which models need a longer timeout?
if tool_name == 'hylaa' and model in NONLINEAR_MODELS + ['pd_lut_linear']:
# unsupported dynamics for hylaa
allowed_results += [Engine.ERROR_UNSUPPORTED]
if tool_name == 'dreach' and model in NONLINEAR_MODELS + ['cont_approx', 'pd_lut_linear']:
# unsupported dynamics
allowed_results += [Engine.ERROR_UNSUPPORTED]
if tool_name == 'spaceex' and model in NONLINEAR_MODELS:
# unsupported dynamics
allowed_results += [Engine.ERROR_UNSUPPORTED]
# if tool_name == 'my_broken_tool' and model in ['foo', 'bar']:
# allowed_results += [Engine.ERROR_UNSUPPORTED, Engine.ERROR_TOOL, Engine.ERROR_CONVERSION]
if tool_name == 'hylaa' and model == 'toy_diverging':
# Hylaa doesn't abort the computation for this diverging automaton
allowed_results += [Engine.TIMEOUT_TOOL]
elif 'toy' in model:
# for the trivial examples, give plenty of time,
# do not allow a timeout
current_timeout += 60
else:
# for nontrivial examples, timeout is okay; we are happy if the tool starts without crashing
# FIXME: is this really what we want?
allowed_results += [Engine.TIMEOUT_TOOL]
print("{}/{} Running {} with {} and timeout {}".format(index, total, model, tool_name, current_timeout))
sys.stdout.flush()
# enable 'parse_output' if the tool overrides the parse_output() method
tool_supports_parse_output = (hypy.TOOLS[tool_name].parse_output.__code__ is not hybridpy.hybrid_tool.HybridTool.parse_output.__code__)
result = e.run(run_tool=SHOULD_RUN_TOOLS, timeout=current_timeout, save_stdout=True, image_path=base + ".png", parse_output=tool_supports_parse_output)
if result['code'] not in allowed_results:
message = "Test failed for {}/{} model {} with {}: {}. (If this is not an error, add an ignore rule to src/tests/integration/run_test.py.".format(index, total, model, tool_name, result['code'])
log = "\n".join(result['stdout'])
rv = (message, log)
quit_flag.value = 1
elif result['code'] != Engine.SUCCESS:
print("Notice: Ignoring test result for {}/{} model {} with {}: {}".format(index, total, model, tool_name, result['code']))
print("{}/{} Finished {} with {}".format(index, total, model, tool_name))
return rv
