def run_drivetrain(theta, scale_init=1.0): 'generate a drivetrain benchmark instance, run it and return the runtime' title = "Drivetrain (Theta={}, InitScale={})".format(theta, scale_init) output_path = "hylaa_drivetrain_theta{}_initscale{}.py".format(theta, str(scale_init).replace(".", "_")) gen_param = '-theta {} -init_scale {} -reverse_errors -switch_time 0.20001'.format(theta, scale_init) e = hypy.Engine('hylaa') e.set_generator('drivetrain', gen_param) e.set_output(output_path) e.add_pass("sub_constants", "") e.add_pass("simplify", "-p") print 'Running ' + title res = e.run(print_stdout=False, parse_output=True) print 'Finished ' + title if res['code'] != hypy.Engine.SUCCESS: raise RuntimeError('Error generating ' + title + ': ' + str(res['code'])) runtime = res['tool_time'] safe = res['output']['safe'] return runtime, safe
def plot(plot_param): '''run a tool and make a plot''' name = plot_param[0] model = plot_param[1] tool = plot_param[2] tool_params = plot_param[3] e = hypy.Engine() e.set_save_terminal_output(True) #e.set_save_model_path("out.model") e.set_model(model) e.set_tool(tool) if tool_params is not None: e.set_tool_params(tool_params) e.set_print_terminal_output(True) e.set_output_image(name + '.png') print 'Running ' + name code = e.run() print 'Finished ' + name if code != hypy.RUN_CODES.SUCCESS: print '\n\n-------------------\nError in ' + name + ': ' + str( code) + "; terminal output was:" print e.get_terminal_output() raise RuntimeError('Error in ' + name + ': ' + str(code))
def run_spaceex(model, interval_bounds): """ Run SpaceEx on the given model. @param model: path to model @param interval_bounds: True to compute interval state bounds, False to produce plot image """ hyst_options = '-output_vars *' if interval_bounds else '-output-format GEN' # we need to explicitly specify the output variables because all except the first two are lost somewhere in Hyst # (Design limitation in Hyst, not easy to fix? AutomatonSettings.plotVariableNames has fixed length 2, output_vars uses that by default) e = hypy.Engine('spaceex', hyst_options) e.set_input(model) # sets input model path image_path = None if interval_bounds else (get_script_dir() + "/output_spaceex/" + os.path.basename(model)[:-4] + "__spaceex_y_vs_tau.png") res = e.run(parse_output=True, image_path=image_path) #PrettyPrinter().pprint(res) assert res['code'] == hypy.Engine.SUCCESS if not res['output']['fixpoint']: res_print = deepcopy(res) res_print['tool_stdout'] = '<omitted>' res_print['stdout'] = '<omitted>' PrettyPrinter().pprint(res_print) raise Exception( "SpaceEx did not find fixpoint - not proven stable. This should not happen (except if you modified the model)" ) if interval_bounds: print('State limits:', res['output']['variables']) else: print("Plot saved to {}".format(image_path))
def run_flowstar(model, time, timestep): """ run flowstar with given model file, final time and timestep """ print("\ntimestep: {}, final time: {}".format(timestep, time)) e = hypy.Engine('flowstar', '-step {ts} -time {t}'.format(ts=timestep, t=time)) e.set_input(model) modelname = model.split("/")[-1].split(".")[0] filename = get_script_dir( ) + '/output_flowstar/' + modelname + '__timestep_{}_time_{}'.format( timestep, time) e.set_output(filename + '.flowstar') result = e.run(image_path=filename + '.png', parse_output=True, timeout=2 * 60 * 60) print(result['code']) print("writing outputs to {}".format(filename)) with open(filename + ".output.txt", "w") as f: f.write(PrettyPrinter().pformat(result)) if time <= 10 and timestep >= 1e-3: assert result['code'] == hypy.Engine.SUCCESS, 'hyst or flowstar failed' print("Runtime: {} s".format(result['time'])) if result['code'] == hypy.Engine.SUCCESS: bounds = gnuplot_oct_data_to_bounds( result['output']['gnuplot_oct_data'], xlabel='tau', ylabel='y') print("state bounds computed by flowstar:" + str(bounds)) if bounds['y'][1] > 10: print("does not converge -- state bounds far too high") elif time >= 20: print( "may be converging -- to be sure, check with greater final time" )
def plot(plot_param): '''run a tool and make a plot''' name = plot_param[0] model = plot_param[1] tool_name = plot_param[2] tool_param = plot_param[3] pass_name = plot_param[4] pass_param = plot_param[5] e = hypy.Engine(tool_name, tool_param) e.set_input(model) if pass_name is not None: e.add_pass(pass_name, pass_param) print 'Running ' + name code = e.run(image_path=name + ".png", print_stdout=False)['code'] print 'Finished ' + name if code != hypy.Engine.SUCCESS: print '\n\n-------------------\nError in ' + name + ': ' + str(code) raise RuntimeError('Error in ' + name + ': ' + str(code))
def gen_nav(name, a_matrix, i_list, width, start_point, time, noise): 'generate a navigation benchmark instance and plot a simulation' gen_param = '-matrix ' + str(a_matrix[0][0]) + ' ' + str(a_matrix[0][1]) + \ ' ' + str(a_matrix[1][0]) + ' ' + str(a_matrix[1][1]) + ' -i_list ' for i in i_list: gen_param += str(i) + ' ' (start_x, start_y) = start_point gen_param += '-width ' + str(width) + ' -startx ' + str(start_x) + ' -starty ' + str(start_y) + \ ' -noise ' + str(noise) tool_param = "-corners True -legend False -rand 100 -time {} -title {}".format( time, name) e = hypy.Engine('pysim', tool_param) e.set_generator('nav', gen_param) e.set_output(name + '.py') print('Running ' + name) e.run(print_stdout=True, image_path=name + '.png') print('Finished ' + name) res = e.run() if res['code'] != hypy.Engine.SUCCESS: raise RuntimeError('Error in ' + name + ': ' + str(res['code']))
def run_without_pi(): 'run without pi pass' e = hypy.Engine('flowstar') e.set_input('neuron.xml') code = e.run(image_path='original.png')['code'] if code != hypy.Engine.SUCCESS: raise RuntimeError('Error:' + str(code))
def run(unsafe_condition, order, step_size): '''run flowstar with the given unsafe condition, order, and step size returns a 2-tuple: (is_safe, runtime_seconds) ''' # we will add an error condition, and then find parameters in Flow* that prove safety with the error condition input_model_file = 'vanderpol.xml' output_model_file = 'out_flowstar.model' flowstar_hyst_param = '-orders {} -step {} -unsafe "{}"'.format(order, step_size, unsafe_condition) e = hypy.Engine('flowstar', flowstar_hyst_param) e.set_input(input_model_file) print_stdout = False image_path = None #### enable these for debugging #### #e.set_output('out.model') # output the generated Flow* model to this path #e.set_verbose(True) # print hyst verbose output #print_stdout=True # print hyst/tool output #image_path='out.png' # output image path (requires GIMP is setup according to Hyst README) print "{}".format(step_size), result = e.run(parse_output=True, image_path=image_path, print_stdout=print_stdout) # result is a dictionary object with the following keys: # 'code' - exit code - engine.SUCCESS if successful, an engine.ERROR_* code otherwise # 'hyst_time' - time in seconds for hyst to run, only returned if run_hyst == True # 'tool_time' - time in seconds for tool(+image) to run, only returned if run_tool == True # 'time' - total run time in seconds # 'stdout' - the list of lines anything produced to stdout, only returned if save_stdout == True # 'tool_stdout' - the list of lines the tool produced to stdout, only returned if save_stdout == True # 'hypy_stdout' - the list of lines hypy produces to stdout, only returned if save_stdout == True # 'output' - tool-specific processed output object, only returned if successful and parse_output == True if result['code'] != hypy.Engine.SUCCESS: raise RuntimeError('Hypy Error: {}'.format(result['code'])) runtime = result['tool_time'] output = result['output'] #The output object is an ordered dictionary, with: # 'terminated' -> True/False <-- did errors occur during computation (was 'terminated' printed?) # 'mode_times' -> [(mode1, time1), ...] <-- list of reach-tmes computed in each mode, in order # 'result' -> 'UNKNOWN'/'SAFE'/None <-- if unsafe set is used and 'terminated' is false, this stores # the text after "Result: " in stdout # 'safe' -> True iff 'result' == 'SAFE' # 'gnuplot_oct_data' -> the octogon data in the tool's gnuplot output # 'reachable_taylor_models' -> a list of TaylorModel objects parsed from out.flow is_safe = output['result'] == "SAFE" print "({}) ".format(output['result']), return is_safe, runtime
def run_with_pi(): 'run with pi pass' e = hypy.Engine('flowstar') e.add_pass('pi_sim', '-times 2.0 1.0') e.set_input('neuron.xml') code = e.run(image_path='pi.png')['code'] if code != hypy.Engine.SUCCESS: raise RuntimeError(str(code))
def test_simple(self): '''run the simple example from the hyst readme''' model = get_script_dir() + "/../../../examples/toy/toy.xml" e = hypy.Engine('pysim') e.set_input(model) # sets input model path res = e.run() self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def test_pysim(self): '''run the simple example from the hyst readme''' model = get_script_dir() + "/../../../examples/toy/toy.xml" e = hypy.Engine('pysim') e.set_input(model) # sets input model path res = e.run(parse_output=True) self.assertEqual(res['code'], hypy.Engine.SUCCESS) np.testing.assert_allclose(res['output']['interval_bounds'], np.array([[0, 9],[0, 20], [0, 20]]))
def run_without_pi(): 'run without pi pass' e = hypy.Engine() e.set_print_terminal_output(False) e.set_model('neuron.xml') e.set_tool('flowstar') e.set_output_image('original.png') code = e.run() if code != hypy.RUN_CODES.SUCCESS: raise RuntimeError('Error:' + str(code))
def run_single((index, total, path, tool_tuple, quit_flag)): '''run a single model with a single tool. file is the path to the model file tool is an entry of the TOOLS list ''' if not quit_flag.value == 0: return None rv = None filename = os.path.split(path)[1] model = os.path.splitext(filename)[0] (tool_name, tool_param) = tool_tuple base = RESULT_PATH + '/' + model + '_' + tool_name e = hypy.Engine(tool_name, tool_param) e.set_input(path) e.set_output(base + hypy.TOOLS[tool_name].default_ext()) to = TIMEOUT print "{}/{} Running {} with {} and timeout {}".format( index, total, model, tool_name, to) sys.stdout.flush() result = e.run(run_tool=SHOULD_RUN_TOOLS, timeout=to, save_stdout=True, image_path=base + ".png") if result['code'] in [ Engine.ERROR_TOOL, Engine.ERROR_CONVERSION, Engine.TIMEOUT_CONVERSION ]: message = "Test failed for {}/{} model {} with {}: {}".format( index, total, model, tool_name, result['code']) log = "\n".join(result['stdout']) rv = (message, log) quit_flag.value = 1 print "{}/{} Finished {} with {}".format(index, total, model, tool_name) return rv
def run_with_pi(): 'run with pi pass' hyst_params = ['-pass_pi_sim', '-times 2.0 1.0'] e = hypy.Engine() e.set_print_terminal_output(False) e.set_model('neuron.xml') e.set_tool('flowstar') e.set_tool_params(hyst_params) e.set_output_image('pi.png') code = e.run() if code != hypy.RUN_CODES.SUCCESS: raise RuntimeError('Error:' + str(code))
def run_single((index, total, path, tool, quit_flag)): '''run a single model with a single tool. file is the path to the model file tool is an entry of the TOOLS list ''' if not quit_flag.value == 0: return None rv = None filename = os.path.split(path)[1] model = os.path.splitext(filename)[0] tool_name = tool['name'] param = tool.get('param') index_str = str(index+1) + "/" + str(total) print index_str + " Running " + model + " with " + tool_name sys.stdout.flush() e = hypy.Engine() e.set_timeout(TIMEOUT) e.set_save_terminal_output(True) e.set_model(path) e.set_tool(tool_name) if not param is None: e.set_tool_params(['-tp', param]) base = RESULT_PATH + '/' + model + '_' + tool_name e.set_output_image(base + '.png') e.set_save_model_path(base + hypy.TOOLS[tool_name].default_ext()) code = e.run(run_tool=SHOULD_RUN_TOOLS) if code in hypy.get_error_run_codes(): message = "Test failed for " + index_str + " model " + model + " with " + tool_name + ": " + str(code) log = e.get_terminal_output() rv = (message, log) quit_flag.value = 1 print index_str + " Finished " + model + " with " + tool_name return rv
def test_build_nondeterministic(self): 'generate a model with nondeterministic dynamics' m = BuildGenArgsBuilder(["t", "x"]) m.add_init_condition("on", "t == 0 && x == 0") m.set_time_bound(1.0) m.add_mode("on", "true", ["1", " x + [0.1, 0.2]"]) e = hypy.Engine('flowstar') e.set_verbose(True) e.set_generator('build', m.get_generator_param()) res = e.run(run_tool=False, print_stdout=True) self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def run_single((index, total, path, tool_tuple, quit_flag)): '''run a single model with a single tool. file is the path to the model file tool is an entry of the TOOLS list ''' if not quit_flag.value == 0: return None rv = None filename = os.path.split(path)[1] model = os.path.splitext(filename)[0] (tool_name, tool_param) = tool_tuple index_str = str(index + 1) + "/" + str(total) print index_str + " Running " + model + " with " + tool_name sys.stdout.flush() base = RESULT_PATH + '/' + model + '_' + tool_name e = hypy.Engine(tool_name, tool_param) e.set_input(path) e.set_output(base + hypy.TOOLS[tool_name].default_ext()) result = e.run(run_tool=SHOULD_RUN_TOOLS, timeout=TIMEOUT, save_stdout=True, image_path=base + ".png") if result['code'] in [ Engine.ERROR_TOOL, Engine.ERROR_CONVERSION, Engine.TIMEOUT_CONVERSION ]: message = "Test failed for " + index_str + " model " + model + " with " + tool_name + ": " + str( result['code']) log = "\n".join(result['stdout']) rv = (message, log) quit_flag.value = 1 print index_str + " Finished " + model + " with " + tool_name return rv
def test_build_model_run_hyst(self): 'generate a model and run it through hypy' m = BuildGenArgsBuilder(["t", "x"]) m.add_init_condition("on", "t == 0 && 41 <= x <= 42") m.add_error_condition("on", "x >= 80") m.set_time_bound(4.5) m.add_mode("on", "true", ["1.2 * x", "x * t"]) m.add_mode("off", "t <= 10", ["0", "0"]) m.add_transition("on", "off", "x <= t", ["t", "1"]) e = hypy.Engine('pysim') e.set_generator('build', m.get_generator_param()) res = e.run(run_tool=False) self.assertEqual(res['code'], hypy.Engine.SUCCESS)
def test_plot_limits(self): '''test plot creation with xlim and ylim arguments''' model = get_script_dir() + "/../../../examples/toy/toy.xml" e = hypy.Engine('pysim') e.set_input(model) # sets input model path tmpdir = None try: tmpdir = tempfile.mkdtemp() image_path = tmpdir + "/myimage.png" res = e.run(image_path=image_path, xlim=[1, 5], ylim=[3, 7]) self.assertEqual(res['code'], hypy.Engine.SUCCESS) self.assertTrue(os.path.exists(image_path)) # Note: the resulting image contents are not tested finally: if tmpdir: shutil.rmtree(tmpdir)
def test_simple(self): '''run the simple example from the hyst readme''' model = "/home/stan/repositories/hyst/examples/toy/toy.xml" out_image = "toy_output.png" tool = "pysim" # pysim is the built-in simulator; try flowstar or spaceex e = hypy.Engine() e.set_model(model) # sets input model path e.set_tool(tool) # sets tool name to use #e.set_print_terminal_output(True) # print output to terminal? #e.set_save_model_path(converted_model_path) # save converted model? #e.set_output_image(out_image) # sets output image path #e.set_tool_params(["-tp", "jumps=2"]) # sets parameters for hyst conversion code = e.run(make_image=False) self.assertEqual(code, hypy.RUN_CODES.SUCCESS)
def plot(pass_params): 'make the plot' pass_params += ' -noerror' # skip error modes since we're plotting e = hypy.Engine('spaceex') e.set_input('vanderpol_althoff.xml') e.set_output('out_vanderpol_hybridized_plot.xml') e.add_pass('hybridizemt', pass_params) start = time.time() print 'Running computation + plot... ', code = e.run(image_path='out_vanderpol_hybridized_plot.png')['code'] dif = time.time() - start print 'done. Finished in ' + str(dif) + ' seconds' if code != hypy.Engine.SUCCESS: raise RuntimeError('Error: ' + str(code))
def check_dcem(pass_params): '''check for domain contraction error modes (dcem)''' e = hypy.Engine('spaceex') e.set_input('vanderpol_althoff.xml') e.add_pass('hybridizemt', pass_params) start = time.time() print 'Running DCEM checking computation... ', result = e.run(parse_output=True) dif = time.time() - start print 'done. Finished in ' + str(dif) + ' seconds' if result['output']['safe'] != True: raise RuntimeError("Domain contraction error modes (DCEMs) were reachable.") else: print "Success: DCEMs were not reachable." if result['code'] != hypy.Engine.SUCCESS: raise RuntimeError("Hyst resturn code was: " + str(result['code']))
def spaceex_single(abortmin, abortmax, agg="none", tol=0.1, directions='box', timeout=5): ''' run spaceex with the given settings. runs only one time Switching to passive occurs at time [abortmin, abortmax] returns ('safe', runtime_sec), ('unsafe', runtime_sec) ''' init = 'abortmin=={} & abortmax=={} & t==0 & vx==0 & vy==0 & '.format(abortmin, abortmax) + \ '-925<=x<=-875 & -425<=y<=-375 & Tmax==250 & loc()==P2"' params = '-output-format none -aggregation {} -flowpipe_tol {} -directions {}'.format( agg, tol, directions) e = hypy.Engine('spaceex', params) e.set_verbose(True) e.set_input('abort.xml') e.set_init(init) e.set_output('out.xml') print "Running Spaceex with r={} and params {}...".format(70-abortmin, params), result = e.run(parse_output=True, print_stdout=False, timeout=2*timeout) # use 2*timeout as a hard timeout code = result['code'] if code == hypy.Engine.TIMEOUT_TOOL: rv = ['unsafe', None] # no time means it was a timeout elif code != hypy.Engine.SUCCESS: raise RuntimeError('Error: ' + str(code)) elif result['output']['safe']: rv = ['safe', result['tool_time']] else: rv = ['unsafe', result['tool_time']] print rv return rv
def plot_vanderpol(hyst_params): '''plot widths using vanderpol althoff ''' name = 'vanderpol_althoff' e = hypy.Engine() e.set_tool_params(hyst_params) e.set_save_model_path('out_vanderpol_hybridized_plot.xml') e.set_output_image('out_vanderpol_hybridized_plot.png') e.set_print_terminal_output(True) e.set_model(name + '.xml') e.set_tool('spaceex') start = time.time() print 'Running computation... ', code = e.run(make_image=True) dif = time.time() - start print 'done. Finished in ' + str(dif) + ' seconds' if code != hypy.RUN_CODES.SUCCESS: raise RuntimeError('Error in ' + name + ': ' + str(code))
def gen_drivetrain_pysim(theta): 'generate a drivetrain benchmark instance and plot a simulation' title = "Drivetrain (Theta={})".format(theta) image_path = "pysim_drivetrain_theta{}.png".format(theta) output_path = "pysim_drivetrain{}.py".format(theta) gen_param = '-theta {} -init_points 10'.format(theta) tool_param = "-title \"{}\" -xdim 0 -ydim 2".format(title) e = hypy.Engine('pysim', tool_param) e.set_generator('drivetrain', gen_param) #e.set_output(output_path) #e.set_verbose(True) #e.add_pass("sub_constants", "") #e.add_pass("simplify", "-p") print 'Running ' + title res = e.run(print_stdout=True, image_path=image_path) print 'Finished ' + title if res['code'] != hypy.Engine.SUCCESS: raise RuntimeError('Error in ' + title + ': ' + str(res['code']))
def run_analysis(self, name, outdir="./output/"): ''' Analyse system with SpaceEx (practical stability via fixpoint computation) and pysim (Simulation). This also generates plots and saves the model files used for analysis. @param name System name for plots and other output files @param outdir Output directory for plots and other output files ''' print("") print("====================") print("Analysing System: " + name) sys.stdout.flush() # NOTE: Hyst's SpaceEx-conversion doesn't like dashes in filenames. (Bugfix submitted: https://github.com/verivital/hyst/pull/43 ) model_file = system.to_spaceex_files(outdir + name) system.global_time = True model_file_global_time = system.to_spaceex_files( outdir + name + "__reachability_with_time_") system.spaceex_only_simulate = True system.to_spaceex_files(outdir + name + "__simulation_with_time_") system.spaceex_only_simulate = False system.use_urgent_semantics_and_pseudorandom_sequence = True model_file_pysim = system.to_spaceex_files(outdir + name + "__for_pysim__.") # Nominal case? if system.is_nominal_timing(): system.results[ 'stability_eigenvalues'] = system.nominal_case_stability() else: # not the nominal case, cannot test stability via eigenvalues of nominal case system.results['stability_eigenvalues'] = 'N/A' if not hypy_available: logging.warning( "hypy is unavailable. Not running SpaceEx and other tools.") return # PySim config # number of random initial states # (mostly for illustration: usually these are not near maximum x_p, and therefore the resulting trajectories are not useful to determine the interval bounds) pysim_rand_points = 50 if '--fast' in sys.argv: pysim_rand_points = 3 # additionally use initial states at corners? # Disabled if the number of states is large pysim_initstate_corners = True if (system.n_p + system.n_d + system.m + system.p) > 8 or '--fast' in sys.argv: pysim_initstate_corners = False pysim_options = '-star True -corners {corners} -rand {rand}'.format( corners=pysim_initstate_corners, rand=pysim_rand_points) + ' -xdim {x} -ydim 2' # Note for xdim/ydim: order of states in SpaceEx file: 0 = tau, 1 = t, 2...2+n-1 = x_p_1...n, ..., random_state_1 ... _(m+p) e = hypy.Engine('pysim', pysim_options.format(x=1)) e.set_input(model_file_pysim) e.set_output(outdir + name + "_pysim.py") res = e.run(parse_output=True, image_path=model_file_pysim + "_pysim_plot_xp1_over_t.png") system.results['pysim_hypy'] = res assert res['code'] == hypy.Engine.SUCCESS, "running pysim failed" #PrettyPrinter(depth=3).pprint(res) pysim_state_bounds = res['output']['interval_bounds'] #print("PySim min/max states, including simulation-internal states (global time, random number)") #print(pysim_state_bounds) print( "PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}, u_{1...m}, y_{1...p})" ) pysim_state_bounds = pysim_state_bounds[ [0] + range(2, 2 + system.n_p + system.n_d + system.m + system.p), :] system.results['pysim_state_bounds'] = pysim_state_bounds print(pysim_state_bounds) # PySim: plot over tau e = hypy.Engine('pysim', pysim_options.format(x=0)) e.set_input(model_file_pysim) assert e.run(image_path=model_file_pysim + "_pysim_plot_xp1_over_tau.png" )['code'] == hypy.Engine.SUCCESS, "running pysim failed" # SpaceEx: interval bounds e = hypy.Engine('spaceex', '-output-format INTV -output_vars *') e.set_input(model_file) # sets input model path res = e.run(parse_output=True, timeout=10 if "--fast" in sys.argv else 7200) system.results['spaceex_hypy'] = res print("Result: {} after {} s".format(res['code'], res['time'])) def spaceex_bounds_to_array(res): variables = res['output']['variables'] return np.block([[minmax[0], minmax[1]] for minmax in variables.itervalues()]) #PrettyPrinter().pprint(res) if res['code'] == 'Timeout (Tool)': self.results['stability_spaceex'] = 'timeout' elif res['code'] != hypy.Engine.SUCCESS: # Tool crashed print("SpaceEx or Hyst crashed: {}".format(res['code'])) last_stdout_lines = res.get('tool_stdout', [])[-10:] if 'Error detected in file bflib/sgf.c at line 99' in last_stdout_lines: print("SpaceEx failed in GLPK library (bflib/sgf.c)") self.results['stability_spaceex'] = 'crash (GLPK)' elif any([ 'Segmentation fault (core dumped)' in line for line in last_stdout_lines ]): print("SpaceEx died with Segmentation fault") self.results['stability_spaceex'] = 'crash' elif '[stderr] caused by: glpk 4.55returned status 1(GLP_UNDEF).' in last_stdout_lines: print( "SpaceEx failed with GLPK-related error message GLP_UNDEF") self.results['stability_spaceex'] = 'error (GLPK)' elif '[stderr] caused by: Support function evaluation requested for an unbounded set:' in last_stdout_lines: print("SpaceEx failed: unbounded set") self.results['stability_spaceex'] = 'unbounded' else: print("Unknown error") self.results['stability_spaceex'] = 'unknown failure' print("stdout was: ...") print("\n".join(last_stdout_lines)) else: #res_print=deepcopy(res) #res_print['tool_stdout']='<omitted>' #res_print['stdout']='<omitted>' #PrettyPrinter().pprint(res_print) found_fixpoint = res['output']['fixpoint'] fixpoint_warning = "(incomplete result because no fixpoint was found)" if not found_fixpoint else "" print("SpaceEx min/max state bounds " + fixpoint_warning) spaceex_state_bounds = spaceex_bounds_to_array(res) print(spaceex_state_bounds) print( "K-factor (by which factor must box(simulation) be scaled to be a superset of box(analysis)? 1 = optimal, >1 = analaysis is probably pessimistic)" ) def kFactor(analysis_bounds, simulation_bounds): return np.max( np.max(np.abs(analysis_bounds), axis=1) / np.max(np.abs(simulation_bounds), axis=1)) k = kFactor(spaceex_state_bounds, pysim_state_bounds) print("{} {}".format(k, fixpoint_warning)) if found_fixpoint: # a fixpoint was found: System is practically stable (neglecting floating point inaccuracy) # and the state is within the computed bounds. self.results['stability_spaceex'] = 'stable' print( "Stable (SpaceEx found fixpoint -- the above results are strict bounds)" ) self.results['k'] = k self.results[ 'spaceex_pessimistic_state_bounds'] = spaceex_state_bounds else: self.results['k_lower_bound'] = k if k > 1000: print( "SpaceEx iteration is diverging (K>1000 without finding a fixpoint)" ) self.results['stability_spaceex'] = 'diverging' else: print( "Unknown (SpaceEx did not find fixpoint within given number of iterations -- the above results are no strict bounds!)" ) self.results[ 'stability_spaceex'] = 'unknown, max iterations reached' # SpaceEx: plot over tau # hypy doesn't support multiple output formats, so we need to rerun SpaceEx. e = hypy.Engine('spaceex', '-output-format GEN -output_vars tau,x_p_1') e.set_input(model_file) assert e.run( image_path=model_file + "__spaceex_plot_xp1_over_tau.png" )['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot" # SpaceEx: plot over global time e = hypy.Engine('spaceex', '-output-format GEN -output_vars t,x_p_1') e.set_input(model_file_global_time) assert e.run( image_path=model_file_global_time + "__spaceex_plot_xp1_over_t.png" )['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
def run_analysis(self, name, outdir, extra_plots=True, print_header=True): ''' Analyse system with SpaceEx (practical stability via fixpoint computation) and pysim (Simulation). This also generates plots and saves the model files used for analysis. @param name Title (SpaceEx "system name") for plots and other output files @param outdir Output directory for plots and other output files WARNING: this will modify parameters in self.s (TODO fix that) ''' if print_header: print("") print("====================") print("Analysing System: " + name) if self.s.continuize: print("Continuization enabled: Assuming a fixed bound for delta_p, delta_c, which is determined in an outer loop.") sys.stdout.flush() model_file = self.to_spaceex_files(outdir + name) # FIXME: this will modify self.s (which is okay for how this function currently used, but may be annoying in the future) self.s.global_time = True model_file_global_time = self.to_spaceex_files(outdir + name + "__reachability_with_time_") self.s.spaceex_only_simulate=True self.to_spaceex_files(outdir + name + "__simulation_with_time_") self.s.spaceex_only_simulate=False self.s.use_urgent_semantics_and_pseudorandom_sequence = True model_file_pysim = self.to_spaceex_files(outdir + name + "__for_pysim__.") self.s.global_time = False # Nominal case? if self.s.is_nominal_timing(): self.results['stability_eigenvalues'] = self.s.nominal_case_stability() else: # not the nominal case, cannot test stability via eigenvalues of nominal case self.results['stability_eigenvalues'] = 'N/A' # PySim config # number of random initial states # (mostly for illustration: usually these are not near maximum x_p, and therefore the resulting trajectories are not useful to determine the interval bounds) pysim_rand_points=50 if '--fast' in sys.argv: pysim_rand_points=3 # additionally use initial states at corners? # Disabled if the number of states is large pysim_initstate_corners = True if (self.s.n_p + self.s.n_d + self.s.m + self.s.p) > 8 or '--fast' in sys.argv: pysim_initstate_corners = False pysim_options = '-legend False -title " " -star True -corners {corners} -rand {rand}'.format(corners=pysim_initstate_corners, rand=pysim_rand_points) + ' -xdim {x} -ydim {y}' # order of states in SpaceEx file: if not self.s.continuize: #normally: 0 = tau, 1 = t, 2...2+n-1 = x_p_1...n, ..., random_state_1 ... _(m+p) idx_t = 1 idx_xp1 = 2 else: # continuized: xp x_c_tilde delta_p delta_c t idx_xp1 = 0 idx_t = 2 * (self.s.n_p + self.s.n_d) idx_xd1 = idx_xp1 + self.s.n_p e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xp1)) e.set_input(model_file_pysim) e.set_output(outdir + name + "_pysim.py") xlim = [0, (self.s.spaceex_iterations_for_global_time or self.s.spaceex_iterations) * self.s.T] ylim_xp = [(self.s.plot_ylim_xp[i] if self.s.plot_ylim_xp else None) for i in range(self.s.n_p)] ylim_xd = [(self.s.plot_ylim_xd[i] if self.s.plot_ylim_xd else None) for i in range(self.s.n_d)] res = e.run(parse_output=True, image_path = model_file_pysim + "_pysim_plot_xp1_over_t.png", xlim=xlim, ylim=ylim_xp[0]) self.results['pysim_hypy'] = res assert res['code'] == hypy.Engine.SUCCESS, "running pysim failed:" + repr(res) #PrettyPrinter(depth=3).pprint(res) pysim_state_bounds = res['output']['interval_bounds'] #print("PySim min/max states, including simulation-internal states (global time, random number)") #print(pysim_state_bounds) if self.s.continuize: assert self.s.immediate_ctrl print("PySim min/max states (x_p_{1...n_p}, x_d_{1...n_d}, delta_p, delta_c, t") elif self.s.immediate_ctrl: print("PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}") pysim_state_bounds = pysim_state_bounds[[0] + list(range(2, 2 + self.s.n_p + self.s.n_d)), :] else: print("PySim min/max states (tau, x_p_{1...n_p}, x_d_{1...n_d}, u_{1...m}, y_{1...p})") pysim_state_bounds = pysim_state_bounds[[0] + list(range(2, 2 + self.s.n_p + self.s.n_d + self.s.m + self.s.p)), :] self.results['pysim_state_bounds'] = pysim_state_bounds print(pysim_state_bounds) if extra_plots and not self.s.continuize: # PySim: plot over tau e = hypy.Engine('pysim', pysim_options.format(x=0, y=2)) e.set_input(model_file_pysim) assert e.run(image_path = model_file_pysim + "_pysim_plot_xp1_over_tau.png")['code'] == hypy.Engine.SUCCESS, "running pysim failed" if extra_plots: # Pysim: plot all states over t # note that xp1 was already plotted earlier. # xp2 ... xpN for i in range(1, self.s.n_p): e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xp1 + i)) e.set_input(model_file_pysim) assert e.run(image_path = model_file_pysim + f"_pysim_plot_xp{i+1}_over_t.png", xlim=xlim, ylim=ylim_xp[i])['code'] == hypy.Engine.SUCCESS, "running pysim failed" # xd1 ... xdN for i in range(0, self.s.n_d): e = hypy.Engine('pysim', pysim_options.format(x=idx_t, y=idx_xd1 + i)) e.set_input(model_file_pysim) assert e.run(image_path = model_file_pysim + f"_pysim_plot_xd{i+1}_over_t.png", xlim=xlim, ylim=ylim_xd[i])['code'] == hypy.Engine.SUCCESS, "running pysim failed" # SpaceEx: interval bounds e = hypy.Engine('spaceex', '-output-format INTV -output_vars *') e.set_input(model_file) # sets input model path res = e.run(parse_output=True, timeout=10 if "--fast" in sys.argv else self.s.spaceex_timeout) self.results['spaceex_hypy'] = res print("Result: {} after {} s".format(res['code'], res['time'])) #PrettyPrinter().pprint(res) if res['code'] == 'Timeout (Tool)': self.results['stability_spaceex'] = 'timeout' elif res['code'] != hypy.Engine.SUCCESS: # Tool crashed print("SpaceEx or Hyst crashed: {}".format(res['code'])) last_stdout_lines = res.get('tool_stdout', [])[-10:] if 'Error detected in file bflib/sgf.c at line 99' in last_stdout_lines: print("SpaceEx failed in GLPK library (bflib/sgf.c)") self.results['stability_spaceex'] = 'crash (GLPK)' elif any(['Segmentation fault (core dumped)' in line for line in last_stdout_lines]): print("SpaceEx died with Segmentation fault") self.results['stability_spaceex'] = 'crash' elif '[stderr] caused by: glpk 4.55returned status 1(GLP_UNDEF).' in last_stdout_lines: print("SpaceEx failed with GLPK-related error message GLP_UNDEF") self.results['stability_spaceex'] = 'error (GLPK)' elif '[stderr] caused by: Support function evaluation requested for an unbounded set:' in last_stdout_lines: print("SpaceEx failed: unbounded set") self.results['stability_spaceex'] = 'unbounded' else: print("Unknown error") self.results['stability_spaceex'] = 'unknown failure' print("stdout was: ...") print("\n".join(last_stdout_lines)) else: #res_print=deepcopy(res) #res_print['tool_stdout']='<omitted>' #res_print['stdout']='<omitted>' #PrettyPrinter().pprint(res_print) spaceex_state_bounds = spaceex_bounds_to_array(res['output']['variables']) found_fixpoint = res['output']['fixpoint'] fixpoint_warning = "(incomplete result because no fixpoint was found)" if not found_fixpoint else "" inf_warning = "\n (note that +inf/-inf is shown for bounded disturbance variables due to technical reasons)" if (self.s.continuize and np.any(np.isinf(spaceex_state_bounds))) else "" print("SpaceEx min/max state bounds " + fixpoint_warning + inf_warning) print(spaceex_state_bounds) if self.s.continuize: k = float('nan') else: print("K-factor (by which factor must box(simulation) be scaled to be a superset of box(analysis)? 1 = optimal, >1 = analaysis is probably pessimistic)") def kFactor(analysis_bounds, simulation_bounds): return np.max(np.max(np.abs(analysis_bounds),axis=1) / np.max(np.abs(simulation_bounds),axis=1)) k = kFactor(spaceex_state_bounds, pysim_state_bounds) print("{} {}".format(k, fixpoint_warning)) if found_fixpoint: # a fixpoint was found: System is practically stable (neglecting floating point inaccuracy) # and the state is within the computed bounds. self.results['stability_spaceex'] = 'stable' print("Stable (SpaceEx found fixpoint -- the above results are strict bounds" + (" if the assumed disturbance interval is correct " if self.s.continuize else "") + ")") self.results['k'] = k self.results['spaceex_pessimistic_state_bounds'] = spaceex_state_bounds else: self.results['k_lower_bound'] = k if k>1000: print("SpaceEx iteration is diverging (K>1000 without finding a fixpoint)") self.results['stability_spaceex'] = 'diverging' else: print("Unknown (SpaceEx did not find fixpoint within given number of iterations -- the above results are no strict bounds!)") self.results['stability_spaceex'] = 'unknown, max iterations reached' if extra_plots and not self.s.continuize: # SpaceEx: plot over tau # hypy doesn't support multiple output formats, so we need to rerun SpaceEx. e = hypy.Engine('spaceex', '-output-format GEN -output_vars tau,x_p_1') e.set_input(model_file) assert e.run(image_path = model_file + "__spaceex_plot_xp1_over_tau.png")['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot" if extra_plots: # SpaceEx: plot over global time e = hypy.Engine('spaceex', '-output-format GEN -output_vars t,x_p_1') e.set_input(model_file_global_time) assert e.run(image_path = model_file_global_time + "__spaceex_plot_xp1_over_t.png", xlim=xlim, ylim=ylim_xp[0])['code'] == hypy.Engine.SUCCESS, "SpaceEx failed to generate plot"
def measure(): '''run the measurements''' timeout_secs = 15 num_trials = 10 tools = [] labels = [] tool_params = [] input_xml = [] # tools.append('flowstar') # labels.append('Flowstar') # tool_params.append('-orders 1') # input_xml.append('io.xml') # tools.append('spaceex') # labels.append('SpaceEx') # tool_params.append('-scenario supp -skiptol -flowpipe_tol 0 -output-format TXT') # input_xml.append('io.xml') tools.append('hylaa') labels.append('Hylaa') tool_params.append('-settings settings.print_output=False') input_xml.append('io.xml') tools.append('hylaa') labels.append('Warm') tool_params.append('-settings settings.print_output=False ' + 'settings.opt_decompose_lp=False') input_xml.append('io.xml') tools.append('hylaa') labels.append('Decomp') tool_params.append('-settings settings.print_output=False ' + 'settings.opt_warm_start_lp=False') input_xml.append('io.xml') tools.append('hylaa') labels.append('Basic') tool_params.append( '-settings settings.print_output=False ' + 'settings.opt_decompose_lp=False settings.opt_warm_start_lp=False') input_xml.append('io.xml') tools.append('hylaa') labels.append('NoInput') tool_params.append('-settings settings.print_output=False') input_xml.append('ha.xml') assert len(labels) == len(tools) start = time.time() for i in xrange(len(tools)): tool = tools[i] label = labels[i] with open('out/result_{}.dat'.format(label), 'w') as f: step_size = 0.2 while True: timeout = False total_secs = 0.0 measured_secs = [] for _ in xrange(num_trials): params = "{} -step {:.9f}".format(tool_params[i], step_size) e = hypy.Engine(tool, params) e.set_input(input_xml[i]) #e.set_output('out/temp.flowstar') print "Running '{}' with step_size = {:.9f}...".format( label, step_size) #e.set_verbose(True) res = e.run(print_stdout=True, run_tool=True) if res['code'] == hypy.Engine.SUCCESS: runtime = res['tool_time'] print "Completed after {:2f} seconds".format(runtime) measured_secs.append(runtime) total_secs += runtime else: raise RuntimeError("Running Tool Failed: {}".format( res['code'])) measured_secs.sort() #avg_runtime = (total_secs - measured_secs[0] - measured_secs[-1]) / (num_trials - 2) #lower_bound = measured_secs[1] #upper_bound = measured_secs[-2] avg_runtime = total_secs / num_trials lower_bound = measured_secs[0] upper_bound = measured_secs[-1] print "Average runtime: {:2f}, range = [{:2f}, {:2f}]".format( avg_runtime, lower_bound, upper_bound) max_time = 6.28 num_steps = int(max_time / step_size) f.write('{} {} {} {}\n'.format(num_steps, avg_runtime, lower_bound, upper_bound)) if avg_runtime > timeout_secs: break step_size /= 1.3 dif = time.time() - start print "Measurement Completed in {:2f} seconds".format(dif)
def run_single(xxx_todo_changeme): '''run a single model with a single tool. file is the path to the model file tool is an entry of the TOOLS list ''' (index, total, path, tool_tuple, quit_flag) = xxx_todo_changeme if not quit_flag.value == 0: return None rv = None filename = os.path.split(path)[1] model = os.path.splitext(filename)[0] (tool_name, tool_param) = tool_tuple base = RESULT_PATH + '/' + model + '_' + tool_name e = hypy.Engine(tool_name, tool_param) e.set_input(path) e.set_output(base + hypy.TOOLS[tool_name].default_ext()) current_timeout = TIMEOUT allowed_results = [Engine.SUCCESS] NONLINEAR_MODELS = ['biology7d', 'biology9d', 'brusselator', 'coupled_vanderpol', 'vanderpol', 'lorenz', 'stable_3d', 'neuron'] # Ignore rules: Which errors are okay? Which models need a longer timeout? if tool_name == 'hylaa' and model in NONLINEAR_MODELS + ['pd_lut_linear']: # unsupported dynamics for hylaa allowed_results += [Engine.ERROR_UNSUPPORTED] if tool_name == 'dreach' and model in NONLINEAR_MODELS + ['cont_approx', 'pd_lut_linear']: # unsupported dynamics allowed_results += [Engine.ERROR_UNSUPPORTED] if tool_name == 'spaceex' and model in NONLINEAR_MODELS: # unsupported dynamics allowed_results += [Engine.ERROR_UNSUPPORTED] # if tool_name == 'my_broken_tool' and model in ['foo', 'bar']: # allowed_results += [Engine.ERROR_UNSUPPORTED, Engine.ERROR_TOOL, Engine.ERROR_CONVERSION] if tool_name == 'hylaa' and model == 'toy_diverging': # Hylaa doesn't abort the computation for this diverging automaton allowed_results += [Engine.TIMEOUT_TOOL] elif 'toy' in model: # for the trivial examples, give plenty of time, # do not allow a timeout current_timeout += 60 else: # for nontrivial examples, timeout is okay; we are happy if the tool starts without crashing # FIXME: is this really what we want? allowed_results += [Engine.TIMEOUT_TOOL] print("{}/{} Running {} with {} and timeout {}".format(index, total, model, tool_name, current_timeout)) sys.stdout.flush() # enable 'parse_output' if the tool overrides the parse_output() method tool_supports_parse_output = (hypy.TOOLS[tool_name].parse_output.__code__ is not hybridpy.hybrid_tool.HybridTool.parse_output.__code__) result = e.run(run_tool=SHOULD_RUN_TOOLS, timeout=current_timeout, save_stdout=True, image_path=base + ".png", parse_output=tool_supports_parse_output) if result['code'] not in allowed_results: message = "Test failed for {}/{} model {} with {}: {}. (If this is not an error, add an ignore rule to src/tests/integration/run_test.py.".format(index, total, model, tool_name, result['code']) log = "\n".join(result['stdout']) rv = (message, log) quit_flag.value = 1 elif result['code'] != Engine.SUCCESS: print("Notice: Ignoring test result for {}/{} model {} with {}: {}".format(index, total, model, tool_name, result['code'])) print("{}/{} Finished {} with {}".format(index, total, model, tool_name)) return rv