def is_benchmark_supported(benchmark : Benchmark): """returns True if the provided benchmark is supported by the tool and if the given benchmark should appear on the generated benchmark list""" if benchmark.is_pta() and benchmark.is_prism(): # The PTAs from Prism are not supported because either # modest can't apply digital clocks semantic due to open constraints, or # modest puts branch-rewards on the models (these are not supported by Storm) return False if benchmark.is_prism_inf() and benchmark.is_ctmc(): # Storm does not support the CTMCs with infinite state-spaces return False # do not include models with state space largern than 50 Mio if benchmark.get_num_states_tweak() is not None and benchmark.get_num_states_tweak() > 50000000: return False # do not select dfts with a file parameter "R" that is set to true if benchmark.is_galileo(): for p in benchmark.get_file_parameters(): if p["name"] == "R" and p["value"] == True: return False if benchmark.is_ctmc(): return True return False
def get_invocations(benchmark: Benchmark): """ Returns a list of invocations that invoke the tool for the given benchmark. It can be assumed that the current directory is the directory from which execute_invocations.py is executed. For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters. If this benchmark is not supported, an empty list has to be returned. """ if not is_benchmark_supported(benchmark): return [] short = benchmark.get_model_short_name() prop = benchmark.get_property_name() prop_type = benchmark.get_short_property_type() params = benchmark.get_parameter_values_string() instance = short + "." + params size = benchmark.get_num_states_tweak() invocations = [] benchmark_settings = "--props " + benchmark.get_property_name() if benchmark.get_open_parameter_def_string() != "": benchmark_settings += " -E " + benchmark.get_open_parameter_def_string( ) default_base = "modes/modest modes --unsafe --max-run-length 0 " + benchmark.get_janifilename( ) + " " + benchmark_settings + " -O out.txt Minimal" # # Track "probably-epsilon-correct" # precision = "5e-2" default_cmd = default_base + " --width $PRECISION --relative-width" if benchmark.is_dtmc() or benchmark.is_ctmc(): add_invocations(invocations, "probably-epsilon-correct", default_cmd.replace("$PRECISION", precision)) # # Track "often-epsilon-correct" # precision = "1e-3" if benchmark.is_dtmc() or benchmark.is_ctmc(): add_invocations(invocations, "often-epsilon-correct", default_cmd.replace("$PRECISION", precision)) # # Track "often-epsilon-correct-10-min" # if benchmark.is_dtmc() or benchmark.is_ctmc(): default_cmd = default_base + " -N 2147483647" else: default_cmd = default_base + " --width 2e-2 --relative-width --lss Interruptible 1000000 -L 1000" if benchmark.is_pta(): default_cmd += " --digital-clocks" add_invocations(invocations, "often-epsilon-correct-10-min", default_cmd) # # Done # return invocations
def get_invocations(benchmark: Benchmark): """ Returns a list of invocations that invoke the tool for the given benchmark. It can be assumed that the current directory is the directory from which execute_invocations.py is executed. For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters. If this benchmark is not supported, an empty list has to be returned. """ if not is_benchmark_supported(benchmark): return [] # Gather options that are needed for this particular benchmark for any invocation of PRISM benchmark_instance = get_prism_invocation_model_prop_instance(benchmark) invocations = [] basic_args = "{}".format(prism_mem_args) # epsilon-correct (all models but PTAs), default settings if (benchmark.get_model_type() != "pta"): # Use interval iteration generally (or uniformisation for time-bounded CTMCs) default_args = "-ii" # Choose engine heuristically default_args += " -heuristic speed" # Required precision (default anyway) default_args += " -e 1e-6" # Usual II settings when there is plenty of memory default_args += " -ddextraactionvars 100" # Increase maxiters (since QComp has a timeout anyway) default_args += " -maxiters 1000000" default_inv = Invocation() default_inv.identifier = "default" default_inv.track_id = "epsilon-correct" default_inv.add_command(prism_bin + " " + basic_args + " " + default_args + " " + benchmark_instance) invocations.append(default_inv) # epsilon-correct (all models but PTAs), specific settings if (benchmark.get_model_type() != "pta"): # Choose method/engine # Use interval iteration generally (or uniformisation for time-bounded CTMCs) if benchmark.get_model_short_name() == "haddad-monmege": specific_args = "-exact" elif (benchmark.get_num_states_tweak() == None or benchmark.get_num_states_tweak() >= 20000000): specific_args = "-ii -mtbdd" else: specific_args = "-ii -heuristic speed" # Required precision (default anyway) specific_args += " -e 1e-6" # Usual II settings when there is plenty of memory specific_args += " -ddextraactionvars 100" # Increase maxiters (since QComp has a timeout anyway) specific_args += " -maxiters 1000000" specific_inv = Invocation() specific_inv.identifier = "specific" specific_inv.track_id = "epsilon-correct" specific_inv.add_command(prism_bin + " " + basic_args + " " + specific_args + " " + benchmark_instance) invocations.append(specific_inv) # often-epsilon-correct (all models), default settings if (True): # Choose engine heuristically default_args = "-heuristic speed" # Required precision (just use default 1e-6, as agreed for QComp'19) default_args += " -e 1e-6" # Increase maxiters (since QComp has a timeout anyway) default_args += " -maxiters 1000000" default_inv = Invocation() default_inv.identifier = "default" default_inv.track_id = "often-epsilon-correct" default_inv.add_command(prism_bin + " " + basic_args + " " + default_args + " " + benchmark_instance) invocations.append(default_inv) # often-epsilon-correct (all models), specific settings if (True): # Choose method/engine if benchmark.get_model_short_name() == "haddad-monmege": specific_args = "-exact" elif (benchmark.get_num_states_tweak() == None or benchmark.get_num_states_tweak() >= 20000000): specific_args = "-mtbdd" else: specific_args = "-heuristic speed" # Required precision (just use default 1e-6, as agreed for QComp'19) specific_args += " -e 1e-6" # Increase maxiters (since QComp has a timeout anyway) specific_args += " -maxiters 1000000" specific_inv = Invocation() specific_inv.identifier = "specific" specific_inv.track_id = "often-epsilon-correct" specific_inv.add_command(prism_bin + " " + basic_args + " " + specific_args + " " + benchmark_instance) invocations.append(specific_inv) # probably-epsilon-correct (all models but PTAs), default settings if (benchmark.get_model_type() != "pta"): # Use interval iteration generally (or uniformisation for time-bounded CTMCs) if (benchmark.get_model_type() == "ctmc" and benchmark.is_time_bounded_probabilistic_reachability()): default_args = "" else: default_args = "-ii -e 5e-2" # Choose engine heuristically default_args += " -heuristic speed" # Usual II settings when there is plenty of memory default_args += " -ddextraactionvars 100" # Increase maxiters (since QComp has a timeout anyway) default_args += " -maxiters 1000000" default_inv = Invocation() default_inv.identifier = "default" default_inv.track_id = "probably-epsilon-correct" default_inv.add_command(prism_bin + " " + basic_args + " " + default_args + " " + benchmark_instance) invocations.append(default_inv) # probably-epsilon-correct (all models but PTAs), specific settings if (benchmark.get_model_type() != "pta"): # Choose method/engine # Use interval iteration generally (or uniformisation for time-bounded CTMCs) if benchmark.get_model_short_name() == "haddad-monmege": specific_args = "-exact" elif (benchmark.get_model_type() == "ctmc" and benchmark.is_time_bounded_probabilistic_reachability()): specific_args = "" elif (benchmark.get_num_states_tweak() == None or benchmark.get_num_states_tweak() >= 20000000): specific_args = "-ii -e 5e-2 -mtbdd" else: specific_args = "-ii -e 5e-2 -heuristic speed" # Usual II settings when there is plenty of memory specific_args += " -ddextraactionvars 100" # Increase maxiters (since QComp has a timeout anyway) specific_args += " -maxiters 1000000" specific_inv = Invocation() specific_inv.identifier = "specific" specific_inv.track_id = "probably-epsilon-correct" specific_inv.add_command(prism_bin + " " + basic_args + " " + specific_args + " " + benchmark_instance) invocations.append(specific_inv) return invocations
def get_invocations(benchmark: Benchmark): """ Returns a list of invocations that invoke the tool for the given benchmark. It can be assumed that the current directory is the directory from which execute_invocations.py is executed. For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters. If this benchmark is not supported, an empty list has to be returned. """ if not is_benchmark_supported(benchmark): return [] prec = dict() prec["epsilon-correct"] = "0.000001" prec["probably-epsilon-correct"] = "0.05" prec["often-epsilon-correct"] = "0.001" prec["often-epsilon-correct-10-min"] = "0.001" result = [] for track in prec.keys(): benchmark_settings = "./pet.sh reachability --precision {} --relative-error --only-result -m {} -p {} --property {}".format( prec[track], benchmark.get_prism_program_filename(), benchmark.get_prism_property_filename(), benchmark.get_property_name(), ) if benchmark.get_open_parameter_def_string() != "": benchmark_settings += " --const {}".format( benchmark.get_open_parameter_def_string()) if ("haddad" in benchmark.get_prism_program_filename() or "gathering" in benchmark.get_prism_program_filename()): benchmark_settings = "./fix-syntax " + benchmark_settings # default settings PET eps-corr default_inv = Invocation() default_inv.identifier = "default" default_inv.note = "Default settings." default_inv.track_id = track default_inv.add_command(benchmark_settings) result += [default_inv] if (track == "epsilon-correct" or benchmark.get_model_type() == "ctmc" or "haddad" in benchmark.get_prism_program_filename() or "csma" in benchmark.get_prism_program_filename() or "wlan" in benchmark.get_prism_program_filename() or "gathering" in benchmark.get_prism_program_filename()): # smc is prob eps correct, cannot handle ctmc and haddad monmege cannot be parsed by it continue if benchmark.get_num_states_tweak() is None: # need this info continue smc_settings = "./smc.sh {} {} -prop {} -heuristic RTDP_ADJ -RTDP_ADJ_OPTS 1 -colourParams S:{},Av:10,e:{},d:0.05,p:0.05,post:64".format( benchmark.get_prism_program_filename(), benchmark.get_prism_property_filename(), benchmark.get_property_name(), benchmark.get_num_states_tweak(), prec[track], ) if benchmark.get_open_parameter_def_string() != "": smc_settings += " -const {}".format( benchmark.get_open_parameter_def_string()) if ("haddad" in benchmark.get_prism_program_filename() or "gathering" in benchmark.get_prism_program_filename()): smc_settings = "./fix-syntax " + smc_settings # SMC invocations SMC_inv = Invocation() SMC_inv.identifier = "specific" SMC_inv.note = "Statistical model checking with limited information (no transition probabilities)" SMC_inv.track_id = track SMC_inv.add_command(smc_settings) result += [SMC_inv] return result
def get_invocations(benchmark : Benchmark): """ Returns a list of invocations that invoke the tool for the given benchmark. It can be assumed that the current directory is the directory from which execute_invocations.py is executed. For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters. If this benchmark is not supported, an empty list has to be returned. """ if not is_benchmark_supported(benchmark): return [] short = benchmark.get_model_short_name() prop = benchmark.get_property_name() prop_type = benchmark.get_short_property_type() params = benchmark.get_parameter_values_string() instance = short + "." + params size = benchmark.get_num_states_tweak() invocations = [] benchmark_settings = "" if benchmark.get_open_parameter_def_string() != "": benchmark_settings += "-E " + benchmark.get_open_parameter_def_string() + " " benchmark_settings += "--props " + benchmark.get_property_name() default_base = "mcsta/modest mcsta " + benchmark.get_janifilename() + " " + benchmark_settings + " -O out.txt Minimal --unsafe --es" specific_base = default_base + tweak_memory(benchmark) # specific settings default_base += " -S Memory" # # Track "floating-point-correct" # skip = False precision = "0" default_cmd = default_base + " --no-partial-results" specific_cmd = specific_base + " --no-partial-results" if prop_type == "S" or (benchmark.is_ma() or benchmark.is_ctmc()) and benchmark.is_time_bounded_probabilistic_reachability(): # long-run average or time-bounded on MA/CTMC: no fp-exact algorithm available skip = True elif prop_type == "P": # probabilistic reachability: try value iteration until fp-fixpoint default_cmd += " --p0 --p1 --epsilon 0 --absolute-epsilon" specific_cmd += " --p0 --p1 --epsilon 0 --absolute-epsilon" elif prop_type == "E": # expected reward: try value iteration until fp-fixpoint default_cmd += " --epsilon 0 --absolute-epsilon" specific_cmd += " --epsilon 0 --absolute-epsilon" elif prop_type == "Pb": # state elimination is fp-exact default_cmd += " --reward-bounded-alg StateElimination" if "-S Memory" in specific_cmd: specific_cmd += " --reward-bounded-alg StateElimination" else: specific_cmd += " --epsilon 0 --absolute-epsilon" if not skip: add_invocations(invocations, "floating-point-correct", default_cmd, tweak(benchmark, specific_cmd)) # # Track "epsilon-correct" # skip = False precision = "1e-6" default_cmd = default_base + " --no-partial-results" specific_cmd = specific_base + " --no-partial-results" if prop_type == "S": # long-run average: default is the sound algorithm based on value iteration default_cmd += " --width $PRECISION --relative-width" specific_cmd += " --width $PRECISION --relative-width" elif (benchmark.is_ma() or benchmark.is_ctmc()) and benchmark.is_time_bounded_probabilistic_reachability(): # time-bounded probability for CTMC and MA: default is sound Unif+ default_cmd += " --width $PRECISION --relative-width" specific_cmd += " --width $PRECISION --relative-width" elif benchmark.is_pta() and prop_type == "Pb": # time-bounded reachability for PTA: state elimination recommended default_cmd += " --reward-bounded-alg StateElimination" if "-S Memory" in specific_cmd: specific_cmd += " --reward-bounded-alg StateElimination" else: specific_cmd += " --alg IntervalIteration --width $PRECISION --relative-width" elif prop_type == "Pb": # reward-bounded probability: default is unsound VI, so need to change to II (SVI and OVI not yet implemented for this case) default_cmd += " --alg IntervalIteration --width $PRECISION --relative-width" specific_cmd += " --alg IntervalIteration --width $PRECISION --relative-width" else: # unbounded probability or expected reward: default is unsound VI, so need to change to OVI default_cmd += " --alg OptimisticValueIteration --epsilon $PRECISION --width $PRECISION --relative-width" specific_cmd += " --alg OptimisticValueIteration --epsilon $PRECISION --width $PRECISION --relative-width" if prop_type == "P" and benchmark.is_dtmc() or benchmark.is_ctmc(): # for unbounded probabilities on DTMC and CTMC: use 0/1 preprocessing default_cmd += " --p0 --p1" specific_cmd += " --p0 --p1" if not skip: add_invocations(invocations, "epsilon-correct", default_cmd.replace("$PRECISION", precision), tweak(benchmark, specific_cmd).replace("$PRECISION", precision)) # # Track "probably-epsilon-correct" # skip = False precision = "5e-2" if not skip: add_invocations(invocations, "probably-epsilon-correct", default_cmd.replace("$PRECISION", precision), tweak(benchmark, specific_cmd).replace("$PRECISION", precision)) # # Track "often-epsilon-correct" # skip = False precision = "1e-3" default_cmd = default_base + " --no-partial-results" specific_cmd = specific_base + " --no-partial-results" if prop_type == "S": # long-run average: default is the sound algorithm based on value iteration default_cmd += " --width $PRECISION --relative-width" specific_cmd += " --width $PRECISION --relative-width" elif (benchmark.is_ma() or benchmark.is_ctmc()) and benchmark.is_time_bounded_probabilistic_reachability(): # time-bounded probability for CTMC and MA: default is sound Unif+ default_cmd += " --width $PRECISION --relative-width" specific_cmd += " --width $PRECISION --relative-width" elif benchmark.is_pta() and prop_type == "Pb": # time-bounded reachability for PTA: state elimination recommended default_cmd += " --reward-bounded-alg StateElimination" if "-S Memory" in specific_cmd: specific_cmd += " --reward-bounded-alg StateElimination" else: specific_cmd += " --alg IntervalIteration --width $PRECISION --relative-width" elif prop_type == "Pb": # reward-bounded probability: default is unsound VI, which is okay here pass else: # unbounded probability or expected reward: default is unsound VI, which is okay here if prop_type == "P" and benchmark.is_dtmc() or benchmark.is_ctmc(): # for unbounded probabilities on DTMC and CTMC: use 0/1 preprocessing default_cmd += " --p0 --p1" specific_cmd += " --p0 --p1" if not skip: add_invocations(invocations, "often-epsilon-correct", default_cmd.replace("$PRECISION", precision), tweak(benchmark, specific_cmd).replace("$PRECISION", precision)) # # Track "often-epsilon-correct-10-min" # skip = False precision = "0" # so we just run for the full 10 minutes (or until we get an exact result) default_cmd = default_base specific_cmd = specific_base if prop_type == "S": skip = True elif (benchmark.is_ma() or benchmark.is_ctmc()) and benchmark.is_time_bounded_probabilistic_reachability(): default_cmd += " --width 0" specific_cmd += " --width 0" elif prop_type == "Pb": # reward-bounded reachability for DTMC, MDP, and PTA default_cmd += " --reward-bounded-alg StateElimination" if "-S Memory" in specific_cmd: specific_cmd += " --reward-bounded-alg StateElimination" else: specific_cmd += " --epsilon 0" else: default_cmd += " --epsilon 0" specific_cmd += " --epsilon 0" if prop_type == "P" and benchmark.is_dtmc() or benchmark.is_ctmc(): # for unbounded probabilities on DTMC and CTMC: use 0/1 preprocessing default_cmd += " --p0 --p1" specific_cmd += " --p0 --p1" if not skip: add_invocations(invocations, "often-epsilon-correct-10-min", default_cmd, tweak(benchmark, specific_cmd)) # # Done # return invocations