def is_benchmark_supported(benchmark: Benchmark):
"""returns True if the provided benchmark is supported by the tool and if the given benchmark should appear on the generated benchmark list"""
if not benchmark.is_prism() or benchmark.is_prism_inf():
return False
if benchmark.get_model_type() not in {"ctmc", "dtmc", "mdp"}:
return False
if benchmark.get_property_type() not in {
"prob-reach", "prob-reach-step-bounded"
}:
return False
if (benchmark.is_ctmc()
and benchmark.get_property_type() == "prob-reach-step-bounded"):
return False
return True
def get_invocations(benchmark: Benchmark):
"""
Returns a list of invocations that invoke the tool for the given benchmark.
It can be assumed that the current directory is the directory from which execute_invocations.py is executed.
For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters.
If this benchmark is not supported, an empty list has to be returned.
"""
if not is_benchmark_supported(benchmark):
return []
# Gather options that are needed for this particular benchmark for any invocation of PRISM
benchmark_instance = get_prism_invocation_model_prop_instance(benchmark)
invocations = []
basic_args = "{}".format(prism_mem_args)
# epsilon-correct (all models but PTAs), default settings
if (benchmark.get_model_type() != "pta"):
# Use interval iteration generally (or uniformisation for time-bounded CTMCs)
default_args = "-ii"
# Choose engine heuristically
default_args += " -heuristic speed"
# Required precision (default anyway)
default_args += " -e 1e-6"
# Usual II settings when there is plenty of memory
default_args += " -ddextraactionvars 100"
# Increase maxiters (since QComp has a timeout anyway)
default_args += " -maxiters 1000000"
default_inv = Invocation()
default_inv.identifier = "default"
default_inv.track_id = "epsilon-correct"
default_inv.add_command(prism_bin + " " + basic_args + " " +
default_args + " " + benchmark_instance)
invocations.append(default_inv)
# epsilon-correct (all models but PTAs), specific settings
if (benchmark.get_model_type() != "pta"):
# Choose method/engine
# Use interval iteration generally (or uniformisation for time-bounded CTMCs)
if benchmark.get_model_short_name() == "haddad-monmege":
specific_args = "-exact"
elif (benchmark.get_num_states_tweak() == None
or benchmark.get_num_states_tweak() >= 20000000):
specific_args = "-ii -mtbdd"
else:
specific_args = "-ii -heuristic speed"
# Required precision (default anyway)
specific_args += " -e 1e-6"
# Usual II settings when there is plenty of memory
specific_args += " -ddextraactionvars 100"
# Increase maxiters (since QComp has a timeout anyway)
specific_args += " -maxiters 1000000"
specific_inv = Invocation()
specific_inv.identifier = "specific"
specific_inv.track_id = "epsilon-correct"
specific_inv.add_command(prism_bin + " " + basic_args + " " +
specific_args + " " + benchmark_instance)
invocations.append(specific_inv)
# often-epsilon-correct (all models), default settings
if (True):
# Choose engine heuristically
default_args = "-heuristic speed"
# Required precision (just use default 1e-6, as agreed for QComp'19)
default_args += " -e 1e-6"
# Increase maxiters (since QComp has a timeout anyway)
default_args += " -maxiters 1000000"
default_inv = Invocation()
default_inv.identifier = "default"
default_inv.track_id = "often-epsilon-correct"
default_inv.add_command(prism_bin + " " + basic_args + " " +
default_args + " " + benchmark_instance)
invocations.append(default_inv)
# often-epsilon-correct (all models), specific settings
if (True):
# Choose method/engine
if benchmark.get_model_short_name() == "haddad-monmege":
specific_args = "-exact"
elif (benchmark.get_num_states_tweak() == None
or benchmark.get_num_states_tweak() >= 20000000):
specific_args = "-mtbdd"
else:
specific_args = "-heuristic speed"
# Required precision (just use default 1e-6, as agreed for QComp'19)
specific_args += " -e 1e-6"
# Increase maxiters (since QComp has a timeout anyway)
specific_args += " -maxiters 1000000"
specific_inv = Invocation()
specific_inv.identifier = "specific"
specific_inv.track_id = "often-epsilon-correct"
specific_inv.add_command(prism_bin + " " + basic_args + " " +
specific_args + " " + benchmark_instance)
invocations.append(specific_inv)
# probably-epsilon-correct (all models but PTAs), default settings
if (benchmark.get_model_type() != "pta"):
# Use interval iteration generally (or uniformisation for time-bounded CTMCs)
if (benchmark.get_model_type() == "ctmc"
and benchmark.is_time_bounded_probabilistic_reachability()):
default_args = ""
else:
default_args = "-ii -e 5e-2"
# Choose engine heuristically
default_args += " -heuristic speed"
# Usual II settings when there is plenty of memory
default_args += " -ddextraactionvars 100"
# Increase maxiters (since QComp has a timeout anyway)
default_args += " -maxiters 1000000"
default_inv = Invocation()
default_inv.identifier = "default"
default_inv.track_id = "probably-epsilon-correct"
default_inv.add_command(prism_bin + " " + basic_args + " " +
default_args + " " + benchmark_instance)
invocations.append(default_inv)
# probably-epsilon-correct (all models but PTAs), specific settings
if (benchmark.get_model_type() != "pta"):
# Choose method/engine
# Use interval iteration generally (or uniformisation for time-bounded CTMCs)
if benchmark.get_model_short_name() == "haddad-monmege":
specific_args = "-exact"
elif (benchmark.get_model_type() == "ctmc"
and benchmark.is_time_bounded_probabilistic_reachability()):
specific_args = ""
elif (benchmark.get_num_states_tweak() == None
or benchmark.get_num_states_tweak() >= 20000000):
specific_args = "-ii -e 5e-2 -mtbdd"
else:
specific_args = "-ii -e 5e-2 -heuristic speed"
# Usual II settings when there is plenty of memory
specific_args += " -ddextraactionvars 100"
# Increase maxiters (since QComp has a timeout anyway)
specific_args += " -maxiters 1000000"
specific_inv = Invocation()
specific_inv.identifier = "specific"
specific_inv.track_id = "probably-epsilon-correct"
specific_inv.add_command(prism_bin + " " + basic_args + " " +
specific_args + " " + benchmark_instance)
invocations.append(specific_inv)
return invocations
def get_invocations(benchmark: Benchmark):
"""
Returns a list of invocations that invoke the tool for the given benchmark.
It can be assumed that the current directory is the directory from which execute_invocations.py is executed.
For QCOMP 2020, this should return a list of invocations for all tracks in which the tool can take part. For each track an invocation with default settings has to be provided and in addition, an optimized setting (e.g., the fastest engine and/or solution technique for this benchmark) can be specified. Only information about the model type, the property type and the state space size are allowed to be used to tweak the parameters.
If this benchmark is not supported, an empty list has to be returned.
"""
if not is_benchmark_supported(benchmark):
return []
prec = dict()
prec["epsilon-correct"] = "0.000001"
prec["probably-epsilon-correct"] = "0.05"
prec["often-epsilon-correct"] = "0.001"
prec["often-epsilon-correct-10-min"] = "0.001"
result = []
for track in prec.keys():
benchmark_settings = "./pet.sh reachability --precision {} --relative-error --only-result -m {} -p {} --property {}".format(
prec[track],
benchmark.get_prism_program_filename(),
benchmark.get_prism_property_filename(),
benchmark.get_property_name(),
)
if benchmark.get_open_parameter_def_string() != "":
benchmark_settings += " --const {}".format(
benchmark.get_open_parameter_def_string())
if ("haddad" in benchmark.get_prism_program_filename()
or "gathering" in benchmark.get_prism_program_filename()):
benchmark_settings = "./fix-syntax " + benchmark_settings
# default settings PET eps-corr
default_inv = Invocation()
default_inv.identifier = "default"
default_inv.note = "Default settings."
default_inv.track_id = track
default_inv.add_command(benchmark_settings)
result += [default_inv]
if (track == "epsilon-correct" or benchmark.get_model_type() == "ctmc"
or "haddad" in benchmark.get_prism_program_filename()
or "csma" in benchmark.get_prism_program_filename()
or "wlan" in benchmark.get_prism_program_filename()
or "gathering" in benchmark.get_prism_program_filename()):
# smc is prob eps correct, cannot handle ctmc and haddad monmege cannot be parsed by it
continue
if benchmark.get_num_states_tweak() is None:
# need this info
continue
smc_settings = "./smc.sh {} {} -prop {} -heuristic RTDP_ADJ -RTDP_ADJ_OPTS 1 -colourParams S:{},Av:10,e:{},d:0.05,p:0.05,post:64".format(
benchmark.get_prism_program_filename(),
benchmark.get_prism_property_filename(),
benchmark.get_property_name(),
benchmark.get_num_states_tweak(),
prec[track],
)
if benchmark.get_open_parameter_def_string() != "":
smc_settings += " -const {}".format(
benchmark.get_open_parameter_def_string())
if ("haddad" in benchmark.get_prism_program_filename()
or "gathering" in benchmark.get_prism_program_filename()):
smc_settings = "./fix-syntax " + smc_settings
# SMC invocations
SMC_inv = Invocation()
SMC_inv.identifier = "specific"
SMC_inv.note = "Statistical model checking with limited information (no transition probabilities)"
SMC_inv.track_id = track
SMC_inv.add_command(smc_settings)
result += [SMC_inv]
return result
