def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
elif not hasattr(config, 'exp_name'):
config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results[0]
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
create_python_dir("./experiments")
create_clean_python_dir(config.results_dir)
# Copy stdout and stderr to a file "simulator.out"
tee = Tee(open(os.path.join(config.results_dir, "simulator.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Load log configuration.
# N.B. this must be done after Tee()'ing.
if args.log_config:
logging.config.fileConfig(args.log_config)
else:
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
# If specified, set up a config file for each controller
for controller_config in config.simulation_config.controller_configs:
if controller_config.config_template:
controller_config.generate_config_file(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir)
# Copy over config file
config_file = re.sub(r'\.pyc$', '.py', config.__file__)
if os.path.exists(config_file):
canonical_config_file = config.results_dir + "/orig_config.py"
if os.path.abspath(config_file) != os.path.abspath(canonical_config_file):
shutil.copy(config_file, canonical_config_file)
def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
elif not hasattr(config, 'exp_name'):
config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results[0]
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
if not os.path.exists(config.results_dir):
os.makedirs(config.results_dir)
# Make sure there's an __init__.py
module_init_py = os.path.join(config.results_dir, "__init__.py")
if not os.path.exists(module_init_py):
open(module_init_py,"a").close()
# Copy stdout and stderr to a file "simulator.out"
tee = Tee(open(os.path.join(config.results_dir, "simulator.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# If specified, set up a config file for each controller
for controller_config in config.simulation_config.controller_configs:
if controller_config.config_template:
controller_config.generate_config_file(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir)
# Copy over config file
config_file = re.sub(r'\.pyc$', '.py', config.__file__)
if os.path.exists(config_file):
canonical_config_file = config.results_dir + "/orig_config.py"
if os.path.abspath(config_file) != os.path.abspath(canonical_config_file):
shutil.copy(config_file, canonical_config_file)
def play_forward(results_dir, subsequence_id):
# TODO(cs): need to serialize the parameters to Replayer rather than
# wrapping them in a closure... otherwise, can't use RemoteForker
# TODO(aw): MCSFinder needs to configure Simulation to always let DataplaneEvents pass through
create_clean_python_dir(results_dir)
# Copy stdout and stderr to a file "replay.out"
tee = Tee(open(os.path.join(results_dir, "replay.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Set up replayer.
input_logger = InputLogger()
replayer = Replayer(
self.simulation_cfg,
new_dag,
wait_on_deterministic_values=self.wait_on_deterministic_values,
input_logger=input_logger,
allow_unexpected_messages=False,
pass_through_whitelisted_messages=True,
delay_flow_mods=self.delay_flow_mods,
**self.kwargs)
replayer.init_results(results_dir)
self._runtime_stats = RuntimeStats(subsequence_id)
violations = []
simulation = None
try:
simulation = replayer.simulate()
self._track_new_internal_events(simulation, replayer)
# Wait a bit in case the bug takes awhile to happen
self.log("Sleeping %d seconds after run" %
self.end_wait_seconds)
time.sleep(self.end_wait_seconds)
violations = self.invariant_check(simulation)
if violations != []:
input_logger.log_input_event(
InvariantViolation(violations))
except SystemExit:
# One of the invariant checks bailed early. Oddly, this is not an
# error for us, it just means that there were no violations...
# [this logic is arguably broken]
# Return no violations, and let Forker handle system exit for us.
violations = []
finally:
input_logger.close(replayer,
self.simulation_cfg,
skip_mcs_cfg=True)
if simulation is not None:
simulation.clean_up()
tee.close()
if self.strict_assertion_checking:
test_serialize_response(violations,
self._runtime_stats.client_dict())
timed_out_internal = [
e.label for e in new_dag.events if e.timed_out
]
return (violations, self._runtime_stats.client_dict(),
timed_out_internal)
def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
elif not hasattr(config, 'exp_name'):
config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results[0]
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
create_python_dir("./experiments")
create_clean_python_dir(config.results_dir)
# Copy stdout and stderr to a file "simulator.out"
tee = Tee(open(os.path.join(config.results_dir, "simulator.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Load log configuration.
# N.B. this must be done after Tee()'ing.
if args.log_config:
logging.config.fileConfig(args.log_config)
else:
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO)
# If specified, set up a config file for each controller
for controller_config in config.simulation_config.controller_configs:
if controller_config.config_template:
controller_config.generate_config_file(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir)
# Copy over config file
config_file = re.sub(r'\.pyc$', '.py', config.__file__)
if os.path.exists(config_file):
canonical_config_file = config.results_dir + "/orig_config.py"
if os.path.abspath(config_file) != os.path.abspath(
canonical_config_file):
shutil.copy(config_file, canonical_config_file)
def play_forward(results_dir, subsequence_id):
# TODO(cs): need to serialize the parameters to Replayer rather than
# wrapping them in a closure... otherwise, can't use RemoteForker
# TODO(aw): MCSFinder needs to configure Simulation to always let DataplaneEvents pass through
create_clean_python_dir(results_dir)
# Copy stdout and stderr to a file "replay.out"
tee = Tee(open(os.path.join(results_dir, "replay.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Set up replayer.
input_logger = InputLogger()
replayer = Replayer(self.simulation_cfg, new_dag,
wait_on_deterministic_values=self.wait_on_deterministic_values,
input_logger=input_logger,
allow_unexpected_messages=False,
pass_through_whitelisted_messages=True,
delay_flow_mods=self.delay_flow_mods,
**self.kwargs)
replayer.init_results(results_dir)
self._runtime_stats = RuntimeStats(subsequence_id)
violations = []
simulation = None
try:
simulation = replayer.simulate()
self._track_new_internal_events(simulation, replayer)
# Wait a bit in case the bug takes awhile to happen
self.log("Sleeping %d seconds after run" % self.end_wait_seconds)
time.sleep(self.end_wait_seconds)
violations = self.invariant_check(simulation)
if violations != []:
input_logger.log_input_event(InvariantViolation(violations))
except SystemExit:
# One of the invariant checks bailed early. Oddly, this is not an
# error for us, it just means that there were no violations...
# [this logic is arguably broken]
# Return no violations, and let Forker handle system exit for us.
violations = []
finally:
input_logger.close(replayer, self.simulation_cfg, skip_mcs_cfg=True)
if simulation is not None:
simulation.clean_up()
tee.close()
if self.strict_assertion_checking:
test_serialize_response(violations, self._runtime_stats.client_dict())
timed_out_internal = [ e.label for e in new_dag.events if e.timed_out ]
return (violations, self._runtime_stats.client_dict(), timed_out_internal)
def play_forward(results_dir, subsequence_id):
# TODO(cs): need to serialize the parameters to Replayer rather than
# wrapping them in a closure... otherwise, can't use RemoteForker
# TODO(aw): MCSFinder needs to configure Simulation to always let DataplaneEvents pass through
create_clean_python_dir(results_dir)
# Copy stdout and stderr to a file "replay.out"
tee = Tee(open(os.path.join(results_dir, "replay.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Set up replayer.
input_logger = InputLogger()
replayer = Replayer(self.simulation_cfg, new_dag,
input_logger=input_logger,
bug_signature=self.bug_signature,
invariant_check_name=self.invariant_check_name,
**self.kwargs)
replayer.init_results(results_dir)
self._runtime_stats = RuntimeStats(subsequence_id)
simulation = None
try:
simulation = replayer.simulate()
self._track_new_internal_events(simulation, replayer)
except SystemExit:
# One of the invariant checks bailed early. Oddly, this is not an
# error for us, it just means that there were no violations...
# [this logic is arguably broken]
# Return no violations, and let Forker handle system exit for us.
simulation.violation_found = False
finally:
input_logger.close(replayer, self.simulation_cfg, skip_mcs_cfg=True)
if simulation is not None:
simulation.clean_up()
tee.close()
if self.strict_assertion_checking:
test_serialize_response(violations, self._runtime_stats.client_dict())
timed_out_internal = [ e.label for e in new_dag.events if e.timed_out ]
return (simulation.violation_found, self._runtime_stats.client_dict(), timed_out_internal)
def play_forward(results_dir, subsequence_id):
# TODO(cs): need to serialize the parameters to Replayer rather than
# wrapping them in a closure... otherwise, can't use RemoteForker
# TODO(aw): MCSFinder needs to configure Simulation to always let DataplaneEvents pass through
create_clean_python_dir(results_dir)
# Copy stdout and stderr to a file "replay.out"
tee = Tee(open(os.path.join(results_dir, "replay.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Set up replayer.
input_logger = InputLogger()
replayer = Replayer(self.simulation_cfg, new_dag,
input_logger=input_logger,
bug_signature=self.bug_signature,
invariant_check_name=self.invariant_check_name,
**self.kwargs)
replayer.init_results(results_dir)
self._runtime_stats = RuntimeStats(subsequence_id)
simulation = None
try:
simulation = replayer.simulate()
self._track_new_internal_events(simulation, replayer)
except SystemExit:
# One of the invariant checks bailed early. Oddly, this is not an
# error for us, it just means that there were no violations...
# [this logic is arguably broken]
# Return no violations, and let Forker handle system exit for us.
simulation.violation_found = False
finally:
input_logger.close(replayer, self.simulation_cfg, skip_mcs_cfg=True)
if simulation is not None:
simulation.clean_up()
tee.close()
if self.strict_assertion_checking:
test_serialize_response(violations, self._runtime_stats.client_dict())
timed_out_internal = [ e.label for e in new_dag.events if e.timed_out ]
return (simulation.violation_found, self._runtime_stats.client_dict(), timed_out_internal)
def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
elif not hasattr(config, 'exp_name'):
config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
create_python_dir("./experiments")
create_clean_python_dir(config.results_dir)
# Copy stdout and stderr to a file "simulator.out"
tee = Tee(open(os.path.join(config.results_dir, "simulator.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Load log configuration.
# N.B. this must be done after Tee()'ing.
if args.log_config:
logging.config.fileConfig(args.log_config)
else:
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
# If specified, set up a config file for each controller
for controller_config in config.simulation_config.controller_configs:
if controller_config.config_template:
controller_config.generate_config_file(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
additional_metadata = None
if hasattr(config, "get_additional_metadata"):
additional_metadata = config.get_additional_metadata()
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir,
additional_metadata=additional_metadata)
# Copy over config file
config_file = re.sub(r'\.pyc$', '.py', config.__file__)
if os.path.exists(config_file):
canonical_config_file = config.results_dir + "/orig_config.py"
if os.path.abspath(config_file) != os.path.abspath(canonical_config_file):
shutil.copy(config_file, canonical_config_file)
# Check configuration warnings
log = logging.getLogger("setup")
con = config.control_flow.simulation_cfg.controller_configs
def builtin_pox_controller(c):
# pox/ is already accounted for in metadata.
return ("POXController" in str(c.controller_class) and
c.cwd is not None and
re.match("^pox[/]?", c.cwd) is not None)
if (not hasattr(config, "get_additional_metadata") and
find(lambda c: not builtin_pox_controller(c),
config.control_flow.simulation_cfg.controller_configs) is not None):
log.warn('''No get_additional_metadata() defined for config file. See '''
'''config/nox_routing.py for an example.''')
def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
elif not hasattr(config, 'exp_name'):
config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
create_python_dir("./experiments")
create_clean_python_dir(config.results_dir)
# Copy stdout and stderr to a file "simulator.out"
tee = Tee(open(os.path.join(config.results_dir, "simulator.out"), "w"))
tee.tee_stdout()
tee.tee_stderr()
# Load log configuration.
# N.B. this must be done after Tee()'ing.
if args.log_config:
logging.config.fileConfig(args.log_config)
else:
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO)
# If specified, set up a config file for each controller
for controller_config in config.simulation_config.controller_configs:
if controller_config.config_template:
controller_config.generate_config_file(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
additional_metadata = None
if hasattr(config, "get_additional_metadata"):
additional_metadata = config.get_additional_metadata()
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir,
additional_metadata=additional_metadata)
# Copy over config file
config_file = re.sub(r'\.pyc$', '.py', config.__file__)
if os.path.exists(config_file):
canonical_config_file = config.results_dir + "/orig_config.py"
if os.path.abspath(config_file) != os.path.abspath(
canonical_config_file):
shutil.copy(config_file, canonical_config_file)
# Check configuration warnings
log = logging.getLogger("setup")
con = config.control_flow.simulation_cfg.controller_configs
def builtin_pox_controller(c):
# pox/ is already accounted for in metadata.
return ("POXController" in str(c.controller_class)
and c.cwd is not None
and re.match("^pox[/]?", c.cwd) is not None)
if (not hasattr(config, "get_additional_metadata")
and find(lambda c: not builtin_pox_controller(c),
config.control_flow.simulation_cfg.controller_configs)
is not None):
log.warn(
'''No get_additional_metadata() defined for config file. See '''
'''config/nox_routing.py for an example.''')