def handle_int(signal, frame):
import os
from sts.util.rpc_forker import LocalForker
sys.stderr.write("Caught signal %d, stopping sdndebug (pid %d)\n" %
(signal, os.getpid()))
if (simulator.simulation_cfg.current_simulation is not None):
simulator.simulation_cfg.current_simulation.clean_up()
# kill fork()ed procs
LocalForker.kill_all()
sys.exit(13)
def handle_int(signal, frame):
import os
from sts.util.rpc_forker import LocalForker
sys.stderr.write("Caught signal %d, stopping sdndebug (pid %d)\n" %
(signal, os.getpid()))
if (simulator.simulation_cfg.current_simulation is not None):
simulator.simulation_cfg.current_simulation.clean_up()
# kill fork()ed procs
LocalForker.kill_all()
sys.exit(13)
def __init__(self, simulation_cfg, default_wait_time_seconds=0.05,
epsilon_time=0.05, **kwargs):
if len(simulation_cfg.controller_configs) != 1:
raise ValueError("Only one controller supported for snapshotting")
if simulation_cfg.controller_configs[0].sync is not None:
raise ValueError("STSSyncProto currently incompatible with snapshotting")
super(SnapshotPeeker, self).__init__(simulation_cfg,
default_wait_time_seconds=default_wait_time_seconds,
epsilon_time=epsilon_time)
self.forker = LocalForker()
if 'default_dp_permit' in kwargs and not kwargs['default_dp_permit']:
raise ValueError('''Non-default DP Permit not currently supported '''
'''Please implement the TODO near the sleep() call '''
'''in play_forward()''')
kwargs['default_dp_permit'] = True
if 'pass_through_whitelisted_messages' not in kwargs:
kwargs['pass_through_whitelisted_messages'] = False
self.kwargs = kwargs
unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ]
if unknown_kwargs != []:
raise ValueError("Unknown kwargs %s" % str(unknown_kwargs))
def __init__(self, simulation_cfg, default_wait_time_seconds=0.5,
epsilon_time=0.2, default_dp_permit=False,
pass_through_sends=False):
if len(simulation_cfg.controller_configs) != 1:
raise ValueError("Only one controller supported for snapshotting")
if simulation_cfg.controller_configs[0].sync is not None:
raise ValueError("STSSyncProto currently incompatible with snapshotting")
super(SnapshotPeeker, self).__init__(simulation_cfg,
default_wait_time_seconds=default_wait_time_seconds,
epsilon_time=epsilon_time)
self.forker = LocalForker()
if not default_dp_permit:
raise ValueError('''Non-default DP Permit not currently supported '''
'''Please implement the TODO near the sleep() call '''
'''in play_forward()''')
self.default_dp_permit = default_dp_permit
self.pass_through_sends = pass_through_sends
class SnapshotPeeker(Peeker):
''' O(n) peeker that takes controller snapshots at each input, peeks
forward, then restarts the snapshot up until the next input'''
def __init__(self, simulation_cfg, default_wait_time_seconds=0.5,
epsilon_time=0.2, default_dp_permit=False,
pass_through_sends=False):
if len(simulation_cfg.controller_configs) != 1:
raise ValueError("Only one controller supported for snapshotting")
if simulation_cfg.controller_configs[0].sync is not None:
raise ValueError("STSSyncProto currently incompatible with snapshotting")
super(SnapshotPeeker, self).__init__(simulation_cfg,
default_wait_time_seconds=default_wait_time_seconds,
epsilon_time=epsilon_time)
self.forker = LocalForker()
if not default_dp_permit:
raise ValueError('''Non-default DP Permit not currently supported '''
'''Please implement the TODO near the sleep() call '''
'''in play_forward()''')
self.default_dp_permit = default_dp_permit
self.pass_through_sends = pass_through_sends
def setup_simulation(self):
# TODO(cs): currently assumes that STSSyncProto is not used alongside
# snapshotting.
# N.B. bootstrap() does not connect switches to controllers.
# ConnectionToControllers is an input event, which we peek() for like
# any other input event.
simulation = self.simulation_cfg.bootstrap(ReplaySyncCallback(None))
simulation.openflow_buffer.pass_through_whitelisted_messages = True
controller = simulation.controller_manager.controllers[0]
return (simulation, controller)
def peek(self, dag):
'''
If dag.events == [], returns immediately.
If dag.events != [], assumes that isinstance(dag.events[0], ConnectToControllers)
'''
if dag.input_events == []:
return dag
# post: len(dag.input_events) > 0
unsupported_types = [ProcessFlowMod, FailFlowMod, DataplaneDrop]
if find(lambda e: type(e) in unsupported_types, dag.events) is not None:
raise ValueError('''Delayed flow_mods not yet supported. Please '''
'''implement the TODO near the sleep() call in play_forward()''')
if not isinstance(dag.events[0], ConnectToControllers):
raise ValueError("First event must be ConnectToControllers")
simulation = None
try:
# Inferred events includes input events and internal events
inferred_events = []
(simulation, controller) = self.setup_simulation()
# dag.input_events returns only prunable input events. We also need
# include ConnectToControllers, which is a non-prunable input event.
assert(dag.input_events[0] != dag.events[0])
snapshot_inputs = [dag.events[0]] + dag.input_events
# The input event from the previous iteration, followed by the internal
# events that were inferred from the previous peek().
# Since we assume that the first event is always ConnectToControllers,
# there are never internal events (nor input events) preceding the
# initial ConnectToControllers.
# TODO(cs): might not want to contrain the caller's dag in this way.
events_inferred_last_iteration = []
for inject_input_idx in xrange(0, len(snapshot_inputs)):
inject_input = get_inject_input(inject_input_idx, snapshot_inputs)
following_input = get_following_input(inject_input_idx, snapshot_inputs)
expected_internal_events = \
get_expected_internal_events(inject_input, following_input, dag.events)
log.debug("peek()'ing after input %d (%s)" %
(inject_input_idx, str(inject_input)))
inferred_events += events_inferred_last_iteration
if expected_internal_events == []:
# Optimization: if no internal events occured between this input and the
# next, no need to peek(), just replay the next input
log.debug("Optimization: no expected internal events")
Peeker.ambiguous_counts[0.0] += 1
events_inferred_last_iteration = [inject_input]
continue
# We replay events_inferred_last_iteration (internal events preceding
# inject_input), as well as a NOPInput with the same timestamp as inject_input
# to ensure that the timing is the same before peek()ing.
fencepost = NOPInput(time=inject_input.time, round=inject_input.round)
dag_interval = EventDag(events_inferred_last_iteration + [fencepost])
wait_time_seconds = self.get_wait_time_seconds(inject_input, following_input)
found_events = self.find_internal_events(simulation, controller,
dag_interval, inject_input,
wait_time_seconds)
events_inferred_last_iteration = [inject_input]
events_inferred_last_iteration += match_and_filter(found_events, expected_internal_events)
# Don't forget the final iteration's output
inferred_events += events_inferred_last_iteration
finally:
if simulation is not None:
simulation.clean_up()
return EventDag(inferred_events)
def find_internal_events(self, simulation, controller, dag_interval,
inject_input, wait_time_seconds):
assert(dag_interval.events != [])
initial_wait_seconds = (inject_input.time.as_float() -
dag_interval.events[-1].time.as_float())
# TODO(cs): set EventScheduler's epsilon_seconds parameter?
replayer = Replayer(self.simulation_cfg, dag_interval,
pass_through_whitelisted_messages=True,
default_dp_permit=self.default_dp_permit,
initial_wait=initial_wait_seconds)
replayer.simulation = simulation
if self.pass_through_sends:
replayer.set_pass_through_sends(simulation)
replayer.run_simulation_forward()
# Now peek() for internal events following inject_input
return self.snapshot_and_play_forward(simulation, controller,
inject_input, wait_time_seconds)
def snapshot_and_play_forward(self, simulation, controller, inject_input, wait_time_seconds):
snapshotter = Snapshotter(simulation, controller)
snapshotter.snapshot_controller()
# Here we also fork() ourselves (the network simulator), since we need
# switches to respond to messages, and potentially change their state in
# order to properly peek().
# TODO(cs): I believe that to be technically correct we need to use Chandy-Lamport
# here, since STS + POX = a distributed system.
def play_forward_and_marshal():
# Can't marshal the simulation object, so use this method as a closure
# instead.
# N.B. depends on LocalForker() -- cannot be used with RemoteForker().
# TODO(cs): even though DataplaneDrops are technically InputEvents, they
# may time out, and we might do well to try to infer this somehow.
found_events = play_forward(simulation, inject_input, wait_time_seconds)
# TODO(cs): DataplaneDrops are a special case of an InputEvent that may
# time out. We should check whether the input for this peek() was a
# DataplaneDrop, and return whether it timed out.
return [ e.to_json() for e in found_events ]
self.forker.register_task("snapshot_fork_task", play_forward_and_marshal)
# N.B. play_forward cleans up the simulation, including snaphotted controller
unmarshalled_found_events = self.forker.fork("snapshot_fork_task")
found_events = parse(unmarshalled_found_events)
# Finally, bring the controller back to the state it was at just after
# injecting inject_input
# N.B. this must be invoked in the parent process (not a forker.fork()ed
# child), since it mutates class variables in Controller.
snapshotter.snapshot_proceed()
return found_events
def __init__(self,
simulation_cfg,
superlog_path_or_dag,
invariant_check_name=None,
transform_dag=None,
end_wait_seconds=0.5,
mcs_trace_path=None,
extra_log=None,
runtime_stats_path=None,
wait_on_deterministic_values=False,
no_violation_verification_runs=1,
optimized_filtering=False,
forker=LocalForker(),
replay_final_trace=True,
strict_assertion_checking=False,
delay_flow_mods=False,
**kwargs):
super(MCSFinder, self).__init__(simulation_cfg)
# number of subsequences delta debugging has examined so far, for
# distingushing runtime stats from different intermediate runs.
self.subsequence_id = 0
self.mcs_log_tracker = None
self.replay_log_tracker = None
self.mcs_trace_path = mcs_trace_path
self.sync_callback = None
self._log = logging.getLogger("mcs_finder")
if invariant_check_name is None:
raise ValueError("Must specify invariant check")
if invariant_check_name not in name_to_invariant_check:
raise ValueError(
'''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check = name_to_invariant_check[invariant_check_name]
if type(superlog_path_or_dag) == str:
self.superlog_path = superlog_path_or_dag
# The dag is codefied as a list, where each element has
# a list of its dependents
self.dag = EventDag(log_parser.parse_path(self.superlog_path))
else:
self.dag = superlog_path_or_dag
last_invariant_violation = self.dag.get_last_invariant_violation()
if last_invariant_violation is None:
raise ValueError("No invariant violation found in dag...")
violations = last_invariant_violation.violations
if len(violations) > 1:
self.bug_signature = None
while self.bug_signature is None:
msg.interactive(
"\n------------------------------------------\n")
msg.interactive(
"Multiple violations detected! Choose one for MCS Finding:"
)
for i, violation in enumerate(violations):
msg.interactive(" [%d] %s" % (i + 1, violation))
violation_index = msg.raw_input("> ")
if re.match("^\d+$", violation_index) is None or\
int(violation_index) < 1 or\
int(violation_index) > len(violations):
msg.fail("Must provide an integer between 1 and %d!" %
len(violations))
continue
self.bug_signature = violations[int(violation_index) - 1]
else:
self.bug_signature = violations[0]
msg.success(
"\nBug signature to match is %s. Proceeding with MCS finding!\n" %
self.bug_signature)
self.transform_dag = transform_dag
# A second log with just our MCS progress log messages
self._extra_log = extra_log
self.kwargs = kwargs
self.end_wait_seconds = end_wait_seconds
self.wait_on_deterministic_values = wait_on_deterministic_values
# `no' means "number"
self.no_violation_verification_runs = no_violation_verification_runs
self._runtime_stats = RuntimeStats(
self.subsequence_id, runtime_stats_path=runtime_stats_path)
# Whether to try alternate trace splitting techiques besides splitting by time.
self.optimized_filtering = optimized_filtering
self.forker = forker
self.replay_final_trace = replay_final_trace
self.strict_assertion_checking = strict_assertion_checking
self.delay_flow_mods = delay_flow_mods
def __init__(self, simulation_cfg, superlog_path_or_dag,
invariant_check_name="", bug_signature="", transform_dag=None,
mcs_trace_path=None, extra_log=None, runtime_stats_path=None,
max_replays_per_subsequence=1,
optimized_filtering=False, forker=LocalForker(),
replay_final_trace=True, strict_assertion_checking=False,
no_violation_verification_runs=None,
**kwargs):
''' Note that you may pass in any keyword argument for Replayer to
MCSFinder, except 'bug_signature' and 'invariant_check_name' '''
super(MCSFinder, self).__init__(simulation_cfg)
# number of subsequences delta debugging has examined so far, for
# distingushing runtime stats from different intermediate runs.
self.subsequence_id = 0
self.mcs_log_tracker = None
self.replay_log_tracker = None
self.mcs_trace_path = mcs_trace_path
self.sync_callback = None
self._log = logging.getLogger("mcs_finder")
if invariant_check_name == "":
raise ValueError("Must specify invariant check")
if invariant_check_name not in name_to_invariant_check:
raise ValueError('''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
if type(superlog_path_or_dag) == str:
self.superlog_path = superlog_path_or_dag
# The dag is codefied as a list, where each element has
# a list of its dependents
self.dag = EventDag(log_parser.parse_path(self.superlog_path))
else:
self.dag = superlog_path_or_dag
if self.simulation_cfg.ignore_interposition:
filtered_events = [e for e in self.dag.events if type(e) not in all_internal_events]
self.dag = EventDag(filtered_events)
last_invariant_violation = self.dag.get_last_invariant_violation()
if last_invariant_violation is None:
raise ValueError("No invariant violation found in dag...")
violations = last_invariant_violation.violations
self.bug_signature = bug_signature
if len(violations) > 1:
while self.bug_signature == "":
msg.interactive("\n------------------------------------------\n")
msg.interactive("Multiple violations detected! Choose one for MCS Finding:")
for i, violation in enumerate(violations):
msg.interactive(" [%d] %s" % (i+1, violation))
violation_index = msg.raw_input("> ")
if re.match("^\d+$", violation_index) is None or\
int(violation_index) < 1 or\
int(violation_index) > len(violations):
msg.fail("Must provide an integer between 1 and %d!" % len(violations))
continue
self.bug_signature = violations[int(violation_index)-1]
if self.bug_signature == "":
self.bug_signature = violations[0]
msg.success("\nBug signature to match is %s. Proceeding with MCS finding!\n" % self.bug_signature)
self.transform_dag = transform_dag
# A second log with just our MCS progress log messages
self._extra_log = extra_log
self.kwargs = kwargs
unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ]
if unknown_kwargs != []:
raise ValueError("Unknown kwargs %s" % str(unknown_kwargs))
if no_violation_verification_runs is not None:
raise ValueError('''no_violation_verification_runs parameter is deprecated. '''
'''Use max_replays_per_subsequence.''')
self.max_replays_per_subsequence = max_replays_per_subsequence
self._runtime_stats = RuntimeStats(self.subsequence_id, runtime_stats_path=runtime_stats_path)
# Whether to try alternate trace splitting techiques besides splitting by time.
self.optimized_filtering = optimized_filtering
self.forker = forker
self.replay_final_trace = replay_final_trace
self.strict_assertion_checking = strict_assertion_checking
class SnapshotPeeker(Peeker):
''' O(n) peeker that takes controller snapshots at each input, peeks
forward, then restarts the snapshot up until the next input'''
def __init__(self, simulation_cfg, default_wait_time_seconds=0.05,
epsilon_time=0.05, **kwargs):
if len(simulation_cfg.controller_configs) != 1:
raise ValueError("Only one controller supported for snapshotting")
if simulation_cfg.controller_configs[0].sync is not None:
raise ValueError("STSSyncProto currently incompatible with snapshotting")
super(SnapshotPeeker, self).__init__(simulation_cfg,
default_wait_time_seconds=default_wait_time_seconds,
epsilon_time=epsilon_time)
self.forker = LocalForker()
if 'default_dp_permit' in kwargs and not kwargs['default_dp_permit']:
raise ValueError('''Non-default DP Permit not currently supported '''
'''Please implement the TODO near the sleep() call '''
'''in play_forward()''')
kwargs['default_dp_permit'] = True
if 'pass_through_whitelisted_messages' not in kwargs:
kwargs['pass_through_whitelisted_messages'] = False
self.kwargs = kwargs
unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ]
if unknown_kwargs != []:
raise ValueError("Unknown kwargs %s" % str(unknown_kwargs))
def setup_simulation(self):
# TODO(cs): currently assumes that STSSyncProto is not used alongside
# snapshotting.
# N.B. bootstrap() does not connect switches to controllers.
# ConnectionToControllers is an input event, which we peek() for like
# any other input event.
simulation = self.simulation_cfg.bootstrap(ReplaySyncCallback(None))
simulation.openflow_buffer.pass_through_whitelisted_packets = self.kwargs['pass_through_whitelisted_messages']
controller = simulation.controller_manager.controllers[0]
return (simulation, controller)
def peek(self, dag):
'''
If dag.events == [], returns immediately.
If dag.events != [], assumes that isinstance(dag.events[0], ConnectToControllers)
'''
if dag.input_events == []:
return dag
# post: len(dag.input_events) > 0
unsupported_types = [ProcessFlowMod, DataplaneDrop]
if find(lambda e: type(e) in unsupported_types, dag.events) is not None:
raise ValueError('''Delayed flow_mods not yet supported. Please '''
'''implement the TODO near the sleep() call in play_forward()''')
if not isinstance(dag.events[0], ConnectToControllers):
raise ValueError("First event must be ConnectToControllers")
simulation = None
try:
# Inferred events includes input events and internal events
inferred_events = []
(simulation, controller) = self.setup_simulation()
# dag.input_events returns only prunable input events. We also need
# include ConnectToControllers, which is a non-prunable input event.
assert(dag.input_events[0] != dag.events[0])
snapshot_inputs = [dag.events[0]] + dag.input_events
# The input event from the previous iteration, followed by the internal
# events that were inferred from the previous peek().
# Since we assume that the first event is always ConnectToControllers,
# there are never internal events (nor input events) preceding the
# initial ConnectToControllers.
# TODO(cs): might not want to contrain the caller's dag in this way.
events_inferred_last_iteration = []
for inject_input_idx in xrange(0, len(snapshot_inputs)):
inject_input = get_inject_input(inject_input_idx, snapshot_inputs)
following_input = get_following_input(inject_input_idx, snapshot_inputs)
expected_internal_events = \
get_expected_internal_events(inject_input, following_input, dag.events)
log.debug("peek()'ing after input %d (%s)" %
(inject_input_idx, str(inject_input)))
inferred_events += events_inferred_last_iteration
# We replay events_inferred_last_iteration (internal events preceding
# inject_input), as well as a NOPInput with the same timestamp as inject_input
# to ensure that the timing is the same before peek()ing.
fencepost = NOPInput(time=inject_input.time, round=inject_input.round)
dag_interval = EventDag(events_inferred_last_iteration + [fencepost])
if expected_internal_events == []:
# Optimization: if no internal events occured between this input and the
# next, no need to peek(), just bring the simulation's state forward up to
# inject_input
log.debug("Optimization: no expected internal events")
Peeker.ambiguous_counts[0.0] += 1
self.replay_interval(simulation, dag_interval, 0)
events_inferred_last_iteration = [inject_input]
continue
wait_time_seconds = self.get_wait_time_seconds(inject_input, following_input)
(found_events, snapshotter) = self.find_internal_events(simulation, controller,
dag_interval, inject_input,
wait_time_seconds)
events_inferred_last_iteration = [inject_input]
events_inferred_last_iteration += match_and_filter(found_events, expected_internal_events)
snapshotter.snapshot_proceed()
# Don't forget the final iteration's output
inferred_events += events_inferred_last_iteration
finally:
if simulation is not None:
simulation.clean_up()
return EventDag(inferred_events)
def replay_interval(self, simulation, dag_interval, initial_wait_seconds):
assert(dag_interval.events != [])
# TODO(cs): set EventScheduler's epsilon_seconds parameter?
replayer = Replayer(self.simulation_cfg, dag_interval,
initial_wait=initial_wait_seconds,
**self.kwargs)
replayer.simulation = simulation
if 'pass_through_sends' in self.kwargs and self.kwargs['pass_through_sends']:
replayer.set_pass_through_sends(simulation)
replayer.run_simulation_forward()
def find_internal_events(self, simulation, controller, dag_interval,
inject_input, wait_time_seconds):
initial_wait_seconds = (inject_input.time.as_float() -
dag_interval.events[-1].time.as_float())
self.replay_interval(simulation, dag_interval, initial_wait_seconds)
# Now peek() for internal events following inject_input
return self.snapshot_and_play_forward(simulation, controller,
inject_input, wait_time_seconds)
def snapshot_and_play_forward(self, simulation, controller, inject_input, wait_time_seconds):
snapshotter = Snapshotter(simulation, controller)
snapshotter.snapshot_controller()
# Here we also fork() ourselves (the network simulator), since we need
# switches to respond to messages, and potentially change their state in
# order to properly peek().
# TODO(cs): I believe that to be technically correct we need to use Chandy-Lamport
# here, since STS + POX = a distributed system.
def play_forward_and_marshal():
# Can't marshal the simulation object, so use this method as a closure
# instead.
# N.B. depends on LocalForker() -- cannot be used with RemoteForker().
# TODO(cs): even though DataplaneDrops are technically InputEvents, they
# may time out, and we might do well to try to infer this somehow.
found_events = play_forward(simulation, inject_input, wait_time_seconds)
# TODO(cs): DataplaneDrops are a special case of an InputEvent that may
# time out. We should check whether the input for this peek() was a
# DataplaneDrop, and return whether it timed out.
return [ e.to_json() for e in found_events ]
self.forker.register_task("snapshot_fork_task", play_forward_and_marshal)
# N.B. play_forward cleans up the simulation, including snaphotted controller
unmarshalled_found_events = self.forker.fork("snapshot_fork_task")
found_events = parse(unmarshalled_found_events)
# Finally, bring the controller back to the state it was at just after
# injecting inject_input
# N.B. this must be invoked in the parent process (not a forker.fork()ed
# child), since it mutates class variables in Controller.
return (found_events, snapshotter)
