def handle_int(signal, frame): import os from sts.util.rpc_forker import LocalForker sys.stderr.write("Caught signal %d, stopping sdndebug (pid %d)\n" % (signal, os.getpid())) if (simulator.simulation_cfg.current_simulation is not None): simulator.simulation_cfg.current_simulation.clean_up() # kill fork()ed procs LocalForker.kill_all() sys.exit(13)
def __init__(self, simulation_cfg, default_wait_time_seconds=0.05, epsilon_time=0.05, **kwargs): if len(simulation_cfg.controller_configs) != 1: raise ValueError("Only one controller supported for snapshotting") if simulation_cfg.controller_configs[0].sync is not None: raise ValueError("STSSyncProto currently incompatible with snapshotting") super(SnapshotPeeker, self).__init__(simulation_cfg, default_wait_time_seconds=default_wait_time_seconds, epsilon_time=epsilon_time) self.forker = LocalForker() if 'default_dp_permit' in kwargs and not kwargs['default_dp_permit']: raise ValueError('''Non-default DP Permit not currently supported ''' '''Please implement the TODO near the sleep() call ''' '''in play_forward()''') kwargs['default_dp_permit'] = True if 'pass_through_whitelisted_messages' not in kwargs: kwargs['pass_through_whitelisted_messages'] = False self.kwargs = kwargs unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ] if unknown_kwargs != []: raise ValueError("Unknown kwargs %s" % str(unknown_kwargs))
def __init__(self, simulation_cfg, default_wait_time_seconds=0.5, epsilon_time=0.2, default_dp_permit=False, pass_through_sends=False): if len(simulation_cfg.controller_configs) != 1: raise ValueError("Only one controller supported for snapshotting") if simulation_cfg.controller_configs[0].sync is not None: raise ValueError("STSSyncProto currently incompatible with snapshotting") super(SnapshotPeeker, self).__init__(simulation_cfg, default_wait_time_seconds=default_wait_time_seconds, epsilon_time=epsilon_time) self.forker = LocalForker() if not default_dp_permit: raise ValueError('''Non-default DP Permit not currently supported ''' '''Please implement the TODO near the sleep() call ''' '''in play_forward()''') self.default_dp_permit = default_dp_permit self.pass_through_sends = pass_through_sends
class SnapshotPeeker(Peeker): ''' O(n) peeker that takes controller snapshots at each input, peeks forward, then restarts the snapshot up until the next input''' def __init__(self, simulation_cfg, default_wait_time_seconds=0.5, epsilon_time=0.2, default_dp_permit=False, pass_through_sends=False): if len(simulation_cfg.controller_configs) != 1: raise ValueError("Only one controller supported for snapshotting") if simulation_cfg.controller_configs[0].sync is not None: raise ValueError("STSSyncProto currently incompatible with snapshotting") super(SnapshotPeeker, self).__init__(simulation_cfg, default_wait_time_seconds=default_wait_time_seconds, epsilon_time=epsilon_time) self.forker = LocalForker() if not default_dp_permit: raise ValueError('''Non-default DP Permit not currently supported ''' '''Please implement the TODO near the sleep() call ''' '''in play_forward()''') self.default_dp_permit = default_dp_permit self.pass_through_sends = pass_through_sends def setup_simulation(self): # TODO(cs): currently assumes that STSSyncProto is not used alongside # snapshotting. # N.B. bootstrap() does not connect switches to controllers. # ConnectionToControllers is an input event, which we peek() for like # any other input event. simulation = self.simulation_cfg.bootstrap(ReplaySyncCallback(None)) simulation.openflow_buffer.pass_through_whitelisted_messages = True controller = simulation.controller_manager.controllers[0] return (simulation, controller) def peek(self, dag): ''' If dag.events == [], returns immediately. If dag.events != [], assumes that isinstance(dag.events[0], ConnectToControllers) ''' if dag.input_events == []: return dag # post: len(dag.input_events) > 0 unsupported_types = [ProcessFlowMod, FailFlowMod, DataplaneDrop] if find(lambda e: type(e) in unsupported_types, dag.events) is not None: raise ValueError('''Delayed flow_mods not yet supported. Please ''' '''implement the TODO near the sleep() call in play_forward()''') if not isinstance(dag.events[0], ConnectToControllers): raise ValueError("First event must be ConnectToControllers") simulation = None try: # Inferred events includes input events and internal events inferred_events = [] (simulation, controller) = self.setup_simulation() # dag.input_events returns only prunable input events. We also need # include ConnectToControllers, which is a non-prunable input event. assert(dag.input_events[0] != dag.events[0]) snapshot_inputs = [dag.events[0]] + dag.input_events # The input event from the previous iteration, followed by the internal # events that were inferred from the previous peek(). # Since we assume that the first event is always ConnectToControllers, # there are never internal events (nor input events) preceding the # initial ConnectToControllers. # TODO(cs): might not want to contrain the caller's dag in this way. events_inferred_last_iteration = [] for inject_input_idx in xrange(0, len(snapshot_inputs)): inject_input = get_inject_input(inject_input_idx, snapshot_inputs) following_input = get_following_input(inject_input_idx, snapshot_inputs) expected_internal_events = \ get_expected_internal_events(inject_input, following_input, dag.events) log.debug("peek()'ing after input %d (%s)" % (inject_input_idx, str(inject_input))) inferred_events += events_inferred_last_iteration if expected_internal_events == []: # Optimization: if no internal events occured between this input and the # next, no need to peek(), just replay the next input log.debug("Optimization: no expected internal events") Peeker.ambiguous_counts[0.0] += 1 events_inferred_last_iteration = [inject_input] continue # We replay events_inferred_last_iteration (internal events preceding # inject_input), as well as a NOPInput with the same timestamp as inject_input # to ensure that the timing is the same before peek()ing. fencepost = NOPInput(time=inject_input.time, round=inject_input.round) dag_interval = EventDag(events_inferred_last_iteration + [fencepost]) wait_time_seconds = self.get_wait_time_seconds(inject_input, following_input) found_events = self.find_internal_events(simulation, controller, dag_interval, inject_input, wait_time_seconds) events_inferred_last_iteration = [inject_input] events_inferred_last_iteration += match_and_filter(found_events, expected_internal_events) # Don't forget the final iteration's output inferred_events += events_inferred_last_iteration finally: if simulation is not None: simulation.clean_up() return EventDag(inferred_events) def find_internal_events(self, simulation, controller, dag_interval, inject_input, wait_time_seconds): assert(dag_interval.events != []) initial_wait_seconds = (inject_input.time.as_float() - dag_interval.events[-1].time.as_float()) # TODO(cs): set EventScheduler's epsilon_seconds parameter? replayer = Replayer(self.simulation_cfg, dag_interval, pass_through_whitelisted_messages=True, default_dp_permit=self.default_dp_permit, initial_wait=initial_wait_seconds) replayer.simulation = simulation if self.pass_through_sends: replayer.set_pass_through_sends(simulation) replayer.run_simulation_forward() # Now peek() for internal events following inject_input return self.snapshot_and_play_forward(simulation, controller, inject_input, wait_time_seconds) def snapshot_and_play_forward(self, simulation, controller, inject_input, wait_time_seconds): snapshotter = Snapshotter(simulation, controller) snapshotter.snapshot_controller() # Here we also fork() ourselves (the network simulator), since we need # switches to respond to messages, and potentially change their state in # order to properly peek(). # TODO(cs): I believe that to be technically correct we need to use Chandy-Lamport # here, since STS + POX = a distributed system. def play_forward_and_marshal(): # Can't marshal the simulation object, so use this method as a closure # instead. # N.B. depends on LocalForker() -- cannot be used with RemoteForker(). # TODO(cs): even though DataplaneDrops are technically InputEvents, they # may time out, and we might do well to try to infer this somehow. found_events = play_forward(simulation, inject_input, wait_time_seconds) # TODO(cs): DataplaneDrops are a special case of an InputEvent that may # time out. We should check whether the input for this peek() was a # DataplaneDrop, and return whether it timed out. return [ e.to_json() for e in found_events ] self.forker.register_task("snapshot_fork_task", play_forward_and_marshal) # N.B. play_forward cleans up the simulation, including snaphotted controller unmarshalled_found_events = self.forker.fork("snapshot_fork_task") found_events = parse(unmarshalled_found_events) # Finally, bring the controller back to the state it was at just after # injecting inject_input # N.B. this must be invoked in the parent process (not a forker.fork()ed # child), since it mutates class variables in Controller. snapshotter.snapshot_proceed() return found_events
def __init__(self, simulation_cfg, superlog_path_or_dag, invariant_check_name=None, transform_dag=None, end_wait_seconds=0.5, mcs_trace_path=None, extra_log=None, runtime_stats_path=None, wait_on_deterministic_values=False, no_violation_verification_runs=1, optimized_filtering=False, forker=LocalForker(), replay_final_trace=True, strict_assertion_checking=False, delay_flow_mods=False, **kwargs): super(MCSFinder, self).__init__(simulation_cfg) # number of subsequences delta debugging has examined so far, for # distingushing runtime stats from different intermediate runs. self.subsequence_id = 0 self.mcs_log_tracker = None self.replay_log_tracker = None self.mcs_trace_path = mcs_trace_path self.sync_callback = None self._log = logging.getLogger("mcs_finder") if invariant_check_name is None: raise ValueError("Must specify invariant check") if invariant_check_name not in name_to_invariant_check: raise ValueError( '''Unknown invariant check %s.\n''' '''Invariant check name must be defined in config.invariant_checks''', invariant_check_name) self.invariant_check = name_to_invariant_check[invariant_check_name] if type(superlog_path_or_dag) == str: self.superlog_path = superlog_path_or_dag # The dag is codefied as a list, where each element has # a list of its dependents self.dag = EventDag(log_parser.parse_path(self.superlog_path)) else: self.dag = superlog_path_or_dag last_invariant_violation = self.dag.get_last_invariant_violation() if last_invariant_violation is None: raise ValueError("No invariant violation found in dag...") violations = last_invariant_violation.violations if len(violations) > 1: self.bug_signature = None while self.bug_signature is None: msg.interactive( "\n------------------------------------------\n") msg.interactive( "Multiple violations detected! Choose one for MCS Finding:" ) for i, violation in enumerate(violations): msg.interactive(" [%d] %s" % (i + 1, violation)) violation_index = msg.raw_input("> ") if re.match("^\d+$", violation_index) is None or\ int(violation_index) < 1 or\ int(violation_index) > len(violations): msg.fail("Must provide an integer between 1 and %d!" % len(violations)) continue self.bug_signature = violations[int(violation_index) - 1] else: self.bug_signature = violations[0] msg.success( "\nBug signature to match is %s. Proceeding with MCS finding!\n" % self.bug_signature) self.transform_dag = transform_dag # A second log with just our MCS progress log messages self._extra_log = extra_log self.kwargs = kwargs self.end_wait_seconds = end_wait_seconds self.wait_on_deterministic_values = wait_on_deterministic_values # `no' means "number" self.no_violation_verification_runs = no_violation_verification_runs self._runtime_stats = RuntimeStats( self.subsequence_id, runtime_stats_path=runtime_stats_path) # Whether to try alternate trace splitting techiques besides splitting by time. self.optimized_filtering = optimized_filtering self.forker = forker self.replay_final_trace = replay_final_trace self.strict_assertion_checking = strict_assertion_checking self.delay_flow_mods = delay_flow_mods
def __init__(self, simulation_cfg, superlog_path_or_dag, invariant_check_name="", bug_signature="", transform_dag=None, mcs_trace_path=None, extra_log=None, runtime_stats_path=None, max_replays_per_subsequence=1, optimized_filtering=False, forker=LocalForker(), replay_final_trace=True, strict_assertion_checking=False, no_violation_verification_runs=None, **kwargs): ''' Note that you may pass in any keyword argument for Replayer to MCSFinder, except 'bug_signature' and 'invariant_check_name' ''' super(MCSFinder, self).__init__(simulation_cfg) # number of subsequences delta debugging has examined so far, for # distingushing runtime stats from different intermediate runs. self.subsequence_id = 0 self.mcs_log_tracker = None self.replay_log_tracker = None self.mcs_trace_path = mcs_trace_path self.sync_callback = None self._log = logging.getLogger("mcs_finder") if invariant_check_name == "": raise ValueError("Must specify invariant check") if invariant_check_name not in name_to_invariant_check: raise ValueError('''Unknown invariant check %s.\n''' '''Invariant check name must be defined in config.invariant_checks''', invariant_check_name) self.invariant_check_name = invariant_check_name self.invariant_check = name_to_invariant_check[invariant_check_name] if type(superlog_path_or_dag) == str: self.superlog_path = superlog_path_or_dag # The dag is codefied as a list, where each element has # a list of its dependents self.dag = EventDag(log_parser.parse_path(self.superlog_path)) else: self.dag = superlog_path_or_dag if self.simulation_cfg.ignore_interposition: filtered_events = [e for e in self.dag.events if type(e) not in all_internal_events] self.dag = EventDag(filtered_events) last_invariant_violation = self.dag.get_last_invariant_violation() if last_invariant_violation is None: raise ValueError("No invariant violation found in dag...") violations = last_invariant_violation.violations self.bug_signature = bug_signature if len(violations) > 1: while self.bug_signature == "": msg.interactive("\n------------------------------------------\n") msg.interactive("Multiple violations detected! Choose one for MCS Finding:") for i, violation in enumerate(violations): msg.interactive(" [%d] %s" % (i+1, violation)) violation_index = msg.raw_input("> ") if re.match("^\d+$", violation_index) is None or\ int(violation_index) < 1 or\ int(violation_index) > len(violations): msg.fail("Must provide an integer between 1 and %d!" % len(violations)) continue self.bug_signature = violations[int(violation_index)-1] if self.bug_signature == "": self.bug_signature = violations[0] msg.success("\nBug signature to match is %s. Proceeding with MCS finding!\n" % self.bug_signature) self.transform_dag = transform_dag # A second log with just our MCS progress log messages self._extra_log = extra_log self.kwargs = kwargs unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ] if unknown_kwargs != []: raise ValueError("Unknown kwargs %s" % str(unknown_kwargs)) if no_violation_verification_runs is not None: raise ValueError('''no_violation_verification_runs parameter is deprecated. ''' '''Use max_replays_per_subsequence.''') self.max_replays_per_subsequence = max_replays_per_subsequence self._runtime_stats = RuntimeStats(self.subsequence_id, runtime_stats_path=runtime_stats_path) # Whether to try alternate trace splitting techiques besides splitting by time. self.optimized_filtering = optimized_filtering self.forker = forker self.replay_final_trace = replay_final_trace self.strict_assertion_checking = strict_assertion_checking
class SnapshotPeeker(Peeker): ''' O(n) peeker that takes controller snapshots at each input, peeks forward, then restarts the snapshot up until the next input''' def __init__(self, simulation_cfg, default_wait_time_seconds=0.05, epsilon_time=0.05, **kwargs): if len(simulation_cfg.controller_configs) != 1: raise ValueError("Only one controller supported for snapshotting") if simulation_cfg.controller_configs[0].sync is not None: raise ValueError("STSSyncProto currently incompatible with snapshotting") super(SnapshotPeeker, self).__init__(simulation_cfg, default_wait_time_seconds=default_wait_time_seconds, epsilon_time=epsilon_time) self.forker = LocalForker() if 'default_dp_permit' in kwargs and not kwargs['default_dp_permit']: raise ValueError('''Non-default DP Permit not currently supported ''' '''Please implement the TODO near the sleep() call ''' '''in play_forward()''') kwargs['default_dp_permit'] = True if 'pass_through_whitelisted_messages' not in kwargs: kwargs['pass_through_whitelisted_messages'] = False self.kwargs = kwargs unknown_kwargs = [ k for k in kwargs.keys() if k not in Replayer.kwargs ] if unknown_kwargs != []: raise ValueError("Unknown kwargs %s" % str(unknown_kwargs)) def setup_simulation(self): # TODO(cs): currently assumes that STSSyncProto is not used alongside # snapshotting. # N.B. bootstrap() does not connect switches to controllers. # ConnectionToControllers is an input event, which we peek() for like # any other input event. simulation = self.simulation_cfg.bootstrap(ReplaySyncCallback(None)) simulation.openflow_buffer.pass_through_whitelisted_packets = self.kwargs['pass_through_whitelisted_messages'] controller = simulation.controller_manager.controllers[0] return (simulation, controller) def peek(self, dag): ''' If dag.events == [], returns immediately. If dag.events != [], assumes that isinstance(dag.events[0], ConnectToControllers) ''' if dag.input_events == []: return dag # post: len(dag.input_events) > 0 unsupported_types = [ProcessFlowMod, DataplaneDrop] if find(lambda e: type(e) in unsupported_types, dag.events) is not None: raise ValueError('''Delayed flow_mods not yet supported. Please ''' '''implement the TODO near the sleep() call in play_forward()''') if not isinstance(dag.events[0], ConnectToControllers): raise ValueError("First event must be ConnectToControllers") simulation = None try: # Inferred events includes input events and internal events inferred_events = [] (simulation, controller) = self.setup_simulation() # dag.input_events returns only prunable input events. We also need # include ConnectToControllers, which is a non-prunable input event. assert(dag.input_events[0] != dag.events[0]) snapshot_inputs = [dag.events[0]] + dag.input_events # The input event from the previous iteration, followed by the internal # events that were inferred from the previous peek(). # Since we assume that the first event is always ConnectToControllers, # there are never internal events (nor input events) preceding the # initial ConnectToControllers. # TODO(cs): might not want to contrain the caller's dag in this way. events_inferred_last_iteration = [] for inject_input_idx in xrange(0, len(snapshot_inputs)): inject_input = get_inject_input(inject_input_idx, snapshot_inputs) following_input = get_following_input(inject_input_idx, snapshot_inputs) expected_internal_events = \ get_expected_internal_events(inject_input, following_input, dag.events) log.debug("peek()'ing after input %d (%s)" % (inject_input_idx, str(inject_input))) inferred_events += events_inferred_last_iteration # We replay events_inferred_last_iteration (internal events preceding # inject_input), as well as a NOPInput with the same timestamp as inject_input # to ensure that the timing is the same before peek()ing. fencepost = NOPInput(time=inject_input.time, round=inject_input.round) dag_interval = EventDag(events_inferred_last_iteration + [fencepost]) if expected_internal_events == []: # Optimization: if no internal events occured between this input and the # next, no need to peek(), just bring the simulation's state forward up to # inject_input log.debug("Optimization: no expected internal events") Peeker.ambiguous_counts[0.0] += 1 self.replay_interval(simulation, dag_interval, 0) events_inferred_last_iteration = [inject_input] continue wait_time_seconds = self.get_wait_time_seconds(inject_input, following_input) (found_events, snapshotter) = self.find_internal_events(simulation, controller, dag_interval, inject_input, wait_time_seconds) events_inferred_last_iteration = [inject_input] events_inferred_last_iteration += match_and_filter(found_events, expected_internal_events) snapshotter.snapshot_proceed() # Don't forget the final iteration's output inferred_events += events_inferred_last_iteration finally: if simulation is not None: simulation.clean_up() return EventDag(inferred_events) def replay_interval(self, simulation, dag_interval, initial_wait_seconds): assert(dag_interval.events != []) # TODO(cs): set EventScheduler's epsilon_seconds parameter? replayer = Replayer(self.simulation_cfg, dag_interval, initial_wait=initial_wait_seconds, **self.kwargs) replayer.simulation = simulation if 'pass_through_sends' in self.kwargs and self.kwargs['pass_through_sends']: replayer.set_pass_through_sends(simulation) replayer.run_simulation_forward() def find_internal_events(self, simulation, controller, dag_interval, inject_input, wait_time_seconds): initial_wait_seconds = (inject_input.time.as_float() - dag_interval.events[-1].time.as_float()) self.replay_interval(simulation, dag_interval, initial_wait_seconds) # Now peek() for internal events following inject_input return self.snapshot_and_play_forward(simulation, controller, inject_input, wait_time_seconds) def snapshot_and_play_forward(self, simulation, controller, inject_input, wait_time_seconds): snapshotter = Snapshotter(simulation, controller) snapshotter.snapshot_controller() # Here we also fork() ourselves (the network simulator), since we need # switches to respond to messages, and potentially change their state in # order to properly peek(). # TODO(cs): I believe that to be technically correct we need to use Chandy-Lamport # here, since STS + POX = a distributed system. def play_forward_and_marshal(): # Can't marshal the simulation object, so use this method as a closure # instead. # N.B. depends on LocalForker() -- cannot be used with RemoteForker(). # TODO(cs): even though DataplaneDrops are technically InputEvents, they # may time out, and we might do well to try to infer this somehow. found_events = play_forward(simulation, inject_input, wait_time_seconds) # TODO(cs): DataplaneDrops are a special case of an InputEvent that may # time out. We should check whether the input for this peek() was a # DataplaneDrop, and return whether it timed out. return [ e.to_json() for e in found_events ] self.forker.register_task("snapshot_fork_task", play_forward_and_marshal) # N.B. play_forward cleans up the simulation, including snaphotted controller unmarshalled_found_events = self.forker.fork("snapshot_fork_task") found_events = parse(unmarshalled_found_events) # Finally, bring the controller back to the state it was at just after # injecting inject_input # N.B. this must be invoked in the parent process (not a forker.fork()ed # child), since it mutates class variables in Controller. return (found_events, snapshotter)