def __init__(self,
simulation_cfg,
fuzzer_params="config.fuzzer_params",
check_interval=None,
traffic_inject_interval=10,
random_seed=None,
delay=0.1,
steps=None,
input_logger=None,
invariant_check=InvariantChecker.check_correspondence,
halt_on_violation=False,
log_invariant_checks=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.check_interval = check_interval
self.invariant_check = invariant_check
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
self.random = random.Random()
else:
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
# Logical time (round #) for the simulation execution
self.logical_time = 0
def __init__(self, simulation_cfg, input_logger=None,
pass_through_of_messages=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.logical_time = 0
self._input_logger = input_logger
self.traffic_generator = TrafficGenerator(random.Random())
# TODO(cs): we don't actually have a way to support pass_through=False, in
# a reasonable way -- the user has no (easy) way to examine and route
# OpenFlowBuffer's pending_receives and pending_sends.
self.pass_through_of_messages = pass_through_of_messages
def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
check_interval=None, traffic_inject_interval=10, random_seed=None,
delay=0.1, steps=None, input_logger=None,
invariant_check=InvariantChecker.check_correspondence,
halt_on_violation=False, log_invariant_checks=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.check_interval = check_interval
self.invariant_check = invariant_check
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
self.random = random.Random()
else:
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
# Logical time (round #) for the simulation execution
self.logical_time = 0
def __init__(self, simulation_cfg, input_logger=None,
pass_through_of_messages=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.logical_time = 0
self._input_logger = input_logger
self.traffic_generator = TrafficGenerator(random.Random())
# TODO(cs): we don't actually have a way to support pass_through=False, in
# a reasonable way -- the user has no (easy) way to examine and route
# OpenFlowBuffer's pending_receives and pending_sends.
self.pass_through_of_messages = pass_through_of_messages
class Fuzzer(ControlFlow):
'''
Injects input events at random intervals, periodically checking
for invariant violations. (Not the proper use of the term `Fuzzer`)
'''
def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
check_interval=None, traffic_inject_interval=10, random_seed=None,
delay=0.1, steps=None, input_logger=None,
invariant_check=InvariantChecker.check_correspondence,
halt_on_violation=False, log_invariant_checks=True,
delay_startup=True, print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger,
record_deterministic_values=record_deterministic_values)
self.check_interval = check_interval
self.invariant_check = invariant_check
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# Logical time (round #) for the simulation execution
self.logical_time = 0
def _log_input_event(self, event, **kws):
if self._input_logger is not None:
self._input_logger.log_input_event(event, **kws)
def _load_fuzzer_params(self, fuzzer_params_path):
if fuzzer_params_path.endswith('.py'):
fuzzer_params_path = fuzzer_params_path[:-3].replace("/", ".")
try:
self.params = __import__(fuzzer_params_path, globals(), locals(), ["*"])
# TODO(cs): temporary hack until we get determinism figured out
self.params.link_discovery_rate = 0.1
except:
raise IOError("Could not find fuzzer params config file: %s" %
fuzzer_params_path)
def init_results(self, results_dir):
self._input_logger.open(results_dir)
def simulate(self):
"""Precondition: simulation.patch_panel is a buffered patch panel"""
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
assert(isinstance(self.simulation.patch_panel, BufferedPatchPanel))
self.traffic_generator.set_hosts(self.simulation.topology.hosts)
if self._input_logger is not None:
self.simulation_cfg.set_dataplane_trace_path(self._input_logger.dp_trace_path)
return self.loop()
def loop(self):
if self.steps:
end_time = self.logical_time + self.steps
else:
end_time = sys.maxint
exit_code = 0
self.interrupted = False
old_interrupt = None
def interrupt(sgn, frame):
msg.interactive("Interrupting fuzzer, dropping to console (press ^C again to terminate)")
signal.signal(signal.SIGINT, self.old_interrupt)
self.old_interrupt = None
self.interrupted = True
raise KeyboardInterrupt()
self.old_interrupt = signal.signal(signal.SIGINT, interrupt)
try:
# Always connect to controllers explicitly
self.simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
if self.delay_startup:
# Wait until the first OpenFlow message is received
log.info("Waiting until first OpenfFlow message received..")
while self.simulation.god_scheduler.pending_receives() == []:
self.simulation.io_master.select(self.delay)
while self.logical_time < end_time:
self.logical_time += 1
try:
self.trigger_events()
msg.event("Round %d completed." % self.logical_time)
halt = self.maybe_check_invariant()
if halt:
exit_code = 5
break
self.maybe_inject_trace_event()
time.sleep(self.delay)
except KeyboardInterrupt as e:
if self.interrupted:
interactive = Interactive(self.simulation_cfg, self._input_logger)
interactive.simulate(self.simulation, bound_objects=( ('fuzzer', self), ))
self.old_interrupt = signal.signal(signal.SIGINT, interrupt)
else:
raise e
log.info("Terminating fuzzing after %d rounds" % self.logical_time)
if self.print_buffers:
self._print_buffers()
finally:
if self.old_interrupt:
signal.signal(signal.SIGINT, self.old_interrupt)
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return exit_code
def _print_buffers(self):
buffered_events = []
log.debug("Pending Message Receives:")
for p in self.simulation.god_scheduler.pending_receives():
log.debug("- %s", p)
event = ControlMessageReceive(p.dpid, p.controller_id, p.fingerprint)
buffered_events.append(event)
# Note that there shouldn't be any pending state changes in record mode
if self._input_logger is not None:
self._input_logger.dump_buffered_events(buffered_events)
def maybe_check_invariant(self):
if (self.check_interval is not None and
(self.logical_time % self.check_interval) == 0):
# Time to run correspondence!
# TODO(cs): may need to revert to threaded version if runtime is too
# long
def do_invariant_check():
if self.log_invariant_checks:
self._log_input_event(CheckInvariants(invariant_check=self.invariant_check,
fail_on_error=self.halt_on_violation))
controllers_with_violations = self.invariant_check(self.simulation)
if controllers_with_violations != []:
msg.fail("The following controllers had correctness violations!: %s"
% str(controllers_with_violations))
self._log_input_event(InvariantViolation(controllers_with_violations))
if self.halt_on_violation:
return True
else:
msg.interactive("No correctness violations!")
return do_invariant_check()
def maybe_inject_trace_event(self):
if (self.simulation.dataplane_trace and
(self.logical_time % self.traffic_inject_interval) == 0):
dp_event = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def trigger_events(self):
self.check_dataplane()
self.check_tcp_connections()
self.check_message_receipts()
self.check_switch_crashes()
self.check_link_failures()
self.fuzz_traffic()
self.check_controllers()
self.check_migrations()
def check_dataplane(self):
''' Decide whether to delay, drop, or deliver packets '''
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
if self.random.random() < self.params.dataplane_delay_rate:
self.simulation.patch_panel.delay_dp_event(dp_event)
elif self.random.random() < self.params.dataplane_drop_rate:
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
elif not self.simulation.topology.ok_to_send(dp_event):
# Switches have very small buffers! drop it on the floor if the link
# is down
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
else:
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
# TODO(cs): temporary hack until we have determinism figured out
if self.mock_link_discovery and self.random.random() < self.params.link_discovery_rate:
# Pick a random link to be discovered
link = self.random.choice(self.simulation.topology.network_links)
attrs = [link.start_software_switch.dpid, link.start_port.port_no,
link.end_software_switch.dpid, link.end_port.port_no]
# Send it to a random controller
if self.simulation.controller_manager.live_controllers != []:
c = self.random.choice(list(self.simulation.controller_manager.live_controllers))
c.sync_connection.send_link_notification(attrs)
self._log_input_event(LinkDiscovery(c.uuid, attrs))
def check_tcp_connections(self):
''' Decide whether to block or unblock control channels '''
for (switch, connection) in self.simulation.topology.unblocked_controller_connections:
if self.random.random() < self.params.controlplane_block_rate:
self.simulation.topology.block_connection(connection)
self._log_input_event(ControlChannelBlock(switch.dpid,
connection.get_controller_id()))
for (switch, connection) in self.simulation.topology.blocked_controller_connections:
if self.random.random() < self.params.controlplane_unblock_rate:
self.simulation.topology.unblock_connection(connection)
self._log_input_event(ControlChannelUnblock(switch.dpid,
controller_id=connection.get_controller_id()))
def check_message_receipts(self):
for pending_receipt in self.simulation.god_scheduler.pending_receives():
# TODO(cs): this is a really dumb way to fuzz packet receipt scheduling
if self.random.random() < self.params.ofp_message_receipt_rate:
self.simulation.god_scheduler.schedule(pending_receipt)
self._log_input_event(ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint))
def check_switch_crashes(self):
''' Decide whether to crash or restart switches, links and controllers '''
def crash_switches():
crashed_this_round = set()
for software_switch in list(self.simulation.topology.live_switches):
if self.random.random() < self.params.switch_failure_rate:
crashed_this_round.add(software_switch)
self.simulation.topology.crash_switch(software_switch)
self._log_input_event(SwitchFailure(software_switch.dpid))
return crashed_this_round
def restart_switches(crashed_this_round):
# Make sure we don't try to connect to dead controllers
down_controller_ids = map(lambda c: c.uuid,
self.simulation.controller_manager.down_controllers)
for software_switch in list(self.simulation.topology.failed_switches):
if software_switch in crashed_this_round:
continue
if self.random.random() < self.params.switch_recovery_rate:
connected = self.simulation.topology\
.recover_switch(software_switch,
down_controller_ids=down_controller_ids)
if connected:
self._log_input_event(SwitchRecovery(software_switch.dpid))
crashed_this_round = crash_switches()
try:
restart_switches(crashed_this_round)
except TimeoutError:
log.warn("Unable to connect to controllers")
def check_link_failures(self):
def sever_links():
# TODO(cs): model administratively down links? (OFPPC_PORT_DOWN)
cut_this_round = set()
for link in list(self.simulation.topology.live_links):
if self.random.random() < self.params.link_failure_rate:
cut_this_round.add(link)
self.simulation.topology.sever_link(link)
self._log_input_event(LinkFailure(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
return cut_this_round
def repair_links(cut_this_round):
for link in list(self.simulation.topology.cut_links):
if link in cut_this_round:
continue
if self.random.random() < self.params.link_recovery_rate:
self.simulation.topology.repair_link(link)
self._log_input_event(LinkRecovery(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
cut_this_round = sever_links()
repair_links(cut_this_round)
def fuzz_traffic(self):
if not self.simulation.dataplane_trace:
# randomly generate messages from switches
for host in self.simulation.topology.hosts:
if self.random.random() < self.params.traffic_generation_rate:
if len(host.interfaces) > 0:
msg.event("injecting a random packet")
traffic_type = "icmp_ping"
# Generates a packet, and feeds it to the software_switch
dp_event = self.traffic_generator.generate(traffic_type, host)
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def check_controllers(self):
def crash_controllers():
crashed_this_round = set()
for controller in self.simulation.controller_manager.live_controllers:
if self.random.random() < self.params.controller_crash_rate:
crashed_this_round.add(controller)
controller.kill()
self._log_input_event(ControllerFailure(controller.uuid))
return crashed_this_round
def reboot_controllers(crashed_this_round):
for controller in self.simulation.controller_manager.down_controllers:
if controller in crashed_this_round:
continue
if self.random.random() < self.params.controller_recovery_rate:
controller.start()
self._log_input_event(ControllerRecovery(controller.uuid))
crashed_this_round = crash_controllers()
reboot_controllers(crashed_this_round)
def check_migrations(self):
for access_link in list(self.simulation.topology.access_links):
if self.random.random() < self.params.host_migration_rate:
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
live_edge_switches = list(self.simulation.topology.live_edge_switches)
if len(live_edge_switches) > 0:
new_switch = random.choice(live_edge_switches)
new_switch_dpid = new_switch.dpid
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no,
access_link.host.name))
class Fuzzer(ControlFlow):
'''
Injects input events at random intervals, periodically checking
for invariant violations. (Not the proper use of the term `Fuzzer`)
'''
def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
check_interval=None, traffic_inject_interval=10, random_seed=None,
delay=0.1, steps=None, input_logger=None,
invariant_check=InvariantChecker.check_correspondence,
halt_on_violation=False, log_invariant_checks=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.check_interval = check_interval
self.invariant_check = invariant_check
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
self.random = random.Random()
else:
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
# Logical time (round #) for the simulation execution
self.logical_time = 0
def _log_input_event(self, event, **kws):
if self._input_logger is not None:
self._input_logger.log_input_event(event, **kws)
def _load_fuzzer_params(self, fuzzer_params_path):
try:
self.params = __import__(fuzzer_params_path, globals(), locals(), ["*"])
except:
raise IOError("Could not find logging config file: %s" %
fuzzer_params_path)
def simulate(self):
"""Precondition: simulation.patch_panel is a buffered patch panel"""
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
assert(isinstance(self.simulation.patch_panel, BufferedPatchPanel))
self.loop()
def loop(self):
if self.steps:
end_time = self.logical_time + self.steps
else:
end_time = sys.maxint
try:
while self.logical_time < end_time:
self.logical_time += 1
self.trigger_events()
msg.event("Round %d completed." % self.logical_time)
halt = self.maybe_check_invariant()
if halt:
break
self.maybe_inject_trace_event()
time.sleep(self.delay)
finally:
if self._input_logger is not None:
self._input_logger.close(self.simulation_cfg)
def maybe_check_invariant(self):
if (self.check_interval is not None and
(self.logical_time % self.check_interval) == 0):
# Time to run correspondence!
# TODO(cs): may need to revert to threaded version if runtime is too
# long
def do_invariant_check():
if self.log_invariant_checks:
self._log_input_event(CheckInvariants(invariant_check=self.invariant_check,
fail_on_error=self.halt_on_violation))
controllers_with_violations = self.invariant_check(self.simulation)
if controllers_with_violations != []:
msg.fail("The following controllers had correctness violations!: %s"
% str(controllers_with_violations))
self._log_input_event(InvariantViolation(controllers_with_violations))
if self.halt_on_violation:
return True
else:
msg.interactive("No correctness violations!")
return do_invariant_check()
def maybe_inject_trace_event(self):
if (self.simulation.dataplane_trace and
(self.logical_time % self.traffic_inject_interval) == 0):
dp_event = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def trigger_events(self):
self.check_dataplane()
self.check_tcp_connections()
self.check_message_receipts()
self.check_switch_crashes()
self.check_link_failures()
self.fuzz_traffic()
self.check_controllers()
self.check_migrations()
def check_dataplane(self):
''' Decide whether to delay, drop, or deliver packets '''
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
if self.random.random() < self.params.dataplane_delay_rate:
self.simulation.patch_panel.delay_dp_event(dp_event)
elif self.random.random() < self.params.dataplane_drop_rate:
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
elif not self.simulation.topology.ok_to_send(dp_event):
# Switches have very small buffers! drop it on the floor if the link
# is down
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
else:
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
def check_tcp_connections(self):
''' Decide whether to block or unblock control channels '''
for (switch, connection) in self.simulation.topology.unblocked_controller_connections:
if self.random.random() < self.params.controlplane_block_rate:
self.simulation.topology.block_connection(connection)
self._log_input_event(ControlChannelBlock(switch.dpid,
connection.get_controller_id()))
for (switch, connection) in self.simulation.topology.blocked_controller_connections:
if self.random.random() < self.params.controlplane_unblock_rate:
self.simulation.topology.unblock_connection(connection)
self._log_input_event(ControlChannelUnblock(switch.dpid,
controller_id=connection.get_controller_id()))
def check_message_receipts(self):
for pending_receipt in self.simulation.god_scheduler.pending_receives():
# TODO(cs): this is a really dumb way to fuzz packet receipt scheduling
if self.random.random() < self.params.ofp_message_receipt_rate:
self.simulation.god_scheduler.schedule(pending_receipt)
self._log_input_event(ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint))
def check_switch_crashes(self):
''' Decide whether to crash or restart switches, links and controllers '''
def crash_switches():
crashed_this_round = set()
for software_switch in list(self.simulation.topology.live_switches):
if self.random.random() < self.params.switch_failure_rate:
crashed_this_round.add(software_switch)
self.simulation.topology.crash_switch(software_switch)
self._log_input_event(SwitchFailure(software_switch.dpid))
return crashed_this_round
def restart_switches(crashed_this_round):
# Make sure we don't try to connect to dead controllers
down_controller_ids = map(lambda c: c.uuid,
self.simulation.controller_manager.down_controllers)
for software_switch in list(self.simulation.topology.failed_switches):
if software_switch in crashed_this_round:
continue
if self.random.random() < self.params.switch_recovery_rate:
connected = self.simulation.topology\
.recover_switch(software_switch,
down_controller_ids=down_controller_ids)
if connected:
self._log_input_event(SwitchRecovery(software_switch.dpid))
crashed_this_round = crash_switches()
try:
restart_switches(crashed_this_round)
except TimeoutError:
log.warn("Unable to connect to controllers")
def check_link_failures(self):
def sever_links():
# TODO(cs): model administratively down links? (OFPPC_PORT_DOWN)
cut_this_round = set()
for link in list(self.simulation.topology.live_links):
if self.random.random() < self.params.link_failure_rate:
cut_this_round.add(link)
self.simulation.topology.sever_link(link)
self._log_input_event(LinkFailure(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
return cut_this_round
def repair_links(cut_this_round):
for link in list(self.simulation.topology.cut_links):
if link in cut_this_round:
continue
if self.random.random() < self.params.link_recovery_rate:
self.simulation.topology.repair_link(link)
self._log_input_event(LinkRecovery(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
cut_this_round = sever_links()
repair_links(cut_this_round)
def fuzz_traffic(self):
if not self.simulation.dataplane_trace:
# randomly generate messages from switches
for host in self.simulation.topology.hosts:
if self.random.random() < self.params.traffic_generation_rate:
if len(host.interfaces) > 0:
msg.event("injecting a random packet")
traffic_type = "icmp_ping"
# Generates a packet, and feeds it to the software_switch
dp_event = self.traffic_generator.generate(traffic_type, host)
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def check_controllers(self):
def crash_controllers():
crashed_this_round = set()
for controller in self.simulation.controller_manager.live_controllers:
if self.random.random() < self.params.controller_crash_rate:
crashed_this_round.add(controller)
controller.kill()
self._log_input_event(ControllerFailure(controller.uuid))
return crashed_this_round
def reboot_controllers(crashed_this_round):
for controller in self.simulation.controller_manager.down_controllers:
if controller in crashed_this_round:
continue
if self.random.random() < self.params.controller_recovery_rate:
controller.start()
self._log_input_event(ControllerRecovery(controller.uuid))
crashed_this_round = crash_controllers()
reboot_controllers(crashed_this_round)
def check_migrations(self):
for access_link in list(self.simulation.topology.access_links):
if self.random.random() < self.params.host_migration_rate:
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
live_edge_switches = list(self.simulation.topology.live_edge_switches)
if len(live_edge_switches) > 0:
new_switch = random.choice(live_edge_switches)
new_switch_dpid = new_switch.dpid
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no))
def __init__(self,
simulation_cfg,
fuzzer_params="config.fuzzer_params",
check_interval=None,
traffic_inject_interval=10,
random_seed=None,
delay=0.1,
steps=None,
input_logger=None,
invariant_check_name="InvariantChecker.check_correspondence",
halt_on_violation=False,
log_invariant_checks=True,
delay_startup=True,
print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False,
never_drop_whitelisted_packets=True,
initialization_rounds=0):
'''
Options:
- fuzzer_params: path to event probabilities
- check_interval: the period for checking invariants, in terms of
logical rounds
- traffic_inject_interval: how often to inject dataplane trace packets
- random_seed: optionally set the seed of the random number generator
- delay: how long to sleep between each logical round
- input_logger: None, or a InputLogger instance
- invariant_check_name: the name of the invariant check, from
config/invariant_checks.py
- halt_on_violation: whether to stop after a bug has been detected
- log_invariant_checks: whether to log InvariantCheck events
- delay_startup: whether to until the first OpenFlow message is received
before proceeding with fuzzing
- print_buffers: whether to print the remaining contents of the
dataplane/controlplane buffers at the end of the execution
- record_deterministic_values: whether to record gettimeofday requests
for replay
- mock_link_discovery: optional module for POX to experiment with
better determinism -- tell POX exactly when links should be discovered
- initialization_rounds: if non-zero, will wait the specified rounds to
let the controller discover the topology before injecting inputs
'''
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(
input_logger,
record_deterministic_values=record_deterministic_values)
self.check_interval = check_interval
if self.check_interval is None:
print >> sys.stderr, "Fuzzer Warning: Check interval is not specified... not checking invariants"
if invariant_check_name not in name_to_invariant_check:
raise ValueError(
'''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# How many rounds to let the controller initialize:
# send one round of packets directed at the source host itself (to facilitate
# learning), then send all-to-all packets until all pairs have been
# pinged. Tell MCSFinder not to prune initial inputs during this period.
self.initialization_rounds = initialization_rounds
# If initialization_rounds isn't 0, also make sure to send all-to-all
# pings before starting any events
self._pending_all_to_all = initialization_rounds != 0
# Our current place in the all-to-all cycle. Stop when == len(hosts)
self._all_to_all_iterations = 0
# How often (in terms of logical rounds) to inject all-to-all packets
self._all_to_all_interval = 5
self.blocked_controller_pairs = []
self.unblocked_controller_pairs = []
# Logical time (round #) for the simulation execution
self.logical_time = 0
self.never_drop_whitelisted_packets = never_drop_whitelisted_packets
# Determine whether to use delayed and randomized flow mod processing
# (Set by fuzzer_params, not by an optional __init__ argument)
self.delay_flow_mods = False
class Fuzzer(ControlFlow):
'''
Injects input events at random intervals, periodically checking
for invariant violations. (Not the proper use of the term `Fuzzer`)
'''
def __init__(self,
simulation_cfg,
fuzzer_params="config.fuzzer_params",
check_interval=None,
traffic_inject_interval=10,
random_seed=None,
delay=0.1,
steps=None,
input_logger=None,
invariant_check_name="InvariantChecker.check_correspondence",
halt_on_violation=False,
log_invariant_checks=True,
delay_startup=True,
print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False,
never_drop_whitelisted_packets=True,
initialization_rounds=0):
'''
Options:
- fuzzer_params: path to event probabilities
- check_interval: the period for checking invariants, in terms of
logical rounds
- traffic_inject_interval: how often to inject dataplane trace packets
- random_seed: optionally set the seed of the random number generator
- delay: how long to sleep between each logical round
- input_logger: None, or a InputLogger instance
- invariant_check_name: the name of the invariant check, from
config/invariant_checks.py
- halt_on_violation: whether to stop after a bug has been detected
- log_invariant_checks: whether to log InvariantCheck events
- delay_startup: whether to until the first OpenFlow message is received
before proceeding with fuzzing
- print_buffers: whether to print the remaining contents of the
dataplane/controlplane buffers at the end of the execution
- record_deterministic_values: whether to record gettimeofday requests
for replay
- mock_link_discovery: optional module for POX to experiment with
better determinism -- tell POX exactly when links should be discovered
- initialization_rounds: if non-zero, will wait the specified rounds to
let the controller discover the topology before injecting inputs
'''
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(
input_logger,
record_deterministic_values=record_deterministic_values)
self.check_interval = check_interval
if self.check_interval is None:
print >> sys.stderr, "Fuzzer Warning: Check interval is not specified... not checking invariants"
if invariant_check_name not in name_to_invariant_check:
raise ValueError(
'''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# How many rounds to let the controller initialize:
# send one round of packets directed at the source host itself (to facilitate
# learning), then send all-to-all packets until all pairs have been
# pinged. Tell MCSFinder not to prune initial inputs during this period.
self.initialization_rounds = initialization_rounds
# If initialization_rounds isn't 0, also make sure to send all-to-all
# pings before starting any events
self._pending_all_to_all = initialization_rounds != 0
# Our current place in the all-to-all cycle. Stop when == len(hosts)
self._all_to_all_iterations = 0
# How often (in terms of logical rounds) to inject all-to-all packets
self._all_to_all_interval = 5
self.blocked_controller_pairs = []
self.unblocked_controller_pairs = []
# Logical time (round #) for the simulation execution
self.logical_time = 0
self.never_drop_whitelisted_packets = never_drop_whitelisted_packets
# Determine whether to use delayed and randomized flow mod processing
# (Set by fuzzer_params, not by an optional __init__ argument)
self.delay_flow_mods = False
def _log_input_event(self, event, **kws):
if self._input_logger is not None:
if self._initializing():
# Tell MCSFinder never to prune this event
event.prunable = False
event.round = self.logical_time
self._input_logger.log_input_event(event, **kws)
def _load_fuzzer_params(self, fuzzer_params_path):
if fuzzer_params_path.endswith('.py'):
fuzzer_params_path = fuzzer_params_path[:-3].replace("/", ".")
try:
self.params = __import__(fuzzer_params_path, globals(), locals(),
["*"])
# TODO(cs): temporary hack until we get determinism figured out
self.params.link_discovery_rate = 0.1
except:
raise IOError("Could not find fuzzer params config file: %s" %
fuzzer_params_path)
def _compute_unblocked_controller_pairs(self):
sorted_controllers = sorted(
self.simulation.controller_manager.controllers,
key=lambda c: c.cid)
unblocked_pairs = []
for i in xrange(0, len(sorted_controllers)):
for j in xrange(i + 1, len(sorted_controllers)):
c1 = sorted_controllers[i]
c2 = sorted_controllers[j]
# Make sure all controller pairs are unblocked on startup
c1.unblock_peer(c2)
c2.unblock_peer(c1)
unblocked_pairs.append((c1.cid, c2.cid))
return unblocked_pairs
def init_results(self, results_dir):
if self._input_logger:
self._input_logger.open(results_dir)
params_file = re.sub(r'\.pyc$', '.py', self.params.__file__)
# Move over our fuzzer params
# TODO(cs): need to modify copied config file to point to the new fuzzer
# params
if os.path.exists(params_file):
new_params_file = os.path.join(results_dir,
os.path.basename(params_file))
if os.path.abspath(params_file) != os.path.abspath(
new_params_file):
shutil.copy(params_file, new_params_file)
def _initializing(self):
return self.logical_time < self.initialization_rounds or self._pending_all_to_all
def simulate(self):
"""Precondition: simulation.patch_panel is a buffered patch panel"""
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
assert (isinstance(self.simulation.patch_panel, BufferedPatchPanel))
self.traffic_generator.set_topology(self.simulation.topology)
self.unblocked_controller_pairs = self._compute_unblocked_controller_pairs(
)
self.delay_flow_mods = self.params.ofp_cmd_passthrough_rate != 0.0
self.fail_flow_mods = self.params.ofp_flow_mod_failure_rate != 0.0
if self.delay_flow_mods:
for switch in self.simulation.topology.switches:
assert (isinstance(switch, FuzzSoftwareSwitch))
if self.delay_flow_mods:
switch.use_delayed_commands()
switch.randomize_flow_mods()
return self.loop()
def loop(self):
if self.steps:
end_time = self.logical_time + self.steps
else:
end_time = sys.maxint
self.interrupted = False
self.old_interrupt = None
def interrupt(sgn, frame):
msg.interactive(
"Interrupting fuzzer, dropping to console (press ^C again to terminate)"
)
signal.signal(signal.SIGINT, self.old_interrupt)
self.old_interrupt = None
self.interrupted = True
raise KeyboardInterrupt()
self.old_interrupt = signal.signal(signal.SIGINT, interrupt)
try:
# Always connect to controllers explicitly
self.simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
if self.delay_startup:
# Wait until the first OpenFlow message is received
log.info("Waiting until first OpenFlow message received..")
while self.simulation.openflow_buffer.pending_receives() == []:
self.simulation.io_master.select(self.delay)
sent_self_packets = False
while self.logical_time < end_time:
self.logical_time += 1
try:
if not self._initializing():
self.trigger_events()
halt = self.maybe_check_invariant()
if halt:
self.simulation.set_exit_code(5)
break
self.maybe_inject_trace_event()
else: # Initializing
self.check_pending_messages(pass_through=True)
if not sent_self_packets and (
self.logical_time %
self._all_to_all_interval) == 0:
# Only need to send self packets once
self._send_initialization_packets(
send_to_self=True)
sent_self_packets = True
elif self.logical_time > self.initialization_rounds:
# All-to-all mode
if (self.logical_time %
self._all_to_all_interval) == 0:
self._send_initialization_packets(
send_to_self=False)
self._all_to_all_iterations += 1
if self._all_to_all_iterations > len(
self.simulation.topology.hosts):
log.info("Done initializing")
self._pending_all_to_all = False
self.check_dataplane(pass_through=True)
msg.event("Round %d completed." % self.logical_time)
time.sleep(self.delay)
except KeyboardInterrupt as e:
if self.interrupted:
interactive = Interactive(self.simulation_cfg,
self._input_logger)
interactive.simulate(self.simulation,
bound_objects=(('fuzzer',
self), ))
self.old_interrupt = signal.signal(
signal.SIGINT, interrupt)
else:
raise e
log.info("Terminating fuzzing after %d rounds" % self.logical_time)
if self.print_buffers:
self._print_buffers()
finally:
if self.old_interrupt:
signal.signal(signal.SIGINT, self.old_interrupt)
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return self.simulation
def _send_initialization_packet(self, host, send_to_self=False):
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(
traffic_type, host, send_to_self=send_to_self)
self._log_input_event(
TrafficInjection(dp_event=dp_event, host_id=host.hid))
def _send_initialization_packets(self, send_to_self=False):
for host in self.simulation.topology.hosts:
self._send_initialization_packet(host, send_to_self=send_to_self)
def _print_buffers(self):
# TODO(cs): this method should also be added to Interactive.
# Note that there shouldn't be any pending state changes in record mode,
# only pending message sends/receives.
buffered_events = []
log.info("Pending Messages:")
for event_type, pending2conn_messages in [
(ControlMessageReceive,
self.simulation.openflow_buffer.pendingreceive2conn_messages),
(ControlMessageSend,
self.simulation.openflow_buffer.pendingsend2conn_messages)
]:
for p, conn_messages in pending2conn_messages.iteritems():
log.info("- %r", p)
for _, ofp_message in conn_messages:
b64_packet = base64_encode(ofp_message)
event = event_type(p.dpid,
p.controller_id,
p.fingerprint,
b64_packet=b64_packet)
buffered_events.append(event)
if self._input_logger is not None:
self._input_logger.dump_buffered_events(buffered_events)
def maybe_check_invariant(self):
if (self.check_interval is not None
and (self.logical_time % self.check_interval) == 0):
# Time to run correspondence!
# TODO(cs): may need to revert to threaded version if runtime is too
# long
def do_invariant_check():
if self.log_invariant_checks:
self._log_input_event(
CheckInvariants(
round=self.logical_time,
invariant_check_name=self.invariant_check_name))
violations = self.invariant_check(self.simulation)
self.simulation.violation_tracker.track(
violations, self.logical_time)
persistent_violations = self.simulation.violation_tracker.persistent_violations
transient_violations = list(
set(violations) - set(persistent_violations))
if violations != []:
msg.fail(
"The following correctness violations have occurred: %s"
% str(violations))
else:
msg.success("No correctness violations!")
if transient_violations != []:
self._log_input_event(
InvariantViolation(transient_violations))
if persistent_violations != []:
msg.fail("Persistent violations detected!: %s" %
str(persistent_violations))
self._log_input_event(
InvariantViolation(persistent_violations,
persistent=True))
if self.halt_on_violation:
return True
return do_invariant_check()
def maybe_inject_trace_event(self):
if (self.simulation.dataplane_trace
and (self.logical_time % self.traffic_inject_interval) == 0):
(dp_event,
host) = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(
TrafficInjection(dp_event=dp_event, host_id=host.hid))
def trigger_events(self):
self.check_dataplane()
self.check_tcp_connections()
self.check_pending_messages()
self.check_pending_commands()
self.check_switch_crashes()
self.check_link_failures()
self.fuzz_traffic()
self.check_controllers()
self.check_migrations()
self.check_intracontroller_blocks()
def check_dataplane(self, pass_through=False):
''' Decide whether to delay, drop, or deliver packets '''
def drop(dp_event):
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(
DataplaneDrop(dp_event.fingerprint,
host_id=dp_event.get_host_id(),
dpid=dp_event.get_switch_id()))
def permit(dp_event):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
def in_whitelist(dp_event):
return (self.never_drop_whitelisted_packets
and OpenFlowBuffer.in_whitelist(dp_event.fingerprint[0]))
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
if pass_through:
permit(dp_event)
elif not self.simulation.topology.ok_to_send(dp_event):
drop(dp_event)
elif (self.random.random() >= self.params.dataplane_drop_rate
or in_whitelist(dp_event)):
permit(dp_event)
else:
drop(dp_event)
# TODO(cs): temporary hack until we have determinism figured out
if self.mock_link_discovery and self.random.random(
) < self.params.link_discovery_rate:
# Pick a random link to be discovered
link = self.random.choice(self.simulation.topology.network_links)
attrs = [
link.start_software_switch.dpid, link.start_port.port_no,
link.end_software_switch.dpid, link.end_port.port_no
]
# Send it to a random controller
live_controllers = self.simulation.controller_manager.live_controllers
if live_controllers != []:
c = self.random.choice(list(live_controllers))
c.sync_connection.send_link_notification(attrs)
self._log_input_event(LinkDiscovery(c.cid, attrs))
def check_tcp_connections(self):
''' Decide whether to block or unblock control channels '''
for (switch, connection
) in self.simulation.topology.unblocked_controller_connections:
if self.random.random() < self.params.controlplane_block_rate:
self.simulation.topology.block_connection(connection)
self._log_input_event(
ControlChannelBlock(switch.dpid,
connection.get_controller_id()))
for (switch, connection
) in self.simulation.topology.blocked_controller_connections:
if self.random.random() < self.params.controlplane_unblock_rate:
self.simulation.topology.unblock_connection(connection)
self._log_input_event(
ControlChannelUnblock(
switch.dpid,
controller_id=connection.get_controller_id()))
def check_pending_messages(self, pass_through=False):
for pending_receipt in self.simulation.openflow_buffer.pending_receives(
):
# TODO(cs): this is a really dumb way to fuzz packet receipt scheduling
if (self.random.random() < self.params.ofp_message_receipt_rate
or pass_through):
message = self.simulation.openflow_buffer.schedule(
pending_receipt)
b64_packet = base64_encode(message)
self._log_input_event(
ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint,
b64_packet=b64_packet))
for pending_send in self.simulation.openflow_buffer.pending_sends():
if (self.random.random() < self.params.ofp_message_send_rate
or pass_through):
message = self.simulation.openflow_buffer.schedule(
pending_send)
b64_packet = base64_encode(message)
self._log_input_event(
ControlMessageSend(pending_send.dpid,
pending_send.controller_id,
pending_send.fingerprint,
b64_packet=b64_packet))
def check_pending_commands(self):
''' If Fuzzer is configured to delay flow mods, this decides whether
each switch is allowed to process a buffered flow mod '''
def should_fail_flow_mod(command):
# super simple filter to tell whether to apply or actively fail a flow_mod
return self.random.random() < self.params.ofp_flow_mod_failure_rate
if self.delay_flow_mods:
for switch in self.simulation.topology.switches:
assert (isinstance(switch, FuzzSoftwareSwitch))
# first decide if we should try to process the next command from the switch
if (self.random.random() < self.params.ofp_cmd_passthrough_rate
) and switch.has_pending_commands():
(cmd, pending_receipt) = switch.get_next_command()
# then check whether we should make the attempt to process the next command fail
if should_fail_flow_mod(cmd):
eventclass = FailFlowMod
else:
eventclass = ProcessFlowMod
switch.process_delayed_command(pending_receipt)
b64_packet = base64_encode(cmd)
self._log_input_event(
eventclass(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint,
b64_packet=b64_packet))
def check_switch_crashes(self):
''' Decide whether to crash or restart switches, links and controllers '''
def crash_switches():
crashed_this_round = set()
for software_switch in list(
self.simulation.topology.live_switches):
if self.random.random() < self.params.switch_failure_rate:
crashed_this_round.add(software_switch)
self.simulation.topology.crash_switch(software_switch)
self._log_input_event(SwitchFailure(software_switch.dpid))
return crashed_this_round
def restart_switches(crashed_this_round):
# Make sure we don't try to connect to dead controllers
down_controller_ids = None
for software_switch in list(
self.simulation.topology.failed_switches):
if software_switch in crashed_this_round:
continue
if self.random.random() < self.params.switch_recovery_rate:
if down_controller_ids is None:
self.simulation.controller_manager.check_controller_status(
)
down_controller_ids = [ c.cid for c in self.simulation.controller_manager.controllers\
if c.state == ControllerState.STARTING or\
c.state == ControllerState.DEAD ]
connected = self.simulation.topology\
.recover_switch(software_switch,
down_controller_ids=down_controller_ids)
if connected:
self._log_input_event(
SwitchRecovery(software_switch.dpid))
crashed_this_round = crash_switches()
try:
restart_switches(crashed_this_round)
except TimeoutError:
log.warn("Unable to connect to controllers")
def check_link_failures(self):
def sever_links():
# TODO(cs): model administratively down links? (OFPPC_PORT_DOWN)
cut_this_round = set()
for link in list(self.simulation.topology.live_links):
if self.random.random() < self.params.link_failure_rate:
cut_this_round.add(link)
self.simulation.topology.sever_link(link)
self._log_input_event(
LinkFailure(link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
return cut_this_round
def repair_links(cut_this_round):
for link in list(self.simulation.topology.cut_links):
if link in cut_this_round:
continue
if self.random.random() < self.params.link_recovery_rate:
self.simulation.topology.repair_link(link)
self._log_input_event(
LinkRecovery(link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
cut_this_round = sever_links()
repair_links(cut_this_round)
def fuzz_traffic(self):
if not self.simulation.dataplane_trace:
# randomly generate messages from switches
for host in self.simulation.topology.hosts:
if self.random.random() < self.params.traffic_generation_rate:
if len(host.interfaces) > 0:
msg.event("Injecting a random packet")
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(
traffic_type, host)
self._log_input_event(
TrafficInjection(dp_event=dp_event))
def check_controllers(self):
def crash_controllers():
crashed_this_round = set()
for controller in self.simulation.controller_manager.live_controllers:
if self.random.random() < self.params.controller_crash_rate:
crashed_this_round.add(controller)
controller.kill()
self._log_input_event(ControllerFailure(controller.cid))
return crashed_this_round
def reboot_controllers(crashed_this_round):
for controller in self.simulation.controller_manager.down_controllers:
if controller in crashed_this_round:
continue
if self.random.random() < self.params.controller_recovery_rate:
controller.restart()
self._log_input_event(ControllerRecovery(controller.cid))
crashed_this_round = crash_controllers()
reboot_controllers(crashed_this_round)
def check_migrations(self):
for access_link in list(self.simulation.topology.access_links):
if self.random.random() < self.params.host_migration_rate:
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
live_edge_switches = list(
self.simulation.topology.live_edge_switches)
if len(live_edge_switches) > 0:
new_switch = random.choice(live_edge_switches)
new_switch_dpid = new_switch.dpid
new_port_no = max(new_switch.ports.keys()) + 1
msg.event("Migrating host %s" % str(access_link.host))
self.simulation.topology.migrate_host(
old_ingress_dpid, old_ingress_port_no, new_switch_dpid,
new_port_no)
self._log_input_event(
HostMigration(old_ingress_dpid, old_ingress_port_no,
new_switch_dpid, new_port_no,
access_link.host.hid))
self._send_initialization_packet(access_link.host,
send_to_self=True)
def check_intracontroller_blocks(self):
blocked_this_round = None
# Block at most one controller pair per round.
if (len(self.unblocked_controller_pairs) > 0 and
self.random.random() < self.params.intracontroller_block_rate):
(cid1, cid2) = self.random.choice(self.unblocked_controller_pairs)
msg.event("Unblocking controllers %s, %s" % (cid1, cid2))
blocked_this_round = (cid1, cid2)
self.unblocked_controller_pairs.remove((cid1, cid2))
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.block_controller_pair(
cid1, cid2)
else:
(c1, c2) = [
self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2]
]
c1.block_peer(c2)
c2.block_peer(c1)
self._log_input_event(BlockControllerPair(cid1, cid2))
if (len(self.blocked_controller_pairs) > 0 and self.random.random() <
self.params.intracontroller_unblock_rate):
(cid1, cid2) = self.random.choice(self.blocked_controller_pairs)
msg.event("Blocking controllers %s, %s" % (cid1, cid2))
self.blocked_controller_pairs.remove((cid1, cid2))
self.unblocked_controller_pairs.append((cid1, cid2))
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.unblock_controller_pair(
cid1, cid2)
else:
(c1, c2) = [
self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2]
]
c1.unblock_peer(c2)
c2.unblock_peer(c1)
self._log_input_event(UnblockControllerPair(cid1, cid2))
if blocked_this_round is not None:
self.blocked_controller_pairs.append(blocked_this_round)
class Interactive(ControlFlow):
'''
Presents an interactive prompt for injecting events and
checking for invariants at the users' discretion
'''
def __init__(self, simulation_cfg, input_logger=None,
pass_through_of_messages=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.logical_time = 0
self._input_logger = input_logger
self.traffic_generator = TrafficGenerator(random.Random())
# TODO(cs): we don't actually have a way to support pass_through=False, in
# a reasonable way -- the user has no (easy) way to examine and route
# OpenFlowBuffer's pending_receives and pending_sends.
self.pass_through_of_messages = pass_through_of_messages
# TODO(cs): future feature: allow the user to interactively choose the order
# events occur for each round, whether to delay, drop packets, fail nodes,
# etc.
# self.failure_lvl = [
# NOTHING, # Everything is handled by the random number generator
# CRASH, # The user only controls node crashes and restarts
# DROP, # The user also controls message dropping
# DELAY, # The user also controls message delays
# EVERYTHING # The user controls everything, including message ordering
# ]
def _log_input_event(self, event, **kws):
# TODO(cs): redundant with Fuzzer._log_input_event
if self._input_logger is not None:
event.round = self.logical_time
self._input_logger.log_input_event(event, **kws)
def init_results(self, results_dir):
if self._input_logger is not None:
self._input_logger.open(results_dir)
def default_connect_to_controllers(self, simulation):
simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
def simulate(self, simulation=None, boot_controllers=default_boot_controllers,
connect_to_controllers=None,
bound_objects=()):
if simulation is None:
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback,
boot_controllers=boot_controllers)
if connect_to_controllers is None:
self.default_connect_to_controllers(self.simulation)
else:
connect_to_controllers(self.simulation)
else:
self.simulation = simulation
if self.pass_through_of_messages:
self.simulation.openflow_buffer.set_pass_through(input_logger=self._input_logger)
self._forwarded_this_step = 0
self.traffic_generator.set_topology(self.simulation.topology)
try:
c = STSConsole(default_command=self.default_command)
c.cmd_group("Simulation State")
c.cmd(self.next_step, "next", alias="n", help_msg="Proceed to next simulation step")
c.cmd(self.quit, "quit", alias="q", help_msg="Quit the simulation")
c.cmd_group("Invariants")
c.cmd(self.invariant_check, "check_invariants", alias="inv", help_msg="Run an invariant check")\
.arg("kind", values=['omega', 'connectivity', 'loops', 'liveness', "python_connectivity", "blackholes"])
c.cmd_group("Dataplane")
c.cmd(self.dataplane_trace_feed, "dp_inject", alias="dpi", help_msg="Inject the next dataplane event from the trace")
c.cmd(self.dataplane_ping, "dp_ping", alias="dpp", help_msg="Generate and inject a new ping packet")
c.cmd(self.dataplane_forward, "dp_forward", alias="dpf", help_msg="Forward a pending dataplane event")
c.cmd(self.dataplane_drop, "dp_drop", alias="dpd", help_msg="Drop a pending dataplane event")
c.cmd(self.dataplane_delay, "dp_delay", alias="dpe", help_msg="Delay a pending dataplane event")
c.cmd_group("Controlling Entitities")
c.cmd(self.list_controllers, "list_controllers", alias="lsc", help_msg="List controllers")
c.cmd(self.kill_controller, "kill_controller", alias="kc", help_msg="Kill a controller")\
.arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.live_controllers))
c.cmd(self.start_controller, "start_controller", alias="sc", help_msg="Restart a controller")\
.arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.down_controllers))
c.cmd(self.list_switches, "list_switches", alias="lss", help_msg="List switches")
c.cmd(self.kill_switch, "kill_switch", alias="ks", help_msg="Kill a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.start_switch, "start_switch", alias="ss", help_msg="Restart a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.cut_link, "cut_link", alias="cl", help_msg="Cut a link")\
.arg("dpid1", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))\
.arg("dpid2", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.repair_link, "repair_link", alias="rl", help_msg="Repair a link")\
.arg("dpid1", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))\
.arg("dpid2", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.show_flow_table, "show_flows", alias="sf", help_msg="Show flowtable of a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.list_hosts, "list_hosts", alias="lhs", help_msg="List hosts")
c.cmd(self.migrate_host, "migrate_host", alias="mh", help_msg="Migrate a host to switch dpid")\
.arg("hid", values=lambda: map(lambda h: h.hid, self.simulation.topology.hosts))\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.block_controller_traffic, "block_controllers", alias="bc", help_msg="Drop traffic between controllers")\
.arg("cid1", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))\
.arg("cid2", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))
c.cmd(self.unblock_controller_traffic, "unblock_controllers", alias="ubc", help_msg="Stop dropping traffic between controllers")\
.arg("cid1", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))\
.arg("cid2", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))
c.cmd_group("Python Objects")
c.register_obj(self.simulation, "simulation", alias="sim", help_msg="access the simulation object")
c.register_obj(self.simulation.topology, "topology", alias="topo", help_msg="access the topology object")
for (name, obj) in bound_objects:
c.register_obj(obj, name, help_msg="access the %s object" % name)
c.run()
finally:
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return self.simulation
def default_command(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) == 1 and self._forwarded_this_step == 0 and self.simulation.topology.ok_to_send(queued[0]):
return "dpf"
else:
return "next"
def quit(self):
print "End console with CTRL-D"
def next_step(self):
time.sleep(0.05)
self.logical_time += 1
self._forwarded_this_step = 0
print "-------------------"
print "Advanced to step %d" % self.logical_time
self.show_queued_events()
def show_queued_events(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) > 0:
print "Queued Dataplane events:"
for (i, e) in enumerate(queued):
print "%d: %s on %s:%s" % (i, e, e.node, e.port)
def list_controllers(self):
cm = self.simulation.controller_manager
live = cm.live_controllers
print "Controllers:"
for c in cm.controllers:
print "%s %s %s %s" % (c.label, c.cid, repr(c), "[ALIVE]" if c in live else "[DEAD]")
def kill_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.kill_controller(c)
self._log_input_event(ControllerFailure(c.cid))
else:
print "Controller with label %s not found" %label
def start_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.reboot_controller(c)
self._log_input_event(ControllerRecovery(c.cid))
else:
print "Controller with label %s not found" %label
def list_switches(self):
topology = self.simulation.topology
live = topology.live_switches
print "Switches:"
for s in topology.switches:
print "%d %s %s" % (s.dpid, repr(s), "[ALIVE]" if s in live else "[DEAD]")
def list_hosts(self):
topology = self.simulation.topology
print "Hosts:"
for h in topology.hosts:
print "%d %s" % (h.hid, str(h),)
def kill_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
topology.crash_switch(switch)
self._log_input_event(SwitchFailure(switch.dpid))
def start_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
down_controller_ids = map(lambda c: c.cid, self.simulation.controller_manager.down_controllers)
topology.recover_switch(switch, down_controller_ids=down_controller_ids)
self._log_input_event(SwitchRecovery(switch.dpid))
def cut_link(self, dpid1, dpid2):
topology = self.simulation.topology
link = topology.get_link(dpid1, dpid2)
topology.sever_link(link)
self._log_input_event(LinkFailure(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
def repair_link(self, dpid1, dpid2):
topology = self.simulation.topology
link = topology.get_link(dpid1, dpid2)
topology.repair_link(link)
self._log_input_event(LinkRecovery(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
def migrate_host(self, hid, dpid):
topology = self.simulation.topology
host = topology.get_host(hid)
# TODO(cs): make this lookup more efficient
access_link = find(lambda a: a.host == host, topology.access_links)
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
new_switch = topology.get_switch(dpid)
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no,
access_link.host.hid))
self._send_initialization_packet(access_link.host, send_to_self=True)
# TODO(cs): ripped directly from fuzzer. Redundant!
def _send_initialization_packet(self, host, send_to_self=False):
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(traffic_type, host, send_to_self=send_to_self)
self._log_input_event(TrafficInjection(dp_event=dp_event, host_id=host.hid))
def show_flow_table(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
dl_types = { 0x0800: "IP",
0x0806: "ARP",
0x8100: "VLAN",
0x88cc: "LLDP",
0x888e: "PAE"
}
nw_protos = { 1 : "ICMP", 6 : "TCP", 17 : "UDP" }
ports = { v: k.replace("OFPP_","") for (k,v) in of.ofp_port_rev_map.iteritems() }
def dl_type(e):
d = e.match.dl_type
if d is None:
return d
else:
return dl_types[d] if d in dl_types else "%x" %d
def nw_proto(e):
p = e.match.nw_proto
return nw_protos[p] if p in nw_protos else p
def action(a):
if isinstance(a, ofp_action_output):
return ports[a.port] if a.port in ports else "output(%d)" % a.port
else:
return str(a)
def actions(e):
if len(e.actions) == 0:
return "(drop)"
else:
return ", ".join(action(a) for a in e.actions)
t = Tabular( ("Prio", lambda e: e.priority),
("in_port", lambda e: e.match.in_port),
("dl_type", dl_type),
("dl_src", lambda e: e.match.dl_src),
("dl_dst", lambda e: e.match.dl_dst),
("nw_proto", nw_proto),
("nw_src", lambda e: e.match.nw_src),
("nw_dst", lambda e: e.match.nw_dst),
("tp_src", lambda e: e.match.tp_src),
("tp_dst", lambda e: e.match.tp_dst),
("actions", actions),
)
t.show(switch.table.entries)
def invariant_check(self, kind):
if kind == "omega" or kind == "o":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_correspondence"))
result = InvariantChecker.check_correspondence(self.simulation)
message = "Controllers with miscorrepondence: "
elif kind == "connectivity" or kind == "c":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_connectivity"))
result = InvariantChecker.check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "python_connectivity" or kind == "pc":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.python_check_connectivity"))
result = InvariantChecker.python_check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "loops" or kind == "lo":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_loops"))
result = InvariantChecker.python_check_loops(self.simulation)
message = "Loops: "
elif kind == "liveness" or kind == "li":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_liveness"))
result = InvariantChecker.check_liveness(self.simulation)
message = "Crashed controllers: "
elif kind == "blackholes" or kind == "bl":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_blackholes"))
result = InvariantChecker.python_check_blackholes(self.simulation)
message = "Blackholes: "
else:
log.warn("Unknown invariant kind...")
self.simulation.violation_tracker.track(result, self.logical_time)
persistent_violations = self.simulation.violation_tracker.persistent_violations
transient_violations = list(set(result) - set(persistent_violations))
msg.interactive(message + str(result))
if transient_violations != []:
self._log_input_event(InvariantViolation(transient_violations))
if persistent_violations != []:
msg.fail("Persistent violations detected!: %s"
% str(persistent_violations))
self._log_input_event(InvariantViolation(persistent_violations, persistent=True))
def dataplane_trace_feed(self):
if self.simulation.dataplane_trace:
(dp_event, host) = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(dp_event=dp_event, host=host.hid))
else:
print "No dataplane trace to inject from."
def dataplane_ping(self, from_hid=None, to_hid=None):
if (from_hid is not None) and \
(from_hid not in self.simulation.topology.hid2host.keys()):
print "Unknown host %s" % from_hid
return
if (to_hid is not None) and \
(to_hid not in self.simulation.topology.hid2host.keys()):
print "Unknown host %s" % to_hid
return
dp_event = self.traffic_generator.generate_and_inject("icmp_ping", from_hid, to_hid,
payload_content=raw_input("Enter payload content:\n"))
self._log_input_event(TrafficInjection(dp_event=dp_event))
def _select_dataplane_event(self, sel=None):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) == 0:
print "No dataplane events queued."
return None
if not sel:
return queued[0]
if sel >= len(queued):
print "Given index out of bounds. Try a value smaller than %d" % len(queued)
return queued[sel]
def dataplane_forward(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
if self.simulation.topology.ok_to_send(dp_event):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
self._forwarded_this_step += 1
else:
print "Not ready to send event %s" % event
def dataplane_drop(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint,
host_id=dp_event.get_host_id(),
dpid=dp_event.get_switch_id()))
def dataplane_delay(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.delay_dp_event(dp_event)
def block_controller_traffic(self, cid1, cid2):
''' Drop all messages sent from controller 1 to controller 2 until
unblock_controller_pair is called. '''
if self.simulation.controller_patch_panel is None:
self.log.warn("The controller patch panel does not exist.")
return
self.simulation.controller_patch_panel.block_controller_pair(cid1, cid2)
self._log_input_event(BlockControllerPair(cid1, cid2))
def unblock_controller_traffic(self, cid1, cid2):
''' Stop dropping messages sent from controller 1 to controller 2 '''
if self.simulation.controller_patch_panel is None:
self.log.warn("The controller patch panel does not exist.")
self.simulation.controller_patch_panel.unblock_controller_pair(cid1, cid2)
self._log_input_event(UnblockControllerPair(cid1, cid2))
def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
check_interval=None, traffic_inject_interval=10, random_seed=None,
delay=0.1, steps=None, input_logger=None,
invariant_check_name="InvariantChecker.check_correspondence",
halt_on_violation=False, log_invariant_checks=True,
delay_startup=True, print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False,
never_drop_whitelisted_packets=True,
initialization_rounds=0):
'''
Options:
- fuzzer_params: path to event probabilities
- check_interval: the period for checking invariants, in terms of
logical rounds
- traffic_inject_interval: how often to inject dataplane trace packets
- random_seed: optionally set the seed of the random number generator
- delay: how long to sleep between each logical round
- input_logger: None, or a InputLogger instance
- invariant_check_name: the name of the invariant check, from
config/invariant_checks.py
- halt_on_violation: whether to stop after a bug has been detected
- log_invariant_checks: whether to log InvariantCheck events
- delay_startup: whether to until the first OpenFlow message is received
before proceeding with fuzzing
- print_buffers: whether to print the remaining contents of the
dataplane/controlplane buffers at the end of the execution
- record_deterministic_values: whether to record gettimeofday requests
for replay
- mock_link_discovery: optional module for POX to experiment with
better determinism -- tell POX exactly when links should be discovered
- initialization_rounds: if non-zero, will wait the specified rounds to
let the controller discover the topology before injecting inputs
'''
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger,
record_deterministic_values=record_deterministic_values)
self.check_interval = check_interval
if self.check_interval is None:
print >> sys.stderr, "Fuzzer Warning: Check interval is not specified... not checking invariants"
if invariant_check_name not in name_to_invariant_check:
raise ValueError('''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# How many rounds to let the controller initialize:
# send one round of packets directed at the source host itself (to facilitate
# learning), then send all-to-all packets until all pairs have been
# pinged. Tell MCSFinder not to prune initial inputs during this period.
self.initialization_rounds = initialization_rounds
# If initialization_rounds isn't 0, also make sure to send all-to-all
# pings before starting any events
self._pending_all_to_all = initialization_rounds != 0
# Our current place in the all-to-all cycle. Stop when == len(hosts)
self._all_to_all_iterations = 0
# How often (in terms of logical rounds) to inject all-to-all packets
self._all_to_all_interval = 5
self.blocked_controller_pairs = []
self.unblocked_controller_pairs = []
# Logical time (round #) for the simulation execution
self.logical_time = 0
self.never_drop_whitelisted_packets = never_drop_whitelisted_packets
# Determine whether to use delayed and randomized flow mod processing
# (Set by fuzzer_params, not by an optional __init__ argument)
self.delay_flow_mods = False
class Fuzzer(ControlFlow):
'''
Injects input events at random intervals, periodically checking
for invariant violations. (Not the proper use of the term `Fuzzer`)
'''
def __init__(self, simulation_cfg, fuzzer_params="config.fuzzer_params",
check_interval=None, traffic_inject_interval=10, random_seed=None,
delay=0.1, steps=None, input_logger=None,
invariant_check_name="InvariantChecker.check_correspondence",
halt_on_violation=False, log_invariant_checks=True,
delay_startup=True, print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False,
never_drop_whitelisted_packets=True,
initialization_rounds=0):
'''
Options:
- fuzzer_params: path to event probabilities
- check_interval: the period for checking invariants, in terms of
logical rounds
- traffic_inject_interval: how often to inject dataplane trace packets
- random_seed: optionally set the seed of the random number generator
- delay: how long to sleep between each logical round
- input_logger: None, or a InputLogger instance
- invariant_check_name: the name of the invariant check, from
config/invariant_checks.py
- halt_on_violation: whether to stop after a bug has been detected
- log_invariant_checks: whether to log InvariantCheck events
- delay_startup: whether to until the first OpenFlow message is received
before proceeding with fuzzing
- print_buffers: whether to print the remaining contents of the
dataplane/controlplane buffers at the end of the execution
- record_deterministic_values: whether to record gettimeofday requests
for replay
- mock_link_discovery: optional module for POX to experiment with
better determinism -- tell POX exactly when links should be discovered
- initialization_rounds: if non-zero, will wait the specified rounds to
let the controller discover the topology before injecting inputs
'''
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger,
record_deterministic_values=record_deterministic_values)
self.check_interval = check_interval
if self.check_interval is None:
print >> sys.stderr, "Fuzzer Warning: Check interval is not specified... not checking invariants"
if invariant_check_name not in name_to_invariant_check:
raise ValueError('''Unknown invariant check %s.\n'''
'''Invariant check name must be defined in config.invariant_checks''',
invariant_check_name)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# How many rounds to let the controller initialize:
# send one round of packets directed at the source host itself (to facilitate
# learning), then send all-to-all packets until all pairs have been
# pinged. Tell MCSFinder not to prune initial inputs during this period.
self.initialization_rounds = initialization_rounds
# If initialization_rounds isn't 0, also make sure to send all-to-all
# pings before starting any events
self._pending_all_to_all = initialization_rounds != 0
# Our current place in the all-to-all cycle. Stop when == len(hosts)
self._all_to_all_iterations = 0
# How often (in terms of logical rounds) to inject all-to-all packets
self._all_to_all_interval = 5
self.blocked_controller_pairs = []
self.unblocked_controller_pairs = []
# Logical time (round #) for the simulation execution
self.logical_time = 0
self.never_drop_whitelisted_packets = never_drop_whitelisted_packets
# Determine whether to use delayed and randomized flow mod processing
# (Set by fuzzer_params, not by an optional __init__ argument)
self.delay_flow_mods = False
def _log_input_event(self, event, **kws):
if self._input_logger is not None:
if self._initializing():
# Tell MCSFinder never to prune this event
event.prunable = False
event.round = self.logical_time
self._input_logger.log_input_event(event, **kws)
def _load_fuzzer_params(self, fuzzer_params_path):
if fuzzer_params_path.endswith('.py'):
fuzzer_params_path = fuzzer_params_path[:-3].replace("/", ".")
try:
self.params = __import__(fuzzer_params_path, globals(), locals(), ["*"])
# TODO(cs): temporary hack until we get determinism figured out
self.params.link_discovery_rate = 0.1
except:
raise IOError("Could not find fuzzer params config file: %s" %
fuzzer_params_path)
def _compute_unblocked_controller_pairs(self):
sorted_controllers = sorted(self.simulation.controller_manager.controllers, key=lambda c: c.cid)
unblocked_pairs = []
for i in xrange(0, len(sorted_controllers)):
for j in xrange(i+1, len(sorted_controllers)):
c1 = sorted_controllers[i]
c2 = sorted_controllers[j]
# Make sure all controller pairs are unblocked on startup
c1.unblock_peer(c2)
c2.unblock_peer(c1)
unblocked_pairs.append((c1.cid, c2.cid))
return unblocked_pairs
def init_results(self, results_dir):
if self._input_logger:
self._input_logger.open(results_dir)
params_file = re.sub(r'\.pyc$', '.py', self.params.__file__)
# Move over our fuzzer params
if os.path.exists(params_file):
new_params_file = os.path.join(results_dir, os.path.basename(params_file))
if os.path.abspath(params_file) != os.path.abspath(new_params_file):
shutil.copy(params_file, new_params_file)
# make sure to modify orig_config.py to point to new fuzzer_params.
orig_config_path = os.path.join(results_dir, "orig_config.py")
if os.path.exists(orig_config_path):
with open(orig_config_path, "a") as out:
# TODO(cs): too lazy for now to re-parse the config file and place
# the fuzzer_params parameter in the correct place. So for now,
# force the human to do it for us.
out.write('''\nraise RuntimeError("Please add this parameter to Fuzzer: '''
'''fuzzer_params='%s'")''' % new_params_file)
def _initializing(self):
return self.logical_time < self.initialization_rounds or self._pending_all_to_all
def simulate(self):
"""Precondition: simulation.patch_panel is a buffered patch panel"""
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
assert(isinstance(self.simulation.patch_panel, BufferedPatchPanel))
self.traffic_generator.set_topology(self.simulation.topology)
self.unblocked_controller_pairs = self._compute_unblocked_controller_pairs()
self.delay_flow_mods = self.params.ofp_cmd_passthrough_rate != 1.0
self.fail_flow_mods = self.params.ofp_flow_mod_failure_rate != 0.0
if self.delay_flow_mods:
for switch in self.simulation.topology.switches:
assert(isinstance(switch, FuzzSoftwareSwitch))
switch.use_delayed_commands()
switch.randomize_flow_mods()
return self.loop()
def loop(self):
if self.steps:
end_time = self.logical_time + self.steps
else:
end_time = sys.maxint
self.interrupted = False
self.old_interrupt = None
def interrupt(sgn, frame):
msg.interactive("Interrupting fuzzer, dropping to console (press ^C again to terminate)")
# If ^C is triggered twice in a row, invoke the original handler.
signal.signal(signal.SIGINT, self.old_interrupt)
self.old_interrupt = None
self.interrupted = True
raise KeyboardInterrupt()
# signal.signal returns the previous interrupt handler.
self.old_interrupt = signal.signal(signal.SIGINT, interrupt)
try:
# Always connect to controllers explicitly
self.simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
if self.delay_startup:
# Wait until the first OpenFlow message is received
log.info("Waiting until first OpenFlow message received..")
while self.simulation.openflow_buffer.pending_receives() == []:
self.simulation.io_master.select(self.delay)
sent_self_packets = False
while self.logical_time < end_time:
self.logical_time += 1
try:
if not self._initializing():
self.trigger_events()
halt = self.maybe_check_invariant()
if halt:
self.simulation.set_exit_code(5)
break
self.maybe_inject_trace_event()
else: # Initializing
self.check_pending_messages(pass_through=True)
if not sent_self_packets and (self.logical_time % self._all_to_all_interval) == 0:
# Only need to send self packets once
self._send_initialization_packets(send_to_self=True)
sent_self_packets = True
elif self.logical_time > self.initialization_rounds:
# All-to-all mode
if (self.logical_time % self._all_to_all_interval) == 0:
self._send_initialization_packets(send_to_self=False)
self._all_to_all_iterations += 1
if self._all_to_all_iterations > len(self.simulation.topology.hosts):
log.info("Done initializing")
self._pending_all_to_all = False
self.check_dataplane(pass_through=True)
msg.event("Round %d completed." % self.logical_time)
# Note that time.sleep triggers a round of select.select()
time.sleep(self.delay)
except KeyboardInterrupt as e:
if self.interrupted:
interactive = Interactive(self.simulation_cfg, self._input_logger)
interactive.simulate(self.simulation, bound_objects=( ('fuzzer', self), ))
# If Interactive terminated due to ^D, return to our replaying loop,
# prepared again to drop into Interactive on ^C.
self.old_interrupt = signal.signal(signal.SIGINT, interrupt)
else:
raise e
log.info("Terminating fuzzing after %d rounds" % self.logical_time)
if self.print_buffers:
self._print_buffers()
finally:
if self.old_interrupt:
signal.signal(signal.SIGINT, self.old_interrupt)
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return self.simulation
def _send_initialization_packet(self, host, send_to_self=False):
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(traffic_type, host, send_to_self=send_to_self)
self._log_input_event(TrafficInjection(dp_event=dp_event, host_id=host.hid))
def _send_initialization_packets(self, send_to_self=False):
for host in self.simulation.topology.hosts:
self._send_initialization_packet(host, send_to_self=send_to_self)
def _print_buffers(self):
# TODO(cs): this method should also be added to Interactive.
# Note that there shouldn't be any pending state changes in record mode,
# only pending message sends/receives.
buffered_events = []
log.info("Pending Messages:")
for event_type, pending2conn_messages in [
(ControlMessageReceive, self.simulation.openflow_buffer.pendingreceive2conn_messages),
(ControlMessageSend, self.simulation.openflow_buffer.pendingsend2conn_messages)]:
for p, conn_messages in pending2conn_messages.iteritems():
log.info("- %r", p)
for _, ofp_message in conn_messages:
b64_packet = base64_encode(ofp_message)
event = event_type(p.dpid, p.controller_id, p.fingerprint, b64_packet=b64_packet)
buffered_events.append(event)
if self._input_logger is not None:
self._input_logger.dump_buffered_events(buffered_events)
def maybe_check_invariant(self):
if (self.check_interval is not None and
(self.logical_time % self.check_interval) == 0):
# Time to run correspondence!
# TODO(cs): may need to revert to threaded version if runtime is too
# long
def do_invariant_check():
if self.log_invariant_checks:
self._log_input_event(CheckInvariants(round=self.logical_time,
invariant_check_name=self.invariant_check_name))
violations = self.invariant_check(self.simulation)
self.simulation.violation_tracker.track(violations, self.logical_time)
persistent_violations = self.simulation.violation_tracker.persistent_violations
transient_violations = list(set(violations) - set(persistent_violations))
if violations != []:
msg.fail("The following correctness violations have occurred: %s"
% str(violations))
else:
msg.success("No correctness violations!")
if transient_violations != []:
self._log_input_event(InvariantViolation(transient_violations))
if persistent_violations != []:
msg.fail("Persistent violations detected!: %s"
% str(persistent_violations))
self._log_input_event(InvariantViolation(persistent_violations, persistent=True))
if self.halt_on_violation:
return True
return do_invariant_check()
def maybe_inject_trace_event(self):
if (self.simulation.dataplane_trace and
(self.logical_time % self.traffic_inject_interval) == 0):
(dp_event, host) = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(dp_event=dp_event,
host_id=host.hid))
def trigger_events(self):
self.check_dataplane()
self.check_tcp_connections()
self.check_pending_messages()
self.check_pending_commands()
self.check_switch_crashes()
self.check_link_failures()
self.fuzz_traffic()
self.check_controllers()
self.check_migrations()
self.check_intracontroller_blocks()
def check_dataplane(self, pass_through=False):
''' Decide whether to delay, drop, or deliver packets '''
def drop(dp_event, log_event=True):
self.simulation.patch_panel.drop_dp_event(dp_event)
if log_event:
self._log_input_event(DataplaneDrop(dp_event.fingerprint,
host_id=dp_event.get_host_id(),
dpid=dp_event.get_switch_id()))
def permit(dp_event):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
def in_whitelist(dp_event):
return (self.never_drop_whitelisted_packets and
OpenFlowBuffer.in_whitelist(dp_event.fingerprint[0]))
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
if pass_through:
permit(dp_event)
elif not self.simulation.topology.ok_to_send(dp_event):
drop(dp_event, log_event=False)
elif (self.random.random() >= self.params.dataplane_drop_rate or in_whitelist(dp_event)):
permit(dp_event)
else:
drop(dp_event)
# TODO(cs): temporary hack until we have determinism figured out
if self.mock_link_discovery and self.random.random() < self.params.link_discovery_rate:
# Pick a random link to be discovered
link = self.random.choice(self.simulation.topology.network_links)
attrs = [link.start_software_switch.dpid, link.start_port.port_no,
link.end_software_switch.dpid, link.end_port.port_no]
# Send it to a random controller
live_controllers = self.simulation.controller_manager.live_controllers
if live_controllers != []:
c = self.random.choice(list(live_controllers))
c.sync_connection.send_link_notification(attrs)
self._log_input_event(LinkDiscovery(c.cid, attrs))
def check_tcp_connections(self):
''' Decide whether to block or unblock control channels '''
for (switch, connection) in self.simulation.topology.unblocked_controller_connections:
if self.random.random() < self.params.controlplane_block_rate:
self.simulation.topology.block_connection(connection)
self._log_input_event(ControlChannelBlock(switch.dpid,
connection.get_controller_id()))
for (switch, connection) in self.simulation.topology.blocked_controller_connections:
if self.random.random() < self.params.controlplane_unblock_rate:
self.simulation.topology.unblock_connection(connection)
self._log_input_event(ControlChannelUnblock(switch.dpid,
controller_id=connection.get_controller_id()))
def check_pending_messages(self, pass_through=False):
for pending_receipt in self.simulation.openflow_buffer.pending_receives():
# TODO(cs): this is a really dumb way to fuzz packet receipt scheduling
if (self.random.random() < self.params.ofp_message_receipt_rate or
pass_through):
message = self.simulation.openflow_buffer.schedule(pending_receipt)
b64_packet = base64_encode(message)
self._log_input_event(ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint,
b64_packet=b64_packet))
for pending_send in self.simulation.openflow_buffer.pending_sends():
if (self.random.random() < self.params.ofp_message_send_rate or
pass_through):
message = self.simulation.openflow_buffer.schedule(pending_send)
b64_packet = base64_encode(message)
self._log_input_event(ControlMessageSend(pending_send.dpid,
pending_send.controller_id,
pending_send.fingerprint,
b64_packet=b64_packet))
def check_pending_commands(self):
''' If Fuzzer is configured to delay flow mods, this decides whether
each switch is allowed to process a buffered flow mod '''
def should_fail_flow_mod(command):
# super simple filter to tell whether to apply or actively fail a flow_mod
return self.random.random() < self.params.ofp_flow_mod_failure_rate
if self.delay_flow_mods:
for switch in self.simulation.topology.switches:
assert(isinstance(switch, FuzzSoftwareSwitch))
# first decide if we should try to process the next command from the switch
if switch.has_pending_commands() and (self.random.random() < self.params.ofp_cmd_passthrough_rate):
(cmd, pending_receipt) = switch.get_next_command()
# then check whether we should make the attempt to process the next command fail
if should_fail_flow_mod(cmd):
eventclass = FailFlowMod
else:
eventclass = ProcessFlowMod
switch.process_delayed_command(pending_receipt)
b64_packet = base64_encode(cmd)
self._log_input_event(eventclass(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint,
b64_packet=b64_packet))
def check_switch_crashes(self):
''' Decide whether to crash or restart switches, links and controllers '''
def crash_switches():
crashed_this_round = set()
for software_switch in list(self.simulation.topology.live_switches):
if self.random.random() < self.params.switch_failure_rate:
crashed_this_round.add(software_switch)
self.simulation.topology.crash_switch(software_switch)
self._log_input_event(SwitchFailure(software_switch.dpid))
return crashed_this_round
def restart_switches(crashed_this_round):
# Make sure we don't try to connect to dead controllers
down_controller_ids = None
for software_switch in list(self.simulation.topology.failed_switches):
if software_switch in crashed_this_round:
continue
if self.random.random() < self.params.switch_recovery_rate:
if down_controller_ids is None:
self.simulation.controller_manager.check_controller_status()
down_controller_ids = [ c.cid for c in self.simulation.controller_manager.controllers\
if c.state == ControllerState.STARTING or\
c.state == ControllerState.DEAD ]
connected = self.simulation.topology\
.recover_switch(software_switch,
down_controller_ids=down_controller_ids)
if connected:
self._log_input_event(SwitchRecovery(software_switch.dpid))
crashed_this_round = crash_switches()
try:
restart_switches(crashed_this_round)
except TimeoutError:
log.warn("Unable to connect to controllers")
def check_link_failures(self):
def sever_links():
# TODO(cs): model administratively down links? (OFPPC_PORT_DOWN)
cut_this_round = set()
for link in list(self.simulation.topology.live_links):
if self.random.random() < self.params.link_failure_rate:
cut_this_round.add(link)
self.simulation.topology.sever_link(link)
self._log_input_event(LinkFailure(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
return cut_this_round
def repair_links(cut_this_round):
for link in list(self.simulation.topology.cut_links):
if link in cut_this_round:
continue
if self.random.random() < self.params.link_recovery_rate:
self.simulation.topology.repair_link(link)
self._log_input_event(LinkRecovery(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
cut_this_round = sever_links()
repair_links(cut_this_round)
def fuzz_traffic(self):
if not self.simulation.dataplane_trace:
# randomly generate messages from switches
for host in self.simulation.topology.hosts:
if self.random.random() < self.params.traffic_generation_rate:
if len(host.interfaces) > 0:
msg.event("Injecting a random packet")
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(traffic_type, host)
self._log_input_event(TrafficInjection(dp_event=dp_event))
def check_controllers(self):
def crash_controllers():
crashed_this_round = set()
for controller in self.simulation.controller_manager.live_controllers:
if self.random.random() < self.params.controller_crash_rate:
crashed_this_round.add(controller)
controller.kill()
self._log_input_event(ControllerFailure(controller.cid))
return crashed_this_round
def reboot_controllers(crashed_this_round):
for controller in self.simulation.controller_manager.down_controllers:
if controller in crashed_this_round:
continue
if self.random.random() < self.params.controller_recovery_rate:
controller.restart()
self._log_input_event(ControllerRecovery(controller.cid))
crashed_this_round = crash_controllers()
reboot_controllers(crashed_this_round)
def check_migrations(self):
for access_link in list(self.simulation.topology.access_links):
if self.random.random() < self.params.host_migration_rate:
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
live_edge_switches = list(self.simulation.topology.live_edge_switches)
if len(live_edge_switches) > 0:
new_switch = random.choice(live_edge_switches)
new_switch_dpid = new_switch.dpid
new_port_no = max(new_switch.ports.keys()) + 1
msg.event("Migrating host %s, New switch %s, New port %s" %
(str(access_link.host),str(new_switch_dpid),str(new_port_no)))
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
new_switch_dpid,
new_port_no,
access_link.host.hid))
self._send_initialization_packet(access_link.host, send_to_self=True)
def check_intracontroller_blocks(self):
blocked_this_round = None
# Block at most one controller pair per round.
if (len(self.unblocked_controller_pairs) > 0 and
self.random.random() < self.params.intracontroller_block_rate):
(cid1, cid2) = self.random.choice(self.unblocked_controller_pairs)
msg.event("Unblocking controllers %s, %s" % (cid1, cid2))
blocked_this_round = (cid1, cid2)
self.unblocked_controller_pairs.remove((cid1, cid2))
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.block_controller_pair(cid1, cid2)
else:
(c1, c2) = [ self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2] ]
c1.block_peer(c2)
c2.block_peer(c1)
self._log_input_event(BlockControllerPair(cid1, cid2))
if (len(self.blocked_controller_pairs) > 0 and
self.random.random() < self.params.intracontroller_unblock_rate):
(cid1, cid2) = self.random.choice(self.blocked_controller_pairs)
msg.event("Blocking controllers %s, %s" % (cid1, cid2))
self.blocked_controller_pairs.remove((cid1, cid2))
self.unblocked_controller_pairs.append((cid1, cid2))
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.unblock_controller_pair(cid1, cid2)
else:
(c1, c2) = [ self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2] ]
c1.unblock_peer(c2)
c2.unblock_peer(c1)
self._log_input_event(UnblockControllerPair(cid1, cid2))
if blocked_this_round is not None:
self.blocked_controller_pairs.append(blocked_this_round)
class Fuzzer(ControlFlow):
'''
Injects input events at random intervals, periodically checking
for invariant violations. (Not the proper use of the term `Fuzzer`)
'''
def __init__(self,
simulation_cfg,
fuzzer_params="config.fuzzer_params",
check_interval=None,
traffic_inject_interval=10,
random_seed=None,
delay=0.1,
steps=None,
input_logger=None,
invariant_check=InvariantChecker.check_correspondence,
halt_on_violation=False,
log_invariant_checks=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.check_interval = check_interval
self.invariant_check = invariant_check
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
self.random = random.Random()
else:
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
# Logical time (round #) for the simulation execution
self.logical_time = 0
def _log_input_event(self, event, **kws):
if self._input_logger is not None:
self._input_logger.log_input_event(event, **kws)
def _load_fuzzer_params(self, fuzzer_params_path):
try:
self.params = __import__(fuzzer_params_path, globals(), locals(),
["*"])
except:
raise IOError("Could not find logging config file: %s" %
fuzzer_params_path)
def simulate(self):
"""Precondition: simulation.patch_panel is a buffered patch panel"""
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
assert (isinstance(self.simulation.patch_panel, BufferedPatchPanel))
self.loop()
def loop(self):
if self.steps:
end_time = self.logical_time + self.steps
else:
end_time = sys.maxint
try:
while self.logical_time < end_time:
self.logical_time += 1
self.trigger_events()
msg.event("Round %d completed." % self.logical_time)
halt = self.maybe_check_invariant()
if halt:
break
self.maybe_inject_trace_event()
time.sleep(self.delay)
finally:
if self._input_logger is not None:
self._input_logger.close(self.simulation_cfg)
def maybe_check_invariant(self):
if (self.check_interval is not None
and (self.logical_time % self.check_interval) == 0):
# Time to run correspondence!
# TODO(cs): may need to revert to threaded version if runtime is too
# long
def do_invariant_check():
if self.log_invariant_checks:
self._log_input_event(
CheckInvariants(invariant_check=self.invariant_check,
fail_on_error=self.halt_on_violation))
controllers_with_violations = self.invariant_check(
self.simulation)
if controllers_with_violations != []:
msg.fail(
"The following controllers had correctness violations!: %s"
% str(controllers_with_violations))
self._log_input_event(
InvariantViolation(controllers_with_violations))
if self.halt_on_violation:
return True
else:
msg.interactive("No correctness violations!")
return do_invariant_check()
def maybe_inject_trace_event(self):
if (self.simulation.dataplane_trace
and (self.logical_time % self.traffic_inject_interval) == 0):
dp_event = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def trigger_events(self):
self.check_dataplane()
self.check_tcp_connections()
self.check_message_receipts()
self.check_switch_crashes()
self.check_link_failures()
self.fuzz_traffic()
self.check_controllers()
self.check_migrations()
def check_dataplane(self):
''' Decide whether to delay, drop, or deliver packets '''
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
if self.random.random() < self.params.dataplane_delay_rate:
self.simulation.patch_panel.delay_dp_event(dp_event)
elif self.random.random() < self.params.dataplane_drop_rate:
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
elif not self.simulation.topology.ok_to_send(dp_event):
# Switches have very small buffers! drop it on the floor if the link
# is down
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
else:
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
def check_tcp_connections(self):
''' Decide whether to block or unblock control channels '''
for (switch, connection
) in self.simulation.topology.unblocked_controller_connections:
if self.random.random() < self.params.controlplane_block_rate:
self.simulation.topology.block_connection(connection)
self._log_input_event(
ControlChannelBlock(switch.dpid,
connection.get_controller_id()))
for (switch, connection
) in self.simulation.topology.blocked_controller_connections:
if self.random.random() < self.params.controlplane_unblock_rate:
self.simulation.topology.unblock_connection(connection)
self._log_input_event(
ControlChannelUnblock(
switch.dpid,
controller_id=connection.get_controller_id()))
def check_message_receipts(self):
for pending_receipt in self.simulation.god_scheduler.pending_receives(
):
# TODO(cs): this is a really dumb way to fuzz packet receipt scheduling
if self.random.random() < self.params.ofp_message_receipt_rate:
self.simulation.god_scheduler.schedule(pending_receipt)
self._log_input_event(
ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint))
def check_switch_crashes(self):
''' Decide whether to crash or restart switches, links and controllers '''
def crash_switches():
crashed_this_round = set()
for software_switch in list(
self.simulation.topology.live_switches):
if self.random.random() < self.params.switch_failure_rate:
crashed_this_round.add(software_switch)
self.simulation.topology.crash_switch(software_switch)
self._log_input_event(SwitchFailure(software_switch.dpid))
return crashed_this_round
def restart_switches(crashed_this_round):
# Make sure we don't try to connect to dead controllers
down_controller_ids = map(
lambda c: c.uuid,
self.simulation.controller_manager.down_controllers)
for software_switch in list(
self.simulation.topology.failed_switches):
if software_switch in crashed_this_round:
continue
if self.random.random() < self.params.switch_recovery_rate:
connected = self.simulation.topology\
.recover_switch(software_switch,
down_controller_ids=down_controller_ids)
if connected:
self._log_input_event(
SwitchRecovery(software_switch.dpid))
crashed_this_round = crash_switches()
try:
restart_switches(crashed_this_round)
except TimeoutError:
log.warn("Unable to connect to controllers")
def check_link_failures(self):
def sever_links():
# TODO(cs): model administratively down links? (OFPPC_PORT_DOWN)
cut_this_round = set()
for link in list(self.simulation.topology.live_links):
if self.random.random() < self.params.link_failure_rate:
cut_this_round.add(link)
self.simulation.topology.sever_link(link)
self._log_input_event(
LinkFailure(link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
return cut_this_round
def repair_links(cut_this_round):
for link in list(self.simulation.topology.cut_links):
if link in cut_this_round:
continue
if self.random.random() < self.params.link_recovery_rate:
self.simulation.topology.repair_link(link)
self._log_input_event(
LinkRecovery(link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
cut_this_round = sever_links()
repair_links(cut_this_round)
def fuzz_traffic(self):
if not self.simulation.dataplane_trace:
# randomly generate messages from switches
for host in self.simulation.topology.hosts:
if self.random.random() < self.params.traffic_generation_rate:
if len(host.interfaces) > 0:
msg.event("injecting a random packet")
traffic_type = "icmp_ping"
# Generates a packet, and feeds it to the software_switch
dp_event = self.traffic_generator.generate(
traffic_type, host)
self._log_input_event(TrafficInjection(),
dp_event=dp_event)
def check_controllers(self):
def crash_controllers():
crashed_this_round = set()
for controller in self.simulation.controller_manager.live_controllers:
if self.random.random() < self.params.controller_crash_rate:
crashed_this_round.add(controller)
controller.kill()
self._log_input_event(ControllerFailure(controller.uuid))
return crashed_this_round
def reboot_controllers(crashed_this_round):
for controller in self.simulation.controller_manager.down_controllers:
if controller in crashed_this_round:
continue
if self.random.random() < self.params.controller_recovery_rate:
controller.start()
self._log_input_event(ControllerRecovery(controller.uuid))
crashed_this_round = crash_controllers()
reboot_controllers(crashed_this_round)
def check_migrations(self):
for access_link in list(self.simulation.topology.access_links):
if self.random.random() < self.params.host_migration_rate:
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
live_edge_switches = list(
self.simulation.topology.live_edge_switches)
if len(live_edge_switches) > 0:
new_switch = random.choice(live_edge_switches)
new_switch_dpid = new_switch.dpid
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(
old_ingress_dpid, old_ingress_port_no, new_switch_dpid,
new_port_no)
self._log_input_event(
HostMigration(old_ingress_dpid, old_ingress_port_no,
new_switch_dpid, new_port_no))
class Interactive(ControlFlow):
'''
Presents an interactive prompt for injecting events and
checking for invariants at the users' discretion
'''
def __init__(self, simulation_cfg, input_logger=None,
pass_through_of_messages=True):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.logical_time = 0
self._input_logger = input_logger
self.traffic_generator = TrafficGenerator(random.Random())
# TODO(cs): we don't actually have a way to support pass_through=False, in
# a reasonable way -- the user has no (easy) way to examine and route
# OpenFlowBuffer's pending_receives and pending_sends.
self.pass_through_of_messages = pass_through_of_messages
# TODO(cs): future feature: allow the user to interactively choose the order
# events occur for each round, whether to delay, drop packets, fail nodes,
# etc.
# self.failure_lvl = [
# NOTHING, # Everything is handled by the random number generator
# CRASH, # The user only controls node crashes and restarts
# DROP, # The user also controls message dropping
# DELAY, # The user also controls message delays
# EVERYTHING # The user controls everything, including message ordering
# ]
def _log_input_event(self, event, **kws):
# TODO(cs): redundant with Fuzzer._log_input_event
if self._input_logger is not None:
event.round = self.logical_time
self._input_logger.log_input_event(event, **kws)
def init_results(self, results_dir):
if self._input_logger is not None:
self._input_logger.open(results_dir)
def default_connect_to_controllers(self, simulation):
simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
def simulate(self, simulation=None, boot_controllers=default_boot_controllers,
connect_to_controllers=None,
bound_objects=()):
if simulation is None:
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback,
boot_controllers=boot_controllers)
if connect_to_controllers is None:
self.default_connect_to_controllers(self.simulation)
else:
connect_to_controllers(self.simulation)
else:
self.simulation = simulation
if self.pass_through_of_messages:
self.simulation.openflow_buffer.set_pass_through(input_logger=self._input_logger)
self._forwarded_this_step = 0
self.traffic_generator.set_topology(self.simulation.topology)
try:
c = STSConsole(default_command=self.default_command)
c.cmd_group("Simulation State")
c.cmd(self.next_step, "next", alias="n", help_msg="Proceed to next simulation step")
c.cmd(self.quit, "quit", alias="q", help_msg="Quit the simulation")
c.cmd_group("Invariants")
c.cmd(self.invariant_check, "check_invariants", alias="inv", help_msg="Run an invariant check")\
.arg("kind", values=['omega', 'connectivity', 'loops', 'liveness', "python_connectivity", "blackholes"])
c.cmd_group("Dataplane")
c.cmd(self.dataplane_trace_feed, "dp_inject", alias="dpi", help_msg="Inject the next dataplane event from the trace")
c.cmd(self.dataplane_ping, "dp_ping", alias="dpp", help_msg="Generate and inject a new ping packet")
c.cmd(self.dataplane_forward, "dp_forward", alias="dpf", help_msg="Forward a pending dataplane event")
c.cmd(self.dataplane_drop, "dp_drop", alias="dpd", help_msg="Drop a pending dataplane event")
c.cmd(self.dataplane_delay, "dp_delay", alias="dpe", help_msg="Delay a pending dataplane event")
c.cmd_group("Controlling Entitities")
c.cmd(self.list_controllers, "list_controllers", alias="lsc", help_msg="List controllers")
c.cmd(self.kill_controller, "kill_controller", alias="kc", help_msg="Kill a controller")\
.arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.live_controllers))
c.cmd(self.start_controller, "start_controller", alias="sc", help_msg="Restart a controller")\
.arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.down_controllers))
c.cmd(self.list_switches, "list_switches", alias="lss", help_msg="List switches")
c.cmd(self.kill_switch, "kill_switch", alias="ks", help_msg="Kill a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.start_switch, "start_switch", alias="ss", help_msg="Restart a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.cut_link, "cut_link", alias="cl", help_msg="Cut a link")\
.arg("dpid1", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))\
.arg("dpid2", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.repair_link, "repair_link", alias="rl", help_msg="Repair a link")\
.arg("dpid1", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))\
.arg("dpid2", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.show_flow_table, "show_flows", alias="sf", help_msg="Show flowtable of a switch")\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.list_hosts, "list_hosts", alias="lhs", help_msg="List hosts")
c.cmd(self.migrate_host, "migrate_host", alias="mh", help_msg="Migrate a host to switch dpid")\
.arg("hid", values=lambda: map(lambda h: h.hid, self.simulation.topology.hosts))\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.block_controller_traffic, "block_controllers", alias="bc", help_msg="Drop traffic between controllers")\
.arg("cid1", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))\
.arg("cid2", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))
c.cmd(self.unblock_controller_traffic, "unblock_controllers", alias="ubc", help_msg="Stop dropping traffic between controllers")\
.arg("cid1", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))\
.arg("cid2", values=lambda: map(lambda c: c.cid, self.simulation.controller_manager.live_controllers))
c.cmd_group("Python Objects")
c.register_obj(self.simulation, "simulation", alias="sim", help_msg="access the simulation object")
c.register_obj(self.simulation.topology, "topology", alias="topo", help_msg="access the topology object")
for (name, obj) in bound_objects:
c.register_obj(obj, name, help_msg="access the %s object" % name)
c.run()
finally:
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return self.simulation
def default_command(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) == 1 and self._forwarded_this_step == 0 and self.simulation.topology.ok_to_send(queued[0]):
return "dpf"
else:
return "next"
def quit(self):
print "End console with CTRL-D"
def next_step(self):
time.sleep(0.05)
self.logical_time += 1
self._forwarded_this_step = 0
print "-------------------"
print "Advanced to step %d" % self.logical_time
self.show_queued_events()
def show_queued_events(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) > 0:
log.debug("Queued Dataplane events:")
for (i, e) in enumerate(queued):
log.debug("%d: %s on %s:%s" % (i, e, e.node, e.port))
def list_controllers(self):
cm = self.simulation.controller_manager
live = cm.live_controllers
print "Controllers:"
for c in cm.controllers:
print "%s %s %s %s" % (c.label, c.cid, repr(c), "[ALIVE]" if c in live else "[DEAD]")
def kill_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.kill_controller(c)
self._log_input_event(ControllerFailure(c.cid))
else:
print "Controller with label %s not found" %label
def start_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.reboot_controller(c)
self._log_input_event(ControllerRecovery(c.cid))
else:
print "Controller with label %s not found" %label
def list_switches(self):
topology = self.simulation.topology
live = topology.live_switches
print "Switches:"
for s in topology.switches:
print "%d %s %s" % (s.dpid, repr(s), "[ALIVE]" if s in live else "[DEAD]")
def list_hosts(self):
topology = self.simulation.topology
print "Hosts:"
for h in topology.hosts:
print "%d %s" % (h.hid, str(h),)
def kill_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
topology.crash_switch(switch)
self._log_input_event(SwitchFailure(switch.dpid))
def start_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
down_controller_ids = map(lambda c: c.cid, self.simulation.controller_manager.down_controllers)
topology.recover_switch(switch, down_controller_ids=down_controller_ids)
self._log_input_event(SwitchRecovery(switch.dpid))
def cut_link(self, dpid1, dpid2):
topology = self.simulation.topology
link = topology.get_link(dpid1, dpid2)
topology.sever_link(link)
self._log_input_event(LinkFailure(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
def repair_link(self, dpid1, dpid2):
topology = self.simulation.topology
link = topology.get_link(dpid1, dpid2)
topology.repair_link(link)
self._log_input_event(LinkRecovery(
link.start_software_switch.dpid,
link.start_port.port_no,
link.end_software_switch.dpid,
link.end_port.port_no))
def migrate_host(self, hid, dpid):
topology = self.simulation.topology
host = topology.get_host(hid)
# TODO(cs): make this lookup more efficient
access_link = find(lambda a: a.host == host, topology.access_links)
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
new_switch = topology.get_switch(dpid)
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no,
access_link.host.hid))
self._send_initialization_packet(access_link.host, send_to_self=True)
# TODO(cs): ripped directly from fuzzer. Redundant!
def _send_initialization_packet(self, host, send_to_self=False):
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate_and_inject(traffic_type, host, send_to_self=send_to_self)
self._log_input_event(TrafficInjection(dp_event=dp_event, host_id=host.hid))
def show_flow_table(self, dpid=None):
topology = self.simulation.topology
if not dpid:
for switch in topology.switches:
msg.interactive("Switch %s" % switch.dpid)
switch.show_flow_table()
return
switch = topology.get_switch(dpid)
switch.show_flow_table()
def invariant_check(self, kind):
if kind == "omega" or kind == "o":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_correspondence"))
result = InvariantChecker.check_correspondence(self.simulation)
message = "Controllers with miscorrepondence: "
elif kind == "connectivity" or kind == "c":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_connectivity"))
result = InvariantChecker.check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "python_connectivity" or kind == "pc":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.python_check_connectivity"))
result = InvariantChecker.python_check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "loops" or kind == "lo":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_loops"))
result = InvariantChecker.python_check_loops(self.simulation)
message = "Loops: "
elif kind == "liveness" or kind == "li":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_liveness"))
result = InvariantChecker.check_liveness(self.simulation)
message = "Crashed controllers: "
elif kind == "blackholes" or kind == "bl":
self._log_input_event(CheckInvariants(round=self.logical_time, invariant_check_name="InvariantChecker.check_blackholes"))
result = InvariantChecker.python_check_blackholes(self.simulation)
message = "Blackholes: "
else:
log.warn("Unknown invariant kind...")
return
self.simulation.violation_tracker.track(result, self.logical_time)
persistent_violations = self.simulation.violation_tracker.persistent_violations
transient_violations = list(set(result) - set(persistent_violations))
msg.interactive(message + str(result))
if transient_violations != []:
self._log_input_event(InvariantViolation(transient_violations))
if persistent_violations != []:
msg.fail("Persistent violations detected!: %s"
% str(persistent_violations))
self._log_input_event(InvariantViolation(persistent_violations, persistent=True))
def dataplane_trace_feed(self):
if self.simulation.dataplane_trace:
(dp_event, host) = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(dp_event=dp_event, host=host.hid))
else:
print "No dataplane trace to inject from."
def dataplane_ping(self, from_hid=None, to_hid=None):
if (from_hid is not None) and \
(from_hid not in self.simulation.topology.hid2host.keys()):
print "Unknown host %s" % from_hid
return
if (to_hid is not None) and \
(to_hid not in self.simulation.topology.hid2host.keys()):
print "Unknown host %s" % to_hid
return
dp_event = self.traffic_generator.generate_and_inject("icmp_ping", from_hid, to_hid,
payload_content=raw_input("Enter payload content:\n"))
self._log_input_event(TrafficInjection(dp_event=dp_event))
def _select_dataplane_event(self, sel=None):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) == 0:
print "No dataplane events queued."
return None
if not sel:
return queued[0]
if sel >= len(queued):
print "Given index out of bounds. Try a value smaller than %d" % len(queued)
return queued[sel]
def dataplane_forward(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
if self.simulation.topology.ok_to_send(dp_event):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
self._forwarded_this_step += 1
else:
print "Not ready to send event %s" % event
def dataplane_drop(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint,
host_id=dp_event.get_host_id(),
dpid=dp_event.get_switch_id()))
def dataplane_delay(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.delay_dp_event(dp_event)
def block_controller_traffic(self, cid1, cid2):
''' Drop all messages sent from controller 1 to controller 2 until
unblock_controller_pair is called. '''
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.block_controller_pair(cid1, cid2)
else:
(c1, c2) = [ self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2] ]
c1.block_peer(c2)
self._log_input_event(BlockControllerPair(cid1, cid2))
def unblock_controller_traffic(self, cid1, cid2):
''' Stop dropping messages sent from controller 1 to controller 2 '''
if self.simulation.controller_patch_panel is not None:
self.simulation.controller_patch_panel.unblock_controller_pair(cid1, cid2)
else:
(c1, c2) = [ self.simulation.controller_manager.get_controller(cid)
for cid in [cid1, cid2] ]
c1.unblock_peer(c2)
self._log_input_event(UnblockControllerPair(cid1, cid2))
def __init__(
self,
simulation_cfg,
fuzzer_params="config.fuzzer_params",
check_interval=None,
traffic_inject_interval=10,
random_seed=None,
delay=0.1,
steps=None,
input_logger=None,
invariant_check_name="InvariantChecker.check_correspondence",
halt_on_violation=False,
log_invariant_checks=True,
delay_startup=True,
print_buffers=True,
record_deterministic_values=False,
mock_link_discovery=False,
initialization_rounds=0,
):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(
input_logger, record_deterministic_values=record_deterministic_values
)
self.check_interval = check_interval
if invariant_check_name not in name_to_invariant_check:
raise ValueError(
"""Unknown invariant check %s.\n"""
"""Invariant check name must be defined in config.invariant_checks""",
invariant_check_name,
)
self.invariant_check_name = invariant_check_name
self.invariant_check = name_to_invariant_check[invariant_check_name]
self.log_invariant_checks = log_invariant_checks
self.traffic_inject_interval = traffic_inject_interval
# Make execution deterministic to allow the user to easily replay
if random_seed is None:
random_seed = random.randint(0, sys.maxint)
self.random_seed = random_seed
self.random = random.Random(random_seed)
self.traffic_generator = TrafficGenerator(self.random)
self.delay = delay
self.steps = steps
self.params = object()
self._load_fuzzer_params(fuzzer_params)
self._input_logger = input_logger
self.halt_on_violation = halt_on_violation
self.delay_startup = delay_startup
self.print_buffers = print_buffers
self.mock_link_discovery = mock_link_discovery
# How many rounds to let the controller initialize:
# send one round of packets directed at the source host itself (to facilitate
# learning), then send all-to-all packets until all pairs have been
# pinged. Tell MCSFinder not to prune initial inputs during this period.
self.initialization_rounds = initialization_rounds
# If initialization_rounds isn't 0, also make sure to send all-to-all
# pings before starting any events
self._pending_all_to_all = initialization_rounds != 0
# Our current place in the all-to-all cycle. Stop when == len(hosts)
self._all_to_all_iterations = 0
# How often (in terms of logical rounds) to inject all-to-all packets
self._all_to_all_interval = 5
# Logical time (round #) for the simulation execution
self.logical_time = 0
def __init__(self, simulation_cfg, input_logger=None): ControlFlow.__init__(self, simulation_cfg) self.sync_callback = RecordingSyncCallback(input_logger) self.logical_time = 0 self._input_logger = input_logger self.traffic_generator = TrafficGenerator(random.Random())
class Interactive(ControlFlow):
'''
Presents an interactive prompt for injecting events and
checking for invariants at the users' discretion
'''
def __init__(self, simulation_cfg, input_logger=None):
ControlFlow.__init__(self, simulation_cfg)
self.sync_callback = RecordingSyncCallback(input_logger)
self.logical_time = 0
self._input_logger = input_logger
self.traffic_generator = TrafficGenerator(random.Random())
# TODO(cs): future feature: allow the user to interactively choose the order
# events occur for each round, whether to delay, drop packets, fail nodes,
# etc.
# self.failure_lvl = [
# NOTHING, # Everything is handled by the random number generator
# CRASH, # The user only controls node crashes and restarts
# DROP, # The user also controls message dropping
# DELAY, # The user also controls message delays
# EVERYTHING # The user controls everything, including message ordering
# ]
def _log_input_event(self, event, **kws):
# TODO(cs): redundant with Fuzzer._log_input_event
if self._input_logger is not None:
event.round = self.logical_time
self._input_logger.log_input_event(event, **kws)
def init_results(self, results_dir):
self._input_logger.open(results_dir)
def simulate(self, simulation=None, bound_objects=()):
if simulation is None:
self.simulation = self.simulation_cfg.bootstrap(self.sync_callback)
# Always connect to controllers explicitly
self.simulation.connect_to_controllers()
self._log_input_event(ConnectToControllers())
else:
self.simulation = simulation
self._forwarded_this_step = 0
try:
c = STSConsole(default_command=self.default_command)
c.cmd_group("Simulation state")
c.cmd(self.next_step, "next", alias="n", help_msg="Proceed to next simulation step")
c.cmd(self.quit, "quit", alias="q", help_msg="Quit the simulation")
c.cmd_group("Invariants")
c.cmd(self.invariant_check, "check_invariants", alias="inv", help_msg="Run an invariant check").arg("kind", values=['omega', 'connectivity', 'loops', 'liveness'])
c.cmd_group("Dataplane")
c.cmd(self.dataplane_trace_feed, "dp_inject", alias="dpi", help_msg="Inject the next dataplane event from the trace")
c.cmd(self.dataplane_forward, "dp_forward", alias="dpf", help_msg="Forward a pending dataplane event")
c.cmd(self.dataplane_drop, "dp_drop", alias="dpd", help_msg="Drop a pending dataplane event")
c.cmd(self.dataplane_delay, "dp_delay", alias="dpe", help_msg="Delay a pending dataplane event")
c.cmd_group("Controlling entitities")
c.cmd(self.list_controllers, "list_controllers", alias="lsc", help_msg="List controllers")
c.cmd(self.kill_controller, "kill_controller", alias="kc", help_msg="Kill a controller").arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.live_controllers))
c.cmd(self.start_controller, "start_controller", alias="sc", help_msg="Restart a controller").arg("label", values=lambda: map(lambda c: c.label, self.simulation.controller_manager.down_controllers))
c.cmd(self.list_switches, "list_switches", alias="lss", help_msg="List switches")
c.cmd(self.kill_switch, "kill_switch", alias="ks", help_msg="Kill a switch").arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.start_switch, "start_switch", alias="ss", help_msg="Restart a switch").arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.show_flow_table, "show_flows", alias="sf", help_msg="Show flowtable of a switch").arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd(self.list_hosts, "list_hosts", alias="lhs", help_msg="List hosts")
c.cmd(self.migrate_host, "migrate_host", alias="mh", help_msg="Migrate a host to switch dpid")\
.arg("hid", values=lambda: map(lambda h: h.hid, self.simulation.topology.hosts))\
.arg("dpid", values=lambda: map(lambda s: s.dpid, self.simulation.topology.switches))
c.cmd_group("Python objects")
c.register_obj(self.simulation, "simulation", help_msg="access the simulation object")
c.register_obj(self.simulation.topology, "topology", alias="topo", help_msg="access the topology object")
for (name, obj) in bound_objects:
c.register_obj(obj, name, help_msg="access the %s object" % name)
c.run()
finally:
if self._input_logger is not None:
self._input_logger.close(self, self.simulation_cfg)
return self.simulation
def default_command(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) == 1 and self._forwarded_this_step == 0 and self.simulation.topology.ok_to_send(queued[0]):
return "dpf"
else:
return "next"
def quit(self):
print "End console with CTRL-D"
def next_step(self):
self.check_pending_messages()
time.sleep(0.05)
self.logical_time += 1
self._forwarded_this_step = 0
print "-------------------"
print "Advanced to step %d" % self.logical_time
self.show_queued_events()
def show_queued_events(self):
queued = self.simulation.patch_panel.queued_dataplane_events
if len(queued) > 0:
print "Queued Dataplane events:"
for (i, e) in enumerate(queued):
print "%d: %s on %s:%s" % (i, e, e.node, e.port)
def list_controllers(self):
cm = self.simulation.controller_manager
live = cm.live_controllers
print "Controllers:"
for c in cm.controllers:
print "%s %s %s %s" % (c.label, c.cid, repr(c), "[ALIVE]" if c in live else "[DEAD]")
def kill_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.kill_controller(c)
self._log_input_event(ControllerFailure(c.cid))
else:
print "Controller with label %s not found" %label
def start_controller(self, label):
cm = self.simulation.controller_manager
c = cm.get_controller_by_label(label)
if c:
print "Killing controller: %s %s" % (label, repr(c))
cm.reboot_controller(c)
self._log_input_event(ControllerRecovery(c.cid))
else:
print "Controller with label %s not found" %label
def list_switches(self):
topology = self.simulation.topology
live = topology.live_switches
print "Switches:"
for s in topology.switches:
print "%d %s %s" % (s.dpid, repr(s), "[ALIVE]" if s in live else "[DEAD]")
def list_hosts(self):
topology = self.simulation.topology
print "Hosts:"
for h in topology.hosts:
print "%d %s" % (h.hid, str(h),)
def kill_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
topology.crash_switch(switch)
self._log_input_event(SwitchFailure(switch.dpid))
def start_switch(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
down_controllers_ids = map(lambda c: c.cid, self.simulation.controller_manager.down_controllers)
topology.recover_switch(switch, down_controller_ids=down_controller_ids)
self._log_input_event(SwitchRecovery(switch.dpid))
def migrate_host(self, hid, dpid):
topology = self.simulation.topology
host = topology.get_host(hid)
# TODO(cs): make this lookup more efficient
access_link = find(lambda a: a.host == host, topology.access_links)
old_ingress_dpid = access_link.switch.dpid
old_ingress_port_no = access_link.switch_port.port_no
new_switch = topology.get_switch(dpid)
new_port_no = max(new_switch.ports.keys()) + 1
self.simulation.topology.migrate_host(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no)
self._log_input_event(HostMigration(old_ingress_dpid,
old_ingress_port_no,
dpid,
new_port_no,
access_link.host.name))
self._send_initialization_packet(access_link.host, self_pkt=True)
# TODO(cs): ripped directly from fuzzer. Redundant!
def _send_initialization_packet(self, host, self_pkt=False):
traffic_type = "icmp_ping"
dp_event = self.traffic_generator.generate(traffic_type, host, self_pkt=self_pkt)
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def show_flow_table(self, dpid):
topology = self.simulation.topology
switch = topology.get_switch(dpid)
dl_types = { 0x0800: "IP",
0x0806: "ARP",
0x8100: "VLAN",
0x88cc: "LLDP",
0x888e: "PAE"
}
nw_protos = { 1 : "ICMP", 6 : "TCP", 17 : "UDP" }
ports = { v: k.replace("OFPP_","") for (k,v) in of.ofp_port_rev_map.iteritems() }
def dl_type(e):
d = e.match.dl_type
if d is None:
return d
else:
return dl_types[d] if d in dl_types else "%x" %d
def nw_proto(e):
p = e.match.nw_proto
return nw_protos[p] if p in nw_protos else p
def action(a):
if isinstance(a, ofp_action_output):
return ports[a.port] if a.port in ports else "output(%d)" % a.port
else:
return str(a)
def actions(e):
if len(e.actions) == 0:
return "(drop)"
else:
return ", ".join(action(a) for a in e.actions)
t = Tabular( ("Prio", lambda e: e.priority),
("in_port", lambda e: e.match.in_port),
("dl_type", dl_type),
("dl_src", lambda e: e.match.dl_src),
("dl_dst", lambda e: e.match.dl_dst),
("nw_proto", nw_proto),
("nw_src", lambda e: e.match.nw_src),
("nw_dst", lambda e: e.match.nw_dst),
("tp_src", lambda e: e.match.tp_src),
("tp_dst", lambda e: e.match.tp_dst),
("actions", actions),
)
t.show(switch.table.entries)
def invariant_check(self, kind):
if kind == "omega" or kind == "o":
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_correspondence"))
result = InvariantChecker.check_correspondence(self.simulation)
message = "Controllers with miscorrepondence: "
elif kind == "connectivity" or kind == "c":
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_connectivity"))
result = InvariantChecker.check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "python_connectivity" or kind == "pc":
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.python_check_connectivity"))
result = InvariantChecker.python_check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif kind == "loops" or kind == "lo":
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_loops"))
result = InvariantChecker.check_loops(self.simulation)
message = "Loops: "
elif kind == "liveness" or kind == "li":
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_liveness"))
result = InvariantChecker.check_loops(self.simulation)
message = "Crashed controllers: "
else:
log.warn("Unknown invariant kind...")
def dataplane_trace_feed(self):
if self.simulation.dataplane_trace:
dp_event = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(), dp_event=dp_event)
def _select_dataplane_event(self, sel=None):
queued = self.simulation.patch_panel.queued_dataplane_events
if not sel:
if len(queued) == 1:
return queued[0]
else:
print "Multiple events queued -- please select one"
return None
else:
return queued[sel]
def dataplane_forward(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
if self.simulation.topology.ok_to_send(dp_event):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
self._forwarded_this_step += 1
else:
print "Not ready to send event %s" % event
def dataplane_drop(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
def dataplane_delay(self, event=None):
dp_event = self._select_dataplane_event(event)
if not dp_event:
return
self.simulation.patch_panel.delay_dp_event(dp_event)
# TODO(cs): remove this code -- no longer used
def loop(self):
try:
while True:
# TODO(cs): print out the state of the network at each timestep? Take a
# verbose flag..
time.sleep(0.05)
self.logical_time += 1
self.invariant_check_prompt()
self.dataplane_trace_prompt()
self.check_dataplane()
self.check_pending_messages()
answer = msg.raw_input('Continue to next round? [Yn]').strip()
if answer != '' and answer.lower() != 'y':
break
finally:
if self._input_logger is not None:
self._input_logger.close(self.simulation_cfg)
def invariant_check_prompt(self):
answer = msg.raw_input('Check Invariants? [Ny]')
if answer != '' and answer.lower() != 'n':
msg.interactive("Which one?")
msg.interactive(" 'o' - omega")
msg.interactive(" 'c' - connectivity")
msg.interactive(" 'lo' - loops")
msg.interactive(" 'li' - controller liveness")
answer = msg.raw_input("> ")
result = None
message = ""
if answer.lower() == 'o':
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_correspondence"))
result = InvariantChecker.check_correspondence(self.simulation)
message = "Controllers with miscorrepondence: "
elif answer.lower() == 'c':
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_connectivity"))
result = self.invariant_checker.check_connectivity(self.simulation)
message = "Disconnected host pairs: "
elif answer.lower() == 'lo':
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_loops"))
result = self.invariant_checker.check_loops(self.simulation)
message = "Loops: "
elif answer.lower() == 'li':
self._log_input_event(CheckInvariants(invariant_check_name="InvariantChecker.check_liveness"))
result = self.invariant_checker.check_loops(self.simulation)
message = "Crashed controllers: "
else:
log.warn("Unknown input...")
if result is None:
return
else:
msg.interactive("%s: %s" % (message, str(result)))
def dataplane_trace_prompt(self):
if self.simulation.dataplane_trace:
while True:
answer = msg.raw_input('Feed in next dataplane event? [Ny]')
if answer != '' and answer.lower() != 'n':
dp_event = self.simulation.dataplane_trace.inject_trace_event()
self._log_input_event(TrafficInjection(), dp_event=dp_event)
else:
break
def check_dataplane(self):
''' Decide whether to delay, drop, or deliver packets '''
if type(self.simulation.patch_panel) == BufferedPatchPanel:
for dp_event in self.simulation.patch_panel.queued_dataplane_events:
done = False
while not done:
done = True
answer = msg.raw_input('Allow [a], or Drop [d] dataplane packet %s? [Ad]' %
dp_event)
if ((answer == '' or answer.lower() == 'a') and
self.simulation.topology.ok_to_send(dp_event)):
self.simulation.patch_panel.permit_dp_event(dp_event)
self._log_input_event(DataplanePermit(dp_event.fingerprint))
elif answer.lower() == 'd':
self.simulation.patch_panel.drop_dp_event(dp_event)
self._log_input_event(DataplaneDrop(dp_event.fingerprint))
else:
log.warn("Unknown input...")
done = False
def check_pending_messages(self):
for pending_receipt in self.simulation.god_scheduler.pending_receives():
# For now, just schedule FIFO.
# TODO(cs): make this interactive
self.simulation.god_scheduler.schedule(pending_receipt)
self._log_input_event(ControlMessageReceive(pending_receipt.dpid,
pending_receipt.controller_id,
pending_receipt.fingerprint))
for pending_send in self.simulation.god_scheduler.pending_sends():
self.simulation.god_scheduler.schedule(pending_send)
self._log_input_event(ControlMessageSend(pending_send.dpid,
pending_send.controller_id,
pending_send.fingerprint))
