def list_event_stream(draw):
stream = ListEventStream(draw(some_events(min_size=1)))
# Get some timestamps stats
events = stream.events
min_ts = min(ev.timestamp for ev in events)
max_ts = max(ev.timestamp for ev in events)
duration = max_ts - min_ts
# Add some random out of order events
start_ts = min_ts - duration
max_size = MAX_EVENTS_SIZE // 5
max_interval = 3 * duration // max_size # So that events are generated in range [min_ts-duration, max_ts+duration]
more_events = draw(
some_events(min_size=1,
max_size=max_size,
min_interval=max_interval // 5,
max_interval=max_interval,
start_ts=start_ts))
shuffle(more_events)
stream.extend(more_events)
# Generate events
input = stream.events
events = draw(sim_event_stream(stream, max_size=MAX_EVENTS_SIZE))
return input, events
def composite_two_normal_event_streams(draw):
input_a = draw(some_events(max_size=MAX_EVENTS_SIZE // 2))
input_b = draw(some_events(max_size=MAX_EVENTS_SIZE // 2))
stream_a = ListEventStream(input_a)
stream_b = ListEventStream(input_b)
stream = CompositeEventStream(stream_a, stream_b)
events = draw(sim_event_stream(stream, max_size=MAX_EVENTS_SIZE))
return input_a, input_b, events
def model_event_stream(draw, model_factory):
input_a = draw(some_events(max_size=MAX_EVENTS_SIZE // 2))
input_b = draw(some_events(max_size=MAX_EVENTS_SIZE // 2))
stream_a = ListEventStream(input_a)
stream_b = ListEventStream(input_b)
stream = CompositeEventStream(stream_a, stream_b)
model = model_factory()
model_with_events = ModelWithExternalEvents(model, stream)
events = process_model(draw, model_with_events, max_size=MAX_EVENTS_SIZE)
return input_a, input_b, events, model
def __init__(self, name: str, node_names: List[str], connections: PoolConnections):
self._name = name
self._node_names = node_names
self._quorum = Quorums(len(node_names))
self._ts = 0
self._corrupted_name = None
self._timer = TimerModel(name)
self._connections = connections
self._view_changer = create_view_changer(self)
self._internal_outbox = ListEventStream()
self.outbox = CompositeEventStream(self._internal_outbox, self._timer.outbox())
self._check_performance_timer = RepeatingTimer(self._timer, 30, self._check_performance)
def composite_normal_and_random_event_stream(draw):
input = draw(some_events(max_size=MAX_EVENTS_SIZE))
stream_a = RandomEventStream(draw, some_event())
stream_b = ListEventStream(input)
stream = CompositeEventStream(stream_a, stream_b)
events = draw(sim_event_stream(stream, max_size=MAX_EVENTS_SIZE))
return input, events
def pool_model_events(draw, node_count):
greek_names = [
'Alpha', 'Beta', 'Gamma', 'Delta', 'Epsilon', 'Zeta', 'Eta', 'Theta'
]
node_names = greek_names[:node_count]
events = []
events.extend(
draw(
sim_events(st.one_of(
outage_event(max_count=1,
node_names=node_names,
min_duration=3),
outage_event(max_count=node_count // 2,
node_names=node_names,
min_duration=3),
restart_event(node_names=node_names)),
min_interval=10)))
events.extend(
draw(
sim_events(corrupt_event(node_names=node_names),
max_size=1,
min_interval=50,
max_interval=1000)))
pool = PoolModel(node_names)
model = ModelWithExternalEvents(pool, ListEventStream(events))
return pool, process_model(draw, model, max_size=1000)
class RandomErrorModel(SimModel):
def __init__(self):
self.processed_events = []
self._events = st.one_of(st.just(ErrorEvent(reason="random")),
some_event())
self._outbox = ListEventStream()
self._error_status = None
def process(self, draw, event: SimEvent):
self.processed_events.append(event)
if isinstance(event.payload, ErrorEvent):
self._error_status = 'Has error'
ts = event.timestamp
delays = draw(st.lists(elements=st.integers(min_value=1, max_value=1000)))
self._outbox.extend(SimEvent(timestamp=ts + delay, payload=draw(self._events)) for delay in delays)
def outbox(self):
return self._outbox
def error_status(self) -> Optional[str]:
pass
class TimerModel(SimModel, TimerService):
TimerEvent = NamedTuple('TimerEvent', [('name', str),
('callback', Callable)])
def __init__(self, name):
self._name = name
self._ts = 0
self._outbox = ListEventStream()
def process(self, draw, event: SimEvent):
self._ts = event.timestamp
if isinstance(event.payload, self.TimerEvent):
if event.payload.name == self._name:
event.payload.callback()
def outbox(self):
return self._outbox
def error_status(self) -> Optional[str]:
pass
def get_current_time(self) -> float:
return self._ts
def schedule(self, delay: float, callback: Callable):
# TODO: Some classes (like ViewChanger) sometimes try to schedule None events o_O
if callback is None:
return
self._outbox.add(
SimEvent(timestamp=self._ts + delay,
payload=self.create_timer_event(callback)))
def cancel(self, callback: Callable):
self._outbox.remove_all(
lambda ev: isinstance(ev.payload, self.TimerEvent) and ev.payload
== self.create_timer_event(callback))
def create_timer_event(self, callback: Callable):
return self.TimerEvent(name=self._name, callback=callback)
def __init__(self, name):
self._name = name
self._ts = 0
self._outbox = ListEventStream()
def __init__(self):
self.processed_events = []
self._events = st.one_of(st.just(ErrorEvent(reason="random")),
some_event())
self._outbox = ListEventStream()
self._error_status = None
def __init__(self):
self.processed_events = []
self._outbox = ListEventStream()
class NodeModel:
def __init__(self, name: str, node_names: List[str], connections: PoolConnections):
self._name = name
self._node_names = node_names
self._quorum = Quorums(len(node_names))
self._ts = 0
self._corrupted_name = None
self._timer = TimerModel(name)
self._connections = connections
self._view_changer = create_view_changer(self)
self._internal_outbox = ListEventStream()
self.outbox = CompositeEventStream(self._internal_outbox, self._timer.outbox())
self._check_performance_timer = RepeatingTimer(self._timer, 30, self._check_performance)
@property
def name(self):
return self._name
@property
def view_no(self):
return self._view_changer.view_no
@property
def is_participating(self):
return not self._view_changer.view_change_in_progress and not self.is_corrupted
@property
def primary_name(self):
return self._primary_name(self._view_changer.view_no)
@property
def next_primary_name(self):
view_no = self._view_changer.view_no
if not self._view_changer.view_change_in_progress:
view_no += 1
return self._primary_name(view_no)
@property
def current_primary_name(self):
view_no = self._view_changer.view_no - 1
if not self._view_changer.view_change_in_progress:
view_no += 1
return self._primary_name(view_no)
@property
def is_primary(self):
return self.name == self.primary_name
@property
def is_primary_disconnected(self):
return self._connections.are_connected(self._ts, (self.name, self.primary_name))
@property
def is_corrupted(self):
return self._corrupted_name == self.name
@property
def connected_nodes(self) -> List[int]:
result = []
for name in self._node_names:
if name == self.name:
continue
if self._connections.are_connected(self._ts, (self.name, name)):
result.append(name)
return result
def restart(self):
pass
def outage(self, node: str):
if node == self.primary_name:
self._view_changer.on_primary_loss()
self._flush_viewchanger_outbox()
def corrupt(self, node: str):
self._corrupted_name = node
if self.name == node:
self._check_performance_timer.stop()
def process(self, draw, event: SimEvent):
self._ts = event.timestamp
self._timer.process(draw, event)
if isinstance(event.payload, CatchupDoneEvent):
if event.payload.node == self.name:
self._view_changer.on_catchup_complete()
if isinstance(event.payload, NetworkEvent):
self.process_network(event.payload)
self._flush_viewchanger_outbox()
def process_network(self, message: NetworkEvent):
if self.name == self._corrupted_name:
return
if message.dst != self.name:
return
self._view_changer.inBoxRouter.handleSync((message.payload, message.src))
def _flush_viewchanger_outbox(self):
if self.name == self._corrupted_name:
return
for msg in self._view_changer.outBox:
self._broadcast(msg)
self._view_changer.outBox.clear()
def _send(self, message, delay=1):
self._internal_outbox.add(SimEvent(timestamp=self._ts + delay, payload=message))
self.outbox.sort()
def _broadcast(self, payload):
for name in self._node_names:
if name == self.name:
continue
self._send(NetworkEvent(src=self.name,
dst=name,
payload=payload))
def _check_performance(self):
if self.primary_name == self._corrupted_name:
self._view_changer.on_master_degradation()
self._flush_viewchanger_outbox()
def _primary_name(self, view_no):
return self._node_names[view_no % len(self._node_names)]