def get_anti_entropy_timeout(self):
"""
Creates the anti-entropy timeout.
In the future this could be random timeout not fixed.
"""
self.timeout = Timer(self.env, self.ae_delay, self.gossip)
return self.timeout.start()
def gossip(self):
"""
Randomly select a neighbor and exchange information about the state
of the latest entries in the log since the last anti-entropy delay.
"""
# Gossip to one node at each location
for location in self.locations:
# Don't gossip to nodes in self!
if location == self.location: continue
# Select a random target to gossip to
target = random.choice(list(self.remotes(location)))
# Log the gossip that's happening
self.sim.logger.debug("{} gossiping {} entries to {}".format(
self, len(self.ae_cache), target))
entries = tuple([
Write(version.name, self, version) for version in self.ae_cache
])
# Send all the values in the cache.
self.send(target, Gossip(entries, len(self.ae_cache), -1))
# Empty the cache on gossip
self.ae_cache = []
# Reset the anti-entropy timer
self.ae_timer = Timer(self.env, self.ae_delay, self.gossip)
self.ae_timer.start()
def __init__(self, simulation, **kwargs):
## Initialize the replica
super(RaftReplica, self).__init__(simulation, **kwargs)
## Initialize Raft Specific settings
self.state = State.FOLLOWER
self.currentTerm = 0
self.votedFor = None
self.log = MultiObjectWriteLog()
self.cache = {}
## Policies
self.read_policy = ReadPolicy.get(kwargs.get('read_policy', READ_POLICY))
self.aggregate_writes = kwargs.get('aggregate_writes', AGGREGATE_WRITES)
## Timers for work
eto = kwargs.get('election_timeout', ELECTION_TIMEOUT)
hbt = kwargs.get('heartbeat_interval', HEARTBEAT_INTERVAL)
self.timeout = ElectionTimer.fromReplica(self, eto)
self.heartbeat = Timer(self.env, hbt, self.on_heartbeat_timeout)
## Leader state
self.nextIndex = None
self.matchIndex = None
def handle_session(self):
"""
Starts a session timer if one isn't running, otherwise resets the
currently running session timer on an additional access.
"""
if not self.session:
self.session = Timer(
self.env, self.session_timeout,
partial(self.on_session_timeout, self.env.now))
else:
self.session = self.session.reset()
def run(self):
"""
We have to check in at every heartbeat interval. If we own a tag then
send a heartbeat message, otherwise just keep quiescing.
"""
while True:
if self.state == State.OWNER:
self.heartbeat = Timer(self.env, self.heartbeat_interval,
self.on_heartbeat_timeout)
yield self.heartbeat.start()
else:
yield self.env.timeout(self.heartbeat_interval)
def on_state_change(self):
"""
Does the same stuff as super, but also - if leader; starts the anti
entropy interval to do gossiping.
"""
super(FloatedRaftReplica, self).on_state_change()
if self.state in (State.FOLLOWER, State.CANDIDATE):
if hasattr(self, 'ae_timer') and self.ae_timer is not None:
# Cancel the anti-entropy timer.
self.ae_timer.stop()
self.ae_timer = None
elif self.state == State.LEADER:
self.ae_timer = Timer(self.env, self.ae_delay, self.gossip)
self.ae_timer.start()
elif self.state == State.READY:
# This happens on the call to super, just ignore for now.
pass
else:
raise SimulationException(
"Unknown Floating Raft State: {!r} set on {}".format(
self.state, self))
def read(self, name, **kwargs):
"""
When a tag replica performs a read it has to decide whether or not to
read locally or to make a remote read across the cluster.
Convert the read into an access, then check if we own the object.
If we do, then return the latest commit.
If we don't and no one else does either, attempt to acquire the tag.
If we don't and someone else does then either drop, wait, or remote.
Current implementation: #2, MR, no remote access.
If someone else owns tag, reads are dropped.
TODO: Remote vs Local Reads
"""
# Create the read event using super.
access = super(TagReplica, self).read(name, **kwargs)
# Record the number of attempts for the access
if access.is_local_to(self): access.attempts += 1
# Increase the session on access.
self.handle_session()
# Are we the owner of this tag?
if self.owns(access.name):
# TODO: Change to last commit!
version = self.log[access.name].lastVersion
# If the version is None, bail since we haven't read anything
if version is None: return access.drop(empty=True)
# Update the version, complete the read, and log the access
access.update(version, completed=True)
access.log(self)
# Return, we're done reading!
return access
# Is there a different owner for the tag?
owner = self.find_owner(access.name)
if owner is not None:
# Right now just drop the read on its face.
self.sim.logger.info("ownership conflict: dropped {} at {}".format(
access, self))
return access.drop()
# We're going to acquire the tag!
else:
# Log the access from this particular replica.
access.log(self)
# We're going to have some read latency, retry the read.
retry = Timer(self.env, self.heartbeat_interval,
lambda: self.read(access)).start()
if access.attempts <= 1 and self.state != State.TAGGING:
# Request the ownership of the tag
self.acquire(access.name)
return access
def write(self, name, **kwargs):
"""
When a replica performs a write it needs to decide if it can write to
the tag locally, can acquire a tag for this object, or if it has to do
something else like drop, wait, or remote write.
If the access is local:
- if the replica owns the tag, append and complete
- if someone else owns the tag then drop, wait, or remote
- if no one owns the tag, then attempt to acquire it
If access is remote:
- if we own the tag, then append but do not complete (at local)
- if someone else owns the tag, log and forward to owner
- if no one owns the tag then respond false
"""
# Create the read event using super.
access = super(TagReplica, self).write(name, **kwargs)
# Increase the session on access.
self.handle_session()
# Determine if the write is local or remote
if access.is_local_to(self):
# Record the number of attempts for the access
access.attempts += 1
# Fetch the latest version from the log.
latest = self.log[access.name].lastVersion
# Perform the write
if latest is None:
version = namespace(access.name)(self)
else:
version = latest.nextv(self)
# Update the access with the latest version
access.update(version)
else:
# If there is no version, raise an exception
if access.version is None:
raise AccessError(
"Attempting a remote write on {} without a version!".
format(self))
# Save the version variable for use below.
version = access.version
# Log the access at this replica
access.log(self)
# Are we the owner of this tag?
if self.owns(access.name):
# Perform the append entries
self.log[name].append(version, self.epoch)
# Update the version to track visibility latency
version.update(self)
# Complete the access if it was local
if access.is_local_to(self): access.complete()
# Now do AppendEntries
# Also interrupt the heartbeat since we just sent AppendEntries
if not settings.simulation.aggregate_writes:
self.send_append_entries()
if self.heartbeat: self.heartbeat.stop()
return access
# Is there a different owner for the tag?
owner = self.find_owner(name)
if owner is not None:
# Right now just drop the write on its face.
self.sim.logger.info("ownership conflict: dropped {} at {}".format(
access, self))
return access.drop()
# We're going to acquire the tag!
else:
# We're going to have some write latency, retry the write.
retry = Timer(self.env, self.heartbeat_interval,
lambda: self.write(access)).start()
# Request the ownership of the tag
self.acquire(access.name)
return access
