def test_append_to_empty():
l = Log()
new_entries = [Entry(1, 'first')]
l.append(0, 1, new_entries)
assert l.entries == [None] + new_entries
def test_append_beyond_last():
l = Log()
new_entries = [Entry(1, 'first')]
with pytest.raises(IndexBeyondLast):
l.append(1, 1, new_entries)
def test_append_wrong_term():
initial_entries = [Entry(1, 'first')]
l = Log(initial_entries)
new_entries = [Entry(2, 'second')]
with pytest.raises(InvalidTerm):
l.append(1, 2, new_entries)
def test_append_to_not_empty_beyond_last():
initial_entries = [Entry(1, 'first')]
l = Log(initial_entries)
new_entries = [Entry(1, 'second')]
with pytest.raises(IndexBeyondLast):
l.append(2, 1, new_entries)
class Shell:
def __init__(self):
self.builtins = Builtins(self)
self.completion = Completion(self)
self.history = History()
self.javascript = Javascript()
self.log = Log()
self.prompt = Prompt()
def execute(self, command):
self.log.append(str(self.prompt) + command)
self.history.append(command)
if command:
# execute builtins command
try:
self.builtins.execute(command.strip())
except self.builtins.UnknownCommandError as e:
self.log.append('websh: command not found: {0}'.format(e.command))
except Exception as e:
print 'Error in builtins: {0}'.format(e)
return json.dumps({'javascript': str(self.javascript),
'log': str(self.log),
'prompt': str(self.prompt)})
def template(self):
# read template file
file = open('data/template.html', 'r')
template = string.Template(file.read())
file.close()
return template.substitute(log = str(self.log),
prompt = str(self.prompt))
def test_append_prev_index_is_negative():
l = Log()
new_entries = [Entry(1, 'first')]
with pytest.raises(NegativeIndex):
l.append(-1, 1, new_entries)
def test_append_ovewrite():
initial_entries = [Entry(1, 'first'), Entry(1, 'second')]
l = Log(initial_entries)
new_entries = [Entry(1, 'third')]
l.append(1, 1, new_entries)
assert l.entries == [None] + initial_entries[:1] + new_entries
def test_append_to_not_empty():
initial_entries = [Entry(1, 'first')]
l = Log(initial_entries)
new_entries = [Entry(1, 'second')]
l.append(1, 1, new_entries)
assert l.entries == [None] + initial_entries + new_entries
def clean(self, ext):
try:
os.remove(self.get(ext))
except FileNotFoundError:
Log.append("Cannot clean " + self.get(ext) + ": Does not exist.")
return False
return True
def convert(self):
try:
subprocess.call("tex3ds " + self.png() + " -o " + self.t3x(),
shell=True)
except subprocess.CalledProcessError:
Log.append("Could not convert " + self.png() + ".")
return False
return True
def state(self):
try:
self.gp.output(self.pin_out, self.gp.HIGH)
time.sleep(1)
input_state = self.gp.input(self.pin_in)
time.sleep(1)
self.gp.output(self.pin_out, self.gp.LOW)
except Exception as ex:
Log.append('water level sensore error')
Log.append(str(ex))
return self.LOW
if input_state == 1:
return self.HIGH
else:
return self.LOW
def search(path=None, file_list=None):
if file_list is None:
file_list = []
if path is None:
path = "game"
try:
for file in listdir(path):
if isfile(path + "/" + file) and file[-4:] == ".png":
no_ext_value = splitext(file)[0]
file_list.append(File(path + "/" + no_ext_value))
elif isdir(path + "/" + file):
search(path + "/" + file, file_list)
return file_list
except FileNotFoundError:
Log.append("Failed to find file listing for " + path)
def __init__(self):
Log.append("Init Control.")
# GPIO settings
gp.setmode(gp.BCM)
gp.setwarnings(False)
# read state file
State.sync()
self.state = State.load_state_from_file()
# init pumps
self.pump_1 = Pump(gp, self.PUMP_1_pin, id=1)
self.pump_2 = Pump(gp, self.PUMP_2_pin, id=2)
# moisture sensors
self.m_sensor_1 = MSensor(gp, self.M_SENSOR_1_pin,
self.M_SENSORS_RELAY_pin)
self.m_sensor_2 = MSensor(gp, self.M_SENSOR_2_pin,
self.M_SENSORS_RELAY_pin)
# water level sensor
self.water_level_sensor = WaterLevel(gp,
self.WATER_LEVEL_SENSOR_pin_out,
self.WATER_LEVEL_SENSOR_pin_in)
def state(self):
# turn on sensor, measure, turn off
try:
self.gp.output(self.relay_pin, self.gp.LOW)
time.sleep(1)
state = self.gp.input(self.sensor_pin)
time.sleep(1)
self.gp.output(self.relay_pin, self.gp.HIGH)
time.sleep(1)
except Exception as ex:
Log.append('moisture sensore error')
Log.append(str(ex))
state = 1
# 0 - over the threshold
# 1 - below the threshold
if state == 0:
return self.WET
else:
return self.DRY
def turn_on(self):
Log.append('Turning on pump... ' + str(self.id))
try:
self.gp.output(self.pin, 0)
Log.append('...turned on')
except Exception as ex:
Log.append('pump \'on\' error ' + str(ex))
def turn_off(self):
Log.append('Turning off pump... ' + str(self.id))
try:
self.gp.output(self.pin, 1)
Log.append('...turned off')
except Exception as ex:
Log.append('pump \'off\' error ' + str(ex))
def sync():
Ftp.download_file(State.state_server_file_path, 'state.json')
time.sleep(0.5)
server_state = State.load_state_from_server_file()
local_state = State.load_state_from_file()
local_state['pump_1_interval'] = server_state['pump_1_interval']
local_state['pump_2_interval'] = server_state['pump_2_interval']
local_state['pump_1_water_amount'] = server_state['pump_1_water_amount']
local_state['pump_2_water_amount'] = server_state['pump_2_water_amount']
local_state['tank_capacity'] = server_state['tank_capacity']
if server_state['tank_refilled'] == 1: # tank refilled
local_state['water_level'] = local_state['tank_capacity']
local_state['tank_refilled'] = 0
local_state['low_water_level_alert'] = 0
Log.append('tank refilled')
if server_state['run_test'] == 2: # test request
if local_state['run_test'] == 0: # request not taken
local_state['run_test'] = 2
elif local_state['run_test'] == 1: # request satisfied
local_state['run_test'] = 0
State.save_state_to_file(local_state)
Ftp.upload_file(State.state_file_path, 'state.json')
def do_convert(file_list, arg):
successful = True
if file_list is None:
return
for i in range(len(file_list)):
if arg["c"] == "":
try:
if file_list[i].convert():
print("\x1b[2K\rConverting " + file_list[i].png() + " (" +
str(i + 1) + " of " + str(len(file_list)) + ")",
end="",
flush=True)
if arg["mv"]:
file_list[i].move()
except subprocess.CalledProcessError:
Log.append("Could not convert file " + file_list[i].png())
except KeyboardInterrupt:
Log.append("Conversion stopped.")
except FileNotFoundError as err:
Log.append("Could not find file " + file_list[i].png() +
"\n\t(" + str(err) + ")")
else:
try:
if file_list[i].clean(arg["c"]):
print("\x1b[2K\rCleaning " + file_list[i].t3x() + " (" +
str(i + 1) + " of " + str(len(file_list)) + ")",
end="",
flush=True)
else:
successful = False
except FileNotFoundError:
Log.append("Could not clean file")
if successful:
print("\nDone.")
class Server(threading.Thread):
def __init__(self, queue, port, id):
self.port = port
self.id = id
self.queue = queue
self.title = constants.TITLE_FOLLOWER
self.channel = network.Network(port, id)
self.channel.start()
self.leader = None
self.running = True
self.connected_servers = []
self.last_heartbeat = 0
self.heartbeat_timeout = 0
self.process_heartbeat()
self.heartbeat_frequency = 0.5
self.election_start_time = 0
self.election_timeout = 0 # Time to wait for heartbeat or voting for a candidate before calling election
self.set_election_timeout()
# Election variables
self.id_received_votes = set() # Id of servers who granted you votes
self.id_refused_votes = set() # Id of servers who refused to vote for you
self.num_received_votes = 0 # Number of votes received in current election
# Persistent state variables
# TODO: PERSIST; On server boot, retrieve information from disk
self.current_term = 0 # Latest term server has seen
self.voted_for = None # CandidateId that received vote in current term
self.log = Log()
self.next_index = None # For leader: indices for updating follower logs
self.latest_index_term = None # For leader: tuples of latest entry index and term for each follower. Used for commit
self.load_state()
threading.Thread.__init__(self)
def set_election_timeout(self):
self.election_timeout = 1.5 * random() + 1.5
def process_heartbeat(self):
self.last_heartbeat = time.time()
self.heartbeat_timeout = 1.5 * random() + 1.5
def request_votes(self):
if not self.log.data:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
msg = RequestVoteMessage(self.id, self.current_term, last_log_index, last_log_term)
for server in self.connected_servers:
self.channel.send(msg, id=host_to_id[server[0]])
# print "Requesting vote from server", host_to_id[server[0]]
print "Vote requests sent to other servers"
def request_remaining_votes(self, id_all_voters):
if not self.log.data:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
msg = RequestVoteMessage(self.id, self.current_term, last_log_index, last_log_term)
for server in self.connected_servers:
server_id = host_to_id[server[0]]
if server_id not in id_all_voters:
self.channel.send(msg, id=server_id)
# print "Requesting vote from server", host_to_id[server[0]]
print "Vote requests sent to remaining servers who have not responded"
def check_status(self):
current_time = time.time()
if self.title == constants.TITLE_LEADER:
# Send AppendEntries to update follower logs
for server in self.connected_servers:
server_id = host_to_id[server[0]]
next_index = self.next_index[server_id]
# Send entries that the server has not received yet, if any
if self.log.last_log_index() >= next_index:
entries = self.construct_entries_list(next_index)
if next_index == 0:
prev_log_index = -1
prev_log_term = -1
else:
prev_log_index = self.log.get(next_index-1).index
prev_log_term = self.log.get(next_index-1).term
msg = AppendEntriesMessage(self.current_term, self.id, prev_log_index,
prev_log_term, entries, self.log.last_commit_index)
self.channel.send(msg, id=server_id)
print "AppendEntries sent to ", server_id
if current_time - self.last_heartbeat >= self.heartbeat_frequency:
self.send_heartbeats()
elif self.title == constants.TITLE_FOLLOWER and current_time - self.last_heartbeat > self.heartbeat_timeout:
# Heartbeat timeout passed as follower: Start election
print "Election timeout as follower. No heartbeat. Become candidate and start new election"
self.start_election()
elif self.title == constants.TITLE_CANDIDATE and current_time - self.election_start_time > self.election_timeout:
# Election timeout passed as candidate, without conclusion of election: Start new election
print "Election timeout as candidate. Election has not yet led to new leader. Starting new election"
self.set_election_timeout()
self.start_election()
elif self.title == constants.TITLE_CANDIDATE and current_time - self.election_start_time < self.election_timeout:
# Election timeout has not passed as candidate
print "As candidate, election timeout has not passed. Request votes from servers that have not responded"
id_all_voters = self.id_received_votes.union(self.id_refused_votes)
self.request_remaining_votes(id_all_voters)
def construct_entries_list(self, index):
entries = []
for i in range(index, len(self.log)):
entries.append(self.log.get(i))
return entries
def start_election(self):
self.title = constants.TITLE_CANDIDATE
self.reset_election_info()
self.current_term += 1
self.save_state()
# TODO: Voted_for must persist
self.voted_for = self.id
self.save_state()
self.update_votes(self.id, True)
self.election_start_time = time.time()
self.check_election_status()
self.request_votes()
def send_heartbeats(self):
heartbeat = AppendEntriesMessage(self.current_term, self.id, -1, -1, [], self.log.last_commit_index)
for server in self.connected_servers:
self.channel.send(heartbeat, id=host_to_id[server[0]])
self.process_heartbeat()
def step_down(self):
# Step down as leader or candidate, convert to follower
# Reset various election variables
if self.title == constants.TITLE_LEADER or self.title == constants.TITLE_CANDIDATE:
self.title = constants.TITLE_FOLLOWER
self.process_heartbeat()
self.reset_election_info()
def grant_vote(self, candidate_id):
# TODO: Voted_for must persist
self.voted_for = candidate_id
self.save_state()
print "Grant vote to", candidate_id
self.channel.send(VoteReplyMessage(self.id, self.current_term, True), id=candidate_id)
def refuse_vote(self, candidate_id):
self.channel.send(VoteReplyMessage(self.id, self.current_term, False), id=candidate_id)
print "Refuse vote to", candidate_id
def majority(self):
return (len(self.connected_servers)+1) / 2 + 1
def check_election_status(self):
if self.num_received_votes >= self.majority():
# Become leader when granted majority of votes
self.become_leader()
def become_leader(self):
self.title = constants.TITLE_LEADER
self.leader = self.id
print "Election won - I am now LEADER"
# TODO: Implement rest of leader initialization
self.next_index = [len(self.log) for _ in range(len(addr_to_id))]
if self.log.last_commit_index == -1:
latest_index = None
else:
latest_index = self.log.last_commit_index
if latest_index is None:
latest_term = 0
elif self.log.contains_at_index(latest_index):
latest_term = self.log.get(latest_index).term
else:
latest_term = 0
self.latest_index_term = [(latest_index, latest_term) for _ in range(len(addr_to_id))]
self.latest_index_term[self.id] = (len(self.log)-1, self.current_term)
self.reset_election_info()
self.send_heartbeats()
def reset_election_info(self):
self.id_received_votes = set()
self.id_refused_votes = set()
self.voted_for = None
self.num_received_votes = 0
# server_id: server that sent vote reply; vote_granted: True if vote granted
def update_votes(self, server_id, vote_granted):
if vote_granted:
print "Received vote from", server_id
self.id_received_votes.add(server_id)
self.num_received_votes = len(self.id_received_votes)
print "Number of received votes is now", self.num_received_votes
else:
print "Denied vote from", server_id
self.id_refused_votes.add(server_id)
def update_commits(self):
index = max(self.next_index)
i_count = 0
t_count = 0
while i_count < self.majority() and index >= 0:
if index < 0:
print "Error: Update_commits: index is less than 0"
index -= 1
t_count = 0
i_count = 0
for (i, t) in self.latest_index_term:
if t == self.current_term:
t_count += 1
if i >= index:
i_count += 1
if t_count >= self.majority() and i_count >= self.majority():
if self.log.last_commit_index < index:
self.log.last_commit_index = index
self.save_state()
elif self.log.last_commit_index > index:
print "Error: Update_commits: new commit index is lower than current commit_index"
for entry in self.log.data:
if not entry.client_ack_sent:
# TODO: Send client ack
ack_message = AcknowledgeMessage(ack=True, msg_id=entry.msg_id)
self.channel.send(ack_message, id=entry.author)
entry.client_ack_sent = True
def run(self):
print "Server with id=", self.id, " up and running"
while self.running:
self.update_connected_servers()
for server in list(addr_to_id.keys()):
# if server not in self.connected_servers and not addr_to_id[server] == id:
if server not in self.channel and not host_to_id[server[0]] == self.id:
connected = self.channel.connect(server)
if connected:
print str("Server: Connected to "+server[0])
if server not in self.connected_servers:
self.connected_servers.append(server)
# print "Connected: ", connected
data = self.channel.receive(RECEIVE_FREQ)
if data:
# print "There is data on channel"
for server_id, msg in data:
self.process_msg(server_id, msg)
else:
self.check_status()
def process_msg(self, sender_id, msg):
#print "Processing message from", sender_id, "of type", msg.type
if msg.type == constants.MESSAGE_TYPE_REQUEST_VOTE:
self.process_request_vote(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_VOTE_REPLY:
self.process_vote_reply(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_REQUEST_LEADER:
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg, id=sender_id)
elif msg.type == constants.MESSAGE_TYPE_LOOKUP:
self.process_lookup(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_POST:
self.process_post(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_APPEND_ENTRIES:
self.process_append_entries(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_ACKNOWLEDGE:
self.process_acknowledge(sender_id, msg)
# Used for testing purposes
elif msg.type == constants.MESSAGE_TYPE_TEXT:
print "From", msg.sender_id, ":", msg.msg
else:
print "Error: Invalid message type"
def process_lookup(self, sender_id, msg):
if self.title == constants.TITLE_LEADER or msg.override:
print "-----> Processing Lookup from client"
posts = self.log.get_committed_entries()
msg = messages.LookupMessage(msg_id=msg.msg_id, post=posts, server_id=self.id)
self.channel.send(msg=msg, id=sender_id)
else:
print "Lookup to leader"
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg=msg, id=sender_id)
def process_post(self, sender_id, msg):
if self.title == constants.TITLE_LEADER:
# TODO: Implement adding entry
# TODO: PERSIST; implement in log class?
entry = Entry(msg.post, sender_id, self.current_term, len(self.log), msg_id=msg.msg_id)
if self.log.append(entry):
self.save_state()
self.latest_index_term[self.id] = (len(self.log) - 1, self.current_term)
print "---->Append entry from client to log"
else:
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg=msg, id=sender_id)
def process_request_vote(self, sender_id, msg):
if not self.log:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
# Handle message
if msg.term < self.current_term:
# If candidate's term is less than my term then refuse vote
print "Refuse vote to server", sender_id, "because I have higher term"
self.refuse_vote(msg.candidate_id)
if msg.term > self.current_term:
# If candidate's term is greater than my term then update current_term (latest term I've encountered),
# Step down if leader or candidate
self.current_term = msg.term
self.save_state()
# TODO: Step down if leader or candidate
self.step_down()
if msg.term >= self.current_term:
# If candidate's term is at least as new as mine and I have granted anyone else a vote
# and candidate's log is at least as complete as mine
# then grant vote
if self.voted_for is None or self.voted_for is msg.candidate_id:
if last_log_term < msg.last_log_term or (
last_log_term == msg.last_log_term and last_log_index <= msg.last_log_index):
self.grant_vote(msg.candidate_id)
else:
# print "Cand term, current_term:", msg.term, self.current_term
# print "Voted for:", self.voted_for
# print "Cand log term, last_log_term", msg.last_log_term, last_log_term
# print "Cand log index, last_log_index", msg.last_log_index, last_log_index
self.refuse_vote(msg.candidate_id)
def process_vote_reply(self, sender_id, msg):
if msg.term > self.current_term and not msg.vote_granted:
# Step down if reply from someone with higher term
# Extra condition for security.
# If responder's term is higher, then vote should not be granted with correct execution
self.current_term = msg.term
self.save_state()
print "Denied vote from", msg.follower_id
self.step_down()
else:
# Take care of grant or refusal of vote
self.update_votes(msg.follower_id, msg.vote_granted)
self.check_election_status()
def process_acknowledge(self, sender_id, msg):
if msg.ack:
print "Process Acknowledge from server. ACK == TRUE"
self.next_index[sender_id] = msg.next_index
self.latest_index_term[sender_id] = msg.latest_index_term
self.update_commits()
else:
print "Process Acknowledge from server. ACK == FALSE"
if self.next_index[sender_id]-1 < 0:
self.next_index[sender_id] = 0
else:
self.next_index[sender_id] -= 1
if msg.term > self.current_term:
self.current_term = msg.term
self.save_state()
self.step_down()
def process_append_entries(self, sender_id, msg):
if len(msg.entries) == 0:
self.process_heartbeat()
if msg.commit_index < len(self.log):
self.log.last_commit_index = msg.commit_index
self.save_state()
self.leader = sender_id
#print "Heartbeat received from server", sender_id
if self.title == constants.TITLE_CANDIDATE or self.title == constants.TITLE_LEADER:
self.step_down()
elif self.title == constants.TITLE_LEADER:
# TODO: If a "leader" receives a heartbeat,
# it might have crashed and joined back in after an election (?)
pass
else:
# TODO: Process AppendEntriesMessage
print "-->Processing AppendEntriesMessage from leader"
self.process_heartbeat()
if msg.term > self.current_term:
self.current_term = msg.term
self.save_state()
if self.title == constants.TITLE_CANDIDATE or self.title == constants.TITLE_LEADER:
self.step_down()
# Reject if my term is greater than leader term
if self.current_term > msg.term:
print "Error: Current term greater than leaders term"
self.channel.send(AcknowledgeMessage(ack=False, term=self.current_term), id=sender_id)
# Accept. Self.log is empty and leader is sending all entries
elif self.log.is_empty() and msg.prev_log_index == -1:
print "Appending entries"
# First entry to append is at index 0
if self.log.append_entries(msg.entries):
self.log.last_commit_index = msg.commit_index
self.save_state()
i = self.log.last_log_index()
t = self.log.get(i).term
self.channel.send(AcknowledgeMessage(
ack=True, next_index=len(self.log), latest_index_term=(i, t)), id=sender_id)
print "Log after appending entries:"
self.log.show_data()
else:
print "DET HER SKAL IKKE SKJE 1"
# Accept. Check if self.log has an element at msg.prev_log_index
elif self.log.contains_at_index(msg.prev_log_index):
# Check if the term corresponds with msg.prev_log_term
if self.log.get(msg.prev_log_index).term == msg.prev_log_term:
if self.log.append_entries(msg.entries):
self.log.last_commit_index = msg.commit_index
self.save_state()
i = self.log.last_log_index()
t = self.log.get(i).term
self.channel.send(
AcknowledgeMessage(ack=True, next_index=len(self.log), latest_index_term=(i, t)), id=sender_id)
print "Log after appending entries:"
self.log.show_data()
else:
print "DET HER SKAL IKKE SKJE NUMMER 2"
else:
self.log.remove(msg.prev_log_index)
self.channel.send(AcknowledgeMessage(ack=False), id=sender_id)
else:
print "Send ACK-False"
self.channel.send(AcknowledgeMessage(ack=False),id=sender_id)
def save_state(self):
storage.save(self.id, self.voted_for, self.current_term, self.log)
def load_state(self):
self.voted_for, self.current_term, self.log = storage.load(self.id)
# print "voted for", self.voted_for
print self.current_term
print self.log
def update_connected_servers(self):
for addr in list(addr_to_id.keys()):
if addr in self.channel.address_to_connection.keys() and addr not in self.connected_servers:
self.connected_servers.append(id)
if addr not in self.channel.address_to_connection.keys() and addr in self.connected_servers:
self.connected_servers.remove(addr)
class RaftServer:
# Miscellaneous constants.
NO_LEADER = -1
def __init__(self, id):
self.id = id # unique id for this RaftServer
self.state_machine = StateMachine()
# Persistent state on a server.
self.current_term = 0
self.voted_for = None # candidate_id that received vote
# in current term
self.log = Log()
# Volatile state on a server.
self.commit_index = 0 # index of highest log entry known
# to be committed
self.last_applied = 0 # index of highest log entry applied
# to state machine
# Volatile state on a leader. (Reinitialized after election.)
# for each server, index of next log
# entry to send to that server:
self.next_index = []
# for each server, index of highest
# log entry known to be replicated on server:
self.match_index = []
self.leader_id = self.NO_LEADER
self.num_votes = 0 # votes for this candidate
self.state = self.State.FOLLOWER
self.heartbeat_thread = None # for leader to assert authority
self.candidate_thread = None # for candidate to grab votes
self.election_timeout = 100 # random number for now
def __del__(self):
if self.heartbeat_thread != None:
self.state = self.State.FOLLOWER # kills thread
self.heartbeat_thread.join() # wait for thread to die
class State(Enum):
FOLLOWER = 1
CANDIDATE = 2
LEADER = 3
def __got_majority_vote(self):
return self.num_votes > Temp.NUM_SERVERS / 2
def __request_votes(self):
# Send out vote requests.
# TODO: reset election timer
# TODO: select new randomized election timeout
# send Requestvote RPCs to all other servers:
# for i in range(Temp.NUM_SERVERS):
# if i != self.id:
# response = Temp.servers[i].request_vote_RPC(
# self.current_term, self.id, self.log.size()-1,
# self.log.latest())
# print("Thread {}: thread {} sent {} back to me".format(
# self.id, i, response))
# Keep spinning UNTIL either:
# a) election timeout
# b) got majority vote
while (not self.__got_majority_vote() and
self.leader_id == self.NO_LEADER):
pass
print("Thread {} broke out candidate loop".format(self.id))
if self.__got_majority_vote(): # this server was elected
self.__become_leader()
elif self.leader_id != self.NO_LEADER: # other server was elected
self.__become_follower()
else: # timed out
self.__start_election()
def __become_candidate(self):
print("Thread {} has become candidate".format(self.id))
# Start candidate thread.
self.candidate_thread = Thread(target=self.__start_election)
self.candidate_thread.start()
# Called if no communication from leader and
# election timeout reached.
def __start_election(self):
self.current_term += 1
self.num_votes = 0
assert(self.state != self.State.LEADER) # can't start election if was Leader
self.state = self.State.CANDIDATE
self.voted_for = self.id
self.leader_id = self.NO_LEADER
self.__request_votes()
# For heartbeat thread.
# If leader, sends heartbeat to all followers,
# then wait until must send next heartbeat.
def __send_heartbeat(self):
while self.state == self.State.LEADER:
# for each follower
# send empty AppendEntries RPC
for i in range(Temp.NUM_SERVERS):
if i != self.id:
prev_log_index = self.next_index[i] - 1
response = Temp.servers[i].append_entries_RPC(self.current_term,
self.id,
# these next two params might be wrong
prev_log_index, self.log.get(prev_log_index).term,
[], # because heartbeat
self.commit_index)
# print("Thread {}: thread {} sent {} back to me".format(
# self.id, i, response))
# print("Thread id={} sending heartbeat".format(self.id))
time.sleep(Temp.HEARTBEAT_PERIOD)
def __become_leader(self):
print("Thread {} has become leader".format(self.id))
self.state = self.State.LEADER
initial_next_index = self.last_applied + 1
self.next_index = [initial_next_index] * Temp.NUM_SERVERS
self.match_index = [0] * Temp.NUM_SERVERS
# Start heartbeat thread.
self.heartbeat_thread = Thread(target=self.__send_heartbeat)
self.heartbeat_thread.start()
def __become_follower(self):
print("Thread {} has become follower".format(self.id))
self.state = self.State.FOLLOWER
if self.heartbeat_thread != None: # kill heartbeat thread (if any)
self.heartbeat_thread.join()
self.heartbeat_thread = None
# TODO: what else?
def __handle_request(self, request):
self.log.append(self.current_term, request.command)
# Used to receive request from client.
# @request must be instance of Request.
def handle_request(self, request):
if self.state == RaftServerState.LEADER:
# print("Thread {} handling request".format(self.id))
self.__handle_request(request)
else:
assert(self.leader_id != self.NO_LEADER)
Temp.servers[self.leader_id].handle_request(request)
# Invoked by the leader on other raft servers to replicate
# log entries. Also used as a heartbeat.
# Simulates an RPC.
# Output: tuple (@term, @success), where @term is the
# current term, and @success indicating success (boolean).
def append_entries_RPC(self, term, leader_id, prev_log_index,
prev_log_term, entries,
leader_commit):
# print("Thread {} handling AppendEntries RPC".format(self.id))
if self.id == leader_id: # if leader sent RPC to itself
raise AssertionError("Leader sent AppendEntries RPC to itself")
if term < self.current_term:
return (self.current_term, False)
if (self.log.len() <= prev_log_index
or self.log.get(prev_log_index).term != prev_log_term):
return (self.current_term, False)
# TODO: #3 and 4 and 5
# if leader_commit > self.commit_index:
# self.commit_index = min(leader_commit, (index of last new entry))
if self.leader_id == self.NO_LEADER: # if hasn't acknowledged this leader
self.leader_id = leader_id
# if self.state != self.State.FOLLOWER:
# self.__become_follower()
return (self.current_term, True)
# Invoked by candidates on other raft servers to gather votes.
def request_vote_RPC(term, candidate_id, last_log_index,
last_log_term):
if term < self.current_term: # requester's log is out of date
return (self.current_term, False)
if self.voted_for == None or self.voted_for == candidate_id:
return (self.current_term, True)
else: # already voted for somene else
return (self.current_term, False)
# TODO: remove
def make_leader(self):
self.__become_leader()
def make_follower(self):
self.__become_follower()
def make_candidate(self):
self.__become_candidate()
def leader_die(self):
self.__become_follower() # stop sending heartbeats
def add_vote(self):
self.num_votes += 1
def run(self):
# check if all necessary variables are assigned in self.state
if self.state['water_level'] is None:
warning = 'Water level is none. Check if tank is full. Aborting.'
Log.append(warning)
print(warning)
return
if self.state[
'pump_1_interval'] is None: ## later --> or self.state['pump_2_interval']
warning = 'pump 1 interval is None.'
Log.append(warning)
print(warning)
return
if self.state['pump_1_water_amount'] is None:
warning = 'pump 1 water amount is None.'
Log.append(warning)
print(warning)
return
while True:
if time.time() - self.last_test_time >= self.TEST_STEP_TIME:
# test
State.sync()
self.state = State.load_state_from_file()
if self.state['run_test'] == 2:
self.run_test()
State.save_state_to_file(self.state)
State.sync()
self.last_test_time = time.time()
if time.time(
) - self.moisture_last_check_time >= self.MOISTURE_CHECK_STEP:
# moisture sensors check
m1_state = self.m_sensor_1.state()
if m1_state == MSensor.DRY:
if self.state['dry_alert_1'] == 0:
self.state['dry_alert_1'] = time.time()
Log.append('Moisturness sensor 1 is dry.')
else:
if self.state['dry_alert_1'] != 0:
Log.append('Moisturness sensor 1 is wet again.')
self.state['dry_alert_1'] = 0
m2_state = self.m_sensor_2.state()
if m2_state == MSensor.DRY:
if self.state['dry_alert_2'] == 0:
self.state['dry_alert_2'] = time.time()
Log.append('Moisturness sensor 2 is dry.')
else:
if self.state['dry_alert_2'] != 0:
Log.append('Moisturness sensor 2 is wet again.')
self.state['dry_alert_2'] = 0
State.save_state_to_file(self.state)
State.sync()
self.moisture_last_check_time = time.time()
if self.state['last_loop_time'] is None or time.time(
) - self.state['last_loop_time'] >= self.STEP_TIME: # perform step
#Log.append('--------------- New Loop ----------------')
State.sync()
self.state = State.load_state_from_file()
# check if watering is scheduled
# PUMP_1
if self.state['pump_1_last_watering'] == 0 or time.time(
) - self.state['pump_1_last_watering'] >= self.state[
'pump_1_interval']:
# watering is needed
Log.append('Pump 1 watering is required.')
if self.state['low_water_level_alert'] == 1:
# send warning to user APK
Log.append("Low water level in main tank.")
if self.state['water_level'] < self.state[
'pump_1_water_amount']:
if self.state[
'water_level'] > 400: # pump whatever is left
if self.pump_1.pump_ml(
self.state['water_level']):
self.pumped(pump_nr=1)
else:
Log.append('Pump 1 watering error.')
else:
# if no water, don't run pumps.
Log.append(
"Water tank is empty. Skipping watering.")
else:
# run pump
if self.pump_1.pump_ml(
self.state['pump_1_water_amount']):
self.pumped(pump_nr=1)
else:
# write to error log
Log.append('Pump 1 watering error.')
else: # no water level alert
# run pump
if self.pump_1.pump_ml(
self.state['pump_1_water_amount']):
self.pumped(pump_nr=1)
else:
# write to an error log
Log.append('Pump 1 watering error.')
# #
# same for PUMP_2
# #
# check water level
if self.water_level_sensor.state() == WaterLevel.LOW:
if self.state['low_water_level_alert'] == 0:
Log.append('Low water level reached.')
self.state['low_water_level_alert'] = 1
self.state['water_level'] = self.WATER_RESERVE_CAPACITY
else:
if self.state['low_water_level_alert'] == 1:
self.tank_refilled(
) # if alert was 1, and now is 0, tank was refilled.
# on loop end.
self.state['last_loop_time'] = time.time()
State.save_state_to_file(self.state)
State.sync()
#Log.append('-------------- End of loop --------------')
else:
time.sleep(10)
def run_test(self):
Log.append('Test run.')
self.pump_1.turn_on_for_time(4)
self.state['run_test'] = 1
State.save_state_to_file(self.state)
def pumped(self, pump_nr):
self.state['water_level'] -= self.state['pump_' + str(pump_nr) +
'_water_amount']
self.state['pump_' + str(pump_nr) + '_last_watering'] = time.time()
Log.append('Pump '+str(pump_nr)+' pumped ' + str(self.state['pump_'+str(pump_nr)+'_water_amount']) + ' ml. ' +\
str(self.state['water_level']) +' ml left in tank.')
def tank_refilled(self):
Log.append('Tank refilled.')
self.state['low_water_level_alert'] = 0
self.state['water_level'] = self.state[
'tank_capacity'] # tank is always refilled fully.
State.save_state_to_file(self.state)
# #
# check water level
if self.water_level_sensor.state() == WaterLevel.LOW:
if self.state['low_water_level_alert'] == 0:
Log.append('Low water level reached.')
self.state['low_water_level_alert'] = 1
self.state['water_level'] = self.WATER_RESERVE_CAPACITY
else:
if self.state['low_water_level_alert'] == 1:
self.tank_refilled(
) # if alert was 1, and now is 0, tank was refilled.
# on loop end.
self.state['last_loop_time'] = time.time()
State.save_state_to_file(self.state)
State.sync()
#Log.append('-------------- End of loop --------------')
else:
time.sleep(10)
if __name__ == '__main__':
if os.path.isfile(
os.path.join(os.path.dirname(__file__), 'data', 'autorun')):
Log.append('Autorun enabled.')
c = Control()
c.run()
else:
Log.append('Autorun disabled.')
class Server(threading.Thread):
def __init__(self, queue, port, id):
self.port = port
self.id = id
self.queue = queue
self.title = constants.TITLE_FOLLOWER
self.channel = network.Network(port, id)
self.channel.start()
self.leader = None
self.running = True
self.connected_servers = []
self.last_heartbeat = 0
self.heartbeat_timeout = 0
self.process_heartbeat()
self.heartbeat_frequency = 0.5
self.election_start_time = 0
self.election_timeout = 0 # Time to wait for heartbeat or voting for a candidate before calling election
self.set_election_timeout()
# Election variables
self.id_received_votes = set() # Id of servers who granted you votes
self.id_refused_votes = set(
) # Id of servers who refused to vote for you
self.num_received_votes = 0 # Number of votes received in current election
# Persistent state variables
# TODO: PERSIST; On server boot, retrieve information from disk
self.current_term = 0 # Latest term server has seen
self.voted_for = None # CandidateId that received vote in current term
self.log = Log()
self.next_index = None # For leader: indices for updating follower logs
self.latest_index_term = None # For leader: tuples of latest entry index and term for each follower. Used for commit
self.load_state()
threading.Thread.__init__(self)
def set_election_timeout(self):
self.election_timeout = 1.5 * random() + 1.5
def process_heartbeat(self):
self.last_heartbeat = time.time()
self.heartbeat_timeout = 1.5 * random() + 1.5
def request_votes(self):
if not self.log.data:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
msg = RequestVoteMessage(self.id, self.current_term, last_log_index,
last_log_term)
for server in self.connected_servers:
self.channel.send(msg, id=host_to_id[server[0]])
# print "Requesting vote from server", host_to_id[server[0]]
print "Vote requests sent to other servers"
def request_remaining_votes(self, id_all_voters):
if not self.log.data:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
msg = RequestVoteMessage(self.id, self.current_term, last_log_index,
last_log_term)
for server in self.connected_servers:
server_id = host_to_id[server[0]]
if server_id not in id_all_voters:
self.channel.send(msg, id=server_id)
# print "Requesting vote from server", host_to_id[server[0]]
print "Vote requests sent to remaining servers who have not responded"
def check_status(self):
current_time = time.time()
if self.title == constants.TITLE_LEADER:
# Send AppendEntries to update follower logs
for server in self.connected_servers:
server_id = host_to_id[server[0]]
next_index = self.next_index[server_id]
# Send entries that the server has not received yet, if any
if self.log.last_log_index() >= next_index:
entries = self.construct_entries_list(next_index)
if next_index == 0:
prev_log_index = -1
prev_log_term = -1
else:
prev_log_index = self.log.get(next_index - 1).index
prev_log_term = self.log.get(next_index - 1).term
msg = AppendEntriesMessage(self.current_term, self.id,
prev_log_index, prev_log_term,
entries,
self.log.last_commit_index)
self.channel.send(msg, id=server_id)
print "AppendEntries sent to ", server_id
if current_time - self.last_heartbeat >= self.heartbeat_frequency:
self.send_heartbeats()
elif self.title == constants.TITLE_FOLLOWER and current_time - self.last_heartbeat > self.heartbeat_timeout:
# Heartbeat timeout passed as follower: Start election
print "Election timeout as follower. No heartbeat. Become candidate and start new election"
self.start_election()
elif self.title == constants.TITLE_CANDIDATE and current_time - self.election_start_time > self.election_timeout:
# Election timeout passed as candidate, without conclusion of election: Start new election
print "Election timeout as candidate. Election has not yet led to new leader. Starting new election"
self.set_election_timeout()
self.start_election()
elif self.title == constants.TITLE_CANDIDATE and current_time - self.election_start_time < self.election_timeout:
# Election timeout has not passed as candidate
print "As candidate, election timeout has not passed. Request votes from servers that have not responded"
id_all_voters = self.id_received_votes.union(self.id_refused_votes)
self.request_remaining_votes(id_all_voters)
def construct_entries_list(self, index):
entries = []
for i in range(index, len(self.log)):
entries.append(self.log.get(i))
return entries
def start_election(self):
self.title = constants.TITLE_CANDIDATE
self.reset_election_info()
self.current_term += 1
self.save_state()
# TODO: Voted_for must persist
self.voted_for = self.id
self.save_state()
self.update_votes(self.id, True)
self.election_start_time = time.time()
self.check_election_status()
self.request_votes()
def send_heartbeats(self):
heartbeat = AppendEntriesMessage(self.current_term, self.id, -1, -1,
[], self.log.last_commit_index)
for server in self.connected_servers:
self.channel.send(heartbeat, id=host_to_id[server[0]])
self.process_heartbeat()
def step_down(self):
# Step down as leader or candidate, convert to follower
# Reset various election variables
if self.title == constants.TITLE_LEADER or self.title == constants.TITLE_CANDIDATE:
self.title = constants.TITLE_FOLLOWER
self.process_heartbeat()
self.reset_election_info()
def grant_vote(self, candidate_id):
# TODO: Voted_for must persist
self.voted_for = candidate_id
self.save_state()
print "Grant vote to", candidate_id
self.channel.send(VoteReplyMessage(self.id, self.current_term, True),
id=candidate_id)
def refuse_vote(self, candidate_id):
self.channel.send(VoteReplyMessage(self.id, self.current_term, False),
id=candidate_id)
print "Refuse vote to", candidate_id
def majority(self):
return (len(self.connected_servers) + 1) / 2 + 1
def check_election_status(self):
if self.num_received_votes >= self.majority():
# Become leader when granted majority of votes
self.become_leader()
def become_leader(self):
self.title = constants.TITLE_LEADER
self.leader = self.id
print "Election won - I am now LEADER"
# TODO: Implement rest of leader initialization
self.next_index = [len(self.log) for _ in range(len(addr_to_id))]
if self.log.last_commit_index == -1:
latest_index = None
else:
latest_index = self.log.last_commit_index
if latest_index is None:
latest_term = 0
elif self.log.contains_at_index(latest_index):
latest_term = self.log.get(latest_index).term
else:
latest_term = 0
self.latest_index_term = [(latest_index, latest_term)
for _ in range(len(addr_to_id))]
self.latest_index_term[self.id] = (len(self.log) - 1,
self.current_term)
self.reset_election_info()
self.send_heartbeats()
def reset_election_info(self):
self.id_received_votes = set()
self.id_refused_votes = set()
self.voted_for = None
self.num_received_votes = 0
# server_id: server that sent vote reply; vote_granted: True if vote granted
def update_votes(self, server_id, vote_granted):
if vote_granted:
print "Received vote from", server_id
self.id_received_votes.add(server_id)
self.num_received_votes = len(self.id_received_votes)
print "Number of received votes is now", self.num_received_votes
else:
print "Denied vote from", server_id
self.id_refused_votes.add(server_id)
def update_commits(self):
index = max(self.next_index)
i_count = 0
t_count = 0
while i_count < self.majority() and index >= 0:
if index < 0:
print "Error: Update_commits: index is less than 0"
index -= 1
t_count = 0
i_count = 0
for (i, t) in self.latest_index_term:
if t == self.current_term:
t_count += 1
if i >= index:
i_count += 1
if t_count >= self.majority() and i_count >= self.majority():
if self.log.last_commit_index < index:
self.log.last_commit_index = index
self.save_state()
elif self.log.last_commit_index > index:
print "Error: Update_commits: new commit index is lower than current commit_index"
for entry in self.log.data:
if not entry.client_ack_sent:
# TODO: Send client ack
ack_message = AcknowledgeMessage(ack=True,
msg_id=entry.msg_id)
self.channel.send(ack_message, id=entry.author)
entry.client_ack_sent = True
def run(self):
print "Server with id=", self.id, " up and running"
while self.running:
self.update_connected_servers()
for server in list(addr_to_id.keys()):
# if server not in self.connected_servers and not addr_to_id[server] == id:
if server not in self.channel and not host_to_id[
server[0]] == self.id:
connected = self.channel.connect(server)
if connected:
print str("Server: Connected to " + server[0])
if server not in self.connected_servers:
self.connected_servers.append(server)
# print "Connected: ", connected
data = self.channel.receive(RECEIVE_FREQ)
if data:
# print "There is data on channel"
for server_id, msg in data:
self.process_msg(server_id, msg)
else:
self.check_status()
def process_msg(self, sender_id, msg):
#print "Processing message from", sender_id, "of type", msg.type
if msg.type == constants.MESSAGE_TYPE_REQUEST_VOTE:
self.process_request_vote(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_VOTE_REPLY:
self.process_vote_reply(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_REQUEST_LEADER:
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg, id=sender_id)
elif msg.type == constants.MESSAGE_TYPE_LOOKUP:
self.process_lookup(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_POST:
self.process_post(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_APPEND_ENTRIES:
self.process_append_entries(sender_id, msg)
elif msg.type == constants.MESSAGE_TYPE_ACKNOWLEDGE:
self.process_acknowledge(sender_id, msg)
# Used for testing purposes
elif msg.type == constants.MESSAGE_TYPE_TEXT:
print "From", msg.sender_id, ":", msg.msg
else:
print "Error: Invalid message type"
def process_lookup(self, sender_id, msg):
if self.title == constants.TITLE_LEADER or msg.override:
print "-----> Processing Lookup from client"
posts = self.log.get_committed_entries()
msg = messages.LookupMessage(msg_id=msg.msg_id,
post=posts,
server_id=self.id)
self.channel.send(msg=msg, id=sender_id)
else:
print "Lookup to leader"
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg=msg, id=sender_id)
def process_post(self, sender_id, msg):
if self.title == constants.TITLE_LEADER:
# TODO: Implement adding entry
# TODO: PERSIST; implement in log class?
entry = Entry(msg.post,
sender_id,
self.current_term,
len(self.log),
msg_id=msg.msg_id)
if self.log.append(entry):
self.save_state()
self.latest_index_term[self.id] = (len(self.log) - 1,
self.current_term)
print "---->Append entry from client to log"
else:
msg = messages.RequestLeaderMessage(leader=self.leader)
self.channel.send(msg=msg, id=sender_id)
def process_request_vote(self, sender_id, msg):
if not self.log:
# Log is empty
last_log_index = -1
last_log_term = -1
else:
last_log_index = self.log.get(-1).index
last_log_term = self.log.get(-1).term
# Handle message
if msg.term < self.current_term:
# If candidate's term is less than my term then refuse vote
print "Refuse vote to server", sender_id, "because I have higher term"
self.refuse_vote(msg.candidate_id)
if msg.term > self.current_term:
# If candidate's term is greater than my term then update current_term (latest term I've encountered),
# Step down if leader or candidate
self.current_term = msg.term
self.save_state()
# TODO: Step down if leader or candidate
self.step_down()
if msg.term >= self.current_term:
# If candidate's term is at least as new as mine and I have granted anyone else a vote
# and candidate's log is at least as complete as mine
# then grant vote
if self.voted_for is None or self.voted_for is msg.candidate_id:
if last_log_term < msg.last_log_term or (
last_log_term == msg.last_log_term
and last_log_index <= msg.last_log_index):
self.grant_vote(msg.candidate_id)
else:
# print "Cand term, current_term:", msg.term, self.current_term
# print "Voted for:", self.voted_for
# print "Cand log term, last_log_term", msg.last_log_term, last_log_term
# print "Cand log index, last_log_index", msg.last_log_index, last_log_index
self.refuse_vote(msg.candidate_id)
def process_vote_reply(self, sender_id, msg):
if msg.term > self.current_term and not msg.vote_granted:
# Step down if reply from someone with higher term
# Extra condition for security.
# If responder's term is higher, then vote should not be granted with correct execution
self.current_term = msg.term
self.save_state()
print "Denied vote from", msg.follower_id
self.step_down()
else:
# Take care of grant or refusal of vote
self.update_votes(msg.follower_id, msg.vote_granted)
self.check_election_status()
def process_acknowledge(self, sender_id, msg):
if msg.ack:
print "Process Acknowledge from server. ACK == TRUE"
self.next_index[sender_id] = msg.next_index
self.latest_index_term[sender_id] = msg.latest_index_term
self.update_commits()
else:
print "Process Acknowledge from server. ACK == FALSE"
if self.next_index[sender_id] - 1 < 0:
self.next_index[sender_id] = 0
else:
self.next_index[sender_id] -= 1
if msg.term > self.current_term:
self.current_term = msg.term
self.save_state()
self.step_down()
def process_append_entries(self, sender_id, msg):
if len(msg.entries) == 0:
self.process_heartbeat()
if msg.commit_index < len(self.log):
self.log.last_commit_index = msg.commit_index
self.save_state()
self.leader = sender_id
#print "Heartbeat received from server", sender_id
if self.title == constants.TITLE_CANDIDATE or self.title == constants.TITLE_LEADER:
self.step_down()
elif self.title == constants.TITLE_LEADER:
# TODO: If a "leader" receives a heartbeat,
# it might have crashed and joined back in after an election (?)
pass
else:
# TODO: Process AppendEntriesMessage
print "-->Processing AppendEntriesMessage from leader"
self.process_heartbeat()
if msg.term > self.current_term:
self.current_term = msg.term
self.save_state()
if self.title == constants.TITLE_CANDIDATE or self.title == constants.TITLE_LEADER:
self.step_down()
# Reject if my term is greater than leader term
if self.current_term > msg.term:
print "Error: Current term greater than leaders term"
self.channel.send(AcknowledgeMessage(ack=False,
term=self.current_term),
id=sender_id)
# Accept. Self.log is empty and leader is sending all entries
elif self.log.is_empty() and msg.prev_log_index == -1:
print "Appending entries"
# First entry to append is at index 0
if self.log.append_entries(msg.entries):
self.log.last_commit_index = msg.commit_index
self.save_state()
i = self.log.last_log_index()
t = self.log.get(i).term
self.channel.send(AcknowledgeMessage(
ack=True,
next_index=len(self.log),
latest_index_term=(i, t)),
id=sender_id)
print "Log after appending entries:"
self.log.show_data()
else:
print "DET HER SKAL IKKE SKJE 1"
# Accept. Check if self.log has an element at msg.prev_log_index
elif self.log.contains_at_index(msg.prev_log_index):
# Check if the term corresponds with msg.prev_log_term
if self.log.get(msg.prev_log_index).term == msg.prev_log_term:
if self.log.append_entries(msg.entries):
self.log.last_commit_index = msg.commit_index
self.save_state()
i = self.log.last_log_index()
t = self.log.get(i).term
self.channel.send(AcknowledgeMessage(
ack=True,
next_index=len(self.log),
latest_index_term=(i, t)),
id=sender_id)
print "Log after appending entries:"
self.log.show_data()
else:
print "DET HER SKAL IKKE SKJE NUMMER 2"
else:
self.log.remove(msg.prev_log_index)
self.channel.send(AcknowledgeMessage(ack=False),
id=sender_id)
else:
print "Send ACK-False"
self.channel.send(AcknowledgeMessage(ack=False), id=sender_id)
def save_state(self):
storage.save(self.id, self.voted_for, self.current_term, self.log)
def load_state(self):
self.voted_for, self.current_term, self.log = storage.load(self.id)
# print "voted for", self.voted_for
print self.current_term
print self.log
def update_connected_servers(self):
for addr in list(addr_to_id.keys()):
if addr in self.channel.address_to_connection.keys(
) and addr not in self.connected_servers:
self.connected_servers.append(id)
if addr not in self.channel.address_to_connection.keys(
) and addr in self.connected_servers:
self.connected_servers.remove(addr)
class Bounded_Semaphore_Thread(Thread):
def __init__(self, target, name='', args=(), DEBUG=False):
self.this_log = Log(prefix='thread_class',
log_dir=LOG_DIR,
DEBUG=DEBUG)
self.this_log.append("created log file")
msg = "thread_class.py --> Bounded_Semaphore_Thread --> inside init"
self.this_log.append(msg)
if not args:
msg = "thread_class.py --> Bounded_Semaphore_Thread --> args: EMPTY!"
self.this_log.append(msg)
msg = "thread_class.py --> Bounded_Semaphore_Thread --> name: %s" % name
self.this_log.append(msg)
msg = "thread_class.py --> Bounded_Semaphore_Thread --> args: %s" % str(
args)
self.this_log.append(msg)
Thread.__init__(self, target=target, name=name, args=args)
def run(self, DEBUG=False):
wait_start_time = datetime.datetime.now()
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s --> waiting: %s" % (
self.getName(), str(datetime.datetime.now().time()))
self.this_log.append(msg)
threadLimiter.acquire()
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s --> approved: %s" % (
self.getName(), str(datetime.datetime.now().time()))
self.this_log.append(msg)
wait_finish_time = datetime.datetime.now()
difference = evaluate_Time_Difference(wait_finish_time,
wait_start_time)
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s -------> wait time: %s" % (
self.getName(), difference)
self.this_log.append(msg)
try:
run_start_time = datetime.datetime.now()
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s --> running: %s" % (
self.getName(), str(datetime.datetime.now().time()))
self.this_log.append(msg)
# http://www.pythonforbeginners.com/super/working-python-super-function
super(Bounded_Semaphore_Thread, self).run()
finally:
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s -> completed: %s" % (
self.getName(), str(datetime.datetime.now().time()))
self.this_log.append(msg)
run_finish_time = datetime.datetime.now()
difference = evaluate_Time_Difference(run_finish_time,
run_start_time)
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s --------> run time: %s" % (
self.getName(), difference)
self.this_log.append(msg)
difference = evaluate_Time_Difference(run_finish_time,
wait_start_time)
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s ------> total time: %s" % (
self.getName(), difference)
print msg
self.this_log.append(msg)
threadLimiter.release()
msg = "thread_class.py --> Bounded_Semaphore_Thread --> %s --> released: %s" % (
self.getName(), str(datetime.datetime.now().time()))
self.this_log.append(msg)
class Server(object):
def __init__(self, peers, host, port):
self.host = host
self.port = port
self.peer_id = '{}:{}'.format(host, port)
self._logger = logging.getLogger(__name__)
self._loop = asyncio.get_event_loop()
self._pool = Pool(self, peers)
# heartbeat constants and bookkeeping variables
self._heartbeat_interval = 1000 # ms
self._last_interval = None
self._min_heartbeat_timeout = 2000 # ms
self._max_heartbeat_timeout = 4000 # ms
self._heartbeat_timeout = None
self._last_heartbeat = None
self.reset_heartbeat()
self.reset_timeout()
self._log = Log(Machine())
self.state = State.FOLLOWER
self.term = 0
self.voted = None
self.votes = set()
self._pending_clients = {}
self.handlers = {'append_entries_req': self.handle_append_entries_req,
'append_entries_resp': self.handle_append_entries_resp,
'request_vote_req': self.handle_request_vote_req,
'request_vote_resp': self.handle_request_vote_resp}
def reset_heartbeat(self):
self._last_heartbeat = self._loop.time()
def reset_interval(self):
self._last_interval = self._loop.time()
def reset_timeout(self):
self._heartbeat_timeout = randint(self._min_heartbeat_timeout,
self._max_heartbeat_timeout) / 1000
@property
def stale(self):
return self._last_heartbeat + self._heartbeat_timeout < self._loop.time()
@staticmethod
def decode(data):
return json.loads(data.decode())
@staticmethod
def encode(data):
return json.dumps(data).encode()
def broadcast(self, request):
for peer in self._pool:
self.send_async(peer, request)
@asyncio.coroutine
def run(self):
self.reset_interval()
while True:
self._logger.debug('state: {}, term: {}'.format(self.state,
self.term))
if self.state == State.LEADER:
self.append_entries()
if self.state in (State.CANDIDATE, State.FOLLOWER) and self.stale:
self.request_vote()
yield from self.wait()
@asyncio.coroutine
def send(self, peer, request):
"""
Send a request to a peer (if available).
"""
transport = yield from peer.get_transport()
if transport:
transport.write(self.encode(request))
def send_async(self, peer, request):
"""
Schedule the execution
"""
asyncio.async(self.send(peer, request))
@asyncio.coroutine
def wait(self):
"""
Wait for the next interval.
"""
tic = self._heartbeat_interval / 1000 - self._loop.time() + self._last_interval
yield from asyncio.sleep(tic)
self.reset_interval()
def to_leader(self):
self.append_entries()
self.state = State.LEADER
self.voted = None
self.votes = set()
for peer in self._pool:
peer.match = -1
peer.next = self._log.index + 1
def to_follower(self, term):
self.state = State.FOLLOWER
self.term = term
self.voted = None
self.votes = set()
def append_entries(self, peer=None):
"""
Append entries RPC.
"""
peers = self._pool.all() if peer is None else [peer]
for peer in peers:
log_entries = self._log[peer.next:]
log_index, log_term, _ = self._log[peer.next - 1]
request = {'rpc': 'append_entries_req',
'peer_id': self.peer_id,
'term': self.term,
'log_commit': self._log.commit,
'log_entries': log_entries,
'log_index': log_index,
'log_term': log_term,
}
self.send_async(peer, request)
self._logger.debug('broadcasting append entries')
def request_vote(self):
"""
Request vote RPC.
"""
self.reset_heartbeat()
self.reset_timeout()
self.state = State.CANDIDATE
self.term += 1
self.voted = self.peer_id
self.votes = set([self.peer_id])
request = {'rpc': 'request_vote_req',
'peer_id': self.peer_id,
'term': self.term,
'log_index': self._log.index,
'log_term': self._log.term,
}
self.broadcast(request)
self._logger.debug('broadcasting request vote')
def handle_peer(self, request):
"""
Dispatch requests to the appropriate handlers.
"""
if self.term < request['term']:
self.to_follower(request['term'])
return self.handlers[request['rpc']](request)
def handle_append_entries_req(self, request):
self._logger.debug('append entries request received')
if request['term'] < self.term:
return
self.reset_heartbeat()
log_index = request['log_index']
log_term = request['log_term']
if not self._log.match(log_index, log_term):
return {'rpc': 'append_entries_resp',
'peer_id': self.peer_id,
'term': self.term,
'log_index': self._log.index,
'success': False
}
log_entries = request['log_entries']
self._log.append(log_index, log_entries)
log_commit = request['log_commit']
if self._log.commit < log_commit:
index = min(self._log.index, log_commit)
self._log.commit = index
self._log.apply(index)
if not log_entries: # no need to answer, the peer might have committed
return # new entries but has certainly not replicated new ones
return {'rpc': 'append_entries_resp',
'peer_id': self.peer_id,
'term': self.term,
'log_index': self._log.index,
'log_term': self._log.term,
'success': True,
}
def handle_append_entries_resp(self, response):
if response['success']:
self._logger.debug('append entries succeeded')
log_index = response['log_index']
log_term = response['log_term']
peer_id = response['peer_id']
self._pool[peer_id].match = log_index
self._pool[peer_id].next = log_index + 1
if (self._log.commit < log_index and
self._pool.ack(log_index) and log_term == self.term):
self._log.commit = log_index
results = self._log.apply(log_index)
self.return_results(results)
else:
peer = self._pool[response['peer_id']]
peer.next -= 1
self.append_entries(peer)
# self._logger.debug('append entries failed')
def handle_request_vote_req(self, request):
self._logger.debug('request vote request received')
if request['term'] < self.term:
return
log_index = request['log_index']
log_term = request['log_term']
peer_id = request['peer_id']
if self.voted in (None, peer_id) and self._log.match(log_index, log_term):
granted = True
self.reset_heartbeat()
else:
granted = False
return {'rpc': 'request_vote_resp',
'peer_id': self.peer_id,
'term': self.term,
'granted': granted,
}
def handle_request_vote_resp(self, response):
if self.term == response['term'] and response['granted']:
self.votes.add(response['peer_id'])
if self._pool.majority(len(self.votes)):
self.to_leader()
def handle_client(self, cmd, transport):
self._log.add(self.term, cmd)
self._pending_clients[(self.term, self._log.index)] = transport
self.append_entries()
def return_results(self, results):
for result in results:
term, index, result = result
transport = self._pending_clients.pop((term, index))
transport.write(self.encode(result))
transport.close()