def put_once(self, key, value):
# return if_succeed, (ip, port)
# if_succeed:
# 0 for succeeded
# 1 for redirect
# 2 for unknown
# RPC Put():
# return if_succeeded
# 1 for none_request
# 2 for cur_server_is_not_leader, tailed with leader_ip, leader_port
# 3 exceed_time_limit
with grpc.insecure_channel(self.leader_ip + ':' +
self.leader_port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
try:
response = stub.Put(
storage_service_pb2.PutRequest(
key=key,
value=value,
serial_no=str(random.randint(0, 10000))))
if response.ret == 0:
return 0, (0, 0) # if_succeed, (ip, port)
elif response.ret == 2 and response.leader_ip != '' and response.leader_port != '':
return 1, (response.leader_ip, response.leader_port)
else:
return 2, (0, 0)
except Exception as e:
print(str(e))
return -1, (0, 0)
def replicate_log_entries_to_one(self, ip, port, receiver_index,
ae_succeed_cnt, lock_ae_succeed_cnt):
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
request, new_nextIndex = self.generate_append_entry_request(
receiver_index)
try:
response = stub.AppendEntries(request,
timeout=float(
self.configs['rpc_timeout']))
except Exception:
self.logger.info('AppendEntries call to node #' +
str(receiver_index) + ' timed out.')
return
if response.success:
self.update_nextIndex_and_matchIndex(receiver_index,
new_nextIndex)
# increment ae_succeed_cnt
with lock_ae_succeed_cnt:
ae_succeed_cnt[0] += 1
elif response.failed_for_term:
self.convert_to_follower(response.term, receiver_index)
else:
# AppendEntries failed because of log inconsistency
self.decrement_nextIndex(receiver_index)
self.replicate_log_entries_to_one(ip, port, receiver_index,
ae_succeed_cnt,
lock_ae_succeed_cnt)
def put(key, value):
ip, port = pickANodeRandomly()
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
response = stub.Put(
storage_service_pb2.PutRequest(key=key, value=value))
if response.ret == 1:
print('Success!')
def get(key):
ip, port = pickANodeRandomly()
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
response = stub.Get(storage_service_pb2.GetRequest(key=key))
if response.ret == 1:
print(response.value)
else:
print('Failed!')
def put(key, value):
ip, port = pickANodeRandomly()
with grpc.insecure_channel(ip+':'+port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
try:
response = stub.Put(storage_service_pb2.PutRequest(key=key, value=value))
if response.ret == 0:
print('Success!')
except Exception as e:
print('RPC call failed!\n' + str(e))
def debug_get_variable(self, variable, ip=None, port=None):
if not ip:
ip = self.leader_ip
port = self.leader_port
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
try:
response = stub.DEBUG_GetVariable(
storage_service_pb2.DEBUG_GetVariable_Resquest(
variable=variable))
print(response.value)
except Exception as e:
print('RPC call failed!\n' + str(e))
def partition_network(self, num_of_nodes_with_leader):
# make the first num_of_nodes_with_leader nodes(or num_of_nodes_with_leader - 1) in the same partition as the leader
all_correct = True
for ip, port in self.configs['nodes']:
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
response = stub.Partition(
storage_service_pb2.PartitionRequest(
num_of_nodes_with_leader=num_of_nodes_with_leader))
if response.ret == 1:
all_correct = False
if not all_correct:
return 1
return 0
def heartbeat_once_to_one(self,
ip,
port,
node_index,
hb_success_error_cnt,
hb_success_error_lock,
is_sync_entry=True):
self.logger.info("sending heartbeat to node_{}_{}_{}".format(
node_index, ip, port))
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
request, new_nextIndex = self.generate_append_entry_request(
node_index)
try:
response = stub.AppendEntries(request,
timeout=float(
self.configs['rpc_timeout']))
except Exception:
self.logger.info(
"(Node#{})Timeout error when heartbeat to {}".format(
self.node_index, node_index))
return
if response.success:
self.update_nextIndex_and_matchIndex(node_index, new_nextIndex)
self.logger.info(
'Heartbeat to {} succeeded.'.format(node_index))
if hb_success_error_lock:
with hb_success_error_lock:
hb_success_error_cnt[0] += 1
elif response.failed_for_term:
# voter's term is larger than the leader, so leader changes to follower
if hb_success_error_lock:
with hb_success_error_lock:
hb_success_error_lock[1] += 1
self.convert_to_follower(response.term, node_index)
else:
# AppendEntries failed because of log inconsistency
if not is_sync_entry:
# when there is no need to sync entries to followers in heartbeart
return
self.logger.error(
'Node #{} When heartbeat to node{}, inconsistency detected.'
.format(self.node_index, node_index))
self.decrement_nextIndex(node_index)
self.heartbeat_once_to_one(ip, port, node_index,
hb_success_error_cnt,
hb_success_error_lock)
def broadcast_to_all_nodes(self, request):
print('Start broadcast to other nodes')
for ip, port in self.configs['nodes']:
if ip != self.myIp or port != self.myPort:
print('Addr to connect: ' + ip + ":" + port)
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
try:
response = stub.Put_from_broadcast(
storage_service_pb2.PutRequestToOtherServer(
key=request.key,
value=request.value,
from_node=str(self.node_index)))
print('Response from port' + str(port) + ":" +
str(response.ret))
except Exception as e:
print('RPC call failed! (broadcast)')
print(e)
def get_once(self, key):
# return if_succeed, value, (ip, port)
# if_succeed:
# 0 for succeeded
# 1 for redirect
# 2 for key_not_exist
# 3 for unknown
with grpc.insecure_channel(self.leader_ip + ':' +
self.leader_port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
try:
response = stub.Get(storage_service_pb2.GetRequest(key=key))
if response.ret == 0:
return 0, response.value, (0, 0)
elif response.ret == 1:
if response.leader_ip != '' and response.leader_port != '':
return 1, 0, (response.leader_ip, response.leader_port)
elif response.ret == 2:
return 2, 0, (0, 0)
else:
return 3, 0, (0, 0)
except Exception as e:
print(str(e))
return -1, 0, (0, 0)
def ask_for_vote_to_one(self, ip, port, node_index):
self.logger.info('{} ask vote from {} in term {}'.format(
self.node_index, node_index, self.currentTerm))
with grpc.insecure_channel(ip + ':' + port) as channel:
stub = storage_service_pb2_grpc.KeyValueStoreStub(channel)
request = self.generate_requestVote_request()
try:
self.logger.info('Node #{} asks for the vote by {}'.format(
self.node_index, node_index))
response = stub.RequestVote(request,
timeout=float(
self.configs['rpc_timeout']))
# when the response packets delay and detour in the network so than this response is an stale
if not response.voteGranted:
return
# print(response.term)
if response.term < self.currentTerm: # expired vote
return
elif response.term > self.currentTerm:
self.convert_to_follower(request.term, node_index)
else:
if response.voteGranted:
with self.lock_persistent_operations:
self.voteCnt += 1
self.logger.info(
"get one vote from node {}, current voteCnt is {}"
.format(node_index, self.voteCnt))
majority_cnt = len(self.configs['nodes']) // 2 + 1
if self.state != 2 and self.voteCnt >= majority_cnt:
self.convert_to_leader()
except Exception as e:
self.logger.info(
"Node #{} Timeout error when requestVote to {}".format(
self.node_index, node_index))