def update(self, key, value):
if self.master_stub is not None:
response = self.master_stub.get_chunk(
MasterServer_pb2.Key(key=key))
if response.code == 200:
# print('connect to chunk {}'.format(response.id))
with grpc.insecure_channel('{}:{}'.format(
response.ip, response.port)) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
if type(value) == list or type(value) == dict:
v_type = 'json'
value = json.dumps(value)
else:
v_type = str(type(value))
value = str(value)
response = stub.update_key(
ChunkServer_pb2.Value(key=key,
value=value,
type=v_type))
if response.code != 200:
print('error: {}'.format(response.msg))
else:
print('error: {}'.format(response.msg))
else:
print('update failed: please reconnect')
def get(self, key):
if self.master_stub is not None:
response = self.master_stub.get_chunk(
MasterServer_pb2.Key(key=key))
if response.code == 200:
# return {'id': response.id, 'ip': response.ip, 'port': response.port}
# print('connect to chunk {}'.format(response.id))
with grpc.insecure_channel('{}:{}'.format(
response.ip, response.port)) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.get_key(ChunkServer_pb2.Key(key=key))
if response.code == 200:
if response.type == 'json':
value = json.loads(response.value)
elif response.type == "<class 'float'>":
value = float(response.value)
elif response.type == "<class 'int'>":
value = int(response.value)
else:
value = str(response.value)
return value
else:
print('error: {}'.format(response.msg))
else:
print('error: {}'.format(response.msg))
else:
print('get failed: please reconnect')
def _select(self):
primary_id = self.id
for mate in self.mates:
with grpc.insecure_channel(
'{}:{}'.format(self.mates[mate]['ip'],
self.mates[mate]['port'])) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.select_primary(
ChunkServer_pb2.Secondary(id=self.id,
ip=self.ip,
port=self.port))
if response.id != self.id:
primary_id = response.id
break
if primary_id == self.id:
self.role = 'primary'
for mate in self.mates:
with grpc.insecure_channel('{}:{}'.format(
self.mates[mate]['ip'],
self.mates[mate]['port'])) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
stub.replace_primary(
ChunkServer_pb2.Primary(id=self.id,
ip=self.ip,
port=self.port))
with grpc.insecure_channel('{}:{}'.format(
self.master_ip, self.master_port)) as channel:
stub = MasterServer_pb2_grpc.MasterServerStub(channel)
stub.replace_chunk(
MasterServer_pb2.Replace(old_chunk=MasterServer_pb2.Chunk(
id=self.primary_id,
ip=self.primary_ip,
port=self.primary_port),
new_chunk=MasterServer_pb2.Chunk(
id=self.id,
ip=self.ip,
port=self.port,
num_keys=len(self.table))))
def _detect_heart(self):
while True:
for secondary in list(self.secondaries.keys()):
try:
with grpc.insecure_channel('{}:{}'.format(
self.secondaries[secondary]['ip'],
self.secondaries[secondary]['port'])) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.heart(ChunkServer_pb2.Empty())
if response.code != 200:
del self.secondaries[secondary]
except Exception as e:
print(e)
del self.secondaries[secondary]
for mate in list(self.mates.keys()):
try:
with grpc.insecure_channel('{}:{}'.format(
self.mates[mate]['ip'],
self.mates[mate]['port'])) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.heart(ChunkServer_pb2.Empty())
if response.code != 200:
del self.mates[mate]
except Exception as e:
print(e)
del self.mates[mate]
try:
with grpc.insecure_channel('{}:{}'.format(
self.master_ip, self.master_port)) as channel:
stub = MasterServer_pb2_grpc.MasterServerStub(channel)
response = stub.heart(MasterServer_pb2.Empty())
if response.code == 200:
self.master_status = True
else:
self.master_status = False
except Exception as e:
print(e)
self.master_status = False
if self.role != 'primary':
try:
with grpc.insecure_channel('{}:{}'.format(
self.primary_ip, self.primary_port)) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.heart(ChunkServer_pb2.Empty())
if response.code != 200:
self._select()
except Exception as e:
print('error: {}'.format(e))
self._select()
time.sleep(settings.Detect_Heart_Time)
def delete(self, key):
if self.master_stub is not None:
response = self.master_stub.delete_key(
MasterServer_pb2.Key(key=key))
if response.code == 200:
with grpc.insecure_channel('{}:{}'.format(
response.ip, response.port)) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.delete_key(ChunkServer_pb2.Key(key=key))
if response.code != 200:
print('error: '.format(response.msg))
else:
print('error: {}'.format(response.msg))
else:
print('delete failed: please reconnect')
def __init__(self,
id,
ip,
port,
master_ip,
master_port,
role='primary',
primary_id=None,
primary_ip=None,
primary_port=None):
self.id = id
self.ip = ip
self.port = port
self.role = role
self.master_ip = master_ip
self.master_port = master_port
self.master_status = True
self.primary_id = primary_id
self.primary_ip = primary_ip
self.primary_port = primary_port
self.secondaries = {}
self.mates = {}
self.table = {} # id:{value:...,type:...}
self.secondaries_lock = threading.Lock()
self.table_lock = threading.Lock()
self.mates_lock = threading.Lock()
if self.role == 'primary':
if os.path.exists(settings.Table_File):
with open(settings.Table_File, 'r') as file:
self.table = json.load(file)
else:
with grpc.insecure_channel('{}:{}'.format(
self.master_ip, self.master_port)) as channel:
stub = MasterServer_pb2_grpc.MasterServerStub(channel)
response = stub.add_chunk(
MasterServer_pb2.Chunk(id=self.id,
ip=self.ip,
port=self.port,
num_keys=len(self.table)))
if response.code != 200:
print('error: {}'.format(response.msg))
r = input('Replace the existed chunk? (y or n)')
if r == 'y':
response = stub.replace_chunk(
MasterServer_pb2.Replace(
old_chunk=MasterServer_pb2.Chunk(
id=self.id),
new_chunk=MasterServer_pb2.Chunk(
id=self.id,
ip=self.ip,
port=self.port,
num_keys=len(self.table))))
if response.code != 200:
print('add chunk failed, error: {}'.format(
response.msg))
exit(0)
else:
exit(0)
else:
with grpc.insecure_channel('{}:{}'.format(
self.primary_ip, self.primary_port)) as channel:
stub = ChunkServer_pb2_grpc.ChunkServerStub(channel)
response = stub.sync_tables(ChunkServer_pb2.Empty())
for item in response:
self.table[item.key] = {
'value': item.value,
'type': item.type
}
response = stub.sync_mates(ChunkServer_pb2.Empty())
for mate in response:
self.mates[mate.id] = {'ip': mate.ip, 'port': mate.port}
response = stub.add_secondary(
ChunkServer_pb2.Secondary(id=self.id,
ip=self.ip,
port=self.port))
if response.code != 200:
print('error: {}'.format(response.msg))
exit(0)