def update(self, key, value): if self.master_stub is not None: response = self.master_stub.get_chunk( MasterServer_pb2.Key(key=key)) if response.code == 200: # print('connect to chunk {}'.format(response.id)) with grpc.insecure_channel('{}:{}'.format( response.ip, response.port)) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) if type(value) == list or type(value) == dict: v_type = 'json' value = json.dumps(value) else: v_type = str(type(value)) value = str(value) response = stub.update_key( ChunkServer_pb2.Value(key=key, value=value, type=v_type)) if response.code != 200: print('error: {}'.format(response.msg)) else: print('error: {}'.format(response.msg)) else: print('update failed: please reconnect')
def get(self, key): if self.master_stub is not None: response = self.master_stub.get_chunk( MasterServer_pb2.Key(key=key)) if response.code == 200: # return {'id': response.id, 'ip': response.ip, 'port': response.port} # print('connect to chunk {}'.format(response.id)) with grpc.insecure_channel('{}:{}'.format( response.ip, response.port)) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.get_key(ChunkServer_pb2.Key(key=key)) if response.code == 200: if response.type == 'json': value = json.loads(response.value) elif response.type == "<class 'float'>": value = float(response.value) elif response.type == "<class 'int'>": value = int(response.value) else: value = str(response.value) return value else: print('error: {}'.format(response.msg)) else: print('error: {}'.format(response.msg)) else: print('get failed: please reconnect')
def _select(self): primary_id = self.id for mate in self.mates: with grpc.insecure_channel( '{}:{}'.format(self.mates[mate]['ip'], self.mates[mate]['port'])) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.select_primary( ChunkServer_pb2.Secondary(id=self.id, ip=self.ip, port=self.port)) if response.id != self.id: primary_id = response.id break if primary_id == self.id: self.role = 'primary' for mate in self.mates: with grpc.insecure_channel('{}:{}'.format( self.mates[mate]['ip'], self.mates[mate]['port'])) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) stub.replace_primary( ChunkServer_pb2.Primary(id=self.id, ip=self.ip, port=self.port)) with grpc.insecure_channel('{}:{}'.format( self.master_ip, self.master_port)) as channel: stub = MasterServer_pb2_grpc.MasterServerStub(channel) stub.replace_chunk( MasterServer_pb2.Replace(old_chunk=MasterServer_pb2.Chunk( id=self.primary_id, ip=self.primary_ip, port=self.primary_port), new_chunk=MasterServer_pb2.Chunk( id=self.id, ip=self.ip, port=self.port, num_keys=len(self.table))))
def _detect_heart(self): while True: for secondary in list(self.secondaries.keys()): try: with grpc.insecure_channel('{}:{}'.format( self.secondaries[secondary]['ip'], self.secondaries[secondary]['port'])) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.heart(ChunkServer_pb2.Empty()) if response.code != 200: del self.secondaries[secondary] except Exception as e: print(e) del self.secondaries[secondary] for mate in list(self.mates.keys()): try: with grpc.insecure_channel('{}:{}'.format( self.mates[mate]['ip'], self.mates[mate]['port'])) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.heart(ChunkServer_pb2.Empty()) if response.code != 200: del self.mates[mate] except Exception as e: print(e) del self.mates[mate] try: with grpc.insecure_channel('{}:{}'.format( self.master_ip, self.master_port)) as channel: stub = MasterServer_pb2_grpc.MasterServerStub(channel) response = stub.heart(MasterServer_pb2.Empty()) if response.code == 200: self.master_status = True else: self.master_status = False except Exception as e: print(e) self.master_status = False if self.role != 'primary': try: with grpc.insecure_channel('{}:{}'.format( self.primary_ip, self.primary_port)) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.heart(ChunkServer_pb2.Empty()) if response.code != 200: self._select() except Exception as e: print('error: {}'.format(e)) self._select() time.sleep(settings.Detect_Heart_Time)
def delete(self, key): if self.master_stub is not None: response = self.master_stub.delete_key( MasterServer_pb2.Key(key=key)) if response.code == 200: with grpc.insecure_channel('{}:{}'.format( response.ip, response.port)) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.delete_key(ChunkServer_pb2.Key(key=key)) if response.code != 200: print('error: '.format(response.msg)) else: print('error: {}'.format(response.msg)) else: print('delete failed: please reconnect')
def __init__(self, id, ip, port, master_ip, master_port, role='primary', primary_id=None, primary_ip=None, primary_port=None): self.id = id self.ip = ip self.port = port self.role = role self.master_ip = master_ip self.master_port = master_port self.master_status = True self.primary_id = primary_id self.primary_ip = primary_ip self.primary_port = primary_port self.secondaries = {} self.mates = {} self.table = {} # id:{value:...,type:...} self.secondaries_lock = threading.Lock() self.table_lock = threading.Lock() self.mates_lock = threading.Lock() if self.role == 'primary': if os.path.exists(settings.Table_File): with open(settings.Table_File, 'r') as file: self.table = json.load(file) else: with grpc.insecure_channel('{}:{}'.format( self.master_ip, self.master_port)) as channel: stub = MasterServer_pb2_grpc.MasterServerStub(channel) response = stub.add_chunk( MasterServer_pb2.Chunk(id=self.id, ip=self.ip, port=self.port, num_keys=len(self.table))) if response.code != 200: print('error: {}'.format(response.msg)) r = input('Replace the existed chunk? (y or n)') if r == 'y': response = stub.replace_chunk( MasterServer_pb2.Replace( old_chunk=MasterServer_pb2.Chunk( id=self.id), new_chunk=MasterServer_pb2.Chunk( id=self.id, ip=self.ip, port=self.port, num_keys=len(self.table)))) if response.code != 200: print('add chunk failed, error: {}'.format( response.msg)) exit(0) else: exit(0) else: with grpc.insecure_channel('{}:{}'.format( self.primary_ip, self.primary_port)) as channel: stub = ChunkServer_pb2_grpc.ChunkServerStub(channel) response = stub.sync_tables(ChunkServer_pb2.Empty()) for item in response: self.table[item.key] = { 'value': item.value, 'type': item.type } response = stub.sync_mates(ChunkServer_pb2.Empty()) for mate in response: self.mates[mate.id] = {'ip': mate.ip, 'port': mate.port} response = stub.add_secondary( ChunkServer_pb2.Secondary(id=self.id, ip=self.ip, port=self.port)) if response.code != 200: print('error: {}'.format(response.msg)) exit(0)