def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
replica_count = packet.parameters.get('replica_count', None)
if key is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Key is not found in request packet!')
if replica_count is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Replica count should be passed to GetKeysInfo operation')
self._validate_key(key)
keys = KeyUtils.get_all_keys(key, replica_count)
is_replica = False
ret_keys = []
for key in keys:
long_key = self._validate_key(key)
range_obj = self.operator.ranges_table.find(long_key)
if not range_obj:
logger.warning('[GetKeysInfoOperation] Internal error: No hash range found for key=%s!'%key)
else:
ret_keys.append((key, is_replica, range_obj.node_address))
is_replica = True
return FabnetPacketResponse(ret_parameters={'keys_info': ret_keys})
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
if key is not None:
self._validate_key(key)
replica_count = int(packet.parameters.get('replica_count', MIN_REPLICA_COUNT))
if replica_count < MIN_REPLICA_COUNT:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Minimum replica count is equal to %s!'%MIN_REPLICA_COUNT)
is_replica = False
keys = KeyUtils.generate_new_keys(self.operator.node_name, replica_count, prime_key=key)
ret_keys = []
for key in keys:
range_obj = self.operator.ranges_table.find(long(key, 16))
if not range_obj:
logger.debug('[PutKeysInfoOperation] Internal error: No hash range found for key=%s!'%key)
else:
ret_keys.append((key, is_replica, range_obj.node_address))
is_replica = True
return FabnetPacketResponse(ret_parameters={'keys_info': ret_keys})
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
primary_key = packet.parameters.get('primary_key', None)
replica_count = packet.parameters.get('replica_count', None)
key = packet.parameters.get('key', None)
checksum = packet.parameters.get('checksum', None)
is_replica = packet.parameters.get('is_replica', False)
user_id = packet.parameters.get('user_id', None)
carefully_save = packet.parameters.get('carefully_save', False)
if not packet.binary_data:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Binary data does not found!')
if key is None:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='key does not found!')
key = KeyUtils.to_hex(key)
data_list = []
if primary_key:
primary_key = KeyUtils.to_hex(primary_key)
if replica_count is None:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='replica count for data block does not found!')
if checksum is None:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='data checksum does not found!')
if user_id is None:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='user_id does not found!')
header = DataBlockHeader.pack(primary_key, replica_count, checksum, user_id)
data_list.append(header)
data_list.append(packet.binary_data)
tempfile_path = self.operator.get_tempfile()
try:
tempfile = TmpFile(tempfile_path, data_list)
self.operator.put_data_block(key, tempfile_path, is_replica, carefully_save)
except FSHashRangesOldDataDetected, err:
return FabnetPacketResponse(ret_code=RC_OLD_DATA, ret_message=str(err))
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
data = packet.binary_data
key = packet.parameters.get('key', None)
if key is not None:
self._validate_key(key)
checksum = packet.parameters.get('checksum', None)
replica_count = int(packet.parameters.get('replica_count', MIN_REPLICA_COUNT))
wait_writes_count = int(packet.parameters.get('wait_writes_count', 1))
if checksum is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Checksum does not found in request packet!')
if wait_writes_count > replica_count:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Cant waiting more replicas than saving!')
if replica_count < MIN_REPLICA_COUNT:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Minimum replica count is equal to %s!'%MIN_REPLICA_COUNT)
data_block = DataBlock(data, checksum)
data_block.validate()
succ_count = 0
is_replica = False
keys = KeyUtils.generate_new_keys(self.operator.node_name, replica_count, prime_key=key)
data, checksum = data_block.pack(keys[0], replica_count)
for key in keys:
range_obj = self.operator.ranges_table.find(long(key, 16))
if not range_obj:
logger.debug('[ClientPutOperation] Internal error: No hash range found for key=%s!'%key)
else:
params = {'key': key, 'checksum': checksum, 'is_replica': is_replica}
if succ_count >= wait_writes_count:
self._init_operation(range_obj.node_address, 'PutDataBlock', params, binary_data=data)
else:
resp = self._init_operation(range_obj.node_address, 'PutDataBlock', params, sync=True, binary_data=data)
if resp.ret_code:
logger.debug('[ClientPutOperation] PutDataBlock error from %s: %s'%(range_obj.node_address, resp.ret_message))
else:
succ_count += 1
is_replica = True
if wait_writes_count < succ_count:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Writing data error!')
return FabnetPacketResponse(ret_parameters={'key': keys[0]})
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
replica_count = packet.parameters.get('replica_count', None)
if key is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Key is not found in request packet!')
if replica_count is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Replica count should be passed to GetData operation')
self._validate_key(key)
keys = KeyUtils.get_all_keys(key, replica_count)
data = None
checksum = None
is_replica = False
for key in keys:
long_key = self._validate_key(key)
range_obj = self.operator.find_range(long_key)
if not range_obj:
logger.warning('[ClientGetOperation] Internal error: No hash range found for key=%s!'%key)
else:
_, _, node_address = range_obj
resp = self._init_operation(node_address, 'GetDataBlock', {'key': key, 'is_replica': is_replica, \
'user_id': packet.session_id}, sync=True)
if resp.ret_code == RC_PERMISSION_DENIED:
return FabnetPacketResponse(ret_code=RC_PERMISSION_DENIED, ret_message=resp.ret_message)
elif resp.ret_code:
logger.warning('[ClientGetOperation] GetDataBlock error from %s: %s'%(node_address, resp.ret_message))
else:
data = resp.binary_data
checksum = resp.ret_parameters['checksum']
break
is_replica = True
if data is None:
return FabnetPacketResponse(ret_code=RC_NO_DATA, ret_message='No data found!')
return FabnetPacketResponse(binary_data=data, ret_parameters={'checksum': checksum})
def __process_data_block(self, key, raw_data, is_replica=False):
self.__processed_local_blocks += 1
try:
raw_header = raw_data.read(DataBlockHeader.HEADER_LEN)
primary_key, replica_count, checksum, user_id, stored_dt = DataBlockHeader.unpack(raw_header)
if not is_replica:
if key != primary_key:
raise Exception("Primary key is invalid: %s != %s" % (key, primary_key))
data_keys = KeyUtils.get_all_keys(primary_key, replica_count)
if is_replica:
if key not in data_keys:
raise Exception("Replica key is invalid: %s" % key)
except Exception, err:
self.__invalid_local_blocks += 1
logger.error("[RepairDataBlocks] %s" % err)
return
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
if key is not None:
self._validate_key(key)
else:
key = KeyUtils.generate_key(self.node_name)
range_obj = self.operator.find_range(key)
if not range_obj:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message=\
'[PutKeysInfoOperation] Internal error: No hash range found for key=%s!'%key)
_, _, node_address = range_obj
return FabnetPacketResponse(ret_parameters={'key_info': (key, node_address)})
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
if key is None:
raise Exception('key parameter is expected!')
replica_count = packet.parameters.get('replica_count', None)
if replica_count is None:
raise Exception('replica_count parameter is expected!')
replica_count = int(replica_count)
self._validate_key(key)
keys = KeyUtils.generate_new_keys(self.node_name, replica_count, prime_key=key)
is_replica = False
for key in keys:
h_range = self.operator.find_range(key)
if not h_range:
return FabnetPacketResponse(ret_code=RC_ERROR, \
ret_message='Internal error: No hash range found for key=%s!'%key)
else:
_, _, node_address = h_range
params = {'key': key, 'is_replica': is_replica, \
'carefully_delete': True, 'user_id': packet.session_id}
resp = self._init_operation(node_address, 'DeleteDataBlock', params, sync=True)
if resp.ret_code != RC_OK:
return FabnetPacketResponse(ret_code=resp.ret_code, \
ret_message='DeleteDataBlock failed at %s: %s'%(node_address, resp.ret_message))
is_replica = True
return FabnetPacketResponse()
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
if not packet.binary_data:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='No binary data found!')
key = packet.parameters.get('key', None)
if key is not None:
self._validate_key(key)
replica_count = int(packet.parameters.get('replica_count', MIN_REPLICA_COUNT))
wait_writes_count = int(packet.parameters.get('wait_writes_count', 1))
if wait_writes_count <= 0:
wait_writes_count = 1
if wait_writes_count > (replica_count+1):
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Cant waiting more replicas than saving!')
if replica_count < MIN_REPLICA_COUNT:
return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Minimum replica count is equal to %s!'%MIN_REPLICA_COUNT)
succ_count = 0
carefully_save = True
is_replica = False
errors = []
local_save = None
keys = KeyUtils.generate_new_keys(self.node_name, replica_count, prime_key=key)
tempfile_path = self.operator.get_tempfile()
tempfile = TmpFile(tempfile_path, packet.binary_data, seek=DataBlockHeader.HEADER_LEN)
checksum = tempfile.checksum()
header = DataBlockHeader.pack(keys[0], replica_count, checksum, packet.session_id)
tempfile.write(header, seek=0)
for key in keys:
h_range = self.operator.find_range(key)
if not h_range:
logger.info('[ClientPutOperation] Internal error: No hash range found for key=%s!'%key)
else:
_, _, node_address = h_range
params = {'key': key, 'is_replica': is_replica, 'replica_count': replica_count, 'carefully_save': carefully_save}
if succ_count >= wait_writes_count:
binary_data_pointer = BinaryDataPointer(tempfile.hardlink(), remove_on_close=True)
self._init_operation(node_address, 'PutDataBlock', params, binary_data=binary_data_pointer)
else:
if self.self_address == node_address and local_save is None:
local_save = (key, is_replica)
succ_count += 1
else:
resp = self._init_operation(node_address, 'PutDataBlock', params, sync=True, binary_data=tempfile.chunks())
if resp.ret_code != RC_OK:
logger.error('[ClientPutOperation] PutDataBlock error from %s: %s'%(node_address, resp.ret_message))
errors.append('From %s: %s'%(node_address, resp.ret_message))
else:
succ_count += 1
is_replica = True
try:
if local_save:
key, is_replica = local_save
self.operator.put_data_block(key, tempfile.file_path(), is_replica, carefully_save)
except Exception, err:
succ_count -= 1
msg = 'Saving data block to local range is failed: %s'%err
logger.error(msg)
errors.append(msg)