def run(self):
while True:
job = PutWorker.QUEUE.get()
try:
if job == QUIT_JOB:
break
f_path, seek, size, file_md = job
f_obj = open(f_path, 'rb')
try:
f_obj.seek(seek)
data = f_obj.read(size)
logger.debug('Read %s %s %s %s'%(file_md.name, seek, size, len(data)))
finally:
f_obj.close()
try:
key, checksum = self.fabnet_gateway.put(data, replica_count=file_md.replica_count)
except Exception, err:
logger.error('Cant put data block from file %s. Wait %s seconds and try again...'%(file_md.name, FG_ERROR_TIMEOUT))
time.sleep(FG_ERROR_TIMEOUT)
PutWorker.QUEUE.put(job)
continue
file_md.append_chunk(key, checksum, seek, size)
if file_md.is_all_chunks():
logger.debug('File %s is uploaded. Updating metadata...'%file_md.name)
self.save_metadata_callback(file_md)
except Exception, err:
logger.error('[PutWorker][%s] %s'%(job, err))
def put(self, data, key=None, replica_count=DEFAULT_REPLICA_COUNT, wait_writes_count=2):
source_checksum = hashlib.sha1(data).hexdigest()
data = self.security_manager.encrypt(data)
checksum = hashlib.sha1(data).hexdigest()
params = {'key':key, 'checksum': checksum, 'wait_writes_count': wait_writes_count}
packet = FabnetPacketRequest(method='ClientPutData', parameters=params, binary_data=data, sync=True)
resp = self.fri_client.call_sync('%s:%s'%(self.fabnet_hostname, FRI_PORT), packet, FRI_CLIENT_TIMEOUT)
if resp.ret_code != 0:
logger.error('ClientPutData error: %s'%resp.ret_message)
raise Exception('ClientPutData error: %s'%resp.ret_message)
primary_key = resp.ret_parameters['key']
return primary_key, source_checksum
def get(self, primary_key, replica_count=DEFAULT_REPLICA_COUNT):
params = {'key': primary_key, 'replica_count': replica_count}
packet = FabnetPacketRequest(method='ClientGetData', parameters=params, sync=True)
resp = self.fri_client.call_sync('%s:%s'%(self.fabnet_hostname, FRI_PORT), packet, FRI_CLIENT_TIMEOUT)
if resp.ret_code == RC_NO_DATA:
logger.error('No data found for key %s'%(primary_key,))
elif resp.ret_code != 0:
logger.error('Get data block error for key %s: %s'%(primary_key, resp.ret_message))
if resp.ret_code == 0:
exp_checksum = resp.ret_parameters['checksum']
data = resp.binary_data
checksum = hashlib.sha1(data).hexdigest()
if exp_checksum != checksum:
logger.error('Currupted data block for key %s from node %s'%(primary_key, node_addr))
else:
data = self.security_manager.decrypt(data)
return data
return None
def run(self):
while True:
job = GetWorker.QUEUE.get()
try:
if job == QUIT_JOB:
break
out_streem, key, replica_count, seek, checksum = job
try:
data = self.fabnet_gateway.get(key, replica_count)
except Exception, err:
logger.error("Cant get data block for key %s. Details: %s" % (key, err))
logger.error("Wait %s seconds and try again..." % (FG_ERROR_TIMEOUT,))
time.sleep(FG_ERROR_TIMEOUT)
GetWorker.QUEUE.put(job)
continue
if checksum != hashlib.sha1(data).hexdigest():
raise Exception("Data block for key %s has invalid checksum!" % key)
out_streem.save_block(seek, data)
except Exception, err:
logger.error("[GetWorker][%s] %s" % (job, err))