def _finalize_block(self, length):
self.block.b.numBytes = length
request = client_proto.UpdateBlockForPipelineRequestProto()
request.block.CopyFrom(self.block.b)
request.clientName = "snakebite"
self.service.updateBlockForPipeline(request)
return None
def getBlockInfo(self, fileName, blockID):
#print 'FileName: %s'%fileName
fileMetaInfo = self.client._get_file_info(fileName)
#print 'FileMetaInfo: %s'%fileMetaInfo
fileDataInfo = fileMetaInfo.fs
fileSize = fileDataInfo.length
fileReplication = fileMetaInfo.fs.block_replication
blockLocationReq = SBClientProto.GetBlockLocationsRequestProto()
blockLocationReq.src = fileName
blockLocationReq.length = fileSize
blockLocationReq.offset = 0L
blockInfoList = self.client.service.getBlockLocations(blockLocationReq)
blockCount = int(fileMetaInfo.fs.length) / int(
fileMetaInfo.fs.blocksize)
if blockCount * fileMetaInfo.fs.blocksize < fileMetaInfo.fs.length:
blockCount = blockCount + 1
blockReplicaInfoList = []
for i in range(0, blockCount - 1):
block = blockInfoList.locations.blocks[i]
if blockID == block.b.blockId:
for replica in range(0, int(fileReplication)):
blockReplicaInfo = {
"replicaID": replica,
"storageID": block.locs[replica].id.storageID,
"ipAddr": block.locs[replica].id.ipAddr,
"hostName": block.locs[replica].id.hostName,
"txPort": block.locs[replica].id.xferPort,
"infoPort": block.locs[replica].id.infoPort,
"ipcPort": block.locs[replica].id.ipcPort,
"blockToken": block.blockToken
}
blockReplicaInfoList.append(blockReplicaInfo)
block = blockInfoList.locations.lastBlock
if blockID == block.b.blockId:
#There is an issue in fetching all replica's Information
#Need Debugging and review of the Snakebite API
#for replica in range(0,int(fileReplication)):
for replica in range(0, int(fileReplication)):
blockReplicaInfo = {
"fileSize": fileSize,
"replicaID": replica,
"storageID": block.locs[replica].id.storageID,
"ipAddr": block.locs[replica].id.ipAddr,
"hostName": block.locs[replica].id.hostName,
"txPort": block.locs[replica].id.xferPort,
"infoPort": block.locs[replica].id.infoPort,
"ipcPort": block.locs[replica].id.ipcPort,
"blockToken": block.blockToken
}
blockReplicaInfoList.append(blockReplicaInfo)
#print blockReplicaInfoList
return blockReplicaInfoList
def find_blocks(client, path):
fileinfo = client._get_file_info(path)
node = fileinfo.fs
length = node.length
request = client_proto.GetBlockLocationsRequestProto()
request.src = path
request.length = length
request.offset = 0L
response = client.service.getBlockLocations(request)
return list(response.locations.blocks)
def find(f, client, data_root='/data/dfs/dn'):
request = client_proto.GetBlockLocationsRequestProto()
request.src = f['path']
request.length = long(f['length'])
request.offset = long(0)
response = client.service.getBlockLocations(request)
return [{
'block': block,
'path': get_local_path(block, data_root),
'hosts': [location.id.ipAddr for location in block.locs]
} for block in response.locations.blocks]
def close(self):
last_block = None
if self.block_writer is not None:
self.block_writer.close()
last_block = self.block.b
# Issue a CompleteRequestProto
request = client_proto.CompleteRequestProto()
request.src = self.path
request.fileId = self.file_id
request.clientName = "snakebite"
if last_block is not None:
request.last.CopyFrom(last_block)
return self.service.complete(request)
def _start_new_block(self):
if self.block_writer is not None:
self.block_writer.close()
previous = None
if self.block is not None:
previous = self.block.b
request = client_proto.AddBlockRequestProto()
request.src = self.path
request.clientName = "snakebite"
if previous is not None:
request.previous.CopyFrom(previous)
response = self.service.addBlock(request)
if response is not None:
self.block = response.block
self.block_writer = BlockWriter(self.block, self.service,
self.blocksize,
self.use_datanode_hostname)
def getFileBlocksList(self, fileName):
blockMetaInfo = {}
blockMetaInfoList = []
fileMetaInfo = self.client._get_file_info(fileName)
fileDataInfo = fileMetaInfo.fs
fileSize = fileDataInfo.length
blockLocationReq = SBClientProto.GetBlockLocationsRequestProto()
blockLocationReq.src = fileName
blockLocationReq.length = fileSize
blockLocationReq.offset = 0L
blockInfoList = self.client.service.getBlockLocations(blockLocationReq)
blockCount = int(fileMetaInfo.fs.length) / int(
fileMetaInfo.fs.blocksize)
if blockCount * fileMetaInfo.fs.blocksize < fileMetaInfo.fs.length:
blockCount = blockCount + 1
for i in range(0, blockCount - 1):
block = blockInfoList.locations.blocks[i]
#print block
blockMetaInfo = {
"blockID": block.b.blockId,
"numBytes": block.b.numBytes,
"poolID": block.b.poolId,
"generationStamp": block.b.generationStamp
}
blockMetaInfoList.append(blockMetaInfo)
block = blockInfoList.locations.lastBlock
blockMetaInfo = {
"blockID": block.b.blockId,
"numBytes": block.b.numBytes,
"poolID": block.b.poolId,
"generationStamp": block.b.generationStamp,
"offset": block.offset
}
blockMetaInfoList.append(blockMetaInfo)
return blockMetaInfoList
def find_out_things(self, path, tail_only=False, check_crc=False):
client = self.client
fileinfo = client._get_file_info(path)
node = fileinfo.fs
length = node.length
print "Length: ", length
request = client_proto.GetBlockLocationsRequestProto()
request.src = path
request.length = length
if tail_only: # Only read last KB
request.offset = max(0, length - 1024)
else:
request.offset = 0L
response = client.service.getBlockLocations(request)
lastblock = response.locations.lastBlock
#if tail_only:
# if lastblock.b.blockId == response.locations.blocks[0].b.blockId:
# num_blocks_tail = 1 # Tail is on last block
# else:
# num_blocks_tail = 2 # Tail is on two blocks
failed_nodes = []
total_bytes_read = 0
for block in response.locations.blocks:
length = block.b.numBytes
pool_id = block.b.poolId
print "Length: %s, pool_id: %s" % (length, pool_id)
offset_in_block = 0
if tail_only:
if num_blocks_tail == 2 and block.b.blockId != lastblock.b.blockId:
offset_in_block = block.b.numBytes - (1024 -
lastblock.b.numBytes)
elif num_blocks_tail == 1:
offset_in_block = max(0, lastblock.b.numBytes - 1024)
# Prioritize locations to read from
locations_queue = Queue.PriorityQueue(
) # Primitive queuing based on a node's past failure
for location in block.locs:
if location.id.storageID in failed_nodes:
locations_queue.put((1, location)) # Priority num, data
else:
locations_queue.put((0, location))
# Read data
successful_read = False
while not locations_queue.empty():
location = locations_queue.get()[1]
host = location.id.ipAddr
port = int(location.id.xferPort)
data_xciever = DataXceiverChannel(host, port)
if data_xciever.connect():
try:
for load in data_xciever.readBlock(
length, pool_id, block.b.blockId,
block.b.generationStamp, offset_in_block,
check_crc):
offset_in_block += len(load)
total_bytes_read += len(load)
successful_read = True
yield load
except Exception, e:
log.error(e)
if not location.id.storageID in failed_nodes:
failed_nodes.append(location.id.storageID)
successful_read = False
else:
raise Exception
if successful_read:
break
if successful_read is False:
raise Exception("Failure to read block %s" % block.b.blockId)