def read_block(block): location = block.locs[0] host = location.id.ipAddr port = int(location.id.xferPort) data_xciever = DataXceiverChannel(host, port) if not data_xciever.connect(): return offset_in_block = 0 check_crc = False length = block.b.numBytes block_gen = data_xciever.readBlock(length, block.b.poolId, block.b.blockId, block.b.generationStamp, offset_in_block, check_crc) return block_gen
def _read_file(self, path, node, tail_only, check_crc): length = node.length request = client_proto.GetBlockLocationsRequestProto() request.src = path request.length = length if tail_only: # Only read last KB request.offset = max(0, length - 1024) else: request.offset = 0L response = self.service.getBlockLocations(request) if response.locations.fileLength == 0: # Can't read empty file yield "" lastblock = response.locations.lastBlock if tail_only: if lastblock.b.blockId == response.locations.blocks[0].b.blockId: num_blocks_tail = 1 # Tail is on last block else: num_blocks_tail = 2 # Tail is on two blocks failed_nodes = [] total_bytes_read = 0 for block in response.locations.blocks: length = block.b.numBytes pool_id = block.b.poolId offset_in_block = 0 if tail_only: if num_blocks_tail == 2 and block.b.blockId != lastblock.b.blockId: offset_in_block = block.b.numBytes - (1024 - lastblock.b.numBytes) elif num_blocks_tail == 1: offset_in_block = max(0, lastblock.b.numBytes - 1024) # Prioritize locations to read from locations_queue = Queue.PriorityQueue() # Primitive queuing based on a node's past failure for location in block.locs: if location.id.storageID in failed_nodes: locations_queue.put((1, location)) # Priority num, data else: locations_queue.put((0, location)) # Read data successful_read = False while not locations_queue.empty(): location = locations_queue.get()[1] host = location.id.ipAddr port = int(location.id.xferPort) data_xciever = DataXceiverChannel(host, port) if data_xciever.connect(): try: for load in data_xciever.readBlock(length, pool_id, block.b.blockId, block.b.generationStamp, offset_in_block, check_crc): offset_in_block += len(load) total_bytes_read += len(load) successful_read = True yield load except Exception, e: log.error(e) if not location.id.storageID in failed_nodes: failed_nodes.append(location.id.storageID) successful_read = False else: raise Exception if successful_read: break if successful_read is False: raise Exception("Failure to read block %s" % block.b.blockId)
def find_out_things(self, path, tail_only=False, check_crc=False): client = self.client fileinfo = client._get_file_info(path) node = fileinfo.fs length = node.length print "Length: ", length request = client_proto.GetBlockLocationsRequestProto() request.src = path request.length = length if tail_only: # Only read last KB request.offset = max(0, length - 1024) else: request.offset = 0L response = client.service.getBlockLocations(request) lastblock = response.locations.lastBlock #if tail_only: # if lastblock.b.blockId == response.locations.blocks[0].b.blockId: # num_blocks_tail = 1 # Tail is on last block # else: # num_blocks_tail = 2 # Tail is on two blocks failed_nodes = [] total_bytes_read = 0 for block in response.locations.blocks: length = block.b.numBytes pool_id = block.b.poolId print "Length: %s, pool_id: %s" % (length, pool_id) offset_in_block = 0 if tail_only: if num_blocks_tail == 2 and block.b.blockId != lastblock.b.blockId: offset_in_block = block.b.numBytes - (1024 - lastblock.b.numBytes) elif num_blocks_tail == 1: offset_in_block = max(0, lastblock.b.numBytes - 1024) # Prioritize locations to read from locations_queue = Queue.PriorityQueue( ) # Primitive queuing based on a node's past failure for location in block.locs: if location.id.storageID in failed_nodes: locations_queue.put((1, location)) # Priority num, data else: locations_queue.put((0, location)) # Read data successful_read = False while not locations_queue.empty(): location = locations_queue.get()[1] host = location.id.ipAddr port = int(location.id.xferPort) data_xciever = DataXceiverChannel(host, port) if data_xciever.connect(): try: for load in data_xciever.readBlock( length, pool_id, block.b.blockId, block.b.generationStamp, offset_in_block, check_crc): offset_in_block += len(load) total_bytes_read += len(load) successful_read = True yield load except Exception, e: log.error(e) if not location.id.storageID in failed_nodes: failed_nodes.append(location.id.storageID) successful_read = False else: raise Exception if successful_read: break if successful_read is False: raise Exception("Failure to read block %s" % block.b.blockId)