def read_all(self, filename, length=None): # get metadata, then read chunks direct if not self.exists(filename): raise Exception("read error, file does not exist: %s" % filename) chunks = [] chunkuuids = self.master.get_chunkuuids(filename) chunkservers = self.master.get_chunkservers() for chunkuuid in chunkuuids: read_copies = 0 chunkloc = self.master.get_chunkloc(chunkuuid) chunk = chunkservers[chunkloc].read(chunkuuid) # verify data if chunk is not None: if chunkuuid == ccasutil.hashdata(chunk): read_copies += 1 else: if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % ( chunkservers[chunkloc].local_filesystem_root, chunkuuid) for i in chunkservers: # retry on another chunkserver but let master decide the location # retryloc = i retryloc = self.master.get_retryloc(chunkuuid) while not chunkservers[retryloc].enabled: retryloc = self.master.get_retryloc(chunkuuid) chunk = chunkservers[retryloc].read(chunkuuid) if chunk is not None: if chunkuuid == ccasutil.hashdata(chunk): if self.debug > 0: print "Found a good copy at %s%s." % ( chunkservers[retryloc]. local_filesystem_root, chunkuuid) read_copies += 1 chunks.append(chunk) break else: if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % ( chunkservers[retryloc]. local_filesystem_root, chunkuuid) if read_copies > 0: chunks.append(chunk) else: raise Exception( "FAULTED: Chunk %s failed to verify anywhere." % (chunkuuid)) data = reduce(lambda x, y: x + y, chunks) # reassemble in order return data
def read_all(self, filename, length=None): # get metadata, then read chunks direct if not self.exists(filename): raise Exception("read error, file does not exist: %s" % filename) chunks = [] chunkuuids = self.master.get_chunkuuids(filename) chunkservers = self.master.get_chunkservers() for chunkuuid in chunkuuids: read_copies = 0 chunkloc = self.master.get_chunkloc(chunkuuid) chunk = chunkservers[chunkloc].read(chunkuuid) # verify data if chunk is not None: if chunkuuid == ccasutil.hashdata(chunk): read_copies += 1 else: if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % (chunkservers[chunkloc].local_filesystem_root, chunkuuid) for i in chunkservers: # retry on another chunkserver but let master decide the location # retryloc = i retryloc = self.master.get_retryloc(chunkuuid) while not chunkservers[retryloc].enabled: retryloc = self.master.get_retryloc(chunkuuid) chunk = chunkservers[retryloc].read(chunkuuid) if chunk is not None: if chunkuuid == ccasutil.hashdata(chunk): if self.debug > 0: print "Found a good copy at %s%s." % (chunkservers[retryloc].local_filesystem_root, chunkuuid) read_copies += 1 chunks.append(chunk) break else: if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % (chunkservers[retryloc].local_filesystem_root, chunkuuid) if read_copies > 0: chunks.append(chunk) else: raise Exception("FAULTED: Chunk %s failed to verify anywhere." % (chunkuuid)) data = reduce(lambda x, y: x + y, chunks) # reassemble in order return data
def write_one_chunk(self, chunk, chunkservers): write_copies = 0 chunkuuid = ccasutil.hashdata(chunk) chunkloc = self.master.new_chunkloc(chunkuuid) if self.master.write_algorithm == 'stripe': while not chunkservers[chunkloc].enabled: chunkloc = self.master.new_chunkloc(chunkuuid) resp = chunkservers[chunkloc].write(chunkuuid, chunk) if resp is not None: write_copies += 1 else: if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % ( chunkservers[chunkloc].local_filesystem_root, chunkuuid) for i in chunkservers: # retry on another chunkserver but let master decide the location # retryloc = i retryloc = self.master.new_chunkloc(chunkuuid) while not chunkservers[retryloc].enabled: retryloc = self.master.new_chunkloc(chunkuuid) resp = chunkservers[retryloc].write(chunkuuid, chunk) if resp is not None: print "Rewrote to %s%s." % ( chunkservers[retryloc].local_filesystem_root, chunkuuid) write_copies += 1 chunkuuids.append(chunkuuid) break else: if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % ( chunkservers[retryloc].local_filesystem_root, chunkuuid) elif self.master.write_algorithm == 'mirror': for j in range(0, len(chunkservers)): if chunkservers[j].enabled: resp = chunkservers[j].write(chunkuuid, chunk) if resp is not None: write_copies += 1 else: if self.debug > 0: print "Failed to write a copy to %s%s, consider checking the disk." % ( chunkservers[j].local_filesystem_root, chunkuuid) if write_copies < 1: raise Exception("FAULTED: Chunk %s failed to write anywhere." % (chunkuuid)) return chunkuuid, write_copies
def write(self, chunkuuid, chunk): ''' return None on any error ''' if not self.enabled: return None local_filename = self.chunk_filename(chunkuuid) if not os.access(os.path.dirname(local_filename), os.W_OK): os.makedirs(os.path.dirname(local_filename)) # return early if the chunk already exists and we verified it existing_data = self.read(chunkuuid) if existing_data is not None: if chunkuuid == ccasutil.hashdata(existing_data): if self.debug > 1: print '200 Skipping write: Chunk %s already verified on %s' % (chunkuuid, self.local_filesystem_root) return 200 try: with open(local_filename, "wb") as f: f.write(chunk) if self.debug > 1: print '201 Chunk written to %s%s' % (self.local_filesystem_root, chunkuuid) return 201 except: return None
def write(self, chunkuuid, chunk): ''' return None on any error ''' if not self.enabled: return None local_filename = self.chunk_filename(chunkuuid) if not os.access(os.path.dirname(local_filename), os.W_OK): os.makedirs(os.path.dirname(local_filename)) # return early if the chunk already exists and we verified it existing_data = self.read(chunkuuid) if existing_data is not None: if chunkuuid == ccasutil.hashdata(existing_data): if self.debug > 1: print '200 Skipping write: Chunk %s already verified on %s' % ( chunkuuid, self.local_filesystem_root) return 200 try: with open(local_filename, "wb") as f: f.write(chunk) if self.debug > 1: print '201 Chunk written to %s%s' % ( self.local_filesystem_root, chunkuuid) return 201 except: return None
def write_one_chunk(self, chunk, chunkservers): write_copies = 0 chunkuuid = ccasutil.hashdata(chunk) chunkloc = self.master.new_chunkloc(chunkuuid) if self.master.write_algorithm == 'stripe': while not chunkservers[chunkloc].enabled: chunkloc = self.master.new_chunkloc(chunkuuid) resp = chunkservers[chunkloc].write(chunkuuid, chunk) if resp is not None: write_copies += 1 else: if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % (chunkservers[chunkloc].local_filesystem_root, chunkuuid) for i in chunkservers: # retry on another chunkserver but let master decide the location # retryloc = i retryloc = self.master.new_chunkloc(chunkuuid) while not chunkservers[retryloc].enabled: retryloc = self.master.new_chunkloc(chunkuuid) resp = chunkservers[retryloc].write(chunkuuid, chunk) if resp is not None: print "Rewrote to %s%s." % (chunkservers[retryloc].local_filesystem_root, chunkuuid) write_copies += 1 chunkuuids.append(chunkuuid) break else: if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % (chunkservers[retryloc].local_filesystem_root, chunkuuid) elif self.master.write_algorithm == 'mirror': for j in range(0, len(chunkservers)): if chunkservers[j].enabled: resp = chunkservers[j].write(chunkuuid, chunk) if resp is not None: write_copies += 1 else: if self.debug > 0: print "Failed to write a copy to %s%s, consider checking the disk." % (chunkservers[j].local_filesystem_root, chunkuuid) if write_copies < 1: raise Exception("FAULTED: Chunk %s failed to write anywhere." % (chunkuuid)) return chunkuuid, write_copies