def read_all(self,
filename,
length=None): # get metadata, then read chunks direct
if not self.exists(filename):
raise Exception("read error, file does not exist: %s" % filename)
chunks = []
chunkuuids = self.master.get_chunkuuids(filename)
chunkservers = self.master.get_chunkservers()
for chunkuuid in chunkuuids:
read_copies = 0
chunkloc = self.master.get_chunkloc(chunkuuid)
chunk = chunkservers[chunkloc].read(chunkuuid)
# verify data
if chunk is not None:
if chunkuuid == ccasutil.hashdata(chunk):
read_copies += 1
else:
if self.debug > 0:
print "Chunk %s%s failed verification, consider checking the disk." % (
chunkservers[chunkloc].local_filesystem_root,
chunkuuid)
for i in chunkservers:
# retry on another chunkserver but let master decide the location
# retryloc = i
retryloc = self.master.get_retryloc(chunkuuid)
while not chunkservers[retryloc].enabled:
retryloc = self.master.get_retryloc(chunkuuid)
chunk = chunkservers[retryloc].read(chunkuuid)
if chunk is not None:
if chunkuuid == ccasutil.hashdata(chunk):
if self.debug > 0:
print "Found a good copy at %s%s." % (
chunkservers[retryloc].
local_filesystem_root, chunkuuid)
read_copies += 1
chunks.append(chunk)
break
else:
if self.debug > 0:
print "Chunk %s%s failed verification, consider checking the disk." % (
chunkservers[retryloc].
local_filesystem_root, chunkuuid)
if read_copies > 0:
chunks.append(chunk)
else:
raise Exception(
"FAULTED: Chunk %s failed to verify anywhere." %
(chunkuuid))
data = reduce(lambda x, y: x + y, chunks) # reassemble in order
return data
def read_all(self, filename, length=None): # get metadata, then read chunks direct
if not self.exists(filename):
raise Exception("read error, file does not exist: %s" % filename)
chunks = []
chunkuuids = self.master.get_chunkuuids(filename)
chunkservers = self.master.get_chunkservers()
for chunkuuid in chunkuuids:
read_copies = 0
chunkloc = self.master.get_chunkloc(chunkuuid)
chunk = chunkservers[chunkloc].read(chunkuuid)
# verify data
if chunk is not None:
if chunkuuid == ccasutil.hashdata(chunk):
read_copies += 1
else:
if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % (chunkservers[chunkloc].local_filesystem_root, chunkuuid)
for i in chunkservers:
# retry on another chunkserver but let master decide the location
# retryloc = i
retryloc = self.master.get_retryloc(chunkuuid)
while not chunkservers[retryloc].enabled:
retryloc = self.master.get_retryloc(chunkuuid)
chunk = chunkservers[retryloc].read(chunkuuid)
if chunk is not None:
if chunkuuid == ccasutil.hashdata(chunk):
if self.debug > 0: print "Found a good copy at %s%s." % (chunkservers[retryloc].local_filesystem_root, chunkuuid)
read_copies += 1
chunks.append(chunk)
break
else:
if self.debug > 0: print "Chunk %s%s failed verification, consider checking the disk." % (chunkservers[retryloc].local_filesystem_root, chunkuuid)
if read_copies > 0:
chunks.append(chunk)
else:
raise Exception("FAULTED: Chunk %s failed to verify anywhere." % (chunkuuid))
data = reduce(lambda x, y: x + y, chunks) # reassemble in order
return data
def write_one_chunk(self, chunk, chunkservers):
write_copies = 0
chunkuuid = ccasutil.hashdata(chunk)
chunkloc = self.master.new_chunkloc(chunkuuid)
if self.master.write_algorithm == 'stripe':
while not chunkservers[chunkloc].enabled:
chunkloc = self.master.new_chunkloc(chunkuuid)
resp = chunkservers[chunkloc].write(chunkuuid, chunk)
if resp is not None:
write_copies += 1
else:
if self.debug > 0:
print "Failed to write %s%s, consider checking the disk." % (
chunkservers[chunkloc].local_filesystem_root,
chunkuuid)
for i in chunkservers:
# retry on another chunkserver but let master decide the location
# retryloc = i
retryloc = self.master.new_chunkloc(chunkuuid)
while not chunkservers[retryloc].enabled:
retryloc = self.master.new_chunkloc(chunkuuid)
resp = chunkservers[retryloc].write(chunkuuid, chunk)
if resp is not None:
print "Rewrote to %s%s." % (
chunkservers[retryloc].local_filesystem_root,
chunkuuid)
write_copies += 1
chunkuuids.append(chunkuuid)
break
else:
if self.debug > 0:
print "Failed to write %s%s, consider checking the disk." % (
chunkservers[retryloc].local_filesystem_root,
chunkuuid)
elif self.master.write_algorithm == 'mirror':
for j in range(0, len(chunkservers)):
if chunkservers[j].enabled:
resp = chunkservers[j].write(chunkuuid, chunk)
if resp is not None:
write_copies += 1
else:
if self.debug > 0:
print "Failed to write a copy to %s%s, consider checking the disk." % (
chunkservers[j].local_filesystem_root,
chunkuuid)
if write_copies < 1:
raise Exception("FAULTED: Chunk %s failed to write anywhere." %
(chunkuuid))
return chunkuuid, write_copies
def write(self, chunkuuid, chunk):
''' return None on any error '''
if not self.enabled: return None
local_filename = self.chunk_filename(chunkuuid)
if not os.access(os.path.dirname(local_filename), os.W_OK):
os.makedirs(os.path.dirname(local_filename))
# return early if the chunk already exists and we verified it
existing_data = self.read(chunkuuid)
if existing_data is not None:
if chunkuuid == ccasutil.hashdata(existing_data):
if self.debug > 1: print '200 Skipping write: Chunk %s already verified on %s' % (chunkuuid, self.local_filesystem_root)
return 200
try:
with open(local_filename, "wb") as f:
f.write(chunk)
if self.debug > 1: print '201 Chunk written to %s%s' % (self.local_filesystem_root, chunkuuid)
return 201
except:
return None
def write(self, chunkuuid, chunk):
''' return None on any error '''
if not self.enabled: return None
local_filename = self.chunk_filename(chunkuuid)
if not os.access(os.path.dirname(local_filename), os.W_OK):
os.makedirs(os.path.dirname(local_filename))
# return early if the chunk already exists and we verified it
existing_data = self.read(chunkuuid)
if existing_data is not None:
if chunkuuid == ccasutil.hashdata(existing_data):
if self.debug > 1:
print '200 Skipping write: Chunk %s already verified on %s' % (
chunkuuid, self.local_filesystem_root)
return 200
try:
with open(local_filename, "wb") as f:
f.write(chunk)
if self.debug > 1:
print '201 Chunk written to %s%s' % (
self.local_filesystem_root, chunkuuid)
return 201
except:
return None
def write_one_chunk(self, chunk, chunkservers):
write_copies = 0
chunkuuid = ccasutil.hashdata(chunk)
chunkloc = self.master.new_chunkloc(chunkuuid)
if self.master.write_algorithm == 'stripe':
while not chunkservers[chunkloc].enabled:
chunkloc = self.master.new_chunkloc(chunkuuid)
resp = chunkservers[chunkloc].write(chunkuuid, chunk)
if resp is not None:
write_copies += 1
else:
if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % (chunkservers[chunkloc].local_filesystem_root, chunkuuid)
for i in chunkservers:
# retry on another chunkserver but let master decide the location
# retryloc = i
retryloc = self.master.new_chunkloc(chunkuuid)
while not chunkservers[retryloc].enabled:
retryloc = self.master.new_chunkloc(chunkuuid)
resp = chunkservers[retryloc].write(chunkuuid, chunk)
if resp is not None:
print "Rewrote to %s%s." % (chunkservers[retryloc].local_filesystem_root, chunkuuid)
write_copies += 1
chunkuuids.append(chunkuuid)
break
else:
if self.debug > 0: print "Failed to write %s%s, consider checking the disk." % (chunkservers[retryloc].local_filesystem_root, chunkuuid)
elif self.master.write_algorithm == 'mirror':
for j in range(0, len(chunkservers)):
if chunkservers[j].enabled:
resp = chunkservers[j].write(chunkuuid, chunk)
if resp is not None:
write_copies += 1
else:
if self.debug > 0: print "Failed to write a copy to %s%s, consider checking the disk." % (chunkservers[j].local_filesystem_root, chunkuuid)
if write_copies < 1:
raise Exception("FAULTED: Chunk %s failed to write anywhere." % (chunkuuid))
return chunkuuid, write_copies
