def test_consistency(self):
#Test the upload and download consistency:
print "Test consistency of upload and download."
tempfilepath = self.setting.mirrordir + "/" + tempfilename
#Copy infile to tempfile in mirror dir:
shutil.copyfile(infilename, tempfilepath)
filesize = os.path.getsize(tempfilepath)
#Upload tempfile:
metadata = common.FileMetadata(tempfilename, filesize,
self.setting.totalnode,
self.setting.coding)
workflow.uploadFile(self.setting, metadata)
print "Upload finishes."
#Clean temporary directories:
clean.cleanAll()
print "Clean finishes."
#Download tempfile:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
self.setting.coding)
workflow.downloadFile(self.setting, metadata)
print "Download finishes."
#Copy tempfile to outfile:
shutil.copyfile(tempfilepath, outfilename)
#Clean data in cloud and temporary directories:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
self.setting.coding)
workflow.deleteFile(self.setting, metadata)
clean.cleanAll()
#Check if infile is same as outfile:
print "test file difference"
self.assertEqual(filecmp.cmp(infilename, outfilename), 1)
#Delete outfile:
os.unlink(outfilename)
def downloadFile(setting, metadata):
'''Workflow of Download file.'''
if setting.deduplication == False:
#Acquire read lock if using ZooKeeper:
#if setting.zookeeper: setting.zk.getReadLock(metadata) #cq
#Download metadata:
storage.downloadMetadata(setting, metadata)
#Update metadata for requesting chunks for decode:
coding.updateMetadataForDecode(setting, metadata, 'download')
#Download chunk files:
storage.downloadFile(setting, metadata)
#Release read lock:
#if setting.zookeeper: setting.zk.releaseReadLock(metadata) #cq
#Decode file:
coding.decodeFile(setting, metadata, 'download')
else:
retState = storage.detectPointer(setting, metadata)
if retState == False:
'''The file detected is non_dup or org file'''
storage.downloadMetadata(setting, metadata)
coding.updateMetadataForDecode(setting, metadata, 'download')
storage.downloadFile(setting, metadata)
coding.decodeFile(setting, metadata, 'download')
else:
'''The file detected is a leaf'''
storage.downloadPointer(setting, metadata)
targetFile = storage.getPointerContent(setting, metadata)
targetMetadata = common.FileMetadata(targetFile, 0,
setting.totalnode,
setting.coding)
storage.downloadMetadata(setting, targetMetadata)
coding.updateMetadataForDecode(setting, targetMetadata, 'download')
storage.downloadFile(setting, targetMetadata)
coding.decodeFile(setting, targetMetadata, 'download')
storage.genPtLocalFile(setting, targetMetadata, metadata)
def updatePointers(setting, oldMetadata, newMetadata):
'''replace pointers to old file name with pointers to new file name both locally and on clouds'''
pointerdir = setting.pointerdir
oldFilename = oldMetadata.filename
newFilename = newMetadata.filename
print "update pointers from old %s to new %s" % (oldFilename, newFilename)
for root, dirs, files in os.walk(pointerdir):
for file in files:
print "iterate file %s" % (file)
curPtFilePath = os.path.join(root, file)
print "file path %s" % (curPtFilePath)
fhandle = open(curPtFilePath, 'r')
content = fhandle.read()
fhandle.close()
print "content %s, oldFilename %s" % (content, oldFilename)
print content == oldFilename
if content == oldFilename:
tmpFilename = file[:file.rindex('.pt')]
tmpMeta = common.FileMetadata(tmpFilename, 0,
setting.totalnode,
setting.coding)
deletePointer(setting, tmpMeta)
createPointer(setting, tmpMeta, newFilename)
uploadPointer(setting, tmpMeta)
def infoFile(setting, filename):
'''Display information of a file.'''
metadata = common.FileMetadata(filename, 0, 0, 0)
storage.downloadMetadata(setting, metadata)
#Display file info:
print metadata.filename,
print metadata.filesize,
print metadata.coding
def gencoeff(path, parity_coeff, failed_node, setting):
chunks_vectors = []
chunks = []
#Setup parameters before using function from codings/fmsrtwo.py
n, k = setting.totalnode, setting.datanode
metadata = common.FileMetadata('', 0, n, 'fmsrtwo')
nativeBlockNum = fmsrutil.getNativeBlockNum(n, k)
parityBlockNum = fmsrutil.getParityBlockNum(n, k)
metadata.totalchunk = parityBlockNum
for i in xrange(metadata.totalchunk):
metadata.chunkInfo.append(common.ChunkMetadata(i))
for i in xrange(metadata.totalnode):
metadata.fileNodeInfo.append(common.FileNodeMetadata(i))
setting.healthynode = n - 1
metadata.totalnode = n
metadata.parityCoeff = parity_coeff
for node in setting.nodeInfo:
node.healthy = True
setting.nodeInfo[failed_node].healthy = False
#Get coeff
ret = fmsrtwo.updateMetadataForDecode(setting, metadata, 'repair')
if ret == False:
print >> sys.stderr, "gencoeff failed for node " + str(failed_node)
new_parity_coeff = [
metadata.enc_matrix[i][j] for i in xrange(parityBlockNum)
for j in xrange(nativeBlockNum)
]
for i in xrange(metadata.totalchunk):
if metadata.chunkInfo[i].action == 'download':
chunks.append(i)
chunks_vectors = metadata.repairCodingCoeff
#Write to file
with open(path, 'wb') as f:
f.write(chr(len(new_parity_coeff) & 0b11111111))
f.write(chr((len(new_parity_coeff) >> 8) & 0b11111111))
for i in new_parity_coeff:
f.write(chr(i & 0b11111111))
f.write(chr(len(chunks) & 0b11111111))
for i in chunks:
f.write(chr(i & 0b11111111))
f.write(chr(len(chunks_vectors) & 0b11111111))
for chunk_vectors in chunks_vectors:
f.write(chr(len(chunk_vectors) & 0b11111111))
for i in chunk_vectors:
f.write(chr(i & 0b11111111))
setting.nodeInfo[failed_node].healthy = True
def __init__(self, path, flags, *mode):
'''File open. Download file from cloud for read mode.'''
self.file = os.fdopen(os.open("." + path, flags, *mode),
flag2mode(flags))
self.fd = self.file.fileno()
self.path = path
print "open file ", path
#Set direct_io and keep_cache options as required by fuse:
self.direct_io = True
self.keep_cache = False
#Construct file metadata:
filename = path[path.index("/") + 1:]
filesize = os.path.getsize("." + path)
self.metadata = common.FileMetadata(filename, filesize,
setting.totalnode,
setting.coding)
if ("r" in self.file.mode) or ("+" in self.file.mode):
#Download file from clouds to mirror dir:
workflow.downloadFile(setting, self.metadata)
def unlink(self, path):
'''Remove file.'''
setting = common.Setting()
setting.read(
self.settingpath
) #cq update setting before unlink. repair operation may change setting.cfg
filename = path[path.index("/") + 1:]
metadata = common.FileMetadata(filename, 0, setting.totalnode,
setting.coding)
#Delete chunks and metadata of the file on cloud:
workflow.deleteFile(setting, metadata)
#Delete in chunkdir:
#Delete chunks:
for chunk in metadata.chunkInfo:
if chunk.chunktype != 'replica':
try:
os.unlink(setting.chunkdir + "/" + chunk.chunkpath)
except:
pass
#Delete big-chunks:
for bigchunk in metadata.fileNodeInfo:
try:
os.unlink(setting.chunkdir + "/" + bigchunk.bigchunkpath)
except:
pass
if setting.coding == 'replication':
try:
os.unlink(setting.chunkdir + '/' + metadata.filename +
'.node0')
except:
pass
#Delete in metadatadir:
try:
if setting.coding != 'replication': #cq there is no .meta file in replication
os.unlink(setting.metadatadir + "/" + filename + ".meta")
os.unlink(setting.metadatadir + "/" + filename + ".metadata")
except:
pass
#Delete file in mirror dir
os.unlink("." + path)
def deleteFile(setting, metadata):
'''Workflow of Delete file.'''
if setting.deduplication == False:
#Acquire write lock if using ZooKeeper:
#if setting.zookeeper: setting.zk.getWriteLock(metadata) #cq
#Download metadata:
storage.downloadMetadata(setting, metadata)
#Delete chunk files and metadata:
storage.deleteChunkAndMetadata(setting, metadata)
#Release write lock:
#if setting.zookeeper: setting.zk.releaseWriteLock(metadata) #cq
else:
orgState = storage.detectOrgState(setting, metadata)
if orgState == 3:
# non_dup
print "remove non_dup"
#Download metadata:
storage.downloadMetadata(setting, metadata)
#Delete chunk files and metadata:
storage.deleteChunkAndMetadata(setting, metadata)
elif orgState == 2:
# leaf
print "remove leaf"
storage.deletePointer(setting, metadata)
elif orgState == 1:
# org
print "remove org"
storage.downloadMetadata(setting, metadata)
storage.deleteChunkAndMetadata(setting, metadata)
dupFileName = storage.findOnePointer(setting, metadata)
dupMetadata = common.FileMetadata(dupFileName, 0,
setting.totalnode,
setting.coding)
storage.deletePointer(setting, dupMetadata)
coding.encodeFile(setting, dupMetadata)
storage.uploadFileAndMetadata(setting, dupMetadata, 'upload')
storage.updatePointers(setting, metadata, dupMetadata)
def rebuildFile(settingOld, settingNew, filename):
'''Rebuild a file.'''
metadata = common.FileMetadata(filename, 0, 0, 0)
#Rebuild file:
workflow.rebuildFile(settingOld, settingNew, metadata, filename)
def uploadFile(setting, metadata):
'''Workflow of Upload file.'''
if setting.deduplication == False:
#Encode file:
print "deduplication False, workflow upload file"
coding.encodeFile(setting, metadata)
#Acquire write lock if using ZooKeeper:
#if setting.zookeeper: setting.zk.getWriteLock(metadata) #cq
#Upload chunk files and metadata:
storage.uploadFileAndMetadata(setting, metadata, 'upload')
#Release write lock:
#if setting.zookeeper: setting.zk.releaseWriteLock(metadata) #cq
else:
print "deduplicatoin True, workflow upload file"
orgState = storage.detectOrgState(setting, metadata)
print "orgState %d" % (orgState)
currState = storage.detectCurrState(setting, metadata)
print "currState %d" % (currState[0])
if orgState == 3 and currState[0] == 0:
# non_dup => non_dup or new => non_dup
print 'orgState %d to currState %d' % (orgState, currState[0])
coding.encodeFile(setting, metadata)
storage.uploadFileAndMetadata(setting, metadata, 'upload')
elif orgState == 3 and currState[0] == 1:
# non_dup => leaf or new => leaf
print 'orgState %d to currState %d' % (orgState, currState[0])
detState = storage.detectReplica(setting, metadata)
if detState == True:
deleteFile(setting, metadata)
storage.createPointer(setting, metadata, currState[1])
storage.uploadPointer(setting, metadata)
elif orgState == 2 and currState[0] == 1:
# leaf => leaf
print 'orgState %d to currState %d' % (orgState, currState[0])
storage.deletePointer(setting, metadata)
storage.createPointer(setting, metadata, currState[1])
storage.uploadPointer(setting, metadata)
elif orgState == 2 and currState[0] == 0:
# leaf => non_dup
print 'orgState %d to currState %d' % (orgState, currState[0])
storage.deletePointer(setting, metadata)
coding.encodeFile(setting, metadata)
storage.uploadFileAndMetadata(setting, metadata, 'upload')
elif orgState == 1 and currState[0] == 0:
# org => non_dup
print 'orgState %d to currState %d' % (orgState, currState[0])
coding.encodeFile(setting, metadata)
storage.uploadFileAndMetadata(setting, metadata, 'upload')
dupFileName = storage.findOnePointer(setting, metadata)
dupMetadata = common.FileMetadata(dupFileName, 0,
setting.totalnode,
setting.coding)
storage.deletePointer(setting, dupMetadata)
coding.encodeFile(setting, dupMetadata)
storage.uploadFileAndMetadata(setting, dupMetadata, 'upload')
storage.updatePointers(setting, metadata, dupMetadata)
elif orgState == 1 and currState[0] == 1:
# org => leaf
print 'orgState %d to currState %d' % (orgState, currState[0])
deleteFile(setting, metadata)
storage.createPointer(setting, metadata, currState[1])
storage.uploadPointer(setting, metadata)
'''