def test_consistency(self):
# Test the upload and download consistency:
print "Test consistency of upload and download."
tempfilepath = self.setting.mirrordir + "/" + tempfilename
# Copy infile to tempfile in mirror dir:
shutil.copyfile(infilename, tempfilepath)
filesize = os.path.getsize(tempfilepath)
# Upload tempfile:
metadata = common.FileMetadata(tempfilename, filesize, self.setting.totalnode, self.setting.coding)
workflow.uploadFile(self.setting, metadata)
print "Upload finishes."
# Clean temporary directories:
clean.cleanAll()
print "Clean finishes."
# Download tempfile:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode, self.setting.coding)
workflow.downloadFile(self.setting, metadata)
print "Download finishes."
# Copy tempfile to outfile:
shutil.copyfile(tempfilepath, outfilename)
# Clean data in cloud and temporary directories:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode, self.setting.coding)
workflow.deleteFile(self.setting, metadata)
clean.cleanAll()
# Check if infile is same as outfile:
print "test file difference"
self.assertEqual(filecmp.cmp(infilename, outfilename), 1)
# Delete outfile:
os.unlink(outfilename)
def test_consistency(self):
#Test the upload and download consistency:
print "Test consistency of upload and download."
tempfilepath = self.setting.mirrordir + "/" + tempfilename
#Copy infile to tempfile in mirror dir:
shutil.copyfile(infilename, tempfilepath)
filesize = os.path.getsize(tempfilepath)
#Upload tempfile:
metadata = common.FileMetadata(tempfilename, filesize,
self.setting.totalnode,
self.setting.coding)
workflow.uploadFile(self.setting, metadata)
print "Upload finishes."
#Clean temporary directories:
clean.cleanAll()
print "Clean finishes."
#Download tempfile:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
self.setting.coding)
workflow.downloadFile(self.setting, metadata)
print "Download finishes."
#Copy tempfile to outfile:
shutil.copyfile(tempfilepath, outfilename)
#Clean data in cloud and temporary directories:
metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
self.setting.coding)
workflow.deleteFile(self.setting, metadata)
clean.cleanAll()
#Check if infile is same as outfile:
print "test file difference"
self.assertEqual(filecmp.cmp(infilename, outfilename), 1)
#Delete outfile:
os.unlink(outfilename)
def __init__(self, *args, **kw):
'''Initialize Fuse class object.'''
global settingpath
Fuse.__init__(self, *args, **kw)
#Read file system settings:
setting.read(settingpath)
self.root = setting.mirrordir
self.settingpath = os.path.join(os.path.dirname(setting.mirrordir),
'setting.cfg')
settingpath = self.settingpath
#Clean temporary directories:
clean.cleanAll()
"""
#Connect to ZooKeeper servers if needed:
if setting.zookeeper:
self.zkLock = zk.ZooKeeperLock(setting)
setting.zk = self.zkLock
"""
#Check health of nodes:
storage.checkHealth(setting)
#Print system setting and status:
#for i in range(setting.totalnode):
# print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
#print "HealthyNodes = " + str(setting.healthynode) + "\n"
#Handle auto-repair:
if (setting.autorepair == True) and (setting.healthynode < setting.totalnode) \
and (setting.totalnode - setting.healthynode <= setting.totalsparenode):
print "Auto-repair starts."
globalstate = "repair"
rebuild.rebuildNode()
setting.__init__()
setting.read(settingpath)
globalstate = "normal"
print "Auto-repair finishes."
#Update settings for ZooKeeper
zkLock.updateSetting(setting)
setting.zk = self.zkLock
#Check health of nodes after repair:
storage.checkHealth(setting)
#Print system setting and status:
for i in range(setting.totalnode):
print "Node " + str(i) + " healthy = " + str(
setting.nodeInfo[i].healthy) + "\n"
print "HealthyNodes = " + str(setting.healthynode) + "\n"
#It is necessary to reset the storage queue for later initStorages():
storage.resetStorageQueue()
def __init__(self, *args, **kw):
'''Initialize Fuse class object.'''
global settingpath
Fuse.__init__(self, *args, **kw)
#Read file system settings:
setting.read(settingpath)
self.root = setting.mirrordir
self.settingpath = os.path.join(os.path.dirname(setting.mirrordir),'setting.cfg')
settingpath = self.settingpath
#Clean temporary directories:
clean.cleanAll()
"""
#Connect to ZooKeeper servers if needed:
if setting.zookeeper:
self.zkLock = zk.ZooKeeperLock(setting)
setting.zk = self.zkLock
"""
#Check health of nodes:
storage.checkHealth(setting)
#Print system setting and status:
#for i in range(setting.totalnode):
# print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
#print "HealthyNodes = " + str(setting.healthynode) + "\n"
#Handle auto-repair:
if (setting.autorepair == True) and (setting.healthynode < setting.totalnode) \
and (setting.totalnode - setting.healthynode <= setting.totalsparenode):
print "Auto-repair starts."
globalstate = "repair"
rebuild.rebuildNode()
setting.__init__()
setting.read(settingpath)
globalstate = "normal"
print "Auto-repair finishes."
#Update settings for ZooKeeper
zkLock.updateSetting(setting)
setting.zk = self.zkLock
#Check health of nodes after repair:
storage.checkHealth(setting)
#Print system setting and status:
for i in range(setting.totalnode):
print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
print "HealthyNodes = " + str(setting.healthynode) + "\n"
#It is necessary to reset the storage queue for later initStorages():
storage.resetStorageQueue()
#Get new setting:
settingNew = getNewSetting(settingOld, settingpath)
#Rebuild files recursively:
for targetFile in fileList:
if (filter == []) or (targetFile in filter):
if rebuildthreads > 0:
rebuildQueue.put(
('rebuildFile', (settingOld, settingNew, targetFile)))
else:
rebuildFile(settingOld, settingNew, targetFile)
if rebuildthreads > 0:
waitForRebuild()
#Backup old setting file, write new setting file:
#print "Rebuild finishes. Writing new setting file."
shutil.copyfile(settingpath, settingpath + ".old")
settingNew.write(settingpath)
if __name__ == '__main__':
'''Rebuild all files if no argument. '''
'''Rebuild filter files if filter argument exists.'''
'''The filter argument is a list of strings.'''
filter = []
if len(sys.argv) > 1:
filter = sys.argv[1:]
#Clean temporary directories:
clean.cleanAll()
rebuildNode(filter)
fileList.append(filepath)
#Get new setting:
settingNew = getNewSetting(settingOld, settingpath)
#Rebuild files recursively:
for targetFile in fileList:
if (filter == []) or (targetFile in filter):
if rebuildthreads > 0:
rebuildQueue.put(('rebuildFile',(settingOld, settingNew, targetFile)))
else:
rebuildFile(settingOld, settingNew, targetFile)
if rebuildthreads > 0:
waitForRebuild()
#Backup old setting file, write new setting file:
#print "Rebuild finishes. Writing new setting file."
shutil.copyfile(settingpath, settingpath+".old")
settingNew.write(settingpath)
if __name__ == '__main__':
'''Rebuild all files if no argument. '''
'''Rebuild filter files if filter argument exists.'''
'''The filter argument is a list of strings.'''
filter = []
if len(sys.argv) > 1:
filter = sys.argv[1:]
#Clean temporary directories:
clean.cleanAll()
rebuildNode(filter)