Ejemplo n.º 1
0
 def test_consistency(self):
     # Test the upload and download consistency:
     print "Test consistency of upload and download."
     tempfilepath = self.setting.mirrordir + "/" + tempfilename
     # Copy infile to tempfile in mirror dir:
     shutil.copyfile(infilename, tempfilepath)
     filesize = os.path.getsize(tempfilepath)
     # Upload tempfile:
     metadata = common.FileMetadata(tempfilename, filesize, self.setting.totalnode, self.setting.coding)
     workflow.uploadFile(self.setting, metadata)
     print "Upload finishes."
     # Clean temporary directories:
     clean.cleanAll()
     print "Clean finishes."
     # Download tempfile:
     metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode, self.setting.coding)
     workflow.downloadFile(self.setting, metadata)
     print "Download finishes."
     # Copy tempfile to outfile:
     shutil.copyfile(tempfilepath, outfilename)
     # Clean data in cloud and temporary directories:
     metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode, self.setting.coding)
     workflow.deleteFile(self.setting, metadata)
     clean.cleanAll()
     # Check if infile is same as outfile:
     print "test file difference"
     self.assertEqual(filecmp.cmp(infilename, outfilename), 1)
     # Delete outfile:
     os.unlink(outfilename)
Ejemplo n.º 2
0
 def test_consistency(self):
     #Test the upload and download consistency:
     print "Test consistency of upload and download."
     tempfilepath = self.setting.mirrordir + "/" + tempfilename
     #Copy infile to tempfile in mirror dir:
     shutil.copyfile(infilename, tempfilepath)
     filesize = os.path.getsize(tempfilepath)
     #Upload tempfile:
     metadata = common.FileMetadata(tempfilename, filesize,
                                    self.setting.totalnode,
                                    self.setting.coding)
     workflow.uploadFile(self.setting, metadata)
     print "Upload finishes."
     #Clean temporary directories:
     clean.cleanAll()
     print "Clean finishes."
     #Download tempfile:
     metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
                                    self.setting.coding)
     workflow.downloadFile(self.setting, metadata)
     print "Download finishes."
     #Copy tempfile to outfile:
     shutil.copyfile(tempfilepath, outfilename)
     #Clean data in cloud and temporary directories:
     metadata = common.FileMetadata(tempfilename, 0, self.setting.totalnode,
                                    self.setting.coding)
     workflow.deleteFile(self.setting, metadata)
     clean.cleanAll()
     #Check if infile is same as outfile:
     print "test file difference"
     self.assertEqual(filecmp.cmp(infilename, outfilename), 1)
     #Delete outfile:
     os.unlink(outfilename)
Ejemplo n.º 3
0
    def __init__(self, *args, **kw):
        '''Initialize Fuse class object.'''
        global settingpath
        Fuse.__init__(self, *args, **kw)

        #Read file system settings:
        setting.read(settingpath)
        self.root = setting.mirrordir
        self.settingpath = os.path.join(os.path.dirname(setting.mirrordir),
                                        'setting.cfg')
        settingpath = self.settingpath
        #Clean temporary directories:
        clean.cleanAll()
        """
        #Connect to ZooKeeper servers if needed:
        if setting.zookeeper:
            self.zkLock = zk.ZooKeeperLock(setting)
            setting.zk = self.zkLock
        """
        #Check health of nodes:
        storage.checkHealth(setting)
        #Print system setting and status:
        #for i in range(setting.totalnode):
        #    print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
        #print "HealthyNodes = " + str(setting.healthynode) + "\n"

        #Handle auto-repair:
        if (setting.autorepair == True) and (setting.healthynode < setting.totalnode) \
                and (setting.totalnode - setting.healthynode <= setting.totalsparenode):
            print "Auto-repair starts."
            globalstate = "repair"
            rebuild.rebuildNode()
            setting.__init__()
            setting.read(settingpath)
            globalstate = "normal"
            print "Auto-repair finishes."
            #Update settings for ZooKeeper
            zkLock.updateSetting(setting)
            setting.zk = self.zkLock
            #Check health of nodes after repair:
            storage.checkHealth(setting)
            #Print system setting and status:
            for i in range(setting.totalnode):
                print "Node " + str(i) + " healthy = " + str(
                    setting.nodeInfo[i].healthy) + "\n"
            print "HealthyNodes = " + str(setting.healthynode) + "\n"
            #It is necessary to reset the storage queue for later initStorages():
            storage.resetStorageQueue()
Ejemplo n.º 4
0
    def __init__(self, *args, **kw):
        '''Initialize Fuse class object.'''
        global settingpath
        Fuse.__init__(self, *args, **kw)

        #Read file system settings:
        setting.read(settingpath)
        self.root = setting.mirrordir
        self.settingpath = os.path.join(os.path.dirname(setting.mirrordir),'setting.cfg')
        settingpath = self.settingpath
        #Clean temporary directories:
        clean.cleanAll()
        """
        #Connect to ZooKeeper servers if needed:
        if setting.zookeeper:
            self.zkLock = zk.ZooKeeperLock(setting)
            setting.zk = self.zkLock
        """
        #Check health of nodes:
        storage.checkHealth(setting)
        #Print system setting and status:
        #for i in range(setting.totalnode):
        #    print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
        #print "HealthyNodes = " + str(setting.healthynode) + "\n"

        #Handle auto-repair:
        if (setting.autorepair == True) and (setting.healthynode < setting.totalnode) \
                and (setting.totalnode - setting.healthynode <= setting.totalsparenode):
            print "Auto-repair starts."
            globalstate = "repair"
            rebuild.rebuildNode()
            setting.__init__()
            setting.read(settingpath)
            globalstate = "normal"
            print "Auto-repair finishes."
            #Update settings for ZooKeeper
            zkLock.updateSetting(setting)
            setting.zk = self.zkLock
            #Check health of nodes after repair:
            storage.checkHealth(setting)
            #Print system setting and status:
            for i in range(setting.totalnode):
                print "Node " + str(i) + " healthy = " + str(setting.nodeInfo[i].healthy) + "\n"
            print "HealthyNodes = " + str(setting.healthynode) + "\n"
            #It is necessary to reset the storage queue for later initStorages():
            storage.resetStorageQueue()
Ejemplo n.º 5
0
    #Get new setting:
    settingNew = getNewSetting(settingOld, settingpath)

    #Rebuild files recursively:
    for targetFile in fileList:
        if (filter == []) or (targetFile in filter):
            if rebuildthreads > 0:
                rebuildQueue.put(
                    ('rebuildFile', (settingOld, settingNew, targetFile)))
            else:
                rebuildFile(settingOld, settingNew, targetFile)
    if rebuildthreads > 0:
        waitForRebuild()

    #Backup old setting file, write new setting file:
    #print "Rebuild finishes. Writing new setting file."
    shutil.copyfile(settingpath, settingpath + ".old")
    settingNew.write(settingpath)


if __name__ == '__main__':
    '''Rebuild all files if no argument. '''
    '''Rebuild filter files if filter argument exists.'''
    '''The filter argument is a list of strings.'''
    filter = []
    if len(sys.argv) > 1:
        filter = sys.argv[1:]
    #Clean temporary directories:
    clean.cleanAll()
    rebuildNode(filter)
Ejemplo n.º 6
0
            fileList.append(filepath)
    #Get new setting:
    settingNew = getNewSetting(settingOld, settingpath)

    #Rebuild files recursively:
    for targetFile in fileList:
        if (filter == []) or (targetFile in filter):
            if rebuildthreads > 0:        
                rebuildQueue.put(('rebuildFile',(settingOld, settingNew, targetFile)))
            else:
                rebuildFile(settingOld, settingNew, targetFile)
    if rebuildthreads > 0:
        waitForRebuild()

    #Backup old setting file, write new setting file:
    #print "Rebuild finishes. Writing new setting file."
    shutil.copyfile(settingpath, settingpath+".old")
    settingNew.write(settingpath)


if __name__ == '__main__':
    '''Rebuild all files if no argument. '''
    '''Rebuild filter files if filter argument exists.'''
    '''The filter argument is a list of strings.'''
    filter = []
    if len(sys.argv) > 1:
        filter = sys.argv[1:]
    #Clean temporary directories:
    clean.cleanAll()
    rebuildNode(filter)