Beispiel #1
0
 def __init__(self, configfile):
     self.configfile = configfile
     
     self.paths = paths = []
     i = 1
     while self.configfile.has_option('disks', 'disk%s' % i):
         prefix = self.configfile.get('disks', 'prefix%s' % i)
         for path in glob.glob(self.configfile.get('disks', 'disk%s' % i)):
             paths.append((path.rstrip(os.sep), prefix.strip('/')))
         i += 1
     
     reactor.suggestThreadPoolSize(len(paths) + 2)
     self.filesystem = FileSystem(paths, self.configfile.get('general', 'dbfile'),
                                  list(getPlugins(IMetadata, sdfs.plugins)))
     
     if not self.plugins:
         self.plugins = list(getPlugins(IURIHandler, sdfs.plugins))
         for plugin in self.plugins:
             log.msg('Found plugin for scheme %s' % plugin.scheme)
     
     self.initialize_all_plugins()
     
     self.rescan()
 def setUp(self):
     self.tmpfolder = tmpfolder = os.path.join(os.getcwd(), self.mktemp())
     os.mkdir(tmpfolder)
     
     self.dbfolder = dbfolder = os.path.join(os.getcwd(), self.mktemp())
     os.mkdir(dbfolder)
     
     self.folder1 = os.path.join(tmpfolder, 'tmp1') + os.sep
     self.folder2 = os.path.join(tmpfolder, 'tmp2') + os.sep
         
     os.mkdir(self.folder1)
     os.mkdir(self.folder2)
         
     os.mkdir(os.path.join(self.folder1, 'folder1'))
     os.mkdir(os.path.join(self.folder1, 'folder2'))
     os.mkdir(os.path.join(self.folder2, 'folder2'))
     os.mkdir(os.path.join(self.folder2, 'folder3'))
     
     self._make_file(os.path.join(self.folder1, 'folder1', 'file1'), 5)
     
     self._make_file(os.path.join(self.folder1, 'folder2', 'file1'), 10)
     self._make_file(os.path.join(self.folder1, 'folder2', 'file2'), 10)
     self._make_file(os.path.join(self.folder2, 'folder2', 'file3'), 10)
     self._make_file(os.path.join(self.folder1, 'file1'), 20)
     
     os.mkdir(os.path.join(self.folder2, 'folder3', 'folder1'))
     os.mkdir(os.path.join(self.folder2, 'folder3', 'folder2'))
     os.mkdir(os.path.join(self.folder2, 'folder3', 'folder3'))
     self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file1'), 5)
     self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file2'), 5)
     self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file3'), 5)
     self._make_file(os.path.join(self.folder2, 'folder3', 'folder2', 'file1'), 5)
     self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file1'), 5)
     
     self.filesystem = FileSystem([(self.folder1.rstrip('/'), '/sdfs/'), (self.folder2.rstrip('/'), '/sdfs/')], os.path.join(dbfolder, 'db.db'), [])
     self.filesystem.fsh.get_time = lambda:1000
     return self.filesystem.rescan()
Beispiel #3
0
class SDFSServer(object):
    """General outward facing filesharing API"""
    
    plugins = None
    filesystem = None
    configfile = None
    scanning = False
    last_scan = None
    scan_defer = None
    
    def __init__(self, configfile):
        self.configfile = configfile
        
        self.paths = paths = []
        i = 1
        while self.configfile.has_option('disks', 'disk%s' % i):
            prefix = self.configfile.get('disks', 'prefix%s' % i)
            for path in glob.glob(self.configfile.get('disks', 'disk%s' % i)):
                paths.append((path.rstrip(os.sep), prefix.strip('/')))
            i += 1
        
        reactor.suggestThreadPoolSize(len(paths) + 2)
        self.filesystem = FileSystem(paths, self.configfile.get('general', 'dbfile'),
                                     list(getPlugins(IMetadata, sdfs.plugins)))
        
        if not self.plugins:
            self.plugins = list(getPlugins(IURIHandler, sdfs.plugins))
            for plugin in self.plugins:
                log.msg('Found plugin for scheme %s' % plugin.scheme)
        
        self.initialize_all_plugins()
        
        self.rescan()
        
    def initialize_all_plugins(self):
        for plugin in set(getPlugins(ISDFSPlugin, sdfs.plugins)):
            log.msg('Found plugin %s, initializing' % plugin.name)
            self.initialize_plugin(plugin)
    
    def shutdown(self):
        self.uninitialize_plugins()
        return self.filesystem.close()
    
    def rescan(self):
        """
        Rescans the whole filesystem, looking from changes compared
        to how the database currently matches the actual filesystem.
        
        Returns a deferred, fired when scan is done.
        """
        
        self.scanning = True
        def done_scanning(ignored):
            self.scanning = False
            self.last_scan = datetime.now()
        
        self.scan_defer = self.filesystem.rescan().addCallback(done_scanning)
    
    def server_status(self):
        """
        Returns current server status which is only affected by
        an ongoing scan.
        """
        if self.scanning:
            status = 'scanning'
        else:
            status = 'idle'
        
        return {
            'date': datetime.now().isoformat(),
            'status': status
        }
    
    def uninitialize_plugins(self):
        for plugin in list(getPlugins(ISDFSPlugin, sdfs.plugins)):
            if hasattr(plugin, 'is_initialized'):
                delattr(plugin, 'is_initialized')
    
    def initialize_plugin(self, plugin):
        if not hasattr(plugin, 'is_initialized'):
            setattr(plugin, 'filesystem', self.filesystem)
            setattr(plugin, 'is_initialized', True)
            setattr(plugin, 'paths', self.paths)
            setattr(plugin, 'configfile', self.configfile)
            setattr(plugin, 'config_section', 'plugin-%s' %
                plugin.name)
            if hasattr(plugin, 'initialize'):
                plugin.initialize()
            
            if ICrontab.providedBy(plugin):
                result = plugin.schedule()
                if result:
                    crontab_syntax, f = result
                    crontab = ScheduledCall(f)
                    crontab.start(CronSchedule(crontab_syntax))
                    
    
    def get_plugin(self, scheme):
        """
        Takes a protocol scheme and returns the plugin matching it.
        The function ensures that the plugin is initalized.
        """
        for plugin in self.plugins:
            if plugin.scheme == scheme:
                self.initialize_plugin(plugin)
                return plugin
        
        raise UnknownSchemeException()
class FileSystemScannerTest(unittest.TestCase):
    def _make_file(self, path, size):
        f = open(path, 'w')
        f.write('-'*size)
        f.close()
    
    def setUp(self):
        self.tmpfolder = tmpfolder = os.path.join(os.getcwd(), self.mktemp())
        os.mkdir(tmpfolder)
        
        self.dbfolder = dbfolder = os.path.join(os.getcwd(), self.mktemp())
        os.mkdir(dbfolder)
        
        self.folder1 = os.path.join(tmpfolder, 'tmp1') + os.sep
        self.folder2 = os.path.join(tmpfolder, 'tmp2') + os.sep
            
        os.mkdir(self.folder1)
        os.mkdir(self.folder2)
            
        os.mkdir(os.path.join(self.folder1, 'folder1'))
        os.mkdir(os.path.join(self.folder1, 'folder2'))
        os.mkdir(os.path.join(self.folder2, 'folder2'))
        os.mkdir(os.path.join(self.folder2, 'folder3'))
        
        self._make_file(os.path.join(self.folder1, 'folder1', 'file1'), 5)
        
        self._make_file(os.path.join(self.folder1, 'folder2', 'file1'), 10)
        self._make_file(os.path.join(self.folder1, 'folder2', 'file2'), 10)
        self._make_file(os.path.join(self.folder2, 'folder2', 'file3'), 10)
        self._make_file(os.path.join(self.folder1, 'file1'), 20)
        
        os.mkdir(os.path.join(self.folder2, 'folder3', 'folder1'))
        os.mkdir(os.path.join(self.folder2, 'folder3', 'folder2'))
        os.mkdir(os.path.join(self.folder2, 'folder3', 'folder3'))
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file1'), 5)
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file2'), 5)
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file3'), 5)
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder2', 'file1'), 5)
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file1'), 5)
        
        self.filesystem = FileSystem([(self.folder1.rstrip('/'), '/sdfs/'), (self.folder2.rstrip('/'), '/sdfs/')], os.path.join(dbfolder, 'db.db'), [])
        self.filesystem.fsh.get_time = lambda:1000
        return self.filesystem.rescan()
    
    @defer.inlineCallbacks
    def tearDown(self):
        yield self.filesystem.close()
        shutil.rmtree(self.tmpfolder)
        shutil.rmtree(self.dbfolder)
    
    def testMetadataExists(self):
        self.failIfIdentical(self.filesystem.get_metadata('/sdfs/folder1/'), None, 'No metadata found when there should be.')
    
    def testMetadataDoesNotExist(self):
        self.failUnlessIdentical(self.filesystem.get_metadata('/sdfs/folder10/'), None, 'Metadata found when there should be none.')
    
    def testMultipleCyclesRescan(self):
        import sdfs.fs
        sdfs.fs.COMMIT_COUNTER = 1
        
        yield self.filesystem.rescan()
        
        sdfs.fs.COMMIT_COUNTER = 10000
        
    def testListPrefixedSubFolder(self):
        path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
        self.failUnlessEqual(len(listing), 4,
                             "Wrong number of items listed in subfolder")
        self.failUnlessEqual(path, 'sdfs',
                             "Wrong path returned")
        
        for metadata in listing:
            if metadata['type'] == DatabaseType.FILE:
                self.failUnlessIn('size', metadata, 'No size for file')
                self.failUnlessIn('date', metadata, 'No date for file')
                break
        else:
            self.fail("No files found in folder when there should be")
    
    def testMetadataRoot(self):
        self.failUnlessEqual(self.filesystem.get_metadata(''),
                             {'date': 1000, 'modified': 1000, 'name': '', 'virtual_path': ''},
                             'Metadata for root is bonkers')
    
    def testListRoot(self):
        path, listing = self.filesystem.list_dir('/').items()[0]
        self.failUnlessEqual(len(listing), 1,
                             "Wrong number of items listed in root")
        self.failUnlessEqual(path, '',
                             "Wrong path returned")
        
        metadata = listing[0]
        self.failUnlessEqual(metadata['type'], DatabaseType.DIRECTORY,
                             "Wrong type detected for folder")
        self.failUnlessEqual(metadata['name'], 'sdfs',
                             "Wrong folder name saved")
        
        self.failUnlessIn('date', metadata, 'Date not in dir listing')
        
    def testListSubSubFolderFromMultipleDisks(self):
        path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
        self.failUnlessEqual(len(listing), 3,
                             "Wrong number of items listed in multidisk folder")
    
    @defer.inlineCallbacks
    def testDeleteFolder(self):
        shutil.rmtree(os.path.join(self.folder1, 'folder1'))
        
        yield self.filesystem.rescan()
        
        path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
        self.failUnlessEqual(len(listing), 3,
                            "Deleting folder does not work")
    
    @defer.inlineCallbacks
    def testFileModified(self):
        self.filesystem.fsh.get_time = lambda:2000
        
        self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file10'), 5)
        yield self.filesystem.rescan()
        
        result = self.filesystem.list_dir('/sdfs/')
        for item in result['sdfs']:
            if item['name'] == 'file1':
                self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on file')
            elif item['name'] == 'folder3':
                self.failIfEqual(item['date'], item['modified'], 'Not updated modified date on folder')
            elif item['name'] == 'folder2':
                self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on folder')
    
    @defer.inlineCallbacks
    def testDeleteNestedFolder(self):
        path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
        self.failUnlessEqual(len(listing), 4,
                            "Original state was wrong")
        
        shutil.rmtree(os.path.join(self.folder2, 'folder3'))
        
        yield self.filesystem.rescan()
        path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
        self.failUnlessEqual(len(listing), 3,
                            "Deleting nested folder folder does not work")
            
    @defer.inlineCallbacks
    def testDeleteFile(self):
        path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
        self.failUnlessEqual(len(listing), 3, "Wrong number of files before deleting")
        
        os.remove(os.path.join(self.folder1, 'folder2', 'file1'))
        
        yield self.filesystem.rescan()
        try:
            path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
        except:
            self.fail('Exception while listing folder')
        else:
            self.failUnlessEqual(len(listing), 2, "Deleting file does not work")
    
    def testListUnknownPath(self):
        path, listing = self.filesystem.list_dir('unknown/path').items()[0]
        self.failUnlessEqual(path, 'unknown/path', 'Failed to list unknown path')
        self.failUnlessEqual(listing, None, 'Failed to list unknown path')
    
    def testGetUnknownFile(self):
        self.failUnlessIdentical(self.filesystem.get_file('unknown/path'), None, 'Failed to get unknown file')
    
    def testGetFile(self):
        file_endswith = '/tmp1/file1'
        f = self.filesystem.get_file('sdfs/file1')
        if not f.endswith(file_endswith):
            self.fail('Failed to get file, was "%s", expected the following ending: "%s"' % (f, file_endswith))
    
    @defer.inlineCallbacks
    def testListEmptyFolderWithSpace(self):
        d = os.path.join(self.folder1, 'folder 2')
        os.mkdir(d)
        yield self.filesystem.rescan()
        self.failUnless(self.filesystem.list_dir('sdfs/folder 2')['sdfs/folder 2'] != None,
            'Unable to list folders with space in them')
    
    @defer.inlineCallbacks
    def testAddNewFile(self):
        f = os.path.join(self.folder1, 'folder2', 'file50')
        self._make_file(f, 10)
        yield self.filesystem.add_file(f)
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50'],
            'Unable to manually add a file')
    
    @defer.inlineCallbacks
    def testAddNewFileAndFolder(self):
        f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
        os.mkdir(f)
        f = os.path.join(f, 'file50')
        self._make_file(f, 10)
        yield self.filesystem.add_file(f)
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50'],
            'Unable to manually add a file with folder')
    
    @defer.inlineCallbacks
    def testAddOldFile(self):
        f = os.path.join(self.folder1, 'folder2', 'file50')
        self._make_file(f, 10)
        yield self.filesystem.add_file(f, 500)
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50' and x['date'] == 500],
            'Unable to manually add a file with old time')
        
        self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder2' and x['date'] == 1000],
            'Changed age of subfolder when it was newer')
    
    @defer.inlineCallbacks
    def testAddOldFileAndFolder(self):
        f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
        os.mkdir(f)
        f = os.path.join(f, 'file50')
        self._make_file(f, 10)
        yield self.filesystem.add_file(f, 500)
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == 500],
            'Unable to manually add a file with folder and old age')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == 500],
            'Unable to manually add a folder with correct old age')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['date'] == 1000],
            'Changed foldertime to the past')
    
    @defer.inlineCallbacks
    def testAddNewNewFileAndFolder(self):
        f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
        os.mkdir(f)
        f = os.path.join(f, 'file50')
        self._make_file(f, 10)
        yield self.filesystem.add_file(f, 3000)
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == x['modified'] == 3000],
            'Unable to manually add a file with folder and new new age')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == x['modified'] == 3000],
            'Unable to manually add a folder with correct new new age')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['modified'] == 3000 and x['date'] == 1000],
            'Did not change foldertime to the future')
    
    @defer.inlineCallbacks
    def testTmpFolderNotScanned(self):
        d = os.path.join(self.folder2, '.tmp')
        os.mkdir(d)
        d = os.path.join(d, 'folder2')
        os.mkdir(d)
        f = os.path.join(d, 'file50')
        self._make_file(f, 10)
        yield self.filesystem.rescan()
        self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == '.tmp'],
            '.tmp folder was scanned when it should not')
        self.failIf(self.filesystem.list_dir('sdfs/.tmp')['sdfs/.tmp'],
            '.tmp folder should not be scanned')
        self.failIf(self.filesystem.list_dir('sdfs/.tmp/folder2')['sdfs/.tmp/folder2'],
            '.tmp subfolder should not be scanned')
    
    @defer.inlineCallbacks
    def testResurrectFile(self):
        f = os.path.join(self.folder1, 'folder1', 'file1')
        os.remove(f)
        self.filesystem.set_metadata(DatabaseType.FILE, 'sdfs/folder1/file1', 'test_metadata', 'someval')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
            'Unable to set metadata for test')
        yield self.filesystem.rescan()
        self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
            'Deleted file was found')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1' and x['test_metadata'] == 'someval'],
            'Deleted file was not found using show deleted')
        self._make_file(f, 5)
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
            'Undeleted file was not found')
        self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
            'Retained metadata, should not')
    
    @defer.inlineCallbacks
    def testResurrectFolder(self):
        d = os.path.join(self.folder1, 'folder1')
        shutil.rmtree(d)
        self.filesystem.set_metadata(DatabaseType.DIRECTORY, 'sdfs/folder1', 'test_metadata', 'someval')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
            'Unable to set metadata for test')
        yield self.filesystem.rescan()
        self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
            'Unable to delete folder')
        self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
            'Unable to see delete folder')
        os.mkdir(d)
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
            'Undeleted file was not found')
        self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
            'Retained metadata, should not')
    
    @defer.inlineCallbacks
    def testGarbageCollect(self):
        f = os.path.join(self.folder1, 'folder1', 'file1')
        os.remove(f)
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
            'Deleted file was not found using show deleted')
        self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 2)
        yield self.filesystem.rescan()
        self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
            'File should have been GCed')
        
        d = os.path.join(self.folder1, 'folder1')
        shutil.rmtree(d)
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
            'Unable to see delete folder')

        self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 4)
        yield self.filesystem.rescan()
        
        self.failIf([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
            'Folder should have been GCed')
    
    @defer.inlineCallbacks
    def testDeleteDualRescan(self):
        f = os.path.join(self.folder1, 'folder1', 'file1')
        os.remove(f)
        yield self.filesystem.rescan()
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
            'Deleted file was not found using show deleted')
    
    @defer.inlineCallbacks
    def testDeleteFileModified(self):
        self.filesystem.fsh.get_time = lambda:2000
        os.remove(os.path.join(self.folder1, 'folder1', 'file1'))
        yield self.filesystem.rescan()
        self.failUnless(self.filesystem.get_metadata('sdfs/folder1').get('modified') == 2000,
            'Modified did not propagate to folder containing file')

    @defer.inlineCallbacks
    def testMoveFileOtherDisk(self):
        os.rename(os.path.join(self.folder1, 'folder2', 'file2'), os.path.join(self.folder2, 'folder2', 'file2'))
        yield self.filesystem.rescan()
        self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file2'],
            'Disk moved to other disk did not show up')