def __init__(self, configfile):
self.configfile = configfile
self.paths = paths = []
i = 1
while self.configfile.has_option('disks', 'disk%s' % i):
prefix = self.configfile.get('disks', 'prefix%s' % i)
for path in glob.glob(self.configfile.get('disks', 'disk%s' % i)):
paths.append((path.rstrip(os.sep), prefix.strip('/')))
i += 1
reactor.suggestThreadPoolSize(len(paths) + 2)
self.filesystem = FileSystem(paths, self.configfile.get('general', 'dbfile'),
list(getPlugins(IMetadata, sdfs.plugins)))
if not self.plugins:
self.plugins = list(getPlugins(IURIHandler, sdfs.plugins))
for plugin in self.plugins:
log.msg('Found plugin for scheme %s' % plugin.scheme)
self.initialize_all_plugins()
self.rescan()
def setUp(self):
self.tmpfolder = tmpfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(tmpfolder)
self.dbfolder = dbfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(dbfolder)
self.folder1 = os.path.join(tmpfolder, 'tmp1') + os.sep
self.folder2 = os.path.join(tmpfolder, 'tmp2') + os.sep
os.mkdir(self.folder1)
os.mkdir(self.folder2)
os.mkdir(os.path.join(self.folder1, 'folder1'))
os.mkdir(os.path.join(self.folder1, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3'))
self._make_file(os.path.join(self.folder1, 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder1, 'folder2', 'file1'), 10)
self._make_file(os.path.join(self.folder1, 'folder2', 'file2'), 10)
self._make_file(os.path.join(self.folder2, 'folder2', 'file3'), 10)
self._make_file(os.path.join(self.folder1, 'file1'), 20)
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder1'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder3'))
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file2'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file3'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder2', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file1'), 5)
self.filesystem = FileSystem([(self.folder1.rstrip('/'), '/sdfs/'), (self.folder2.rstrip('/'), '/sdfs/')], os.path.join(dbfolder, 'db.db'), [])
self.filesystem.fsh.get_time = lambda:1000
return self.filesystem.rescan()
class SDFSServer(object):
"""General outward facing filesharing API"""
plugins = None
filesystem = None
configfile = None
scanning = False
last_scan = None
scan_defer = None
def __init__(self, configfile):
self.configfile = configfile
self.paths = paths = []
i = 1
while self.configfile.has_option('disks', 'disk%s' % i):
prefix = self.configfile.get('disks', 'prefix%s' % i)
for path in glob.glob(self.configfile.get('disks', 'disk%s' % i)):
paths.append((path.rstrip(os.sep), prefix.strip('/')))
i += 1
reactor.suggestThreadPoolSize(len(paths) + 2)
self.filesystem = FileSystem(paths, self.configfile.get('general', 'dbfile'),
list(getPlugins(IMetadata, sdfs.plugins)))
if not self.plugins:
self.plugins = list(getPlugins(IURIHandler, sdfs.plugins))
for plugin in self.plugins:
log.msg('Found plugin for scheme %s' % plugin.scheme)
self.initialize_all_plugins()
self.rescan()
def initialize_all_plugins(self):
for plugin in set(getPlugins(ISDFSPlugin, sdfs.plugins)):
log.msg('Found plugin %s, initializing' % plugin.name)
self.initialize_plugin(plugin)
def shutdown(self):
self.uninitialize_plugins()
return self.filesystem.close()
def rescan(self):
"""
Rescans the whole filesystem, looking from changes compared
to how the database currently matches the actual filesystem.
Returns a deferred, fired when scan is done.
"""
self.scanning = True
def done_scanning(ignored):
self.scanning = False
self.last_scan = datetime.now()
self.scan_defer = self.filesystem.rescan().addCallback(done_scanning)
def server_status(self):
"""
Returns current server status which is only affected by
an ongoing scan.
"""
if self.scanning:
status = 'scanning'
else:
status = 'idle'
return {
'date': datetime.now().isoformat(),
'status': status
}
def uninitialize_plugins(self):
for plugin in list(getPlugins(ISDFSPlugin, sdfs.plugins)):
if hasattr(plugin, 'is_initialized'):
delattr(plugin, 'is_initialized')
def initialize_plugin(self, plugin):
if not hasattr(plugin, 'is_initialized'):
setattr(plugin, 'filesystem', self.filesystem)
setattr(plugin, 'is_initialized', True)
setattr(plugin, 'paths', self.paths)
setattr(plugin, 'configfile', self.configfile)
setattr(plugin, 'config_section', 'plugin-%s' %
plugin.name)
if hasattr(plugin, 'initialize'):
plugin.initialize()
if ICrontab.providedBy(plugin):
result = plugin.schedule()
if result:
crontab_syntax, f = result
crontab = ScheduledCall(f)
crontab.start(CronSchedule(crontab_syntax))
def get_plugin(self, scheme):
"""
Takes a protocol scheme and returns the plugin matching it.
The function ensures that the plugin is initalized.
"""
for plugin in self.plugins:
if plugin.scheme == scheme:
self.initialize_plugin(plugin)
return plugin
raise UnknownSchemeException()
class FileSystemScannerTest(unittest.TestCase):
def _make_file(self, path, size):
f = open(path, 'w')
f.write('-'*size)
f.close()
def setUp(self):
self.tmpfolder = tmpfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(tmpfolder)
self.dbfolder = dbfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(dbfolder)
self.folder1 = os.path.join(tmpfolder, 'tmp1') + os.sep
self.folder2 = os.path.join(tmpfolder, 'tmp2') + os.sep
os.mkdir(self.folder1)
os.mkdir(self.folder2)
os.mkdir(os.path.join(self.folder1, 'folder1'))
os.mkdir(os.path.join(self.folder1, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3'))
self._make_file(os.path.join(self.folder1, 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder1, 'folder2', 'file1'), 10)
self._make_file(os.path.join(self.folder1, 'folder2', 'file2'), 10)
self._make_file(os.path.join(self.folder2, 'folder2', 'file3'), 10)
self._make_file(os.path.join(self.folder1, 'file1'), 20)
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder1'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder3'))
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file2'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file3'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder2', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file1'), 5)
self.filesystem = FileSystem([(self.folder1.rstrip('/'), '/sdfs/'), (self.folder2.rstrip('/'), '/sdfs/')], os.path.join(dbfolder, 'db.db'), [])
self.filesystem.fsh.get_time = lambda:1000
return self.filesystem.rescan()
@defer.inlineCallbacks
def tearDown(self):
yield self.filesystem.close()
shutil.rmtree(self.tmpfolder)
shutil.rmtree(self.dbfolder)
def testMetadataExists(self):
self.failIfIdentical(self.filesystem.get_metadata('/sdfs/folder1/'), None, 'No metadata found when there should be.')
def testMetadataDoesNotExist(self):
self.failUnlessIdentical(self.filesystem.get_metadata('/sdfs/folder10/'), None, 'Metadata found when there should be none.')
def testMultipleCyclesRescan(self):
import sdfs.fs
sdfs.fs.COMMIT_COUNTER = 1
yield self.filesystem.rescan()
sdfs.fs.COMMIT_COUNTER = 10000
def testListPrefixedSubFolder(self):
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 4,
"Wrong number of items listed in subfolder")
self.failUnlessEqual(path, 'sdfs',
"Wrong path returned")
for metadata in listing:
if metadata['type'] == DatabaseType.FILE:
self.failUnlessIn('size', metadata, 'No size for file')
self.failUnlessIn('date', metadata, 'No date for file')
break
else:
self.fail("No files found in folder when there should be")
def testMetadataRoot(self):
self.failUnlessEqual(self.filesystem.get_metadata(''),
{'date': 1000, 'modified': 1000, 'name': '', 'virtual_path': ''},
'Metadata for root is bonkers')
def testListRoot(self):
path, listing = self.filesystem.list_dir('/').items()[0]
self.failUnlessEqual(len(listing), 1,
"Wrong number of items listed in root")
self.failUnlessEqual(path, '',
"Wrong path returned")
metadata = listing[0]
self.failUnlessEqual(metadata['type'], DatabaseType.DIRECTORY,
"Wrong type detected for folder")
self.failUnlessEqual(metadata['name'], 'sdfs',
"Wrong folder name saved")
self.failUnlessIn('date', metadata, 'Date not in dir listing')
def testListSubSubFolderFromMultipleDisks(self):
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Wrong number of items listed in multidisk folder")
@defer.inlineCallbacks
def testDeleteFolder(self):
shutil.rmtree(os.path.join(self.folder1, 'folder1'))
yield self.filesystem.rescan()
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Deleting folder does not work")
@defer.inlineCallbacks
def testFileModified(self):
self.filesystem.fsh.get_time = lambda:2000
self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file10'), 5)
yield self.filesystem.rescan()
result = self.filesystem.list_dir('/sdfs/')
for item in result['sdfs']:
if item['name'] == 'file1':
self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on file')
elif item['name'] == 'folder3':
self.failIfEqual(item['date'], item['modified'], 'Not updated modified date on folder')
elif item['name'] == 'folder2':
self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on folder')
@defer.inlineCallbacks
def testDeleteNestedFolder(self):
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 4,
"Original state was wrong")
shutil.rmtree(os.path.join(self.folder2, 'folder3'))
yield self.filesystem.rescan()
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Deleting nested folder folder does not work")
@defer.inlineCallbacks
def testDeleteFile(self):
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
self.failUnlessEqual(len(listing), 3, "Wrong number of files before deleting")
os.remove(os.path.join(self.folder1, 'folder2', 'file1'))
yield self.filesystem.rescan()
try:
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
except:
self.fail('Exception while listing folder')
else:
self.failUnlessEqual(len(listing), 2, "Deleting file does not work")
def testListUnknownPath(self):
path, listing = self.filesystem.list_dir('unknown/path').items()[0]
self.failUnlessEqual(path, 'unknown/path', 'Failed to list unknown path')
self.failUnlessEqual(listing, None, 'Failed to list unknown path')
def testGetUnknownFile(self):
self.failUnlessIdentical(self.filesystem.get_file('unknown/path'), None, 'Failed to get unknown file')
def testGetFile(self):
file_endswith = '/tmp1/file1'
f = self.filesystem.get_file('sdfs/file1')
if not f.endswith(file_endswith):
self.fail('Failed to get file, was "%s", expected the following ending: "%s"' % (f, file_endswith))
@defer.inlineCallbacks
def testListEmptyFolderWithSpace(self):
d = os.path.join(self.folder1, 'folder 2')
os.mkdir(d)
yield self.filesystem.rescan()
self.failUnless(self.filesystem.list_dir('sdfs/folder 2')['sdfs/folder 2'] != None,
'Unable to list folders with space in them')
@defer.inlineCallbacks
def testAddNewFile(self):
f = os.path.join(self.folder1, 'folder2', 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50'],
'Unable to manually add a file')
@defer.inlineCallbacks
def testAddNewFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50'],
'Unable to manually add a file with folder')
@defer.inlineCallbacks
def testAddOldFile(self):
f = os.path.join(self.folder1, 'folder2', 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 500)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50' and x['date'] == 500],
'Unable to manually add a file with old time')
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder2' and x['date'] == 1000],
'Changed age of subfolder when it was newer')
@defer.inlineCallbacks
def testAddOldFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 500)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == 500],
'Unable to manually add a file with folder and old age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == 500],
'Unable to manually add a folder with correct old age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['date'] == 1000],
'Changed foldertime to the past')
@defer.inlineCallbacks
def testAddNewNewFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 3000)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == x['modified'] == 3000],
'Unable to manually add a file with folder and new new age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == x['modified'] == 3000],
'Unable to manually add a folder with correct new new age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['modified'] == 3000 and x['date'] == 1000],
'Did not change foldertime to the future')
@defer.inlineCallbacks
def testTmpFolderNotScanned(self):
d = os.path.join(self.folder2, '.tmp')
os.mkdir(d)
d = os.path.join(d, 'folder2')
os.mkdir(d)
f = os.path.join(d, 'file50')
self._make_file(f, 10)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == '.tmp'],
'.tmp folder was scanned when it should not')
self.failIf(self.filesystem.list_dir('sdfs/.tmp')['sdfs/.tmp'],
'.tmp folder should not be scanned')
self.failIf(self.filesystem.list_dir('sdfs/.tmp/folder2')['sdfs/.tmp/folder2'],
'.tmp subfolder should not be scanned')
@defer.inlineCallbacks
def testResurrectFile(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
self.filesystem.set_metadata(DatabaseType.FILE, 'sdfs/folder1/file1', 'test_metadata', 'someval')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
'Unable to set metadata for test')
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was found')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1' and x['test_metadata'] == 'someval'],
'Deleted file was not found using show deleted')
self._make_file(f, 5)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
'Undeleted file was not found')
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
'Retained metadata, should not')
@defer.inlineCallbacks
def testResurrectFolder(self):
d = os.path.join(self.folder1, 'folder1')
shutil.rmtree(d)
self.filesystem.set_metadata(DatabaseType.DIRECTORY, 'sdfs/folder1', 'test_metadata', 'someval')
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
'Unable to set metadata for test')
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
'Unable to delete folder')
self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Unable to see delete folder')
os.mkdir(d)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
'Undeleted file was not found')
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
'Retained metadata, should not')
@defer.inlineCallbacks
def testGarbageCollect(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was not found using show deleted')
self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 2)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'File should have been GCed')
d = os.path.join(self.folder1, 'folder1')
shutil.rmtree(d)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Unable to see delete folder')
self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 4)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Folder should have been GCed')
@defer.inlineCallbacks
def testDeleteDualRescan(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
yield self.filesystem.rescan()
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was not found using show deleted')
@defer.inlineCallbacks
def testDeleteFileModified(self):
self.filesystem.fsh.get_time = lambda:2000
os.remove(os.path.join(self.folder1, 'folder1', 'file1'))
yield self.filesystem.rescan()
self.failUnless(self.filesystem.get_metadata('sdfs/folder1').get('modified') == 2000,
'Modified did not propagate to folder containing file')
@defer.inlineCallbacks
def testMoveFileOtherDisk(self):
os.rename(os.path.join(self.folder1, 'folder2', 'file2'), os.path.join(self.folder2, 'folder2', 'file2'))
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file2'],
'Disk moved to other disk did not show up')