class FileSystemScannerTest(unittest.TestCase):
def _make_file(self, path, size):
f = open(path, 'w')
f.write('-'*size)
f.close()
def setUp(self):
self.tmpfolder = tmpfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(tmpfolder)
self.dbfolder = dbfolder = os.path.join(os.getcwd(), self.mktemp())
os.mkdir(dbfolder)
self.folder1 = os.path.join(tmpfolder, 'tmp1') + os.sep
self.folder2 = os.path.join(tmpfolder, 'tmp2') + os.sep
os.mkdir(self.folder1)
os.mkdir(self.folder2)
os.mkdir(os.path.join(self.folder1, 'folder1'))
os.mkdir(os.path.join(self.folder1, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3'))
self._make_file(os.path.join(self.folder1, 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder1, 'folder2', 'file1'), 10)
self._make_file(os.path.join(self.folder1, 'folder2', 'file2'), 10)
self._make_file(os.path.join(self.folder2, 'folder2', 'file3'), 10)
self._make_file(os.path.join(self.folder1, 'file1'), 20)
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder1'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder2'))
os.mkdir(os.path.join(self.folder2, 'folder3', 'folder3'))
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file2'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder1', 'file3'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder2', 'file1'), 5)
self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file1'), 5)
self.filesystem = FileSystem([(self.folder1.rstrip('/'), '/sdfs/'), (self.folder2.rstrip('/'), '/sdfs/')], os.path.join(dbfolder, 'db.db'), [])
self.filesystem.fsh.get_time = lambda:1000
return self.filesystem.rescan()
@defer.inlineCallbacks
def tearDown(self):
yield self.filesystem.close()
shutil.rmtree(self.tmpfolder)
shutil.rmtree(self.dbfolder)
def testMetadataExists(self):
self.failIfIdentical(self.filesystem.get_metadata('/sdfs/folder1/'), None, 'No metadata found when there should be.')
def testMetadataDoesNotExist(self):
self.failUnlessIdentical(self.filesystem.get_metadata('/sdfs/folder10/'), None, 'Metadata found when there should be none.')
def testMultipleCyclesRescan(self):
import sdfs.fs
sdfs.fs.COMMIT_COUNTER = 1
yield self.filesystem.rescan()
sdfs.fs.COMMIT_COUNTER = 10000
def testListPrefixedSubFolder(self):
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 4,
"Wrong number of items listed in subfolder")
self.failUnlessEqual(path, 'sdfs',
"Wrong path returned")
for metadata in listing:
if metadata['type'] == DatabaseType.FILE:
self.failUnlessIn('size', metadata, 'No size for file')
self.failUnlessIn('date', metadata, 'No date for file')
break
else:
self.fail("No files found in folder when there should be")
def testMetadataRoot(self):
self.failUnlessEqual(self.filesystem.get_metadata(''),
{'date': 1000, 'modified': 1000, 'name': '', 'virtual_path': ''},
'Metadata for root is bonkers')
def testListRoot(self):
path, listing = self.filesystem.list_dir('/').items()[0]
self.failUnlessEqual(len(listing), 1,
"Wrong number of items listed in root")
self.failUnlessEqual(path, '',
"Wrong path returned")
metadata = listing[0]
self.failUnlessEqual(metadata['type'], DatabaseType.DIRECTORY,
"Wrong type detected for folder")
self.failUnlessEqual(metadata['name'], 'sdfs',
"Wrong folder name saved")
self.failUnlessIn('date', metadata, 'Date not in dir listing')
def testListSubSubFolderFromMultipleDisks(self):
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Wrong number of items listed in multidisk folder")
@defer.inlineCallbacks
def testDeleteFolder(self):
shutil.rmtree(os.path.join(self.folder1, 'folder1'))
yield self.filesystem.rescan()
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Deleting folder does not work")
@defer.inlineCallbacks
def testFileModified(self):
self.filesystem.fsh.get_time = lambda:2000
self._make_file(os.path.join(self.folder2, 'folder3', 'folder3', 'file10'), 5)
yield self.filesystem.rescan()
result = self.filesystem.list_dir('/sdfs/')
for item in result['sdfs']:
if item['name'] == 'file1':
self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on file')
elif item['name'] == 'folder3':
self.failIfEqual(item['date'], item['modified'], 'Not updated modified date on folder')
elif item['name'] == 'folder2':
self.failUnlessEqual(item['date'], item['modified'], 'Updated modified date on folder')
@defer.inlineCallbacks
def testDeleteNestedFolder(self):
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 4,
"Original state was wrong")
shutil.rmtree(os.path.join(self.folder2, 'folder3'))
yield self.filesystem.rescan()
path, listing = self.filesystem.list_dir('/sdfs/').items()[0]
self.failUnlessEqual(len(listing), 3,
"Deleting nested folder folder does not work")
@defer.inlineCallbacks
def testDeleteFile(self):
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
self.failUnlessEqual(len(listing), 3, "Wrong number of files before deleting")
os.remove(os.path.join(self.folder1, 'folder2', 'file1'))
yield self.filesystem.rescan()
try:
path, listing = self.filesystem.list_dir('/sdfs/folder2/').items()[0]
except:
self.fail('Exception while listing folder')
else:
self.failUnlessEqual(len(listing), 2, "Deleting file does not work")
def testListUnknownPath(self):
path, listing = self.filesystem.list_dir('unknown/path').items()[0]
self.failUnlessEqual(path, 'unknown/path', 'Failed to list unknown path')
self.failUnlessEqual(listing, None, 'Failed to list unknown path')
def testGetUnknownFile(self):
self.failUnlessIdentical(self.filesystem.get_file('unknown/path'), None, 'Failed to get unknown file')
def testGetFile(self):
file_endswith = '/tmp1/file1'
f = self.filesystem.get_file('sdfs/file1')
if not f.endswith(file_endswith):
self.fail('Failed to get file, was "%s", expected the following ending: "%s"' % (f, file_endswith))
@defer.inlineCallbacks
def testListEmptyFolderWithSpace(self):
d = os.path.join(self.folder1, 'folder 2')
os.mkdir(d)
yield self.filesystem.rescan()
self.failUnless(self.filesystem.list_dir('sdfs/folder 2')['sdfs/folder 2'] != None,
'Unable to list folders with space in them')
@defer.inlineCallbacks
def testAddNewFile(self):
f = os.path.join(self.folder1, 'folder2', 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50'],
'Unable to manually add a file')
@defer.inlineCallbacks
def testAddNewFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50'],
'Unable to manually add a file with folder')
@defer.inlineCallbacks
def testAddOldFile(self):
f = os.path.join(self.folder1, 'folder2', 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 500)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file50' and x['date'] == 500],
'Unable to manually add a file with old time')
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder2' and x['date'] == 1000],
'Changed age of subfolder when it was newer')
@defer.inlineCallbacks
def testAddOldFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 500)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == 500],
'Unable to manually add a file with folder and old age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == 500],
'Unable to manually add a folder with correct old age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['date'] == 1000],
'Changed foldertime to the past')
@defer.inlineCallbacks
def testAddNewNewFileAndFolder(self):
f = os.path.join(self.folder2, 'folder3', 'folder2', 'folder4')
os.mkdir(f)
f = os.path.join(f, 'file50')
self._make_file(f, 10)
yield self.filesystem.add_file(f, 3000)
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2/folder4')['sdfs/folder3/folder2/folder4'] if x['name'] == 'file50' and x['date'] == x['modified'] == 3000],
'Unable to manually add a file with folder and new new age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3/folder2')['sdfs/folder3/folder2'] if x['name'] == 'folder4' and x['date'] == x['modified'] == 3000],
'Unable to manually add a folder with correct new new age')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder3')['sdfs/folder3'] if x['name'] == 'folder2' and x['modified'] == 3000 and x['date'] == 1000],
'Did not change foldertime to the future')
@defer.inlineCallbacks
def testTmpFolderNotScanned(self):
d = os.path.join(self.folder2, '.tmp')
os.mkdir(d)
d = os.path.join(d, 'folder2')
os.mkdir(d)
f = os.path.join(d, 'file50')
self._make_file(f, 10)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == '.tmp'],
'.tmp folder was scanned when it should not')
self.failIf(self.filesystem.list_dir('sdfs/.tmp')['sdfs/.tmp'],
'.tmp folder should not be scanned')
self.failIf(self.filesystem.list_dir('sdfs/.tmp/folder2')['sdfs/.tmp/folder2'],
'.tmp subfolder should not be scanned')
@defer.inlineCallbacks
def testResurrectFile(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
self.filesystem.set_metadata(DatabaseType.FILE, 'sdfs/folder1/file1', 'test_metadata', 'someval')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
'Unable to set metadata for test')
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was found')
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1' and x['test_metadata'] == 'someval'],
'Deleted file was not found using show deleted')
self._make_file(f, 5)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1'],
'Undeleted file was not found')
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1')['sdfs/folder1'] if x['name'] == 'file1' and x.get('test_metadata', None) == 'someval'],
'Retained metadata, should not')
@defer.inlineCallbacks
def testResurrectFolder(self):
d = os.path.join(self.folder1, 'folder1')
shutil.rmtree(d)
self.filesystem.set_metadata(DatabaseType.DIRECTORY, 'sdfs/folder1', 'test_metadata', 'someval')
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
'Unable to set metadata for test')
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
'Unable to delete folder')
self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Unable to see delete folder')
os.mkdir(d)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1'],
'Undeleted file was not found')
self.failIf([x for x in self.filesystem.list_dir('sdfs')['sdfs'] if x['name'] == 'folder1' and x.get('test_metadata', None) == 'someval'],
'Retained metadata, should not')
@defer.inlineCallbacks
def testGarbageCollect(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was not found using show deleted')
self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 2)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'File should have been GCed')
d = os.path.join(self.folder1, 'folder1')
shutil.rmtree(d)
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Unable to see delete folder')
self.filesystem.fsh.get_time = lambda:(GC_DELETED_FILES * 4)
yield self.filesystem.rescan()
self.failIf([x for x in self.filesystem.list_dir('sdfs', show_deleted=True)['sdfs'] if x['name'] == 'folder1'],
'Folder should have been GCed')
@defer.inlineCallbacks
def testDeleteDualRescan(self):
f = os.path.join(self.folder1, 'folder1', 'file1')
os.remove(f)
yield self.filesystem.rescan()
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder1', show_deleted=True)['sdfs/folder1'] if x['name'] == 'file1'],
'Deleted file was not found using show deleted')
@defer.inlineCallbacks
def testDeleteFileModified(self):
self.filesystem.fsh.get_time = lambda:2000
os.remove(os.path.join(self.folder1, 'folder1', 'file1'))
yield self.filesystem.rescan()
self.failUnless(self.filesystem.get_metadata('sdfs/folder1').get('modified') == 2000,
'Modified did not propagate to folder containing file')
@defer.inlineCallbacks
def testMoveFileOtherDisk(self):
os.rename(os.path.join(self.folder1, 'folder2', 'file2'), os.path.join(self.folder2, 'folder2', 'file2'))
yield self.filesystem.rescan()
self.failUnless([x for x in self.filesystem.list_dir('sdfs/folder2')['sdfs/folder2'] if x['name'] == 'file2'],
'Disk moved to other disk did not show up')
