def movedir(fs1,
fs2,
overwrite=False,
ignore_errors=False,
chunk_size=64 * 1024):
"""Moves contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used
"""
if isinstance(fs1, tuple):
fs1, dir1 = fs1
fs1 = fs1.opendir(dir1)
if isinstance(fs2, tuple):
fs2, dir2 = fs2
fs2.makedir(dir2, allow_recreate=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS()
mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2)
mount_fs.movedir('src',
'dst',
overwrite=overwrite,
ignore_errors=ignore_errors,
chunk_size=chunk_size)
def init_media(self):
if 'media' not in self.filesystems:
return
if not self.media_urls:
if not self.media_app:
raise errors.StartupFailedError("no 'url' or 'app' specified in [media] section")
if self.media_app not in self.apps:
startup_log.warning('app set in [media]/app has not been installed ({})'.format(self.media_app))
return
try:
self.media_urls = [self.apps[self.media_app].mounts[0][1]]
except:
raise errors.StartupFailedError('unable to detect media url! (specify in [media]/url)')
for i, _url in enumerate(self.media_urls):
startup_log.debug('media url #%s is %s', i, _url)
media_fs = self.filesystems['media']
media_mount_fs = MountFS()
for app in itervalues(self.apps):
for media_name, media_sub_fs in iteritems(app.lib.media):
name = "%s_%s" % (app.name, media_name)
media_path = "%s-%s" % (app.name, media_name)
app.media[media_name] = media_path
if name in self.filesystems:
mount_media = self.filesystems[name]
else:
mount_media = media_sub_fs
if name not in self.filesystems:
self.filesystems[name] = mount_media
media_mount_fs.mountdir(media_path, mount_media)
media_fs.addfs("media", media_mount_fs)
def copydir(fs1,
fs2,
create_destination=True,
ignore_errors=False,
chunk_size=64 * 1024):
"""Copies contents of a directory from one filesystem to another.
:param fs1: Source filesystem, or a tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used
"""
if isinstance(fs1, tuple):
fs1, dir1 = fs1
fs1 = fs1.opendir(dir1)
if isinstance(fs2, tuple):
fs2, dir2 = fs2
if create_destination:
fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2)
mount_fs.copydir('src',
'dst',
overwrite=True,
ignore_errors=ignore_errors,
chunk_size=chunk_size)
def test_empty(self):
"""Test MountFS with nothing mounted."""
mount_fs = MountFS()
self.assertEqual(mount_fs.getinfo(''), {})
self.assertEqual(mount_fs.getxattr('', 'yo'), None)
self.assertEqual(mount_fs.listdir(), [])
self.assertEqual(list(mount_fs.ilistdir()), [])
def test_auto_close(self):
"""Test MountFS auto close is working"""
mount_fs = MountFS()
m1 = MemoryFS()
m2 = MemoryFS()
mount_fs.mount("/m1", m1)
mount_fs.mount("/m2", m2)
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
mount_fs.close()
self.assertTrue(m1.isclosed())
self.assertTrue(m2.isclosed())
def test_no_auto_close(self):
"""Test MountFS auto close can be disabled"""
mount_fs = MountFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
mount_fs.mount("/m1", m1)
mount_fs.mount("/m2", m2)
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
mount_fs.close()
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
def test_no_auto_close(self):
"""Test MultiFS auto close can be disabled"""
multi_fs = MountFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.mount('/m1', m1)
multi_fs.mount('/m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(not m1.closed)
self.assert_(not m2.closed)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MountFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.mount('/m1', m1)
multi_fs.mount('/m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(m1.closed)
self.assert_(m2.closed)
def test_listdir(self):
mount_fs = MountFS()
self.assertEqual(mount_fs.listdir("/"), [])
m1 = MemoryFS()
m3 = MemoryFS()
m4 = TempFS()
mount_fs.mount("/m1", m1)
mount_fs.mount("/m2", "temp://")
mount_fs.mount("/m3", m3)
with self.assertRaises(MountError):
mount_fs.mount("/m3/foo", m4)
self.assertEqual(sorted(mount_fs.listdir("/")), ["m1", "m2", "m3"])
m3.makedir("foo")
self.assertEqual(sorted(mount_fs.listdir("/m3")), ["foo"])
def open_slf_for_copy(src_path, dest_path, slf_name):
"""Opens an SLF files for reading and returns a MountFS"""
slf_file = src_path / (slf_name + ".slf")
slf_fs = SlfFS(str(slf_file))
output_dir = dest_path / slf_name
output_dir.mkdir(exist_ok=True)
out_fs = OSFS(output_dir)
combined_fs = MountFS()
combined_fs.mountdir('slf', slf_fs)
combined_fs.mountdir('out', out_fs)
return combined_fs
def test_listdir(self):
mount_fs = MountFS()
self.assertEqual(mount_fs.listdir('/'), [])
m1 = MemoryFS()
m2 = TempFS()
m3 = MemoryFS()
m4 = TempFS()
mount_fs.mount('/m1', m1)
mount_fs.mount('/m2', m2)
mount_fs.mount('/m3', m3)
with self.assertRaises(MountError):
mount_fs.mount('/m3/foo', m4)
self.assertEqual(sorted(mount_fs.listdir('/')), ['m1', 'm2', 'm3'])
m3.makedir('foo')
self.assertEqual(sorted(mount_fs.listdir('/m3')), ['foo'])
def __init__(self, datapath='data', log=None, level=None):
if not log:
log = 'stream'
if not level:
level = 1
if log == 'stream':
logginghandler = Logger()
logginghandler.create_stream_logger(level=level)
#Defining Pathes
workingpath = os.path.dirname(os.path.realpath(__file__))
self.workingpath = workingpath
logger.info('Startup Core')
logger.info('Workingpath: %s' % workingpath)
### Config
if datapath:
self.config = Config(datapath, 'MediaProxy')
else:
self.config = Config(workingpath, 'MediaProxy')
if not self.config.mode:
logger.info('Fresh Start, No Config File')
else:
logger.info('Config in state: %s' % self.config.mode)
### Import dependencies requires extlibs
from .plugin_manager import Plugins
from .database import Database
from fs.mountfs import MountFS
### Plugins
if self.config.mode:
self.plugins = Plugins(
self.config.cfg,
loadpathes=[os.path.join(datapath, 'plugins')])
self.filesystem = MountFS()
###Database
self.database = Database(self.config)
self.database.load()
self.channels_populate()
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.mountfs import MountFS
from configparser import ConfigParser
cfg = ConfigParser()
if '#' in fs_path:
path, section = fs_path.split('#', 1)
else:
path = fs_path
section = 'fs'
cfg.readfp(registry.open(path))
mount_fs = MountFS()
for mount_point, mount_path in cfg.items(section):
mount_fs.mount(mount_point, registry.opendir(mount_path, create_dir=create_dir))
return mount_fs, ''
def movedir(fs1,
fs2,
create_destination=True,
ignore_errors=False,
chunk_size=64 * 1024):
"""Moves contents of a directory from one filesystem to another.
:param fs1: A tuple of (<filesystem>, <directory path>)
:param fs2: Destination filesystem, or a tuple of (<filesystem>, <directory path>)
:param create_destination: If True, the destination will be created if it doesn't exist
:param ignore_errors: If True, exceptions from file moves are ignored
:param chunk_size: Size of chunks to move if a simple copy is used
"""
if not isinstance(fs1, tuple):
raise ValueError(
"first argument must be a tuple of (<filesystem>, <path>)")
fs1, dir1 = fs1
parent_fs1 = fs1
parent_dir1 = dir1
fs1 = fs1.opendir(dir1)
print fs1
if parent_dir1 in ('', '/'):
raise RemoveRootError(dir1)
if isinstance(fs2, tuple):
fs2, dir2 = fs2
if create_destination:
fs2.makedir(dir2, allow_recreate=True, recursive=True)
fs2 = fs2.opendir(dir2)
mount_fs = MountFS(auto_close=False)
mount_fs.mount('src', fs1)
mount_fs.mount('dst', fs2)
mount_fs.copydir('src',
'dst',
overwrite=True,
ignore_errors=ignore_errors,
chunk_size=chunk_size)
parent_fs1.removedir(parent_dir1, force=True)
def test_mountfile(self):
"""Test mounting a file"""
quote = b"""If you wish to make an apple pie from scratch, you must first invent the universe."""
mem_fs = MemoryFS()
mem_fs.makedir('foo')
mem_fs.setcontents('foo/bar.txt', quote)
foo_dir = mem_fs.opendir('foo')
mount_fs = MountFS()
mount_fs.mountfile('bar.txt', foo_dir.open, foo_dir.getinfo)
self.assert_(mount_fs.isdir('/'))
self.assert_(mount_fs.isdir('./'))
self.assert_(mount_fs.isdir(''))
# Check we can see the mounted file in the dir list
self.assertEqual(mount_fs.listdir(), ["bar.txt"])
self.assert_(not mount_fs.exists('nobodyhere.txt'))
self.assert_(mount_fs.exists('bar.txt'))
self.assert_(mount_fs.isfile('bar.txt'))
self.assert_(not mount_fs.isdir('bar.txt'))
# Check open and getinfo callables
self.assertEqual(mount_fs.getcontents('bar.txt'), quote)
self.assertEqual(mount_fs.getsize('bar.txt'), len(quote))
# Check changes are written back
mem_fs.setcontents('foo/bar.txt', 'baz')
self.assertEqual(mount_fs.getcontents('bar.txt'), b'baz')
self.assertEqual(mount_fs.getsize('bar.txt'), len('baz'))
# Check changes are written to the original fs
self.assertEqual(mem_fs.getcontents('foo/bar.txt'), b'baz')
self.assertEqual(mem_fs.getsize('foo/bar.txt'), len('baz'))
# Check unmount
self.assert_(mount_fs.unmount("bar.txt"))
self.assertEqual(mount_fs.listdir(), [])
self.assert_(not mount_fs.exists('bar.txt'))
# Check unount a second time is a null op, and returns False
self.assertFalse(mount_fs.unmount("bar.txt"))
def channel_get(self, ctype=None):
if not ctype:
return self.filesystem
else:
from fs.mountfs import MountFS
channel = MountFS()
for path, fs in self.filesystem.mounts:
if fs.exists(u'/'):
if 'mediaproxy.media' in fs.getinfo(u'/').raw:
logger.info(
'Check Channel: %s - %s' %
(path,
fs.getinfo(u'/').raw['mediaproxy.media']['type']))
if fs.getinfo(
u'/').raw['mediaproxy.media']['type'] == ctype:
channel.mount(path, fs)
return channel
def DEFAULT_FILE_STORAGE_FS():
_m = MountFS()
_m.mount('pytigon', OSFS(settings.ROOT_PATH))
_m.mount('static', OSFS(settings.STATIC_ROOT))
_m.mount('app', OSFS(settings.LOCAL_ROOT_PATH))
_m.mount('data', OSFS(settings.DATA_PATH))
try:
_m.mount('temp', OSFS(settings.TEMP_PATH))
except:
pass
try:
_m.mount('media', OSFS(settings.MEDIA_ROOT))
except:
pass
try:
_m.mount('upload', OSFS(settings.UPLOAD_PATH))
_m.mount('filer_public',
OSFS(os.path.join(settings.UPLOAD_PATH, "filer_public")))
_m.mount('filer_private',
OSFS(os.path.join(settings.UPLOAD_PATH, "filer_private")))
_m.mount(
'filer_public_thumbnails',
OSFS(os.path.join(settings.UPLOAD_PATH,
"filer_public_thumbnails")))
_m.mount(
'filer_private_thumbnails',
OSFS(os.path.join(settings.UPLOAD_PATH,
"filer_private_thumbnails")))
except:
pass
if sys.argv and sys.argv[0].endswith('pytigon'):
if platform_name() == 'Windows':
_m.mount('osfs', OSFS("c:\\"))
else:
_m.mount('osfs', OSFS("/"))
return _m
def make_fs(self):
fs = MountFS()
mem_fs = MemoryFS()
fs.mount("/", mem_fs)
return fs
'id/'||id AS path,
length(testscript) AS size
FROM tasks_task)
UNION
(SELECT
id AS fid,
'task/'||url AS path,
length(testscript) AS size
FROM tasks_task);
""",
readcmd="SELECT testscript FROM tasks_task WHERE id=%s",
writecmd="UPDATE tasks_task SET testscript=%s WHERE id=%s",
)
# create putkafs -- a virtual filesystem with filesystems above mounted onto its folders
putkafs = MountFS()
putkafs.mountdir('uploads', fs_uploads)
putkafs.mountdir('att', fs_attachments)
putkafs.mountdir('testscript', fs_testscripts)
if opts.unmount:
print 'Unmounting putkafs from %s' % opts.mountpoint
ret = os.system('fusermount -u %s' % opts.mountpoint)
sys.exit(ret)
else:
# redirect stdout, stderr to a temporary file
if opts.bg:
sys.stdout.flush()
sys.stderr.flush()
_out, _err = os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
def test_desc(self):
mount_fs = MountFS()
mount_fs.desc("/")
def test_mount_self(self):
mount_fs = MountFS()
with self.assertRaises(ValueError):
mount_fs.mount("/", mount_fs)
def test_empty(self):
"""Test MountFS with nothing mounted."""
mount_fs = MountFS()
self.assertEqual(mount_fs.listdir("/"), [])
def make_fs(self):
fs = MountFS()
mem_fs = MemoryFS()
fs.mount('/foo', mem_fs)
return fs.opendir('foo')
def test_bad_mount(self):
mount_fs = MountFS()
with self.assertRaises(TypeError):
mount_fs.mount('foo', 5)
with self.assertRaises(TypeError):
mount_fs.mount('foo', b'bar')
def test_bad_mount(self):
mount_fs = MountFS()
with self.assertRaises(TypeError):
mount_fs.mount("foo", 5)
with self.assertRaises(TypeError):
mount_fs.mount("foo", b"bar")
def make_fs(self):
fs = MountFS()
mem_fs = MemoryFS()
fs.mount("/foo", mem_fs)
return fs.opendir("foo")
