def main():
""" fuse mount """
options = parse_args()
global USE_APPSTREAM
USE_APPSTREAM = not options.no_appstream
init_logging(options.debug)
fuse_options = set(pyfuse3.default_options)
if options.debug_fuse:
fuse_options.add('debug')
options.mountpoint = Path(options.mountpoint).resolve()
options.mountpoint.mkdir(parents=True, exist_ok=True)
virtual_fs = AlpmFs(path=str(options.mountpoint))
pyfuse3.init(virtual_fs, str(options.mountpoint), fuse_options)
try:
trio.run(pyfuse3.main)
except KeyboardInterrupt:
print(f"\n\nfusermount -u {options.mountpoint}\n")
except:
pyfuse3.close(unmount=True)
print(f"\n\nfusermount -u {options.mountpoint} ok\n")
raise
pyfuse3.close(unmount=True)
def start(self):
"""Starts a pyfuse3 filesystem with different options.
Raises
------
FUSEError
If a FUSEError occurs it will close the pyfuse3 filesystem
"""
fuse_log = _logging.create_logger("pyfuse3", self.fs.debug)
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=' + self.fs.__class__.__name__)
if self.fs.debug:
fuse_options.add('debug')
pyfuse3.init(self.fs, self.fs.mount_point, fuse_options)
try:
trio.run(pyfuse3.main)
except FUSEError:
fuse_log.warning("FUSEError occured")
pyfuse3.close(unmount=False)
except Exception as e:
fuse_log.error("[%s]: %s", type(e).__name__, e)
pyfuse3.close(unmount=False)
except BaseException as be:
fuse_log.error(be.__name__)
fuse_log.error(be.args)
fuse_log.error("BaseException occured")
pyfuse3.close(unmount=False)
finally:
pyfuse3.close()
def main():
options = parse_args(sys.argv[1:])
init_logging(options.debug)
operations = Operations(options.source)
log.debug('Mounting...')
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=passthroughfs')
fuse_options.add('allow_other')
if options.debug_fuse:
fuse_options.add('debug')
fuse_options.discard('default_permissions')
pyfuse3.init(operations, options.mountpoint, fuse_options)
try:
log.debug('Entering main loop..')
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
log.debug('Unmounting..')
pyfuse3.close()
async def main(swhids: List[CoreSWHID], root_path: Path,
conf: Dict[str, Any]) -> None:
""" swh-fuse CLI entry-point """
# Use pyfuse3 asyncio layer to match the rest of Software Heritage codebase
pyfuse3_asyncio.enable()
async with FuseCache(conf["cache"]) as cache:
fs = Fuse(root_path, cache, conf)
# Initially populate the cache
for swhid in swhids:
try:
await fs.get_metadata(swhid)
except Exception as err:
fs.logger.exception("Cannot prefetch object %s: %s", swhid,
err)
fuse_options = set(pyfuse3.default_options)
fuse_options.add("fsname=swhfs")
try:
pyfuse3.init(fs, root_path, fuse_options)
await pyfuse3.main()
except Exception as err:
fs.logger.error("Error running FUSE: %s", err)
finally:
fs.shutdown()
pyfuse3.close(unmount=True)
def run_fuse_mount(ops, options, mount_opts):
'''Performs FUSE mount'''
mount_opts = ['fsname=gridfs'] + mount_opts
opts = dict((opt.split('=', 1) if '=' in opt else (opt, None)
for opt in mount_opts))
# strip invalid keys
ignored_keys = [
'debug', 'foreground', 'log_level', 'log_file', 'workers', 'single'
]
valid_keys = [k for k in opts if k not in ignored_keys]
mount_opts = set(pyfuse3.default_options)
for k in valid_keys:
if opts[k] is not None:
mount_opts.add('='.join([k, opts[k]]))
else:
mount_opts.add(k)
# handle some key options here
if 'log_level' in opts:
try:
log_level = opts['log_level'].upper()
try:
log_level = int(log_level)
except ValueError:
pass
logging.getLogger().setLevel(getattr(logging, log_level))
except (TypeError, ValueError) as error:
logging.warning('Unable to set log_level to {}: {}'.format(
opts['log_level'], error))
# start gridfs bindings and run fuse process
pyfuse3.init(ops, options.mount_point, mount_opts)
# ensure that is single is given then it evaluates to true
if 'single' in opts and opts['single'] is None:
opts['single'] = True
# debug clobbers other log settings such as log_level
if 'debug' in opts:
logging.basicConfig(
format=
'[%(asctime)s] pid=%(process)s {%(module)s:%(funcName)s():%(lineno)d} %(levelname)s - %(message)s',
level=logging.DEBUG)
# TODO: Find way of capturing CTRL+C and calling pyfuse3.close() when in foreground
# Note: This maybe a bug in pyfuse3
workers = opts.get('workers',
opts.get('single',
1)) # fudge for backwards compatibility
try:
trio.run(
pyfuse3.main) # maintain compatibility with single/workers kwarg
except KeyboardInterrupt:
pass
finally:
pyfuse3.close()
def main():
# parse command line ###
parser = argparse.ArgumentParser()
parser.add_argument('source', type=str)
parser.add_argument('mountpoint', type=str)
parser.add_argument('--debug', action='store_true')
parser.add_argument('--logfile', type=str)
args = parser.parse_args()
# initialize logger ###
dbglogformatter = logging.Formatter()
dbgloghandler = logging.StreamHandler()
dbgloghandler.setFormatter(dbglogformatter)
dbglog.addHandler(dbgloghandler)
if args.debug:
dbglog.setLevel(logging.DEBUG)
else:
dbglog.setLevel(logging.INFO)
acslog.setLevel(logging.INFO)
acsformatter = logging.Formatter()
acshandler = logging.StreamHandler(sys.stdout)
acshandler.setFormatter(acsformatter)
acslog.addHandler(acshandler)
# check arguments
if not os.path.isdir(args.source):
dbglog.error('Given source path <{}> is not valid directory.'.format(
args.source))
sys.exit(1)
if not os.path.isdir(args.mountpoint):
dbglog.error(
'Given mountpoint path <{}> is not valid directory'.format(
args.mountpoint))
sys.exit(1)
# setting process environment
os.umask(0) # to respect file permission which user specified
# start filesystem ###
fsops = fusefs.Fusebox(os.path.abspath(args.source),
os.path.abspath(args.mountpoint))
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=fusefs')
if args.debug:
fuse_options.add('debug')
fuse_options.add('dev')
fuse_options.add('allow_other')
pyfuse3.init(fsops, args.mountpoint, fuse_options)
try:
trio.run(pyfuse3.main)
finally:
pyfuse3.close(unmount=True)
if args.logfile:
export_logfile(fsops, args.logfile)
sys.exit(0)
async def mount(client,
id,
destination: str,
offset_id=0,
limit=None,
filter_music=False,
debug_fuse=False,
reverse=False,
updates=False,
fsname="tgfs"):
pyfuse3_asyncio.enable()
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=' + fsname)
if debug_fuse:
fuse_options.add('debug')
# in order to use numeric id
if isinstance(id, int):
await client.get_dialogs()
logging.debug("Querying entity %s" % id)
entity: Entity = await client.get_entity(id)
logging.debug("Got '%s'" % get_display_name(entity))
logging.info(
"Querying %s messages starting with message_id %d, music: %s" %
(limit if limit else "all", offset_id, filter_music))
messages, documents_handles = await client.get_documents(
entity,
limit=limit,
filter_music=filter_music,
offset_id=offset_id,
reverse=reverse)
logging.info("Mounting %d files to %s" %
(len(documents_handles), destination))
# logging.debug("Files: %s" % ([doc['id'] for msg, doc in documents], ))
telegram_fs = TelegramFsAsync()
for msg, dh in zip(messages, documents_handles):
telegram_fs.add_file(msg, dh)
if updates:
client.add_event_handler(
create_new_files_handler(client, telegram_fs, entity), )
pyfuse3.init(telegram_fs, destination, fuse_options)
await pyfuse3.main(min_tasks=10)
def start_charybdisfs(source: str, # noqa: C901 # ignore "is too complex" message
target: str,
debug: bool,
rest_api: bool,
rest_api_port: int,
mount: bool,
static_enospc: bool,
static_enospc_probability: float) -> None:
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG if debug else logging.INFO, format=LOG_FORMAT)
if not rest_api and not mount:
raise click.UsageError(message="can't run --no-rest-api and --no-mount simultaneously")
if debug:
sys.addaudithook(sys_audit_hook)
if static_enospc:
static_enospc_probability = max(0, min(100, round(static_enospc_probability * 100)))
LOGGER.info("Going to add ENOSPC fault for all syscalls with probability %s%%", static_enospc_probability)
enospc_fault = ErrorFault(sys_call=SysCall.ALL, probability=static_enospc_probability, error_no=errno.ENOSPC)
Configuration.add_fault(fault_id=generate_fault_id(), fault=enospc_fault)
LOGGER.debug("Faults added: %s", Configuration.get_all_faults())
if rest_api:
api_server_thread = \
threading.Thread(target=start_charybdisfs_api_server,
kwargs={"port": rest_api_port, },
name="RestServerApi",
daemon=True)
api_server_thread.start()
atexit.register(stop_charybdisfs_api_server)
if mount:
if source is None or target is None:
raise click.BadArgumentUsage("both source and target parameters are required for CharybdisFS mount")
fuse_options = set(pyfuse3.default_options)
fuse_options.add("fsname=charybdisfs")
if debug:
fuse_options.add("debug")
operations = CharybdisOperations(source=source)
pyfuse3.init(operations, target, fuse_options)
atexit.register(pyfuse3.close)
try:
if mount:
trio.run(pyfuse3.main)
else:
api_server_thread.join()
except KeyboardInterrupt:
LOGGER.info("Interrupted by user...")
sys.exit(0)
def _main():
# Parse the arguments
mountpoint, conf, foreground = parse_options()
LOG.info('Mountpoint: %s', mountpoint)
# Required configurations
rootdir = conf.get('DEFAULT', 'rootdir')
if not rootdir:
raise ValueError('Missing rootdir configuration')
rootdir = os.path.expanduser(rootdir)
if not os.path.exists(rootdir):
raise ValueError(f'Rootdir {rootdir} does not exist')
LOG.info('Root dir: %s', rootdir)
# Encryption/Decryption keys
seckey = retrieve_secret_key(conf)
recipients = list(set(build_recipients(conf, seckey))) # remove duplicates
# recipients might be empty if we don't create files
# Default configurations
options = conf.getset('FUSE', 'options', fallback='ro,default_permissions')
LOG.debug('mount options: %s', options)
cache_directories = conf.getboolean('FUSE', 'cache_directories', fallback=True)
extension = conf.get('DEFAULT', 'extension', fallback='.c4gh')
# Build the file system
fs = Crypt4ghFS(rootdir, seckey, recipients, extension, cache_directories)
pyfuse3.init(fs, mountpoint, options)
if not foreground:
LOG.info('Running current process in background')
detach() # daemonize
try:
LOG.debug('Entering main loop')
trio.run(pyfuse3.main)
# This is an infinite loop.
# Ctrl-C / KeyboardInterrupt will be propagated (properly?)
# - https://trio.readthedocs.io/en/stable/reference-core.html
# - https://vorpus.org/blog/control-c-handling-in-python-and-trio/
except Exception as e:
LOG.debug("%r", e)
raise
finally:
LOG.debug('Unmounting')
pyfuse3.close(unmount=True)
# The proper way to exit is to call:
# umount <the-mountpoint>
return 0
def main(mountpoint, root):
passt = Passthrough(root)
fuse_options = set()
fuse_options.add('fsname=passthroughfs')
fuse_options.add('allow_other')
try:
pyfuse3.init(passt,mountpoint,fuse_options)
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
def start_charybdisfs(source: str, target: str, debug: bool) -> None:
operations = CharybdisOperations(source=source)
fuse_options = set(pyfuse3.default_options)
fuse_options.add("fsname=charybdisfs")
if debug:
fuse_options.add("debug")
pyfuse3.init(operations, target, fuse_options)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
pyfuse3.close()
def main():
parser = argparse.ArgumentParser(description='Mount a FapTrack filesystem')
parser.add_argument('mountpoint', type=str, help='the mountpoint')
args = parser.parse_args()
my_opts = set(pyfuse3.default_options)
my_opts.add('allow_root')
my_opts.discard('default_permissions')
pyfuse3.init(VirtualFS(trackFS), args.mountpoint, my_opts)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close()
raise
pyfuse3.close()
def main():
options = parse_args()
init_logging(options.debug)
testfs = ADfs()
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=ad')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(testfs, options.mountpoint, fuse_options)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
pyfuse3.close()
def main():
parser = argparse.ArgumentParser(
description='Mount a FapTrack filesystem',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('mountpoint', type=str, help='the mountpoint')
parser.add_argument('--database',
'-d',
type=str,
help='the sqlite3 database file',
default='faptrack.db')
parser.add_argument('--max_view_time',
'-m',
type=int,
help='the max view time to be recorded',
default=15 * 60)
parser.add_argument('--logging',
'-l',
type=str,
choices=logging._nameToLevel.keys(),
default="WARNING",
help='the logging level')
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging._nameToLevel[args.logging])
my_opts = set(pyfuse3.default_options)
my_opts.add('allow_root')
my_opts.discard('default_permissions')
def conn_fac():
return sqlite3.connect(args.database)
pyfuse3.init(VirtualFS(trackFS(conn_fac, args.max_view_time * (10**9))),
args.mountpoint, my_opts)
try:
asyncio.run(pyfuse3.main())
except:
pyfuse3.close()
raise
pyfuse3.close()
async def _go(self):
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=infofs')
if self.debug:
fuse_options.add('debug')
pyfuse3.init(self, str(self.mountpoint), fuse_options)
try:
async with trio.open_nursery() as nursery:
nursery.start_soon(pyfuse3.main)
for command_spec in self.commands:
nursery.start_soon(self._scheduler,
bytes(command_spec.name,
'utf-8'), command_spec.command,
command_spec.delay, nursery)
except:
raise
finally:
pyfuse3.close()
def start_charybdisfs(source: str, target: str, debug: bool,
enospc_probability) -> None:
logging.basicConfig()
operations = CharybdisOperations(source=source,
enospc_probability=enospc_probability)
fuse_options = set(pyfuse3.default_options)
fuse_options.add("fsname=charybdisfs")
if debug:
fuse_options.add("debug")
pyfuse3.init(operations, target, fuse_options)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
pyfuse3.close()
def main():
parser = argparse.ArgumentParser(description='Mount a FapTrack filesystem')
parser.add_argument('mountpoint', type=str, help='the mountpoint')
parser.add_argument('--database', '-d', type=str, help='the sqlite3 database file', default='faptrack.db')
args = parser.parse_args()
my_opts = set(pyfuse3.default_options)
my_opts.add('allow_root')
my_opts.discard('default_permissions')
def conn_fac():
return sqlite3.connect(args.database)
pyfuse3.init(VirtualFS(trackFS(conn_fac)), args.mountpoint, my_opts)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close()
raise
pyfuse3.close()
def main():
options = parse_args()
init_logging(options.debug)
testfs = TestFs()
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=hello_asyncio')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(testfs, options.mountpoint, fuse_options)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(pyfuse3.main())
except:
pyfuse3.close(unmount=False)
raise
finally:
loop.close()
pyfuse3.close()
def main():
options = parse_args(sys.argv[1:])
init_logging(options.debug)
operations = Operations(options.source)
log.debug('Mounting...')
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=securepassthroughfs')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(operations, options.mountpoint, fuse_options)
try:
log.debug('Entering main loop..')
log.debug('Use fusermount3 -u ' + options.mountpoint + " to umount")
trio.run(pyfuse3.main)
#t.join()
except:
pyfuse3.close(unmount=True)
raise
def run_fs(mountpoint, cross_process):
# Logging (note that we run in a new process, so we can't
# rely on direct log capture and instead print to stdout)
root_logger = logging.getLogger()
formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s '
'%(funcName)s(%(threadName)s): %(message)s',
datefmt="%M:%S")
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
root_logger.addHandler(handler)
root_logger.setLevel(logging.DEBUG)
testfs = Fs(cross_process)
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=pyfuse3_testfs')
pyfuse3.init(testfs, mountpoint, fuse_options)
try:
trio.run(pyfuse3.main)
finally:
pyfuse3.close()
def main():
args = parse_args()
pngfs = pngFS(args)
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=pngFS')
if args.fuse_debug:
fuse_options.add('debug')
pyfuse3.init(pngfs, args.mountpoint, fuse_options)
try:
asyncio.run(pyfuse3.main())
except KeyboardInterrupt:
pass
pngfs.files.clean_files_without_parent()
pngfs.write_to_png()
pyfuse3.close()
def main():
options = parse_args()
init_logging(options.debug)
testfs = TestFs()
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=hello')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(testfs, options.mountpoint, fuse_options)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(pyfuse3.main())
except:
print("Encountered error so will try to unmount")
pyfuse3.close(unmount=True)
raise
else:
print("Unmounted: exiting cleanly with out further unmount")
pyfuse3.close(unmount=False)
finally:
loop.close()
def mount_fuse(source, mountpoint):
init_logging() # Nod debug for now
operations = Operations(str(source))
log.debug('Mounting...')
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=zisofuse')
# if options.debug_fuse:
# fuse_options.add('debug')
pyfuse3.init(operations, str(mountpoint), fuse_options)
try:
log.debug('Entering main loop..')
trio.run(pyfuse3.main)
except KeyboardInterrupt:
pyfuse3.close(unmount=True)
return
except:
pyfuse3.close(unmount=True)
raise
log.debug('Unmounting..')
pyfuse3.close()
def main():
logging.basicConfig()
args = parse_args()
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
argv = args.argv_prefix if args.argv_prefix else ("bash", "-c")
operations = BashFS(argv_prefix=argv, separator=args.separator.encode())
fuse_options = set(pyfuse3.default_options)
fuse_options.add("fsname=bashfs")
fuse_options.discard("default_permissions")
if args.debug_fuse:
fuse_options.add("debug")
pyfuse3.init(operations, args.mountpoint, fuse_options)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=True)
raise
pyfuse3.close()
def main():
options = parse_args(sys.argv[1:])
init_logging(options.debug)
operations = Operations(options.source)
log.debug('Mounting...')
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=passthroughfs')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(operations, options.mountpoint, fuse_options)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(pyfuse3.main())
except:
print("Encountered error so will try to unmount")
pyfuse3.close(unmount=True)
raise
else:
print("Unmounted: exiting cleanly with out further unmount")
pyfuse3.close(unmount=False)
finally:
loop.close()
root_fs = FS(process_course_configs())
except Exception as err:
# We want any exception that is thrown to go through `better_exceptions`, which provides debug information that's **actually useful**
logger.exception(err)
raise
logger.info("canvasfs initialised. Starting filesystem")
mount_target = Path(config()["mount_dir"])
mount_target.mkdir(parents=True, exist_ok=True)
if len(list(mount_target.iterdir())) > 0:
raise ConfigError(
"'mount_dir' must be empty, as its contents will be overwritten when mounted"
)
pyfuse3.init(root_fs, str(mount_target), fuse_options)
async def loop():
# Ensure the files and directories have been fetched from Canvas and setup by the time the filesystem is started
async with trio.open_nursery() as nursery:
for _, sub in root_fs:
nursery.start_soon(sub.build)
logger.success("Filesystem built. Ready to go!")
# Background polling to allow periodic refreshes that can be initiated by each SubFS
async with trio.open_nursery() as nursery:
nursery.start_soon(pyfuse3.main)
for _, sub in root_fs:
nursery.start_soon(sub.poll)
def main():
options = parse_args(sys.argv[1:])
init_logging(options.debug)
cache_size = options.cache_size * 1024 * 1024
degoo_email = options.degoo_email
degoo_pass = options.degoo_pass
degoo_token = options.degoo_token
degoo_refresh_token = options.degoo_refresh_token
degoo_path = options.degoo_path
refresh_interval = options.refresh_interval * 60
disable_refresh = options.disable_refresh
enable_flood_control = options.enable_flood_control
change_hostname = options.change_hostname
mode = options.mode
config_path = options.config_path
log.debug('##### Initializating Degoo drive #####')
log.debug('Local mount point: %s', options.mountpoint)
log.debug('Cache size: %s', str(cache_size) + ' kb')
if degoo_email and degoo_pass:
log.debug('Degoo email: %s', degoo_email)
log.debug('Degoo pass: %s', '*' * len(degoo_pass))
if degoo_token and degoo_refresh_token:
log.debug('Degoo token: %s', '*' * len(degoo_token[:10]))
log.debug('Degoo refresh token: %s',
'*' * len(degoo_refresh_token[:10]))
log.debug('Root Degoo path: %s', degoo_path)
log.debug(
'Refresh interval: %s',
'Disabled' if disable_refresh else str(refresh_interval) + ' seconds')
log.debug('Flood control: %s',
'Enabled' if enable_flood_control else 'Disabled')
if enable_flood_control:
log.debug('Flood sleep time: %s seconds',
str(options.flood_sleep_time))
log.debug('Flood max requests: %s', str(options.flood_max_requests))
log.debug('Flood time check: %s minute(s)',
str(options.flood_time_to_check))
log.debug('Change hostname: %s',
'Disabled' if not change_hostname else DEGOO_HOSTNAME_EU)
log.debug('Mode: %s', mode)
if config_path:
log.debug('Configuration path: %s', config_path)
if options.allow_other:
log.debug('User access: %s', options.allow_other)
Path(options.mountpoint).mkdir(parents=True, exist_ok=True)
operations = Operations(source=degoo_path,
cache_size=cache_size,
flood_sleep_time=options.flood_sleep_time,
flood_time_to_check=options.flood_time_to_check,
flood_max_requests=options.flood_max_requests,
enable_flood_control=enable_flood_control,
change_hostname=change_hostname,
mode=mode)
log.debug('Reading Degoo content from directory %s', degoo_path)
degoo.DegooConfig(config_path,
email=degoo_email,
password=degoo_pass,
token=degoo_token,
refresh_token=degoo_refresh_token)
degoo.API()
operations.load_degoo_content()
log.debug('Mounting...')
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=fusedegoo')
if options.allow_other:
fuse_options.add('allow_other')
mimetypes.init()
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(operations, options.mountpoint, fuse_options)
if not disable_refresh:
t1 = threading.Thread(target=operations.refresh_degoo_content,
args=(refresh_interval, ))
t1.start()
try:
log.debug('Entering main loop..')
trio.run(pyfuse3.main)
except:
print('Unexpected error: ', sys.exc_info()[0])
global is_refresh_enabled
is_refresh_enabled = False
pyfuse3.close(unmount=True)
raise
log.debug('Unmounting..')
pyfuse3.close()
def main():
### parse command line ###
parser = argparse.ArgumentParser()
parser.add_argument('--fusebox_debug', action='store_true')
parser.add_argument('--pyfuse_debug', action='store_true')
parser.add_argument('command')
parser.add_argument('command_args', nargs=argparse.REMAINDER)
#args, cmd = parser.parse_known_args()
args = parser.parse_args()
# assemble command and that arguments
cmd = list()
cmd.append(args.command)
cmd.extend(args.command_args)
### initialize logger ###
dbglogformatter = logging.Formatter()
dbgloghandler = logging.StreamHandler()
dbgloghandler.setFormatter(dbglogformatter)
dbglog.addHandler(dbgloghandler)
if args.fusebox_debug:
dbglog.setLevel(logging.DEBUG)
else:
dbglog.setLevel(logging.INFO)
acslog.setLevel(logging.INFO)
acsformatter = logging.Formatter()
acshandler = logging.StreamHandler(sys.stdout)
acshandler.setFormatter(acsformatter)
acslog.addHandler(acshandler)
# setting process environment
umask_prev = os.umask(0) # to respect file permission which user specified
# initialize filesystem ###
ctx_mountpoint = tempfile.TemporaryDirectory()
atexit.register(
lambda: ctx_mountpoint.cleanup()) # reserve cleanup for mountpoint
mountpoint = ctx_mountpoint.name
fsops = fusefs.Fusebox(os.path.abspath('/'), os.path.abspath(mountpoint))
fsops.auditor.enabled = False
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=fusefs')
if args.pyfuse_debug:
fuse_options.add('debug')
fuse_options.add('dev')
fuse_options.add('allow_other')
# print current status
dbglog.info('*** Fusebox Status ***')
dbglog.info('uid:\t{}'.format(os.getuid()))
dbglog.info('gid:\t{}'.format(os.getgid()))
dbglog.info('pid:\t{}'.format(os.getpid()))
dbglog.info('prev_umask:\t{:04o}'.format(umask_prev))
dbglog.info('cmd:\t{}'.format(cmd))
dbglog.info('acl:\t{}'.format(
'engaged' if fsops.auditor.enabled else 'disengaged'))
dbglog.info('mount:\t{}'.format(mountpoint))
# start filesystem
pyfuse3.init(
fsops, mountpoint, fuse_options
) # From this point, accessing under the mountpoint will be blocked.
tmp_stdin = os.dup(
sys.stdin.fileno()
) # backup stdin fd. since multiprocessing will override stdin in forked process.
os.set_inheritable(tmp_stdin,
True) # make sure to keep fd opened in forked process.
proc_cmd = multiprocessing.Process(target=launcher,
args=(cmd, tmp_stdin, mountpoint))
proc_fusebox = multiprocessing.Process(target=start_sandbox)
del tmp_stdin # forget backuped pipe in parent process as it is not needed.
proc_cmd.start()
proc_fusebox.start()
proc_cmd.join() # to eliminate zombie process
proc_cmd.close()
# given commands are exited.
pyfuse3.close(unmount=True) # release unnecessary resource
proc_fusebox.terminate() # send exit signal to fuse process
proc_fusebox.join()
proc_fusebox.close()
async def main_async(options, stdout_log_handler):
# Get paths
cachepath = options.cachepath
backend_factory = get_backend_factory(options)
backend_pool = BackendPool(backend_factory)
atexit.register(backend_pool.flush)
# Retrieve metadata
with backend_pool() as backend:
(param, db) = get_metadata(backend, cachepath)
#if param['max_obj_size'] < options.min_obj_size:
# raise QuietError('Maximum object size must be bigger than minimum object size.',
# exitcode=2)
# Handle --cachesize
rec_cachesize = options.max_cache_entries * param['max_obj_size'] / 2
avail_cache = shutil.disk_usage(os.path.dirname(cachepath))[2] / 1024
if options.cachesize is None:
options.cachesize = min(rec_cachesize, 0.8 * avail_cache)
log.info('Setting cache size to %d MB', options.cachesize / 1024)
elif options.cachesize > avail_cache:
log.warning('Requested cache size %d MB, but only %d MB available',
options.cachesize / 1024, avail_cache / 1024)
if options.nfs:
# NFS may try to look up '..', so we have to speed up this kind of query
log.info('Creating NFS indices...')
db.execute(
'CREATE INDEX IF NOT EXISTS ix_contents_inode ON contents(inode)')
else:
db.execute('DROP INDEX IF EXISTS ix_contents_inode')
metadata_upload_task = MetadataUploadTask(backend_pool, param, db,
options.metadata_upload_interval)
block_cache = BlockCache(backend_pool, db, cachepath + '-cache',
options.cachesize * 1024,
options.max_cache_entries)
commit_task = CommitTask(block_cache)
operations = fs.Operations(block_cache,
db,
max_obj_size=param['max_obj_size'],
inode_cache=InodeCache(db, param['inode_gen']),
upload_event=metadata_upload_task.event)
block_cache.fs = operations
metadata_upload_task.fs = operations
async with trio.open_nursery() as nursery:
with ExitStack() as cm:
log.info('Mounting %s at %s...', options.storage_url,
options.mountpoint)
try:
pyfuse3.init(operations, options.mountpoint,
get_fuse_opts(options))
except RuntimeError as exc:
raise QuietError(str(exc), exitcode=39)
unmount_clean = False
def unmount():
log.info("Unmounting file system...")
pyfuse3.close(unmount=unmount_clean)
cm.callback(unmount)
if options.fg or options.systemd:
faulthandler.enable()
faulthandler.register(signal.SIGUSR1)
else:
if stdout_log_handler:
logging.getLogger().removeHandler(stdout_log_handler)
crit_log_fd = os.open(
os.path.join(options.cachedir, 'mount.s3ql_crit.log'),
flags=os.O_APPEND | os.O_CREAT | os.O_WRONLY,
mode=0o644)
faulthandler.enable(crit_log_fd)
faulthandler.register(signal.SIGUSR1, file=crit_log_fd)
daemonize(options.cachedir)
mark_metadata_dirty(backend, cachepath, param)
block_cache.init(options.threads)
nursery.start_soon(metadata_upload_task.run,
name='metadata-upload-task')
cm.callback(metadata_upload_task.stop)
nursery.start_soon(commit_task.run, name='commit-task')
cm.callback(commit_task.stop)
exc_info = setup_exchook()
if options.systemd:
import systemd.daemon
systemd.daemon.notify('READY=1')
try:
ret = await pyfuse3.main()
finally:
await operations.destroy()
await block_cache.destroy(options.keep_cache)
if ret is not None:
raise RuntimeError('Received signal %d, terminating' % (ret, ))
# Re-raise if main loop terminated due to exception in other thread
if exc_info:
(exc_inst, exc_tb) = exc_info
raise exc_inst.with_traceback(exc_tb)
log.info("FUSE main loop terminated.")
unmount_clean = True
# At this point, there should be no other threads left
# Do not update .params yet, dump_metadata() may fail if the database is
# corrupted, in which case we want to force an fsck.
if operations.failsafe:
log.warning('File system errors encountered, marking for fsck.')
param['needs_fsck'] = True
with backend_pool() as backend:
seq_no = get_seq_no(backend)
if metadata_upload_task.db_mtime == os.stat(cachepath +
'.db').st_mtime:
log.info('File system unchanged, not uploading metadata.')
del backend['s3ql_seq_no_%d' % param['seq_no']]
param['seq_no'] -= 1
save_params(cachepath, param)
elif seq_no == param['seq_no']:
param['last-modified'] = time.time()
dump_and_upload_metadata(backend, db, param)
save_params(cachepath, param)
else:
log.error(
'Remote metadata is newer than local (%d vs %d), '
'refusing to overwrite!', seq_no, param['seq_no'])
log.error(
'The locally cached metadata will be *lost* the next time the file system '
'is mounted or checked and has therefore been backed up.')
for name in (cachepath + '.params', cachepath + '.db'):
for i in range(4)[::-1]:
if os.path.exists(name + '.%d' % i):
os.rename(name + '.%d' % i, name + '.%d' % (i + 1))
os.rename(name, name + '.0')
log.info('Cleaning up local metadata...')
db.execute('ANALYZE')
db.execute('VACUUM')
db.close()
log.info('All done.')
default=False,
help='Enable debugging output')
parser.add_argument('--debug-fuse',
action='store_true',
default=False,
help='Enable FUSE debugging output')
return parser.parse_args()
if __name__ == '__main__':
options = parse_args()
init_logging(options.debug)
operations = Operations()
fuse_options = set(pyfuse3.default_options)
fuse_options.add('fsname=tmpfs')
fuse_options.discard('default_permissions')
if options.debug_fuse:
fuse_options.add('debug')
pyfuse3.init(operations, options.mountpoint, fuse_options)
try:
trio.run(pyfuse3.main)
except:
pyfuse3.close(unmount=False)
raise
pyfuse3.close()
