def log_level(config_name: str, level_name: str):
"""Gets or sets the log level."""
from maestral.daemon import MaestralProxy
try:
with MaestralProxy(config_name) as m:
if level_name:
m.log_level = logging._nameToLevel[level_name]
click.echo(f'Log level set to {level_name}.')
else:
level_name = logging.getLevelName(m.log_level)
click.echo(f'Log level: {level_name}')
except Pyro5.errors.CommunicationError:
from maestral.config.main import MaestralConfig
conf = MaestralConfig(config_name)
if level_name:
conf.set('app', 'log_level', logging._nameToLevel[level_name])
click.echo(f'Log level set to {level_name}.')
else:
level_name = logging.getLevelName(conf.get('app', 'log_level'))
click.echo(f'Log level: {level_name}')
def notify_level(config_name: str, level_name: str):
"""Gets or sets the level for desktop notifications."""
from maestral.daemon import MaestralProxy
from maestral.utils.notify import levelNameToNumber, levelNumberToName
try:
with MaestralProxy(config_name) as m:
if level_name:
m.notification_level = levelNameToNumber(level_name)
click.echo(f'Notification level set to {level_name}.')
else:
level_name = levelNumberToName(m.notification_level)
click.echo(f'Notification level: {level_name}.')
except Pyro5.errors.CommunicationError:
from maestral.config.main import MaestralConfig
conf = MaestralConfig(config_name)
if level_name:
conf.set('app', 'notification_level',
levelNameToNumber(level_name))
click.echo(f'Notification level set to {level_name}.')
else:
level_name = levelNumberToName(
conf.get('app', 'notification_level'))
click.echo(f'Notification level: {level_name}.')
def migrate_user_config(config_name):
config_path = get_conf_path(CONFIG_DIR_NAME, create=False)
config_fname = osp.join(config_path, config_name + '.ini')
# load old config non-destructively
try:
old_conf = DefaultsConfig(config_path, config_name, '.ini')
old_conf.read(config_fname, encoding='utf-8')
old_version = old_conf.get(UserConfig.DEFAULT_SECTION_NAME, 'version')
except OSError:
return
if Version(old_version) < Version('11.0.0'):
# get values for moved settings
excluded_folders = old_conf.get('main', 'excluded_folders')
email = old_conf.get('account', 'email')
display_name = old_conf.get('account', 'display_name')
abbreviated_name = old_conf.get('account', 'abbreviated_name')
acc_type = old_conf.get('account', 'type')
usage = old_conf.get('account', 'usage')
usage_type = old_conf.get('account', 'usage_type')
update_notification_last = old_conf.get('app',
'update_notification_last')
latest_release = old_conf.get('app', 'latest_release')
cursor = old_conf.get('internal', 'cursor')
lastsync = old_conf.get('internal', 'lastsync')
recent_changes = old_conf.get('internal', 'recent_changes')
# convert non-string types
update_notification_last = float(update_notification_last)
lastsync = float(lastsync)
recent_changes = ast.literal_eval(recent_changes)
excluded_folders = ast.literal_eval(excluded_folders)
# set state values
state = MaestralState(config_name)
state.set('account', 'email', email)
state.set('account', 'display_name', display_name)
state.set('account', 'abbreviated_name', abbreviated_name)
state.set('account', 'type', acc_type)
state.set('account', 'usage', usage)
state.set('account', 'usage_type', usage_type)
state.set('app', 'update_notification_last', update_notification_last)
state.set('app', 'latest_release', latest_release)
state.set('sync', 'cursor', cursor)
state.set('sync', 'lastsync', lastsync)
state.set('sync', 'recent_changes', recent_changes)
# load actual config to remove obsolete options and add moved ones
conf = MaestralConfig(config_name)
conf.set('main', 'excluded_items', excluded_folders)
# clean up backup and defaults files from previous version of maestral
for file in os.scandir(old_conf._path):
if file.is_file():
if (conf._backup_suffix in file.name
or conf._defaults_name_prefix in file.name):
os.remove(file.path)
logger.info(f'Migrated user config "{config_name}"')
elif Version(old_version) < Version('12.0.0'):
excluded_folders = old_conf.get('main', 'excluded_folders')
excluded_folders = ast.literal_eval(excluded_folders)
conf = MaestralConfig(config_name)
conf.set('main', 'excluded_items', excluded_folders)
class OAuth2Session(object):
"""
OAuth2Session provides OAuth2 login and token store.
"""
oAuth2FlowResult = None
Success = 0
InvalidToken = 1
ConnectionFailed = 2
def __init__(self, config_name='maestral'):
self._conf = MaestralConfig(config_name)
self.account_id = self._conf.get("account", "account_id")
self.access_token = ""
self.auth_flow = None
def load_token(self):
"""
Check if auth key has been saved.
:raises: ``KeyringLocked`` if the system keyring cannot be accessed.
"""
logger.debug("Using keyring: %s" % keyring.get_keyring())
try:
if self.account_id == "":
self.access_token = None
else:
self.access_token = keyring.get_password("Maestral", self.account_id)
return self.access_token
except KeyringLocked:
info = "Please make sure that your keyring is unlocked and restart Maestral."
raise KeyringLocked(info)
def get_auth_url(self):
"""Gets the auth URL to start the OAuth2 implicit grant flow."""
self.auth_flow = DropboxOAuth2FlowImplicit(APP_KEY)
authorize_url = self.auth_flow.start()
return authorize_url
def verify_auth_token(self, token):
"""
Verify the provided authorization token with Dropbox servers.
:return: OAuth2Session.Success, OAuth2Session.InvalidToken, or
OAuth2Session.ConnectionFailed
:rtype: int
"""
if not self.auth_flow:
raise RuntimeError('Auth flow not yet started. Please call \'get_auth_url\'.')
try:
self.oAuth2FlowResult = self.auth_flow.finish(token)
self.access_token = self.oAuth2FlowResult.access_token
self.account_id = self.oAuth2FlowResult.account_id
return self.Success
except DropboxAuthError:
return self.InvalidToken
except ConnectionError:
return self.ConnectionFailed
def link(self):
"""Command line flow to get an auth key from Dropbox and save it in the system
keyring."""
authorize_url = self.get_auth_url()
print("1. Go to: " + authorize_url)
print("2. Click \"Allow\" (you might have to log in first).")
print("3. Copy the authorization token.")
res = 1
while res > 0:
auth_code = input("Enter the authorization token here: ").strip()
res = self.verify_auth_token(auth_code)
if res == 1:
print("Invalid token. Please try again.")
elif res == 2:
print("Could not connect to Dropbox. Please try again.")
self.save_creds()
def save_creds(self):
"""Saves auth key to system keyring."""
self._conf.set("account", "account_id", self.account_id)
try:
keyring.set_password("Maestral", self.account_id, self.access_token)
print(" > Credentials written.")
except KeyringLocked:
logger.error("Could not access the user keyring to save your authentication "
"token. Please make sure that the keyring is unlocked.")
def delete_creds(self):
"""Deletes auth key from system keyring."""
self._conf.set("account", "account_id", "")
try:
keyring.delete_password("Maestral", self.account_id)
print(" > Credentials removed.")
except KeyringLocked:
logger.error("Could not access the user keyring to delete your authentication"
" token. Please make sure that the keyring is unlocked.")
class Maestral(object):
"""
An open source Dropbox client for macOS and Linux to syncing a local folder
with your Dropbox account. All functions and properties return objects or
raise exceptions which can safely serialized, i.e., pure Python types. The only
exception are MaestralApiErrors which have been registered explicitly with the Pyro5
serializer.
"""
_daemon_running = True # for integration with Pyro
def __init__(self, config_name='maestral', run=True):
self._config_name = config_name
self._conf = MaestralConfig(self._config_name)
self._setup_logging()
self.set_share_error_reports(self._conf.get("app", "analytics"))
self.client = MaestralApiClient(config_name=self._config_name)
self.monitor = MaestralMonitor(self.client,
config_name=self._config_name)
self.sync = self.monitor.sync
# periodically check for updates and refresh account info
self.update_thread = Thread(
name="Maestral update check",
target=self._periodic_refresh,
daemon=True,
)
self.update_thread.start()
if run:
self.run()
def run(self):
if self.pending_dropbox_folder(self._config_name):
self.create_dropbox_directory()
self.set_excluded_folders()
self.sync.last_cursor = ""
self.sync.last_sync = 0
# start syncing
self.start_sync()
if NOTIFY_SOCKET and system_notifier: # notify systemd that we have started
logger.debug("Running as systemd notify service")
logger.debug(f"NOTIFY_SOCKET = {NOTIFY_SOCKET}")
system_notifier.notify("READY=1")
if IS_WATCHDOG and system_notifier: # notify systemd periodically if alive
logger.debug("Running as systemd watchdog service")
logger.debug(f"WATCHDOG_USEC = {WATCHDOG_USEC}")
logger.debug(f"WATCHDOG_PID = {WATCHDOG_PID}")
self.watchdog_thread = Thread(
name="Maestral watchdog",
target=self._periodic_watchdog,
daemon=True,
)
self.watchdog_thread.start()
def _setup_logging(self):
log_level = self._conf.get("app", "log_level")
mdbx_logger = logging.getLogger("maestral")
mdbx_logger.setLevel(logging.DEBUG)
log_fmt_long = logging.Formatter(
fmt="%(asctime)s %(name)s %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
log_fmt_short = logging.Formatter(fmt="%(message)s")
# log to file
rfh_log_file = get_log_path("maestral", self._config_name + ".log")
self._log_handler_file = logging.handlers.RotatingFileHandler(
rfh_log_file, maxBytes=10**7, backupCount=1)
self._log_handler_file.setFormatter(log_fmt_long)
self._log_handler_file.setLevel(log_level)
mdbx_logger.addHandler(self._log_handler_file)
# log to journal when launched from systemd
if INVOCATION_ID and journal:
self._log_handler_journal = journal.JournalHandler()
self._log_handler_journal.setFormatter(log_fmt_short)
mdbx_logger.addHandler(self._log_handler_journal)
# log to stdout (disabled by default)
self._log_handler_stream = logging.StreamHandler(sys.stdout)
self._log_handler_stream.setFormatter(log_fmt_long)
self._log_handler_stream.setLevel(100)
mdbx_logger.addHandler(self._log_handler_stream)
# log to cached handlers for GUI and CLI
self._log_handler_info_cache = CachedHandler(maxlen=1)
self._log_handler_info_cache.setLevel(logging.INFO)
self._log_handler_info_cache.setFormatter(log_fmt_short)
mdbx_logger.addHandler(self._log_handler_info_cache)
self._log_handler_error_cache = CachedHandler()
self._log_handler_error_cache.setLevel(logging.ERROR)
self._log_handler_error_cache.setFormatter(log_fmt_short)
mdbx_logger.addHandler(self._log_handler_error_cache)
# log to bugsnag (disabled by default)
self._log_handler_bugsnag = BugsnagHandler()
self._log_handler_bugsnag.setLevel(100)
mdbx_logger.addHandler(self._log_handler_bugsnag)
@property
def config_name(self):
return self._config_name
def set_conf(self, section, name, value):
self._conf.set(section, name, value)
def get_conf(self, section, name):
return self._conf.get(section, name)
def set_log_level(self, level_num):
self._log_handler_file.setLevel(level_num)
self._log_handler_stream.setLevel(level_num)
self._conf.set("app", "log_level", level_num)
def set_log_to_stdout(self, enabled=True):
if enabled:
log_level = self._conf.get("app", "log_level")
self._log_handler_stream.setLevel(log_level)
else:
self._log_handler_stream.setLevel(100)
def set_share_error_reports(self, enabled):
bugsnag.configuration.auto_notify = enabled
bugsnag.configuration.auto_capture_sessions = enabled
self._log_handler_bugsnag.setLevel(logging.ERROR if enabled else 100)
self._conf.set("app", "analytics", enabled)
@staticmethod
def pending_link(config_name):
"""
Bool indicating if auth tokens are stored in the system's keychain. This may raise
a KeyringLocked exception if the user's keychain cannot be accessed. This
exception will not be deserialized by Pyro5. You should check if Maestral is
linked before instantiating a daemon.
:param str config_name: Name of user config to check.
:raises: :class:`keyring.errors.KeyringLocked`
"""
auth_session = OAuth2Session(config_name)
return auth_session.load_token() is None
@staticmethod
def pending_dropbox_folder(config_name):
"""
Bool indicating if a local Dropbox directory has been set.
:param str config_name: Name of user config to check.
"""
conf = MaestralConfig(config_name)
return not osp.isdir(conf.get("main", "path"))
def pending_first_download(self):
"""Bool indicating if the initial download has already occurred."""
return (self._conf.get("internal", "lastsync") == 0
or self._conf.get("internal", "cursor") == "")
@property
def syncing(self):
"""Bool indicating if Maestral is syncing. It will be ``True`` if syncing is
not paused by the user *and* Maestral is connected to the internet."""
return self.monitor.syncing.is_set()
@property
def paused(self):
"""Bool indicating if syncing is paused by the user. This is set by calling
:meth:`pause`."""
return not self.monitor._auto_resume_on_connect
@property
def stopped(self):
"""Bool indicating if syncing is stopped, for instance because of an exception."""
return not self.monitor.running.is_set()
@property
def connected(self):
"""Bool indicating if Dropbox servers can be reached."""
return self.monitor.connected.is_set()
@property
def status(self):
"""Returns a string with the last status message. This can be displayed as
information to the user but should not be relied on otherwise."""
return self._log_handler_info_cache.getLastMessage()
@property
def notify(self):
"""Bool indicating if notifications are enabled or disabled."""
return self.sync.notify.enabled
@notify.setter
def notify(self, boolean):
"""Setter: Bool indicating if notifications are enabled."""
self.sync.notify.enabled = boolean
@property
def dropbox_path(self):
"""Returns the path to the local Dropbox directory. Read only. Use
:meth:`create_dropbox_directory` or :meth:`move_dropbox_directory` to set or
change the Dropbox directory location instead. """
return self.sync.dropbox_path
@property
def excluded_folders(self):
"""Returns a list of excluded folders (read only). Use :meth:`exclude_folder`,
:meth:`include_folder` or :meth:`set_excluded_folders` change which folders are
excluded from syncing."""
return self.sync.excluded_folders
@property
def sync_errors(self):
"""Returns list containing the current sync errors as dicts."""
sync_errors = list(self.sync.sync_errors.queue)
sync_errors_dicts = [maestral_error_to_dict(e) for e in sync_errors]
return sync_errors_dicts
@property
def maestral_errors(self):
"""Returns a list of Maestral's errors as dicts. This does not include lost
internet connections or file sync errors which only emit warnings and are tracked
and cleared separately. Errors listed here must be acted upon for Maestral to
continue syncing.
"""
maestral_errors = [
r.exc_info[1] for r in self._log_handler_error_cache.cached_records
]
maestral_errors_dicts = [
maestral_error_to_dict(e) for e in maestral_errors
]
return maestral_errors_dicts
def clear_maestral_errors(self):
"""Manually clears all Maestral errors. This should be used after they have been
resolved by the user through the GUI or CLI.
"""
self._log_handler_error_cache.clear()
@property
def account_profile_pic_path(self):
"""Returns the path of the current account's profile picture. There may not be
an actual file at that path, if the user did not set a profile picture or the
picture has not yet been downloaded."""
return get_cache_path("maestral",
self._config_name + "_profile_pic.jpeg")
def get_file_status(self, local_path):
"""
Returns the sync status of an individual file.
:param local_path: Path to file on the local drive.
:return: String indicating the sync status. Can be "uploading", "downloading",
"up to date", "error", or "unwatched" (for files outside of the Dropbox
directory).
:rtype: str
"""
if not self.syncing:
return "unwatched"
try:
dbx_path = self.sync.to_dbx_path(local_path)
except ValueError:
return "unwatched"
if local_path in self.monitor.queued_for_upload:
return "uploading"
elif local_path in self.monitor.queued_for_download:
return "downloading"
elif any(local_path == err["local_path"] for err in self.sync_errors):
return "error"
elif self.sync.get_local_rev(dbx_path):
return "up to date"
else:
return "unwatched"
def get_activity(self):
"""
Returns a dictionary with lists of all file currently queued for or being synced.
:rtype: dict(list, list)
"""
PathItem = namedtuple("PathItem", "local_path status")
uploading = []
downloading = []
for path in self.monitor.uploading:
uploading.append(PathItem(path, "uploading"))
for path in self.monitor.queued_for_upload:
uploading.append(PathItem(path, "queued"))
for path in self.monitor.downloading:
downloading.append(PathItem(path, "downloading"))
for path in self.monitor.queued_for_download:
downloading.append(PathItem(path, "queued"))
return dict(uploading=uploading, downloading=downloading)
@handle_disconnect
def get_account_info(self):
"""
Gets account information from Dropbox and returns it as a dictionary.
The entries will either be of type ``str`` or ``bool``.
:returns: Dropbox account information.
:rtype: dict[str, bool]
:raises: :class:`MaestralApiError`
"""
res = self.client.get_account_info()
return dropbox_stone_to_dict(res)
@handle_disconnect
def get_space_usage(self):
"""
Gets the space usage stored by Dropbox and returns it as a dictionary.
The entries will either be of type ``str`` or ``bool``.
:returns: Dropbox account information.
:rtype: dict[str, bool]
"""
res = self.client.get_space_usage()
return dropbox_stone_to_dict(res)
@handle_disconnect
def get_profile_pic(self):
"""
Attempts to download the user's profile picture from Dropbox. The picture saved in
Maestral's cache directory for retrieval when there is no internet connection.
This function will fail silently in case of :class:`MaestralApiError`s.
:returns: Path to saved profile picture or None if no profile picture is set.
"""
try:
res = self.client.get_account_info()
except MaestralApiError:
pass
else:
if res.profile_photo_url:
# download current profile pic
res = requests.get(res.profile_photo_url)
with open(self.account_profile_pic_path, "wb") as f:
f.write(res.content)
return self.account_profile_pic_path
else:
# delete current profile pic
self._delete_old_profile_pics()
@handle_disconnect
def list_folder(self, dbx_path, **kwargs):
"""
List all items inside the folder given by :param:`dbx_path`.
:param dbx_path: Path to folder on Dropbox.
:return: List of Dropbox item metadata as dicts or ``False`` if listing failed
due to connection issues.
:rtype: list[dict]
"""
dbx_path = "" if dbx_path == "/" else dbx_path
res = self.client.list_folder(dbx_path, **kwargs)
entries = [dropbox_stone_to_dict(e) for e in res.entries]
return entries
def _delete_old_profile_pics(self):
# delete all old pictures
for file in os.listdir(get_cache_path("maestral")):
if file.startswith(self._config_name + "_profile_pic"):
try:
os.unlink(osp.join(get_cache_path("maestral"), file))
except OSError:
pass
def rebuild_index(self):
"""
Rebuilds the Maestral index and resumes syncing afterwards if it has been
running.
:raises: :class:`MaestralApiError`
"""
self.monitor.rebuild_rev_file()
def start_sync(self, overload=None):
"""
Creates syncing threads and starts syncing.
"""
self.monitor.start()
def resume_sync(self, overload=None):
"""
Resumes the syncing threads if paused.
"""
self.monitor.resume()
def pause_sync(self, overload=None):
"""
Pauses the syncing threads if running.
"""
self.monitor.pause()
def stop_sync(self, overload=None):
"""
Stops the syncing threads if running, destroys observer thread.
"""
self.monitor.stop()
def unlink(self):
"""
Unlinks the configured Dropbox account but leaves all downloaded files
in place. All syncing metadata will be removed as well. Connection and API errors
will be handled silently but the Dropbox access key will always be removed from
the user's PC.
"""
self.stop_sync()
try:
self.client.unlink()
except (ConnectionError, MaestralApiError):
pass
try:
os.remove(self.sync.rev_file_path)
except OSError:
pass
self.sync.dropbox_path = ""
self.sync.last_cursor = ""
self.sync.last_sync = 0.0
self._conf.reset_to_defaults()
logger.info("Unlinked Dropbox account.")
def exclude_folder(self, dbx_path):
"""
Excludes folder from sync and deletes local files. It is safe to call
this method with folders which have already been excluded.
:param str dbx_path: Dropbox folder to exclude.
:raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox.
:raises: :class:`ConnectionError` if connection to Dropbox fails.
"""
dbx_path = dbx_path.lower().rstrip(osp.sep)
md = self.client.get_metadata(dbx_path)
if not isinstance(md, files.FolderMetadata):
raise ValueError(
"No such folder on Dropbox: '{0}'".format(dbx_path))
# add the path to excluded list
excluded_folders = self.sync.excluded_folders
if dbx_path not in excluded_folders:
excluded_folders.append(dbx_path)
else:
logger.info("Folder was already excluded, nothing to do.")
return
self.sync.excluded_folders = excluded_folders
self.sync.set_local_rev(dbx_path, None)
# remove folder from local drive
local_path = self.sync.to_local_path(dbx_path)
local_path_cased = path_exists_case_insensitive(local_path)
logger.info(f"Deleting folder '{local_path_cased}'.")
if osp.isdir(local_path_cased):
shutil.rmtree(local_path_cased)
def include_folder(self, dbx_path):
"""
Includes folder in sync and downloads in the background. It is safe to
call this method with folders which have already been included, they
will not be downloaded again.
:param str dbx_path: Dropbox folder to include.
:raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox or lies inside
another excluded folder.
:raises: :class:`ConnectionError` if connection to Dropbox fails.
"""
dbx_path = dbx_path.lower().rstrip(osp.sep)
md = self.client.get_metadata(dbx_path)
old_excluded_folders = self.sync.excluded_folders
if not isinstance(md, files.FolderMetadata):
raise ValueError(
"No such folder on Dropbox: '{0}'".format(dbx_path))
for folder in old_excluded_folders:
if is_child(dbx_path, folder):
raise ValueError(
"'{0}' lies inside the excluded folder '{1}'. "
"Please include '{1}' first.".format(dbx_path, folder))
# Get folders which will need to be downloaded, do not attempt to download
# subfolders of `dbx_path` which were already included.
# `new_included_folders` will either be empty (`dbx_path` was already
# included), just contain `dbx_path` itself (the whole folder was excluded) or
# only contain subfolders of `dbx_path` (`dbx_path` was partially included).
new_included_folders = tuple(x for x in old_excluded_folders
if x == dbx_path or is_child(x, dbx_path))
if new_included_folders:
# remove `dbx_path` or all excluded children from the excluded list
excluded_folders = list(
set(old_excluded_folders) - set(new_included_folders))
else:
logger.info("Folder was already included, nothing to do.")
return
self.sync.excluded_folders = excluded_folders
# download folder contents from Dropbox
logger.info(f"Downloading added folder '{dbx_path}'.")
for folder in new_included_folders:
self.sync.queued_folder_downloads.put(folder)
@handle_disconnect
def _include_folder_without_subfolders(self, dbx_path):
"""Sets a folder to included without explicitly including its subfolders. This
is to be used internally, when a folder has been removed from the excluded list,
but some of its subfolders may have been added."""
dbx_path = dbx_path.lower().rstrip(osp.sep)
excluded_folders = self.sync.excluded_folders
if dbx_path not in excluded_folders:
return
excluded_folders.remove(dbx_path)
self.sync.excluded_folders = excluded_folders
self.sync.queued_folder_downloads.put(dbx_path)
@handle_disconnect
def set_excluded_folders(self, folder_list=None):
"""
Sets the list of excluded folders to `folder_list`. If not given, gets all top
level folder paths from Dropbox and asks user to include or exclude. Folders
which are no in `folder_list` but exist on Dropbox will be downloaded.
On initial sync, this does not trigger any downloads.
:param list folder_list: If given, list of excluded folder to set.
:return: List of excluded folders.
:rtype: list
:raises: :class:`MaestralApiError`
"""
if folder_list is None:
excluded_folders = []
# get all top-level Dropbox folders
result = self.client.list_folder("", recursive=False)
# paginate through top-level folders, ask to exclude
for entry in result.entries:
if isinstance(entry, files.FolderMetadata):
yes = click.confirm(
f"Exclude '{entry.path_display}' from sync?")
if yes:
excluded_folders.append(entry.path_lower)
else:
excluded_folders = self.sync.clean_excluded_folder_list(
folder_list)
old_excluded_folders = self.sync.excluded_folders
added_excluded_folders = set(excluded_folders) - set(
old_excluded_folders)
added_included_folders = set(old_excluded_folders) - set(
excluded_folders)
if not self.pending_first_download():
# apply changes
for path in added_excluded_folders:
self.exclude_folder(path)
for path in added_included_folders:
self._include_folder_without_subfolders(path)
self.sync.excluded_folders = excluded_folders
return excluded_folders
def excluded_status(self, dbx_path):
"""
Returns 'excluded', 'partially excluded' or 'included'. This function will not
check if the item actually exists on Dropbox.
:param str dbx_path: Path to item on Dropbox.
:returns: Excluded status.
:rtype: str
"""
dbx_path = dbx_path.lower().rstrip(osp.sep)
excluded_items = self._conf.get("main",
"excluded_folders") + self._conf.get(
"main", "excluded_files")
if dbx_path in excluded_items:
return "excluded"
elif any(is_child(f, dbx_path) for f in excluded_items):
return "partially excluded"
else:
return "included"
@with_sync_paused
def move_dropbox_directory(self, new_path=None):
"""
Change or set local dropbox directory. This moves all local files to
the new location. If a file or folder already exists at this location,
it will be overwritten.
:param str new_path: Full path to local Dropbox folder. If not given, the
user will be prompted to input the path.
"""
# get old and new paths
old_path = self.sync.dropbox_path
if new_path is None:
new_path = self._ask_for_path(self._config_name)
try:
if osp.samefile(old_path, new_path):
return
except FileNotFoundError:
pass
# remove existing items at current location
try:
os.unlink(new_path)
except IsADirectoryError:
shutil.rmtree(new_path, ignore_errors=True)
except FileNotFoundError:
pass
# move folder from old location or create a new one if no old folder exists
if osp.isdir(old_path):
shutil.move(old_path, new_path)
else:
os.makedirs(new_path)
# update config file and client
self.sync.dropbox_path = new_path
@with_sync_paused
def create_dropbox_directory(self, path=None, overwrite=True):
"""
Set a new local dropbox directory.
:param str path: Full path to local Dropbox folder. If not given, the user will be
prompted to input the path.
:param bool overwrite: If ``True``, any existing file or folder at ``new_path``
will be replaced.
"""
# ask for new path
if path is None:
path = self._ask_for_path(self._config_name)
if overwrite:
# remove any old items at the location
try:
shutil.rmtree(path)
except NotADirectoryError:
os.unlink(path)
except FileNotFoundError:
pass
# create new folder
os.makedirs(path, exist_ok=True)
# update config file and client
self.sync.dropbox_path = path
@staticmethod
def _ask_for_path(config_name):
"""
Asks for Dropbox path.
"""
conf = MaestralConfig(config_name)
default = osp.join(get_home_dir(), conf.get("main",
"default_dir_name"))
while True:
msg = f"Please give Dropbox folder location or press enter for default ['{default}']:"
res = input(msg).strip("'\" ")
dropbox_path = osp.expanduser(res or default)
old_path = osp.expanduser(conf.get("main", "path"))
same_path = False
try:
if osp.samefile(old_path, dropbox_path):
same_path = True
except FileNotFoundError:
pass
if osp.exists(dropbox_path) and not same_path:
msg = f"Directory '{dropbox_path}' already exist. Do you want to overwrite it?"
yes = click.confirm(msg)
if yes:
return dropbox_path
else:
pass
else:
return dropbox_path
def to_local_path(self, dbx_path):
return self.sync.to_local_path(dbx_path)
@staticmethod
def check_for_updates():
return check_update_available()
def _periodic_refresh(self):
while True:
# update account info
self.get_account_info()
self.get_space_usage()
self.get_profile_pic()
# check for maestral updates
res = self.check_for_updates()
if not res["error"]:
self._conf.set("app", "latest_release", res["latest_release"])
time.sleep(60 * 60) # 60 min
def _periodic_watchdog(self):
while self.monitor._threads_alive():
system_notifier.notify("WATCHDOG=1")
time.sleep(int(WATCHDOG_USEC) / (2 * 10**6))
def shutdown_pyro_daemon(self):
"""Does nothing except for setting the _daemon_running flag to ``False``. This
will be checked by Pyro periodically to shut down the daemon when requested."""
self._daemon_running = False
if NOTIFY_SOCKET and system_notifier:
# notify systemd that we are shutting down
system_notifier.notify("STOPPING=1")
def _loop_condition(self):
return self._daemon_running
def __del__(self):
try:
self.monitor.stop()
except:
pass
def __repr__(self):
email = self._conf.get("account", "email")
account_type = self._conf.get("account", "type")
return f"<{self.__class__}({email}, {account_type})>"
class MaestralApiClient(object):
"""Client for Dropbox SDK.
This client defines basic methods to wrap Dropbox Python SDK calls, such as creating,
moving, modifying and deleting files and folders on Dropbox and downloading files from
Dropbox.
All Dropbox API errors are caught and handled here. ConnectionErrors will
be caught and handled by :class:`MaestralMonitor` instead.
:param int timeout: Timeout for individual requests in sec. Defaults to 60 sec.
"""
SDK_VERSION = "2.0"
_timeout = 60
def __init__(self, config_name='maestral', timeout=_timeout):
self._conf = MaestralConfig(config_name)
# get Dropbox session
self.auth = OAuth2Session(config_name)
if not self.auth.load_token():
self.auth.link()
self._timeout = timeout
self._last_longpoll = None
self._backoff = 0
self._retry_count = 0
# initialize API client
self.dbx = dropbox.Dropbox(self.auth.access_token,
session=SESSION,
user_agent=USER_AGENT,
timeout=self._timeout)
@to_maestral_error()
def get_account_info(self, dbid=None):
"""
Gets current account information.
:param str dbid: Dropbox ID of account. If not given, will get the info of our own
account.
:returns: :class:`dropbox.users.FullAccount` instance or `None` if failed.
:rtype: dropbox.users.FullAccount
"""
if dbid:
res = self.dbx.users_get_account(dbid)
else:
res = self.dbx.users_get_current_account()
if not dbid:
# save our own account info to config
if res.account_type.is_basic():
account_type = "basic"
elif res.account_type.is_business():
account_type = "business"
elif res.account_type.is_pro():
account_type = "pro"
else:
account_type = ""
self._conf.set("account", "account_id", res.account_id)
self._conf.set("account", "email", res.email)
self._conf.set("account", "display_name", res.name.display_name)
self._conf.set("account", "abbreviated_name",
res.name.abbreviated_name)
self._conf.set("account", "type", account_type)
return res
@to_maestral_error()
def get_space_usage(self):
"""
Gets current account space usage.
:returns: :class:`SpaceUsage` instance or `False` if failed.
:rtype: SpaceUsage
"""
res = self.dbx.users_get_space_usage()
# convert from dropbox.users.SpaceUsage to SpaceUsage
res.__class__ = SpaceUsage
# save results to config
self._conf.set("account", "usage", str(res))
self._conf.set("account", "usage_type", res.allocation_type())
return res
@to_maestral_error()
def unlink(self):
"""
Unlinks the Dropbox account and deletes local sync information.
"""
self.auth.delete_creds()
self.dbx.auth_token_revoke() # should only raise auth errors
@to_maestral_error(dbx_path_arg=1)
def get_metadata(self, dbx_path, **kwargs):
"""
Get metadata for Dropbox entry (file or folder). Returns `None` if no
metadata is available. Keyword arguments are passed on to Dropbox SDK
files_get_metadata call.
:param str dbx_path: Path of folder on Dropbox.
:param kwargs: Keyword arguments for Dropbox SDK files_download_to_file.
:returns: FileMetadata|FolderMetadata entries or `False` if failed.
"""
try:
md = self.dbx.files_get_metadata(dbx_path, **kwargs)
logger.debug(f"Retrieved metadata for '{md.path_display}'")
except dropbox.exceptions.ApiError as exc:
# DropboxAPI error is only raised when the item does not exist on Dropbox
# this is handled on a DEBUG level since we use call `get_metadata` to check
# if a file exists
logger.debug(f"Could not get metadata for '{dbx_path}': {exc}")
md = False
return md
@to_maestral_error(dbx_path_arg=1)
def list_revisions(self, dbx_path, mode="path", limit=10):
"""
Lists all file revisions for the given file.
:param str dbx_path: Path to file on Dropbox.
:param str mode: Must be "path" or "id". If "id", specify the Dropbox file ID
instead of the file path to get revisions across move and rename events.
Defaults to "path".
:param int limit: Number of revisions to list. Defaults to 10.
:returns: :class:`dropbox.files.ListRevisionsResult` instance
"""
mode = dropbox.files.ListRevisionsMode(mode)
return self.dbx.files_list_revisions(dbx_path, mode=mode, limit=limit)
@to_maestral_error(dbx_path_arg=1)
def download(self, dbx_path, dst_path, **kwargs):
"""
Downloads file from Dropbox to our local folder.
:param str dbx_path: Path to file on Dropbox.
:param str dst_path: Path to download destination.
:param kwargs: Keyword arguments for Dropbox SDK files_download_to_file.
:returns: :class:`FileMetadata` or
:class:`FolderMetadata` of downloaded item, `False`
if request fails or `None` if local copy is already in sync.
"""
# create local directory if not present
dst_path_directory = osp.dirname(dst_path)
try:
os.makedirs(dst_path_directory)
except FileExistsError:
pass
md = self.dbx.files_download_to_file(dst_path, dbx_path, **kwargs)
logger.debug(
f"File '{md.path_display}' (rev {md.rev}) was successfully downloaded as '{dst_path}'"
)
return md
@to_maestral_error(dbx_path_arg=2)
def upload(self, local_path, dbx_path, chunk_size_mb=5, **kwargs):
"""
Uploads local file to Dropbox.
:param str local_path: Path of local file to upload.
:param str dbx_path: Path to save file on Dropbox.
:param kwargs: Keyword arguments for Dropbox SDK files_upload.
:param int chunk_size_mb: Maximum size for individual uploads in MB. Must be
smaller than 150 MB.
:returns: Metadata of uploaded file or `False` if upload failed.
"""
chunk_size_mb = min(chunk_size_mb, 150)
chunk_size = chunk_size_mb * 10**6 # convert to bytes
file_size = osp.getsize(local_path)
file_size_str = bytes_to_str(file_size)
uploaded = 0
mtime = osp.getmtime(local_path)
mtime_dt = datetime.datetime(*time.gmtime(mtime)[:6])
with open(local_path, "rb") as f:
if file_size <= chunk_size:
md = self.dbx.files_upload(f.read(),
dbx_path,
client_modified=mtime_dt,
**kwargs)
else:
logger.info(
f"Uploading {bytes_to_str(uploaded)}/{file_size_str}...")
session_start = self.dbx.files_upload_session_start(
f.read(chunk_size))
cursor = dropbox.files.UploadSessionCursor(
session_id=session_start.session_id, offset=f.tell())
commit = dropbox.files.CommitInfo(path=dbx_path,
client_modified=mtime_dt,
**kwargs)
while f.tell() < file_size:
if file_size - f.tell() <= chunk_size:
md = self.dbx.files_upload_session_finish(
f.read(chunk_size), cursor, commit)
logger.info(
f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..."
)
else:
# Note: we currently do not support resuming interrupted uploads.
# However, this can be achieved catching connection errors and
# retrying until the upload succeeds. Incorrect offsets due to
# a dropped package can be corrected by getting the right
# offset from the resulting UploadSessionOffsetError and
# resuming the upload from this point.
self.dbx.files_upload_session_append_v2(
f.read(chunk_size), cursor)
cursor.offset = f.tell()
uploaded += chunk_size
logger.info(
f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..."
)
logger.debug(
f"File '{md.path_display}' (rev {md.rev}) uploaded to Dropbox")
return md
@to_maestral_error(dbx_path_arg=1)
def remove(self, dbx_path, **kwargs):
"""
Removes file / folder from Dropbox.
:param str dbx_path: Path to file on Dropbox.
:param kwargs: Keyword arguments for Dropbox SDK files_delete_v2.
:returns: Metadata of deleted file or ``False`` if the file does not exist on
Dropbox.
:raises: :class:`MaestralApiError`.
"""
# try to move file (response will be metadata, probably)
res = self.dbx.files_delete_v2(dbx_path, **kwargs)
md = res.metadata
logger.debug(f"Item '{dbx_path}' removed from Dropbox")
return md
@to_maestral_error(dbx_path_arg=2)
def move(self, dbx_path, new_path, **kwargs):
"""
Moves/renames files or folders on Dropbox.
:param str dbx_path: Path to file/folder on Dropbox.
:param str new_path: New path on Dropbox to move to.
:param kwargs: Keyword arguments for Dropbox SDK files_move_v2.
:returns: Metadata of moved file/folder.
:raises: :class:`MaestralApiError`
"""
res = self.dbx.files_move_v2(dbx_path,
new_path,
allow_shared_folder=True,
allow_ownership_transfer=True,
**kwargs)
md = res.metadata
logger.debug(
f"Item moved from '{dbx_path}' to '{md.path_display}' on Dropbox")
return md
@to_maestral_error(dbx_path_arg=1)
def make_dir(self, dbx_path, **kwargs):
"""
Creates folder on Dropbox.
:param str dbx_path: Path o fDropbox folder.
:param kwargs: Keyword arguments for Dropbox SDK files_create_folder_v2.
:returns: Metadata of created folder.
:raises: :class:`MaestralApiError`
"""
res = self.dbx.files_create_folder_v2(dbx_path, **kwargs)
md = res.metadata
logger.debug(f"Created folder '{md.path_display}' on Dropbox")
return md
@to_maestral_error(dbx_path_arg=1)
def get_latest_cursor(self,
dbx_path,
include_non_downloadable_files=False,
**kwargs):
"""
Gets the latest cursor for the given folder and subfolders.
:param str dbx_path: Path of folder on Dropbox.
:param bool include_non_downloadable_files: If ``True``, files that cannot be
downloaded (at the moment only G-suite files on Dropbox) will be included.
Defaults to ``False``.
:param kwargs: Other keyword arguments for Dropbox SDK files_list_folder.
:returns: The latest cursor representing a state of a folder and its subfolders.
:rtype: str
:raises: :class:`MaestralApiError`
"""
res = self.dbx.files_list_folder_get_latest_cursor(
dbx_path,
include_non_downloadable_files=include_non_downloadable_files,
recursive=True,
**kwargs,
)
return res.cursor
@to_maestral_error(dbx_path_arg=1)
def list_folder(self,
dbx_path,
retry=3,
include_non_downloadable_files=False,
**kwargs):
"""
Lists contents of a folder on Dropbox as dictionary mapping unicode
file names to FileMetadata|FolderMetadata entries.
:param str dbx_path: Path of folder on Dropbox.
:param int retry: Number of times to try again call fails because cursor is
reset. Defaults to 3.
:param bool include_non_downloadable_files: If ``True``, files that cannot be
downloaded (at the moment only G-suite files on Dropbox) will be included.
Defaults to ``False``.
:param kwargs: Other keyword arguments for Dropbox SDK files_list_folder.
:returns: :class:`dropbox.files.ListFolderResult` instance.
:rtype: :class:`dropbox.files.ListFolderResult`
:raises: :class:`MaestralApiError`
"""
results = []
res = self.dbx.files_list_folder(
dbx_path,
include_non_downloadable_files=include_non_downloadable_files,
**kwargs)
results.append(res)
idx = 0
while results[-1].has_more:
idx += len(results[-1].entries)
logger.info(f"Indexing {idx}...")
try:
more_results = self.dbx.files_list_folder_continue(
results[-1].cursor)
results.append(more_results)
except dropbox.exceptions.DropboxException as exc:
new_exc = api_to_maestral_error(exc, dbx_path)
if isinstance(new_exc,
CursorResetError) and self._retry_count < retry:
# retry up to three times, then raise
self._retry_count += 1
self.list_folder(dbx_path, include_non_downloadable_files,
**kwargs)
else:
self._retry_count = 0
raise new_exc
logger.debug(f"Listed contents of folder '{dbx_path}'")
self._retry_count = 0
return self.flatten_results(results)
@staticmethod
def flatten_results(results):
"""
Flattens a list of :class:`dropbox.files.ListFolderResult` instances
and returns their entries only. Only the last cursor will be kept.
:param list results: List of :class:`dropbox.files.ListFolderResult`
instances.
:returns: Single :class:`dropbox.files.ListFolderResult` instance.
:rtype: :class:`dropbox.files.ListFolderResult`
"""
entries_all = []
for result in results:
entries_all += result.entries
results_flattened = dropbox.files.ListFolderResult(
entries=entries_all, cursor=results[-1].cursor, has_more=False)
return results_flattened
@to_maestral_error()
def wait_for_remote_changes(self, last_cursor, timeout=40):
"""
Waits for remote changes since :param:`last_cursor`. Call this method
after starting the Dropbox client and periodically to get the latest
updates.
:param str last_cursor: Last to cursor to compare for changes.
:param int timeout: Seconds to wait until timeout. Must be between 30 and 480.
:returns: ``True`` if changes are available, ``False`` otherwise.
:rtype: bool
:raises: :class:`MaestralApiError`
"""
if not 30 <= timeout <= 480:
raise ValueError("Timeout must be in range [30, 480]")
logger.debug(
f"Waiting for remote changes since cursor:\n{last_cursor}")
# honour last request to back off
if self._last_longpoll is not None:
while time.time() - self._last_longpoll < self._backoff:
time.sleep(1)
result = self.dbx.files_list_folder_longpoll(last_cursor,
timeout=timeout)
# keep track of last long poll, back off if requested by SDK
if result.backoff:
self._backoff = result.backoff + 5
else:
self._backoff = 0
logger.debug(f"Detected remote changes: {result.changes}")
self._last_longpoll = time.time()
return result.changes # will be True or False
@to_maestral_error()
def list_remote_changes(self, last_cursor):
"""
Lists changes to remote Dropbox since :param:`last_cursor`. Call this
after :method:`wait_for_remote_changes` returns ``True``.
:param str last_cursor: Last to cursor to compare for changes.
:returns: :class:`dropbox.files.ListFolderResult` instance.
:rtype: :class:`dropbox.files.ListFolderResult`
:raises:
"""
results = [self.dbx.files_list_folder_continue(last_cursor)]
while results[-1].has_more:
more_results = self.dbx.files_list_folder_continue(
results[-1].cursor)
results.append(more_results)
# combine all results into one
results = self.flatten_results(results)
logger.debug(f"Listed remote changes: {len(results.entries)} changes")
return results