def log_level(config_name: str, level_name: str): """Gets or sets the log level.""" from maestral.daemon import MaestralProxy try: with MaestralProxy(config_name) as m: if level_name: m.log_level = logging._nameToLevel[level_name] click.echo(f'Log level set to {level_name}.') else: level_name = logging.getLevelName(m.log_level) click.echo(f'Log level: {level_name}') except Pyro5.errors.CommunicationError: from maestral.config.main import MaestralConfig conf = MaestralConfig(config_name) if level_name: conf.set('app', 'log_level', logging._nameToLevel[level_name]) click.echo(f'Log level set to {level_name}.') else: level_name = logging.getLevelName(conf.get('app', 'log_level')) click.echo(f'Log level: {level_name}')
def notify_level(config_name: str, level_name: str): """Gets or sets the level for desktop notifications.""" from maestral.daemon import MaestralProxy from maestral.utils.notify import levelNameToNumber, levelNumberToName try: with MaestralProxy(config_name) as m: if level_name: m.notification_level = levelNameToNumber(level_name) click.echo(f'Notification level set to {level_name}.') else: level_name = levelNumberToName(m.notification_level) click.echo(f'Notification level: {level_name}.') except Pyro5.errors.CommunicationError: from maestral.config.main import MaestralConfig conf = MaestralConfig(config_name) if level_name: conf.set('app', 'notification_level', levelNameToNumber(level_name)) click.echo(f'Notification level set to {level_name}.') else: level_name = levelNumberToName( conf.get('app', 'notification_level')) click.echo(f'Notification level: {level_name}.')
def migrate_user_config(config_name): config_path = get_conf_path(CONFIG_DIR_NAME, create=False) config_fname = osp.join(config_path, config_name + '.ini') # load old config non-destructively try: old_conf = DefaultsConfig(config_path, config_name, '.ini') old_conf.read(config_fname, encoding='utf-8') old_version = old_conf.get(UserConfig.DEFAULT_SECTION_NAME, 'version') except OSError: return if Version(old_version) < Version('11.0.0'): # get values for moved settings excluded_folders = old_conf.get('main', 'excluded_folders') email = old_conf.get('account', 'email') display_name = old_conf.get('account', 'display_name') abbreviated_name = old_conf.get('account', 'abbreviated_name') acc_type = old_conf.get('account', 'type') usage = old_conf.get('account', 'usage') usage_type = old_conf.get('account', 'usage_type') update_notification_last = old_conf.get('app', 'update_notification_last') latest_release = old_conf.get('app', 'latest_release') cursor = old_conf.get('internal', 'cursor') lastsync = old_conf.get('internal', 'lastsync') recent_changes = old_conf.get('internal', 'recent_changes') # convert non-string types update_notification_last = float(update_notification_last) lastsync = float(lastsync) recent_changes = ast.literal_eval(recent_changes) excluded_folders = ast.literal_eval(excluded_folders) # set state values state = MaestralState(config_name) state.set('account', 'email', email) state.set('account', 'display_name', display_name) state.set('account', 'abbreviated_name', abbreviated_name) state.set('account', 'type', acc_type) state.set('account', 'usage', usage) state.set('account', 'usage_type', usage_type) state.set('app', 'update_notification_last', update_notification_last) state.set('app', 'latest_release', latest_release) state.set('sync', 'cursor', cursor) state.set('sync', 'lastsync', lastsync) state.set('sync', 'recent_changes', recent_changes) # load actual config to remove obsolete options and add moved ones conf = MaestralConfig(config_name) conf.set('main', 'excluded_items', excluded_folders) # clean up backup and defaults files from previous version of maestral for file in os.scandir(old_conf._path): if file.is_file(): if (conf._backup_suffix in file.name or conf._defaults_name_prefix in file.name): os.remove(file.path) logger.info(f'Migrated user config "{config_name}"') elif Version(old_version) < Version('12.0.0'): excluded_folders = old_conf.get('main', 'excluded_folders') excluded_folders = ast.literal_eval(excluded_folders) conf = MaestralConfig(config_name) conf.set('main', 'excluded_items', excluded_folders)
class OAuth2Session(object): """ OAuth2Session provides OAuth2 login and token store. """ oAuth2FlowResult = None Success = 0 InvalidToken = 1 ConnectionFailed = 2 def __init__(self, config_name='maestral'): self._conf = MaestralConfig(config_name) self.account_id = self._conf.get("account", "account_id") self.access_token = "" self.auth_flow = None def load_token(self): """ Check if auth key has been saved. :raises: ``KeyringLocked`` if the system keyring cannot be accessed. """ logger.debug("Using keyring: %s" % keyring.get_keyring()) try: if self.account_id == "": self.access_token = None else: self.access_token = keyring.get_password("Maestral", self.account_id) return self.access_token except KeyringLocked: info = "Please make sure that your keyring is unlocked and restart Maestral." raise KeyringLocked(info) def get_auth_url(self): """Gets the auth URL to start the OAuth2 implicit grant flow.""" self.auth_flow = DropboxOAuth2FlowImplicit(APP_KEY) authorize_url = self.auth_flow.start() return authorize_url def verify_auth_token(self, token): """ Verify the provided authorization token with Dropbox servers. :return: OAuth2Session.Success, OAuth2Session.InvalidToken, or OAuth2Session.ConnectionFailed :rtype: int """ if not self.auth_flow: raise RuntimeError('Auth flow not yet started. Please call \'get_auth_url\'.') try: self.oAuth2FlowResult = self.auth_flow.finish(token) self.access_token = self.oAuth2FlowResult.access_token self.account_id = self.oAuth2FlowResult.account_id return self.Success except DropboxAuthError: return self.InvalidToken except ConnectionError: return self.ConnectionFailed def link(self): """Command line flow to get an auth key from Dropbox and save it in the system keyring.""" authorize_url = self.get_auth_url() print("1. Go to: " + authorize_url) print("2. Click \"Allow\" (you might have to log in first).") print("3. Copy the authorization token.") res = 1 while res > 0: auth_code = input("Enter the authorization token here: ").strip() res = self.verify_auth_token(auth_code) if res == 1: print("Invalid token. Please try again.") elif res == 2: print("Could not connect to Dropbox. Please try again.") self.save_creds() def save_creds(self): """Saves auth key to system keyring.""" self._conf.set("account", "account_id", self.account_id) try: keyring.set_password("Maestral", self.account_id, self.access_token) print(" > Credentials written.") except KeyringLocked: logger.error("Could not access the user keyring to save your authentication " "token. Please make sure that the keyring is unlocked.") def delete_creds(self): """Deletes auth key from system keyring.""" self._conf.set("account", "account_id", "") try: keyring.delete_password("Maestral", self.account_id) print(" > Credentials removed.") except KeyringLocked: logger.error("Could not access the user keyring to delete your authentication" " token. Please make sure that the keyring is unlocked.")
class Maestral(object): """ An open source Dropbox client for macOS and Linux to syncing a local folder with your Dropbox account. All functions and properties return objects or raise exceptions which can safely serialized, i.e., pure Python types. The only exception are MaestralApiErrors which have been registered explicitly with the Pyro5 serializer. """ _daemon_running = True # for integration with Pyro def __init__(self, config_name='maestral', run=True): self._config_name = config_name self._conf = MaestralConfig(self._config_name) self._setup_logging() self.set_share_error_reports(self._conf.get("app", "analytics")) self.client = MaestralApiClient(config_name=self._config_name) self.monitor = MaestralMonitor(self.client, config_name=self._config_name) self.sync = self.monitor.sync # periodically check for updates and refresh account info self.update_thread = Thread( name="Maestral update check", target=self._periodic_refresh, daemon=True, ) self.update_thread.start() if run: self.run() def run(self): if self.pending_dropbox_folder(self._config_name): self.create_dropbox_directory() self.set_excluded_folders() self.sync.last_cursor = "" self.sync.last_sync = 0 # start syncing self.start_sync() if NOTIFY_SOCKET and system_notifier: # notify systemd that we have started logger.debug("Running as systemd notify service") logger.debug(f"NOTIFY_SOCKET = {NOTIFY_SOCKET}") system_notifier.notify("READY=1") if IS_WATCHDOG and system_notifier: # notify systemd periodically if alive logger.debug("Running as systemd watchdog service") logger.debug(f"WATCHDOG_USEC = {WATCHDOG_USEC}") logger.debug(f"WATCHDOG_PID = {WATCHDOG_PID}") self.watchdog_thread = Thread( name="Maestral watchdog", target=self._periodic_watchdog, daemon=True, ) self.watchdog_thread.start() def _setup_logging(self): log_level = self._conf.get("app", "log_level") mdbx_logger = logging.getLogger("maestral") mdbx_logger.setLevel(logging.DEBUG) log_fmt_long = logging.Formatter( fmt="%(asctime)s %(name)s %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") log_fmt_short = logging.Formatter(fmt="%(message)s") # log to file rfh_log_file = get_log_path("maestral", self._config_name + ".log") self._log_handler_file = logging.handlers.RotatingFileHandler( rfh_log_file, maxBytes=10**7, backupCount=1) self._log_handler_file.setFormatter(log_fmt_long) self._log_handler_file.setLevel(log_level) mdbx_logger.addHandler(self._log_handler_file) # log to journal when launched from systemd if INVOCATION_ID and journal: self._log_handler_journal = journal.JournalHandler() self._log_handler_journal.setFormatter(log_fmt_short) mdbx_logger.addHandler(self._log_handler_journal) # log to stdout (disabled by default) self._log_handler_stream = logging.StreamHandler(sys.stdout) self._log_handler_stream.setFormatter(log_fmt_long) self._log_handler_stream.setLevel(100) mdbx_logger.addHandler(self._log_handler_stream) # log to cached handlers for GUI and CLI self._log_handler_info_cache = CachedHandler(maxlen=1) self._log_handler_info_cache.setLevel(logging.INFO) self._log_handler_info_cache.setFormatter(log_fmt_short) mdbx_logger.addHandler(self._log_handler_info_cache) self._log_handler_error_cache = CachedHandler() self._log_handler_error_cache.setLevel(logging.ERROR) self._log_handler_error_cache.setFormatter(log_fmt_short) mdbx_logger.addHandler(self._log_handler_error_cache) # log to bugsnag (disabled by default) self._log_handler_bugsnag = BugsnagHandler() self._log_handler_bugsnag.setLevel(100) mdbx_logger.addHandler(self._log_handler_bugsnag) @property def config_name(self): return self._config_name def set_conf(self, section, name, value): self._conf.set(section, name, value) def get_conf(self, section, name): return self._conf.get(section, name) def set_log_level(self, level_num): self._log_handler_file.setLevel(level_num) self._log_handler_stream.setLevel(level_num) self._conf.set("app", "log_level", level_num) def set_log_to_stdout(self, enabled=True): if enabled: log_level = self._conf.get("app", "log_level") self._log_handler_stream.setLevel(log_level) else: self._log_handler_stream.setLevel(100) def set_share_error_reports(self, enabled): bugsnag.configuration.auto_notify = enabled bugsnag.configuration.auto_capture_sessions = enabled self._log_handler_bugsnag.setLevel(logging.ERROR if enabled else 100) self._conf.set("app", "analytics", enabled) @staticmethod def pending_link(config_name): """ Bool indicating if auth tokens are stored in the system's keychain. This may raise a KeyringLocked exception if the user's keychain cannot be accessed. This exception will not be deserialized by Pyro5. You should check if Maestral is linked before instantiating a daemon. :param str config_name: Name of user config to check. :raises: :class:`keyring.errors.KeyringLocked` """ auth_session = OAuth2Session(config_name) return auth_session.load_token() is None @staticmethod def pending_dropbox_folder(config_name): """ Bool indicating if a local Dropbox directory has been set. :param str config_name: Name of user config to check. """ conf = MaestralConfig(config_name) return not osp.isdir(conf.get("main", "path")) def pending_first_download(self): """Bool indicating if the initial download has already occurred.""" return (self._conf.get("internal", "lastsync") == 0 or self._conf.get("internal", "cursor") == "") @property def syncing(self): """Bool indicating if Maestral is syncing. It will be ``True`` if syncing is not paused by the user *and* Maestral is connected to the internet.""" return self.monitor.syncing.is_set() @property def paused(self): """Bool indicating if syncing is paused by the user. This is set by calling :meth:`pause`.""" return not self.monitor._auto_resume_on_connect @property def stopped(self): """Bool indicating if syncing is stopped, for instance because of an exception.""" return not self.monitor.running.is_set() @property def connected(self): """Bool indicating if Dropbox servers can be reached.""" return self.monitor.connected.is_set() @property def status(self): """Returns a string with the last status message. This can be displayed as information to the user but should not be relied on otherwise.""" return self._log_handler_info_cache.getLastMessage() @property def notify(self): """Bool indicating if notifications are enabled or disabled.""" return self.sync.notify.enabled @notify.setter def notify(self, boolean): """Setter: Bool indicating if notifications are enabled.""" self.sync.notify.enabled = boolean @property def dropbox_path(self): """Returns the path to the local Dropbox directory. Read only. Use :meth:`create_dropbox_directory` or :meth:`move_dropbox_directory` to set or change the Dropbox directory location instead. """ return self.sync.dropbox_path @property def excluded_folders(self): """Returns a list of excluded folders (read only). Use :meth:`exclude_folder`, :meth:`include_folder` or :meth:`set_excluded_folders` change which folders are excluded from syncing.""" return self.sync.excluded_folders @property def sync_errors(self): """Returns list containing the current sync errors as dicts.""" sync_errors = list(self.sync.sync_errors.queue) sync_errors_dicts = [maestral_error_to_dict(e) for e in sync_errors] return sync_errors_dicts @property def maestral_errors(self): """Returns a list of Maestral's errors as dicts. This does not include lost internet connections or file sync errors which only emit warnings and are tracked and cleared separately. Errors listed here must be acted upon for Maestral to continue syncing. """ maestral_errors = [ r.exc_info[1] for r in self._log_handler_error_cache.cached_records ] maestral_errors_dicts = [ maestral_error_to_dict(e) for e in maestral_errors ] return maestral_errors_dicts def clear_maestral_errors(self): """Manually clears all Maestral errors. This should be used after they have been resolved by the user through the GUI or CLI. """ self._log_handler_error_cache.clear() @property def account_profile_pic_path(self): """Returns the path of the current account's profile picture. There may not be an actual file at that path, if the user did not set a profile picture or the picture has not yet been downloaded.""" return get_cache_path("maestral", self._config_name + "_profile_pic.jpeg") def get_file_status(self, local_path): """ Returns the sync status of an individual file. :param local_path: Path to file on the local drive. :return: String indicating the sync status. Can be "uploading", "downloading", "up to date", "error", or "unwatched" (for files outside of the Dropbox directory). :rtype: str """ if not self.syncing: return "unwatched" try: dbx_path = self.sync.to_dbx_path(local_path) except ValueError: return "unwatched" if local_path in self.monitor.queued_for_upload: return "uploading" elif local_path in self.monitor.queued_for_download: return "downloading" elif any(local_path == err["local_path"] for err in self.sync_errors): return "error" elif self.sync.get_local_rev(dbx_path): return "up to date" else: return "unwatched" def get_activity(self): """ Returns a dictionary with lists of all file currently queued for or being synced. :rtype: dict(list, list) """ PathItem = namedtuple("PathItem", "local_path status") uploading = [] downloading = [] for path in self.monitor.uploading: uploading.append(PathItem(path, "uploading")) for path in self.monitor.queued_for_upload: uploading.append(PathItem(path, "queued")) for path in self.monitor.downloading: downloading.append(PathItem(path, "downloading")) for path in self.monitor.queued_for_download: downloading.append(PathItem(path, "queued")) return dict(uploading=uploading, downloading=downloading) @handle_disconnect def get_account_info(self): """ Gets account information from Dropbox and returns it as a dictionary. The entries will either be of type ``str`` or ``bool``. :returns: Dropbox account information. :rtype: dict[str, bool] :raises: :class:`MaestralApiError` """ res = self.client.get_account_info() return dropbox_stone_to_dict(res) @handle_disconnect def get_space_usage(self): """ Gets the space usage stored by Dropbox and returns it as a dictionary. The entries will either be of type ``str`` or ``bool``. :returns: Dropbox account information. :rtype: dict[str, bool] """ res = self.client.get_space_usage() return dropbox_stone_to_dict(res) @handle_disconnect def get_profile_pic(self): """ Attempts to download the user's profile picture from Dropbox. The picture saved in Maestral's cache directory for retrieval when there is no internet connection. This function will fail silently in case of :class:`MaestralApiError`s. :returns: Path to saved profile picture or None if no profile picture is set. """ try: res = self.client.get_account_info() except MaestralApiError: pass else: if res.profile_photo_url: # download current profile pic res = requests.get(res.profile_photo_url) with open(self.account_profile_pic_path, "wb") as f: f.write(res.content) return self.account_profile_pic_path else: # delete current profile pic self._delete_old_profile_pics() @handle_disconnect def list_folder(self, dbx_path, **kwargs): """ List all items inside the folder given by :param:`dbx_path`. :param dbx_path: Path to folder on Dropbox. :return: List of Dropbox item metadata as dicts or ``False`` if listing failed due to connection issues. :rtype: list[dict] """ dbx_path = "" if dbx_path == "/" else dbx_path res = self.client.list_folder(dbx_path, **kwargs) entries = [dropbox_stone_to_dict(e) for e in res.entries] return entries def _delete_old_profile_pics(self): # delete all old pictures for file in os.listdir(get_cache_path("maestral")): if file.startswith(self._config_name + "_profile_pic"): try: os.unlink(osp.join(get_cache_path("maestral"), file)) except OSError: pass def rebuild_index(self): """ Rebuilds the Maestral index and resumes syncing afterwards if it has been running. :raises: :class:`MaestralApiError` """ self.monitor.rebuild_rev_file() def start_sync(self, overload=None): """ Creates syncing threads and starts syncing. """ self.monitor.start() def resume_sync(self, overload=None): """ Resumes the syncing threads if paused. """ self.monitor.resume() def pause_sync(self, overload=None): """ Pauses the syncing threads if running. """ self.monitor.pause() def stop_sync(self, overload=None): """ Stops the syncing threads if running, destroys observer thread. """ self.monitor.stop() def unlink(self): """ Unlinks the configured Dropbox account but leaves all downloaded files in place. All syncing metadata will be removed as well. Connection and API errors will be handled silently but the Dropbox access key will always be removed from the user's PC. """ self.stop_sync() try: self.client.unlink() except (ConnectionError, MaestralApiError): pass try: os.remove(self.sync.rev_file_path) except OSError: pass self.sync.dropbox_path = "" self.sync.last_cursor = "" self.sync.last_sync = 0.0 self._conf.reset_to_defaults() logger.info("Unlinked Dropbox account.") def exclude_folder(self, dbx_path): """ Excludes folder from sync and deletes local files. It is safe to call this method with folders which have already been excluded. :param str dbx_path: Dropbox folder to exclude. :raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox. :raises: :class:`ConnectionError` if connection to Dropbox fails. """ dbx_path = dbx_path.lower().rstrip(osp.sep) md = self.client.get_metadata(dbx_path) if not isinstance(md, files.FolderMetadata): raise ValueError( "No such folder on Dropbox: '{0}'".format(dbx_path)) # add the path to excluded list excluded_folders = self.sync.excluded_folders if dbx_path not in excluded_folders: excluded_folders.append(dbx_path) else: logger.info("Folder was already excluded, nothing to do.") return self.sync.excluded_folders = excluded_folders self.sync.set_local_rev(dbx_path, None) # remove folder from local drive local_path = self.sync.to_local_path(dbx_path) local_path_cased = path_exists_case_insensitive(local_path) logger.info(f"Deleting folder '{local_path_cased}'.") if osp.isdir(local_path_cased): shutil.rmtree(local_path_cased) def include_folder(self, dbx_path): """ Includes folder in sync and downloads in the background. It is safe to call this method with folders which have already been included, they will not be downloaded again. :param str dbx_path: Dropbox folder to include. :raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox or lies inside another excluded folder. :raises: :class:`ConnectionError` if connection to Dropbox fails. """ dbx_path = dbx_path.lower().rstrip(osp.sep) md = self.client.get_metadata(dbx_path) old_excluded_folders = self.sync.excluded_folders if not isinstance(md, files.FolderMetadata): raise ValueError( "No such folder on Dropbox: '{0}'".format(dbx_path)) for folder in old_excluded_folders: if is_child(dbx_path, folder): raise ValueError( "'{0}' lies inside the excluded folder '{1}'. " "Please include '{1}' first.".format(dbx_path, folder)) # Get folders which will need to be downloaded, do not attempt to download # subfolders of `dbx_path` which were already included. # `new_included_folders` will either be empty (`dbx_path` was already # included), just contain `dbx_path` itself (the whole folder was excluded) or # only contain subfolders of `dbx_path` (`dbx_path` was partially included). new_included_folders = tuple(x for x in old_excluded_folders if x == dbx_path or is_child(x, dbx_path)) if new_included_folders: # remove `dbx_path` or all excluded children from the excluded list excluded_folders = list( set(old_excluded_folders) - set(new_included_folders)) else: logger.info("Folder was already included, nothing to do.") return self.sync.excluded_folders = excluded_folders # download folder contents from Dropbox logger.info(f"Downloading added folder '{dbx_path}'.") for folder in new_included_folders: self.sync.queued_folder_downloads.put(folder) @handle_disconnect def _include_folder_without_subfolders(self, dbx_path): """Sets a folder to included without explicitly including its subfolders. This is to be used internally, when a folder has been removed from the excluded list, but some of its subfolders may have been added.""" dbx_path = dbx_path.lower().rstrip(osp.sep) excluded_folders = self.sync.excluded_folders if dbx_path not in excluded_folders: return excluded_folders.remove(dbx_path) self.sync.excluded_folders = excluded_folders self.sync.queued_folder_downloads.put(dbx_path) @handle_disconnect def set_excluded_folders(self, folder_list=None): """ Sets the list of excluded folders to `folder_list`. If not given, gets all top level folder paths from Dropbox and asks user to include or exclude. Folders which are no in `folder_list` but exist on Dropbox will be downloaded. On initial sync, this does not trigger any downloads. :param list folder_list: If given, list of excluded folder to set. :return: List of excluded folders. :rtype: list :raises: :class:`MaestralApiError` """ if folder_list is None: excluded_folders = [] # get all top-level Dropbox folders result = self.client.list_folder("", recursive=False) # paginate through top-level folders, ask to exclude for entry in result.entries: if isinstance(entry, files.FolderMetadata): yes = click.confirm( f"Exclude '{entry.path_display}' from sync?") if yes: excluded_folders.append(entry.path_lower) else: excluded_folders = self.sync.clean_excluded_folder_list( folder_list) old_excluded_folders = self.sync.excluded_folders added_excluded_folders = set(excluded_folders) - set( old_excluded_folders) added_included_folders = set(old_excluded_folders) - set( excluded_folders) if not self.pending_first_download(): # apply changes for path in added_excluded_folders: self.exclude_folder(path) for path in added_included_folders: self._include_folder_without_subfolders(path) self.sync.excluded_folders = excluded_folders return excluded_folders def excluded_status(self, dbx_path): """ Returns 'excluded', 'partially excluded' or 'included'. This function will not check if the item actually exists on Dropbox. :param str dbx_path: Path to item on Dropbox. :returns: Excluded status. :rtype: str """ dbx_path = dbx_path.lower().rstrip(osp.sep) excluded_items = self._conf.get("main", "excluded_folders") + self._conf.get( "main", "excluded_files") if dbx_path in excluded_items: return "excluded" elif any(is_child(f, dbx_path) for f in excluded_items): return "partially excluded" else: return "included" @with_sync_paused def move_dropbox_directory(self, new_path=None): """ Change or set local dropbox directory. This moves all local files to the new location. If a file or folder already exists at this location, it will be overwritten. :param str new_path: Full path to local Dropbox folder. If not given, the user will be prompted to input the path. """ # get old and new paths old_path = self.sync.dropbox_path if new_path is None: new_path = self._ask_for_path(self._config_name) try: if osp.samefile(old_path, new_path): return except FileNotFoundError: pass # remove existing items at current location try: os.unlink(new_path) except IsADirectoryError: shutil.rmtree(new_path, ignore_errors=True) except FileNotFoundError: pass # move folder from old location or create a new one if no old folder exists if osp.isdir(old_path): shutil.move(old_path, new_path) else: os.makedirs(new_path) # update config file and client self.sync.dropbox_path = new_path @with_sync_paused def create_dropbox_directory(self, path=None, overwrite=True): """ Set a new local dropbox directory. :param str path: Full path to local Dropbox folder. If not given, the user will be prompted to input the path. :param bool overwrite: If ``True``, any existing file or folder at ``new_path`` will be replaced. """ # ask for new path if path is None: path = self._ask_for_path(self._config_name) if overwrite: # remove any old items at the location try: shutil.rmtree(path) except NotADirectoryError: os.unlink(path) except FileNotFoundError: pass # create new folder os.makedirs(path, exist_ok=True) # update config file and client self.sync.dropbox_path = path @staticmethod def _ask_for_path(config_name): """ Asks for Dropbox path. """ conf = MaestralConfig(config_name) default = osp.join(get_home_dir(), conf.get("main", "default_dir_name")) while True: msg = f"Please give Dropbox folder location or press enter for default ['{default}']:" res = input(msg).strip("'\" ") dropbox_path = osp.expanduser(res or default) old_path = osp.expanduser(conf.get("main", "path")) same_path = False try: if osp.samefile(old_path, dropbox_path): same_path = True except FileNotFoundError: pass if osp.exists(dropbox_path) and not same_path: msg = f"Directory '{dropbox_path}' already exist. Do you want to overwrite it?" yes = click.confirm(msg) if yes: return dropbox_path else: pass else: return dropbox_path def to_local_path(self, dbx_path): return self.sync.to_local_path(dbx_path) @staticmethod def check_for_updates(): return check_update_available() def _periodic_refresh(self): while True: # update account info self.get_account_info() self.get_space_usage() self.get_profile_pic() # check for maestral updates res = self.check_for_updates() if not res["error"]: self._conf.set("app", "latest_release", res["latest_release"]) time.sleep(60 * 60) # 60 min def _periodic_watchdog(self): while self.monitor._threads_alive(): system_notifier.notify("WATCHDOG=1") time.sleep(int(WATCHDOG_USEC) / (2 * 10**6)) def shutdown_pyro_daemon(self): """Does nothing except for setting the _daemon_running flag to ``False``. This will be checked by Pyro periodically to shut down the daemon when requested.""" self._daemon_running = False if NOTIFY_SOCKET and system_notifier: # notify systemd that we are shutting down system_notifier.notify("STOPPING=1") def _loop_condition(self): return self._daemon_running def __del__(self): try: self.monitor.stop() except: pass def __repr__(self): email = self._conf.get("account", "email") account_type = self._conf.get("account", "type") return f"<{self.__class__}({email}, {account_type})>"
class MaestralApiClient(object): """Client for Dropbox SDK. This client defines basic methods to wrap Dropbox Python SDK calls, such as creating, moving, modifying and deleting files and folders on Dropbox and downloading files from Dropbox. All Dropbox API errors are caught and handled here. ConnectionErrors will be caught and handled by :class:`MaestralMonitor` instead. :param int timeout: Timeout for individual requests in sec. Defaults to 60 sec. """ SDK_VERSION = "2.0" _timeout = 60 def __init__(self, config_name='maestral', timeout=_timeout): self._conf = MaestralConfig(config_name) # get Dropbox session self.auth = OAuth2Session(config_name) if not self.auth.load_token(): self.auth.link() self._timeout = timeout self._last_longpoll = None self._backoff = 0 self._retry_count = 0 # initialize API client self.dbx = dropbox.Dropbox(self.auth.access_token, session=SESSION, user_agent=USER_AGENT, timeout=self._timeout) @to_maestral_error() def get_account_info(self, dbid=None): """ Gets current account information. :param str dbid: Dropbox ID of account. If not given, will get the info of our own account. :returns: :class:`dropbox.users.FullAccount` instance or `None` if failed. :rtype: dropbox.users.FullAccount """ if dbid: res = self.dbx.users_get_account(dbid) else: res = self.dbx.users_get_current_account() if not dbid: # save our own account info to config if res.account_type.is_basic(): account_type = "basic" elif res.account_type.is_business(): account_type = "business" elif res.account_type.is_pro(): account_type = "pro" else: account_type = "" self._conf.set("account", "account_id", res.account_id) self._conf.set("account", "email", res.email) self._conf.set("account", "display_name", res.name.display_name) self._conf.set("account", "abbreviated_name", res.name.abbreviated_name) self._conf.set("account", "type", account_type) return res @to_maestral_error() def get_space_usage(self): """ Gets current account space usage. :returns: :class:`SpaceUsage` instance or `False` if failed. :rtype: SpaceUsage """ res = self.dbx.users_get_space_usage() # convert from dropbox.users.SpaceUsage to SpaceUsage res.__class__ = SpaceUsage # save results to config self._conf.set("account", "usage", str(res)) self._conf.set("account", "usage_type", res.allocation_type()) return res @to_maestral_error() def unlink(self): """ Unlinks the Dropbox account and deletes local sync information. """ self.auth.delete_creds() self.dbx.auth_token_revoke() # should only raise auth errors @to_maestral_error(dbx_path_arg=1) def get_metadata(self, dbx_path, **kwargs): """ Get metadata for Dropbox entry (file or folder). Returns `None` if no metadata is available. Keyword arguments are passed on to Dropbox SDK files_get_metadata call. :param str dbx_path: Path of folder on Dropbox. :param kwargs: Keyword arguments for Dropbox SDK files_download_to_file. :returns: FileMetadata|FolderMetadata entries or `False` if failed. """ try: md = self.dbx.files_get_metadata(dbx_path, **kwargs) logger.debug(f"Retrieved metadata for '{md.path_display}'") except dropbox.exceptions.ApiError as exc: # DropboxAPI error is only raised when the item does not exist on Dropbox # this is handled on a DEBUG level since we use call `get_metadata` to check # if a file exists logger.debug(f"Could not get metadata for '{dbx_path}': {exc}") md = False return md @to_maestral_error(dbx_path_arg=1) def list_revisions(self, dbx_path, mode="path", limit=10): """ Lists all file revisions for the given file. :param str dbx_path: Path to file on Dropbox. :param str mode: Must be "path" or "id". If "id", specify the Dropbox file ID instead of the file path to get revisions across move and rename events. Defaults to "path". :param int limit: Number of revisions to list. Defaults to 10. :returns: :class:`dropbox.files.ListRevisionsResult` instance """ mode = dropbox.files.ListRevisionsMode(mode) return self.dbx.files_list_revisions(dbx_path, mode=mode, limit=limit) @to_maestral_error(dbx_path_arg=1) def download(self, dbx_path, dst_path, **kwargs): """ Downloads file from Dropbox to our local folder. :param str dbx_path: Path to file on Dropbox. :param str dst_path: Path to download destination. :param kwargs: Keyword arguments for Dropbox SDK files_download_to_file. :returns: :class:`FileMetadata` or :class:`FolderMetadata` of downloaded item, `False` if request fails or `None` if local copy is already in sync. """ # create local directory if not present dst_path_directory = osp.dirname(dst_path) try: os.makedirs(dst_path_directory) except FileExistsError: pass md = self.dbx.files_download_to_file(dst_path, dbx_path, **kwargs) logger.debug( f"File '{md.path_display}' (rev {md.rev}) was successfully downloaded as '{dst_path}'" ) return md @to_maestral_error(dbx_path_arg=2) def upload(self, local_path, dbx_path, chunk_size_mb=5, **kwargs): """ Uploads local file to Dropbox. :param str local_path: Path of local file to upload. :param str dbx_path: Path to save file on Dropbox. :param kwargs: Keyword arguments for Dropbox SDK files_upload. :param int chunk_size_mb: Maximum size for individual uploads in MB. Must be smaller than 150 MB. :returns: Metadata of uploaded file or `False` if upload failed. """ chunk_size_mb = min(chunk_size_mb, 150) chunk_size = chunk_size_mb * 10**6 # convert to bytes file_size = osp.getsize(local_path) file_size_str = bytes_to_str(file_size) uploaded = 0 mtime = osp.getmtime(local_path) mtime_dt = datetime.datetime(*time.gmtime(mtime)[:6]) with open(local_path, "rb") as f: if file_size <= chunk_size: md = self.dbx.files_upload(f.read(), dbx_path, client_modified=mtime_dt, **kwargs) else: logger.info( f"Uploading {bytes_to_str(uploaded)}/{file_size_str}...") session_start = self.dbx.files_upload_session_start( f.read(chunk_size)) cursor = dropbox.files.UploadSessionCursor( session_id=session_start.session_id, offset=f.tell()) commit = dropbox.files.CommitInfo(path=dbx_path, client_modified=mtime_dt, **kwargs) while f.tell() < file_size: if file_size - f.tell() <= chunk_size: md = self.dbx.files_upload_session_finish( f.read(chunk_size), cursor, commit) logger.info( f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..." ) else: # Note: we currently do not support resuming interrupted uploads. # However, this can be achieved catching connection errors and # retrying until the upload succeeds. Incorrect offsets due to # a dropped package can be corrected by getting the right # offset from the resulting UploadSessionOffsetError and # resuming the upload from this point. self.dbx.files_upload_session_append_v2( f.read(chunk_size), cursor) cursor.offset = f.tell() uploaded += chunk_size logger.info( f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..." ) logger.debug( f"File '{md.path_display}' (rev {md.rev}) uploaded to Dropbox") return md @to_maestral_error(dbx_path_arg=1) def remove(self, dbx_path, **kwargs): """ Removes file / folder from Dropbox. :param str dbx_path: Path to file on Dropbox. :param kwargs: Keyword arguments for Dropbox SDK files_delete_v2. :returns: Metadata of deleted file or ``False`` if the file does not exist on Dropbox. :raises: :class:`MaestralApiError`. """ # try to move file (response will be metadata, probably) res = self.dbx.files_delete_v2(dbx_path, **kwargs) md = res.metadata logger.debug(f"Item '{dbx_path}' removed from Dropbox") return md @to_maestral_error(dbx_path_arg=2) def move(self, dbx_path, new_path, **kwargs): """ Moves/renames files or folders on Dropbox. :param str dbx_path: Path to file/folder on Dropbox. :param str new_path: New path on Dropbox to move to. :param kwargs: Keyword arguments for Dropbox SDK files_move_v2. :returns: Metadata of moved file/folder. :raises: :class:`MaestralApiError` """ res = self.dbx.files_move_v2(dbx_path, new_path, allow_shared_folder=True, allow_ownership_transfer=True, **kwargs) md = res.metadata logger.debug( f"Item moved from '{dbx_path}' to '{md.path_display}' on Dropbox") return md @to_maestral_error(dbx_path_arg=1) def make_dir(self, dbx_path, **kwargs): """ Creates folder on Dropbox. :param str dbx_path: Path o fDropbox folder. :param kwargs: Keyword arguments for Dropbox SDK files_create_folder_v2. :returns: Metadata of created folder. :raises: :class:`MaestralApiError` """ res = self.dbx.files_create_folder_v2(dbx_path, **kwargs) md = res.metadata logger.debug(f"Created folder '{md.path_display}' on Dropbox") return md @to_maestral_error(dbx_path_arg=1) def get_latest_cursor(self, dbx_path, include_non_downloadable_files=False, **kwargs): """ Gets the latest cursor for the given folder and subfolders. :param str dbx_path: Path of folder on Dropbox. :param bool include_non_downloadable_files: If ``True``, files that cannot be downloaded (at the moment only G-suite files on Dropbox) will be included. Defaults to ``False``. :param kwargs: Other keyword arguments for Dropbox SDK files_list_folder. :returns: The latest cursor representing a state of a folder and its subfolders. :rtype: str :raises: :class:`MaestralApiError` """ res = self.dbx.files_list_folder_get_latest_cursor( dbx_path, include_non_downloadable_files=include_non_downloadable_files, recursive=True, **kwargs, ) return res.cursor @to_maestral_error(dbx_path_arg=1) def list_folder(self, dbx_path, retry=3, include_non_downloadable_files=False, **kwargs): """ Lists contents of a folder on Dropbox as dictionary mapping unicode file names to FileMetadata|FolderMetadata entries. :param str dbx_path: Path of folder on Dropbox. :param int retry: Number of times to try again call fails because cursor is reset. Defaults to 3. :param bool include_non_downloadable_files: If ``True``, files that cannot be downloaded (at the moment only G-suite files on Dropbox) will be included. Defaults to ``False``. :param kwargs: Other keyword arguments for Dropbox SDK files_list_folder. :returns: :class:`dropbox.files.ListFolderResult` instance. :rtype: :class:`dropbox.files.ListFolderResult` :raises: :class:`MaestralApiError` """ results = [] res = self.dbx.files_list_folder( dbx_path, include_non_downloadable_files=include_non_downloadable_files, **kwargs) results.append(res) idx = 0 while results[-1].has_more: idx += len(results[-1].entries) logger.info(f"Indexing {idx}...") try: more_results = self.dbx.files_list_folder_continue( results[-1].cursor) results.append(more_results) except dropbox.exceptions.DropboxException as exc: new_exc = api_to_maestral_error(exc, dbx_path) if isinstance(new_exc, CursorResetError) and self._retry_count < retry: # retry up to three times, then raise self._retry_count += 1 self.list_folder(dbx_path, include_non_downloadable_files, **kwargs) else: self._retry_count = 0 raise new_exc logger.debug(f"Listed contents of folder '{dbx_path}'") self._retry_count = 0 return self.flatten_results(results) @staticmethod def flatten_results(results): """ Flattens a list of :class:`dropbox.files.ListFolderResult` instances and returns their entries only. Only the last cursor will be kept. :param list results: List of :class:`dropbox.files.ListFolderResult` instances. :returns: Single :class:`dropbox.files.ListFolderResult` instance. :rtype: :class:`dropbox.files.ListFolderResult` """ entries_all = [] for result in results: entries_all += result.entries results_flattened = dropbox.files.ListFolderResult( entries=entries_all, cursor=results[-1].cursor, has_more=False) return results_flattened @to_maestral_error() def wait_for_remote_changes(self, last_cursor, timeout=40): """ Waits for remote changes since :param:`last_cursor`. Call this method after starting the Dropbox client and periodically to get the latest updates. :param str last_cursor: Last to cursor to compare for changes. :param int timeout: Seconds to wait until timeout. Must be between 30 and 480. :returns: ``True`` if changes are available, ``False`` otherwise. :rtype: bool :raises: :class:`MaestralApiError` """ if not 30 <= timeout <= 480: raise ValueError("Timeout must be in range [30, 480]") logger.debug( f"Waiting for remote changes since cursor:\n{last_cursor}") # honour last request to back off if self._last_longpoll is not None: while time.time() - self._last_longpoll < self._backoff: time.sleep(1) result = self.dbx.files_list_folder_longpoll(last_cursor, timeout=timeout) # keep track of last long poll, back off if requested by SDK if result.backoff: self._backoff = result.backoff + 5 else: self._backoff = 0 logger.debug(f"Detected remote changes: {result.changes}") self._last_longpoll = time.time() return result.changes # will be True or False @to_maestral_error() def list_remote_changes(self, last_cursor): """ Lists changes to remote Dropbox since :param:`last_cursor`. Call this after :method:`wait_for_remote_changes` returns ``True``. :param str last_cursor: Last to cursor to compare for changes. :returns: :class:`dropbox.files.ListFolderResult` instance. :rtype: :class:`dropbox.files.ListFolderResult` :raises: """ results = [self.dbx.files_list_folder_continue(last_cursor)] while results[-1].has_more: more_results = self.dbx.files_list_folder_continue( results[-1].cursor) results.append(more_results) # combine all results into one results = self.flatten_results(results) logger.debug(f"Listed remote changes: {len(results.entries)} changes") return results