Example #1
0
def log_level(config_name: str, level_name: str):
    """Gets or sets the log level."""

    from maestral.daemon import MaestralProxy

    try:
        with MaestralProxy(config_name) as m:
            if level_name:
                m.log_level = logging._nameToLevel[level_name]
                click.echo(f'Log level set to {level_name}.')
            else:
                level_name = logging.getLevelName(m.log_level)
                click.echo(f'Log level:  {level_name}')
    except Pyro5.errors.CommunicationError:
        from maestral.config.main import MaestralConfig
        conf = MaestralConfig(config_name)
        if level_name:
            conf.set('app', 'log_level', logging._nameToLevel[level_name])
            click.echo(f'Log level set to {level_name}.')
        else:
            level_name = logging.getLevelName(conf.get('app', 'log_level'))
            click.echo(f'Log level:  {level_name}')
Example #2
0
def notify_level(config_name: str, level_name: str):
    """Gets or sets the level for desktop notifications."""
    from maestral.daemon import MaestralProxy
    from maestral.utils.notify import levelNameToNumber, levelNumberToName

    try:
        with MaestralProxy(config_name) as m:
            if level_name:
                m.notification_level = levelNameToNumber(level_name)
                click.echo(f'Notification level set to {level_name}.')
            else:
                level_name = levelNumberToName(m.notification_level)
                click.echo(f'Notification level: {level_name}.')
    except Pyro5.errors.CommunicationError:
        from maestral.config.main import MaestralConfig
        conf = MaestralConfig(config_name)
        if level_name:
            conf.set('app', 'notification_level',
                     levelNameToNumber(level_name))
            click.echo(f'Notification level set to {level_name}.')
        else:
            level_name = levelNumberToName(
                conf.get('app', 'notification_level'))
            click.echo(f'Notification level: {level_name}.')
Example #3
0
def migrate_user_config(config_name):
    config_path = get_conf_path(CONFIG_DIR_NAME, create=False)
    config_fname = osp.join(config_path, config_name + '.ini')

    # load old config non-destructively
    try:
        old_conf = DefaultsConfig(config_path, config_name, '.ini')
        old_conf.read(config_fname, encoding='utf-8')
        old_version = old_conf.get(UserConfig.DEFAULT_SECTION_NAME, 'version')
    except OSError:
        return

    if Version(old_version) < Version('11.0.0'):

        # get values for moved settings
        excluded_folders = old_conf.get('main', 'excluded_folders')

        email = old_conf.get('account', 'email')
        display_name = old_conf.get('account', 'display_name')
        abbreviated_name = old_conf.get('account', 'abbreviated_name')
        acc_type = old_conf.get('account', 'type')
        usage = old_conf.get('account', 'usage')
        usage_type = old_conf.get('account', 'usage_type')

        update_notification_last = old_conf.get('app',
                                                'update_notification_last')
        latest_release = old_conf.get('app', 'latest_release')

        cursor = old_conf.get('internal', 'cursor')
        lastsync = old_conf.get('internal', 'lastsync')
        recent_changes = old_conf.get('internal', 'recent_changes')

        # convert non-string types
        update_notification_last = float(update_notification_last)
        lastsync = float(lastsync)
        recent_changes = ast.literal_eval(recent_changes)
        excluded_folders = ast.literal_eval(excluded_folders)

        # set state values
        state = MaestralState(config_name)
        state.set('account', 'email', email)
        state.set('account', 'display_name', display_name)
        state.set('account', 'abbreviated_name', abbreviated_name)
        state.set('account', 'type', acc_type)
        state.set('account', 'usage', usage)
        state.set('account', 'usage_type', usage_type)

        state.set('app', 'update_notification_last', update_notification_last)
        state.set('app', 'latest_release', latest_release)

        state.set('sync', 'cursor', cursor)
        state.set('sync', 'lastsync', lastsync)
        state.set('sync', 'recent_changes', recent_changes)

        # load actual config to remove obsolete options and add moved ones
        conf = MaestralConfig(config_name)
        conf.set('main', 'excluded_items', excluded_folders)

        # clean up backup and defaults files from previous version of maestral
        for file in os.scandir(old_conf._path):
            if file.is_file():
                if (conf._backup_suffix in file.name
                        or conf._defaults_name_prefix in file.name):
                    os.remove(file.path)

        logger.info(f'Migrated user config "{config_name}"')

    elif Version(old_version) < Version('12.0.0'):
        excluded_folders = old_conf.get('main', 'excluded_folders')
        excluded_folders = ast.literal_eval(excluded_folders)
        conf = MaestralConfig(config_name)
        conf.set('main', 'excluded_items', excluded_folders)
Example #4
0
class OAuth2Session(object):
    """
    OAuth2Session provides OAuth2 login and token store.
    """

    oAuth2FlowResult = None

    Success = 0
    InvalidToken = 1
    ConnectionFailed = 2

    def __init__(self, config_name='maestral'):

        self._conf = MaestralConfig(config_name)

        self.account_id = self._conf.get("account", "account_id")
        self.access_token = ""

        self.auth_flow = None

    def load_token(self):
        """
        Check if auth key has been saved.

        :raises: ``KeyringLocked`` if the system keyring cannot be accessed.
        """
        logger.debug("Using keyring: %s" % keyring.get_keyring())
        try:
            if self.account_id == "":
                self.access_token = None
            else:
                self.access_token = keyring.get_password("Maestral", self.account_id)
            return self.access_token
        except KeyringLocked:
            info = "Please make sure that your keyring is unlocked and restart Maestral."
            raise KeyringLocked(info)

    def get_auth_url(self):
        """Gets the auth URL to start the OAuth2 implicit grant flow."""

        self.auth_flow = DropboxOAuth2FlowImplicit(APP_KEY)
        authorize_url = self.auth_flow.start()
        return authorize_url

    def verify_auth_token(self, token):
        """
        Verify the provided authorization token with Dropbox servers.

        :return: OAuth2Session.Success, OAuth2Session.InvalidToken, or
            OAuth2Session.ConnectionFailed
        :rtype: int
        """

        if not self.auth_flow:
            raise RuntimeError('Auth flow not yet started. Please call \'get_auth_url\'.')

        try:
            self.oAuth2FlowResult = self.auth_flow.finish(token)
            self.access_token = self.oAuth2FlowResult.access_token
            self.account_id = self.oAuth2FlowResult.account_id
            return self.Success
        except DropboxAuthError:
            return self.InvalidToken
        except ConnectionError:
            return self.ConnectionFailed

    def link(self):
        """Command line flow to get an auth key from Dropbox and save it in the system
        keyring."""
        authorize_url = self.get_auth_url()
        print("1. Go to: " + authorize_url)
        print("2. Click \"Allow\" (you might have to log in first).")
        print("3. Copy the authorization token.")

        res = 1
        while res > 0:
            auth_code = input("Enter the authorization token here: ").strip()
            res = self.verify_auth_token(auth_code)

            if res == 1:
                print("Invalid token. Please try again.")
            elif res == 2:
                print("Could not connect to Dropbox. Please try again.")

        self.save_creds()

    def save_creds(self):
        """Saves auth key to system keyring."""
        self._conf.set("account", "account_id", self.account_id)
        try:
            keyring.set_password("Maestral", self.account_id, self.access_token)
            print(" > Credentials written.")
        except KeyringLocked:
            logger.error("Could not access the user keyring to save your authentication "
                         "token. Please make sure that the keyring is unlocked.")

    def delete_creds(self):
        """Deletes auth key from system keyring."""
        self._conf.set("account", "account_id", "")
        try:
            keyring.delete_password("Maestral", self.account_id)
            print(" > Credentials removed.")
        except KeyringLocked:
            logger.error("Could not access the user keyring to delete your authentication"
                         " token. Please make sure that the keyring is unlocked.")
Example #5
0
class Maestral(object):
    """
    An open source Dropbox client for macOS and Linux to syncing a local folder
    with your Dropbox account. All functions and properties return objects or
    raise exceptions which can safely serialized, i.e., pure Python types. The only
    exception are MaestralApiErrors which have been registered explicitly with the Pyro5
    serializer.
    """

    _daemon_running = True  # for integration with Pyro

    def __init__(self, config_name='maestral', run=True):

        self._config_name = config_name
        self._conf = MaestralConfig(self._config_name)

        self._setup_logging()
        self.set_share_error_reports(self._conf.get("app", "analytics"))

        self.client = MaestralApiClient(config_name=self._config_name)
        self.monitor = MaestralMonitor(self.client,
                                       config_name=self._config_name)
        self.sync = self.monitor.sync

        # periodically check for updates and refresh account info
        self.update_thread = Thread(
            name="Maestral update check",
            target=self._periodic_refresh,
            daemon=True,
        )
        self.update_thread.start()

        if run:
            self.run()

    def run(self):

        if self.pending_dropbox_folder(self._config_name):
            self.create_dropbox_directory()
            self.set_excluded_folders()

            self.sync.last_cursor = ""
            self.sync.last_sync = 0

        # start syncing
        self.start_sync()

        if NOTIFY_SOCKET and system_notifier:  # notify systemd that we have started
            logger.debug("Running as systemd notify service")
            logger.debug(f"NOTIFY_SOCKET = {NOTIFY_SOCKET}")
            system_notifier.notify("READY=1")

        if IS_WATCHDOG and system_notifier:  # notify systemd periodically if alive
            logger.debug("Running as systemd watchdog service")
            logger.debug(f"WATCHDOG_USEC = {WATCHDOG_USEC}")
            logger.debug(f"WATCHDOG_PID = {WATCHDOG_PID}")

            self.watchdog_thread = Thread(
                name="Maestral watchdog",
                target=self._periodic_watchdog,
                daemon=True,
            )
            self.watchdog_thread.start()

    def _setup_logging(self):

        log_level = self._conf.get("app", "log_level")
        mdbx_logger = logging.getLogger("maestral")
        mdbx_logger.setLevel(logging.DEBUG)

        log_fmt_long = logging.Formatter(
            fmt="%(asctime)s %(name)s %(levelname)s: %(message)s",
            datefmt="%Y-%m-%d %H:%M:%S")
        log_fmt_short = logging.Formatter(fmt="%(message)s")

        # log to file
        rfh_log_file = get_log_path("maestral", self._config_name + ".log")
        self._log_handler_file = logging.handlers.RotatingFileHandler(
            rfh_log_file, maxBytes=10**7, backupCount=1)
        self._log_handler_file.setFormatter(log_fmt_long)
        self._log_handler_file.setLevel(log_level)
        mdbx_logger.addHandler(self._log_handler_file)

        # log to journal when launched from systemd
        if INVOCATION_ID and journal:
            self._log_handler_journal = journal.JournalHandler()
            self._log_handler_journal.setFormatter(log_fmt_short)
            mdbx_logger.addHandler(self._log_handler_journal)

        # log to stdout (disabled by default)
        self._log_handler_stream = logging.StreamHandler(sys.stdout)
        self._log_handler_stream.setFormatter(log_fmt_long)
        self._log_handler_stream.setLevel(100)
        mdbx_logger.addHandler(self._log_handler_stream)

        # log to cached handlers for GUI and CLI
        self._log_handler_info_cache = CachedHandler(maxlen=1)
        self._log_handler_info_cache.setLevel(logging.INFO)
        self._log_handler_info_cache.setFormatter(log_fmt_short)
        mdbx_logger.addHandler(self._log_handler_info_cache)

        self._log_handler_error_cache = CachedHandler()
        self._log_handler_error_cache.setLevel(logging.ERROR)
        self._log_handler_error_cache.setFormatter(log_fmt_short)
        mdbx_logger.addHandler(self._log_handler_error_cache)

        # log to bugsnag (disabled by default)
        self._log_handler_bugsnag = BugsnagHandler()
        self._log_handler_bugsnag.setLevel(100)
        mdbx_logger.addHandler(self._log_handler_bugsnag)

    @property
    def config_name(self):
        return self._config_name

    def set_conf(self, section, name, value):
        self._conf.set(section, name, value)

    def get_conf(self, section, name):
        return self._conf.get(section, name)

    def set_log_level(self, level_num):
        self._log_handler_file.setLevel(level_num)
        self._log_handler_stream.setLevel(level_num)
        self._conf.set("app", "log_level", level_num)

    def set_log_to_stdout(self, enabled=True):

        if enabled:
            log_level = self._conf.get("app", "log_level")
            self._log_handler_stream.setLevel(log_level)
        else:
            self._log_handler_stream.setLevel(100)

    def set_share_error_reports(self, enabled):

        bugsnag.configuration.auto_notify = enabled
        bugsnag.configuration.auto_capture_sessions = enabled
        self._log_handler_bugsnag.setLevel(logging.ERROR if enabled else 100)

        self._conf.set("app", "analytics", enabled)

    @staticmethod
    def pending_link(config_name):
        """
        Bool indicating if auth tokens are stored in the system's keychain. This may raise
        a KeyringLocked exception if the user's keychain cannot be accessed. This
        exception will not be deserialized by Pyro5. You should check if Maestral is
        linked before instantiating a daemon.

        :param str config_name: Name of user config to check.

        :raises: :class:`keyring.errors.KeyringLocked`
        """
        auth_session = OAuth2Session(config_name)
        return auth_session.load_token() is None

    @staticmethod
    def pending_dropbox_folder(config_name):
        """
        Bool indicating if a local Dropbox directory has been set.

        :param str config_name: Name of user config to check.
        """
        conf = MaestralConfig(config_name)
        return not osp.isdir(conf.get("main", "path"))

    def pending_first_download(self):
        """Bool indicating if the initial download has already occurred."""
        return (self._conf.get("internal", "lastsync") == 0
                or self._conf.get("internal", "cursor") == "")

    @property
    def syncing(self):
        """Bool indicating if Maestral is syncing. It will be ``True`` if syncing is
        not paused by the user *and* Maestral is connected to the internet."""
        return self.monitor.syncing.is_set()

    @property
    def paused(self):
        """Bool indicating if syncing is paused by the user. This is set by calling
        :meth:`pause`."""
        return not self.monitor._auto_resume_on_connect

    @property
    def stopped(self):
        """Bool indicating if syncing is stopped, for instance because of an exception."""
        return not self.monitor.running.is_set()

    @property
    def connected(self):
        """Bool indicating if Dropbox servers can be reached."""
        return self.monitor.connected.is_set()

    @property
    def status(self):
        """Returns a string with the last status message. This can be displayed as
        information to the user but should not be relied on otherwise."""
        return self._log_handler_info_cache.getLastMessage()

    @property
    def notify(self):
        """Bool indicating if notifications are enabled or disabled."""
        return self.sync.notify.enabled

    @notify.setter
    def notify(self, boolean):
        """Setter: Bool indicating if notifications are enabled."""
        self.sync.notify.enabled = boolean

    @property
    def dropbox_path(self):
        """Returns the path to the local Dropbox directory. Read only. Use
        :meth:`create_dropbox_directory` or :meth:`move_dropbox_directory` to set or
        change the Dropbox directory location instead. """
        return self.sync.dropbox_path

    @property
    def excluded_folders(self):
        """Returns a list of excluded folders (read only). Use :meth:`exclude_folder`,
        :meth:`include_folder` or :meth:`set_excluded_folders` change which folders are
        excluded from syncing."""
        return self.sync.excluded_folders

    @property
    def sync_errors(self):
        """Returns list containing the current sync errors as dicts."""
        sync_errors = list(self.sync.sync_errors.queue)
        sync_errors_dicts = [maestral_error_to_dict(e) for e in sync_errors]
        return sync_errors_dicts

    @property
    def maestral_errors(self):
        """Returns a list of Maestral's errors as dicts. This does not include lost
        internet connections or file sync errors which only emit warnings and are tracked
        and cleared separately. Errors listed here must be acted upon for Maestral to
        continue syncing.
        """

        maestral_errors = [
            r.exc_info[1] for r in self._log_handler_error_cache.cached_records
        ]
        maestral_errors_dicts = [
            maestral_error_to_dict(e) for e in maestral_errors
        ]
        return maestral_errors_dicts

    def clear_maestral_errors(self):
        """Manually clears all Maestral errors. This should be used after they have been
        resolved by the user through the GUI or CLI.
        """
        self._log_handler_error_cache.clear()

    @property
    def account_profile_pic_path(self):
        """Returns the path of the current account's profile picture. There may not be
        an actual file at that path, if the user did not set a profile picture or the
        picture has not yet been downloaded."""
        return get_cache_path("maestral",
                              self._config_name + "_profile_pic.jpeg")

    def get_file_status(self, local_path):
        """
        Returns the sync status of an individual file.

        :param local_path: Path to file on the local drive.
        :return: String indicating the sync status. Can be "uploading", "downloading",
            "up to date", "error", or "unwatched" (for files outside of the Dropbox
            directory).
        :rtype: str
        """
        if not self.syncing:
            return "unwatched"

        try:
            dbx_path = self.sync.to_dbx_path(local_path)
        except ValueError:
            return "unwatched"

        if local_path in self.monitor.queued_for_upload:
            return "uploading"
        elif local_path in self.monitor.queued_for_download:
            return "downloading"
        elif any(local_path == err["local_path"] for err in self.sync_errors):
            return "error"
        elif self.sync.get_local_rev(dbx_path):
            return "up to date"
        else:
            return "unwatched"

    def get_activity(self):
        """
        Returns a dictionary with lists of all file currently queued for or being synced.

        :rtype: dict(list, list)
        """
        PathItem = namedtuple("PathItem", "local_path status")
        uploading = []
        downloading = []

        for path in self.monitor.uploading:
            uploading.append(PathItem(path, "uploading"))

        for path in self.monitor.queued_for_upload:
            uploading.append(PathItem(path, "queued"))

        for path in self.monitor.downloading:
            downloading.append(PathItem(path, "downloading"))

        for path in self.monitor.queued_for_download:
            downloading.append(PathItem(path, "queued"))

        return dict(uploading=uploading, downloading=downloading)

    @handle_disconnect
    def get_account_info(self):
        """
        Gets account information from Dropbox and returns it as a dictionary.
        The entries will either be of type ``str`` or ``bool``.

        :returns: Dropbox account information.
        :rtype: dict[str, bool]
        :raises: :class:`MaestralApiError`
        """
        res = self.client.get_account_info()
        return dropbox_stone_to_dict(res)

    @handle_disconnect
    def get_space_usage(self):
        """
        Gets the space usage stored by Dropbox and returns it as a dictionary.
        The entries will either be of type ``str`` or ``bool``.

        :returns: Dropbox account information.
        :rtype: dict[str, bool]
        """
        res = self.client.get_space_usage()
        return dropbox_stone_to_dict(res)

    @handle_disconnect
    def get_profile_pic(self):
        """
        Attempts to download the user's profile picture from Dropbox. The picture saved in
        Maestral's cache directory for retrieval when there is no internet connection.
        This function will fail silently in case of :class:`MaestralApiError`s.

        :returns: Path to saved profile picture or None if no profile picture is set.
        """

        try:
            res = self.client.get_account_info()
        except MaestralApiError:
            pass
        else:
            if res.profile_photo_url:
                # download current profile pic
                res = requests.get(res.profile_photo_url)
                with open(self.account_profile_pic_path, "wb") as f:
                    f.write(res.content)
                return self.account_profile_pic_path
            else:
                # delete current profile pic
                self._delete_old_profile_pics()

    @handle_disconnect
    def list_folder(self, dbx_path, **kwargs):
        """
        List all items inside the folder given by :param:`dbx_path`.

        :param dbx_path: Path to folder on Dropbox.
        :return: List of Dropbox item metadata as dicts or ``False`` if listing failed
            due to connection issues.
        :rtype: list[dict]
        """
        dbx_path = "" if dbx_path == "/" else dbx_path
        res = self.client.list_folder(dbx_path, **kwargs)

        entries = [dropbox_stone_to_dict(e) for e in res.entries]

        return entries

    def _delete_old_profile_pics(self):
        # delete all old pictures
        for file in os.listdir(get_cache_path("maestral")):
            if file.startswith(self._config_name + "_profile_pic"):
                try:
                    os.unlink(osp.join(get_cache_path("maestral"), file))
                except OSError:
                    pass

    def rebuild_index(self):
        """
        Rebuilds the Maestral index and resumes syncing afterwards if it has been
        running.

        :raises: :class:`MaestralApiError`
        """

        self.monitor.rebuild_rev_file()

    def start_sync(self, overload=None):
        """
        Creates syncing threads and starts syncing.
        """
        self.monitor.start()

    def resume_sync(self, overload=None):
        """
        Resumes the syncing threads if paused.
        """
        self.monitor.resume()

    def pause_sync(self, overload=None):
        """
        Pauses the syncing threads if running.
        """
        self.monitor.pause()

    def stop_sync(self, overload=None):
        """
        Stops the syncing threads if running, destroys observer thread.
        """
        self.monitor.stop()

    def unlink(self):
        """
        Unlinks the configured Dropbox account but leaves all downloaded files
        in place. All syncing metadata will be removed as well. Connection and API errors
        will be handled silently but the Dropbox access key will always be removed from
        the user's PC.
        """
        self.stop_sync()
        try:
            self.client.unlink()
        except (ConnectionError, MaestralApiError):
            pass

        try:
            os.remove(self.sync.rev_file_path)
        except OSError:
            pass

        self.sync.dropbox_path = ""
        self.sync.last_cursor = ""
        self.sync.last_sync = 0.0

        self._conf.reset_to_defaults()

        logger.info("Unlinked Dropbox account.")

    def exclude_folder(self, dbx_path):
        """
        Excludes folder from sync and deletes local files. It is safe to call
        this method with folders which have already been excluded.

        :param str dbx_path: Dropbox folder to exclude.
        :raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox.
        :raises: :class:`ConnectionError` if connection to Dropbox fails.
        """

        dbx_path = dbx_path.lower().rstrip(osp.sep)

        md = self.client.get_metadata(dbx_path)

        if not isinstance(md, files.FolderMetadata):
            raise ValueError(
                "No such folder on Dropbox: '{0}'".format(dbx_path))

        # add the path to excluded list
        excluded_folders = self.sync.excluded_folders
        if dbx_path not in excluded_folders:
            excluded_folders.append(dbx_path)
        else:
            logger.info("Folder was already excluded, nothing to do.")
            return

        self.sync.excluded_folders = excluded_folders
        self.sync.set_local_rev(dbx_path, None)

        # remove folder from local drive
        local_path = self.sync.to_local_path(dbx_path)
        local_path_cased = path_exists_case_insensitive(local_path)
        logger.info(f"Deleting folder '{local_path_cased}'.")
        if osp.isdir(local_path_cased):
            shutil.rmtree(local_path_cased)

    def include_folder(self, dbx_path):
        """
        Includes folder in sync and downloads in the background. It is safe to
        call this method with folders which have already been included, they
        will not be downloaded again.

        :param str dbx_path: Dropbox folder to include.
        :raises: :class:`ValueError` if ``dbx_path`` is not on Dropbox or lies inside
            another excluded folder.
        :raises: :class:`ConnectionError` if connection to Dropbox fails.
        """

        dbx_path = dbx_path.lower().rstrip(osp.sep)
        md = self.client.get_metadata(dbx_path)

        old_excluded_folders = self.sync.excluded_folders

        if not isinstance(md, files.FolderMetadata):
            raise ValueError(
                "No such folder on Dropbox: '{0}'".format(dbx_path))
        for folder in old_excluded_folders:
            if is_child(dbx_path, folder):
                raise ValueError(
                    "'{0}' lies inside the excluded folder '{1}'. "
                    "Please include '{1}' first.".format(dbx_path, folder))

        # Get folders which will need to be downloaded, do not attempt to download
        # subfolders of `dbx_path` which were already included.
        # `new_included_folders` will either be empty (`dbx_path` was already
        # included), just contain `dbx_path` itself (the whole folder was excluded) or
        # only contain subfolders of `dbx_path` (`dbx_path` was partially included).
        new_included_folders = tuple(x for x in old_excluded_folders
                                     if x == dbx_path or is_child(x, dbx_path))

        if new_included_folders:
            # remove `dbx_path` or all excluded children from the excluded list
            excluded_folders = list(
                set(old_excluded_folders) - set(new_included_folders))
        else:
            logger.info("Folder was already included, nothing to do.")
            return

        self.sync.excluded_folders = excluded_folders

        # download folder contents from Dropbox
        logger.info(f"Downloading added folder '{dbx_path}'.")
        for folder in new_included_folders:
            self.sync.queued_folder_downloads.put(folder)

    @handle_disconnect
    def _include_folder_without_subfolders(self, dbx_path):
        """Sets a folder to included without explicitly including its subfolders. This
        is to be used internally, when a folder has been removed from the excluded list,
        but some of its subfolders may have been added."""

        dbx_path = dbx_path.lower().rstrip(osp.sep)
        excluded_folders = self.sync.excluded_folders

        if dbx_path not in excluded_folders:
            return

        excluded_folders.remove(dbx_path)

        self.sync.excluded_folders = excluded_folders
        self.sync.queued_folder_downloads.put(dbx_path)

    @handle_disconnect
    def set_excluded_folders(self, folder_list=None):
        """
        Sets the list of excluded folders to `folder_list`. If not given, gets all top
        level folder paths from Dropbox and asks user to include or exclude. Folders
        which are no in `folder_list` but exist on Dropbox will be downloaded.

        On initial sync, this does not trigger any downloads.

        :param list folder_list: If given, list of excluded folder to set.
        :return: List of excluded folders.
        :rtype: list
        :raises: :class:`MaestralApiError`
        """

        if folder_list is None:

            excluded_folders = []

            # get all top-level Dropbox folders
            result = self.client.list_folder("", recursive=False)

            # paginate through top-level folders, ask to exclude
            for entry in result.entries:
                if isinstance(entry, files.FolderMetadata):
                    yes = click.confirm(
                        f"Exclude '{entry.path_display}' from sync?")
                    if yes:
                        excluded_folders.append(entry.path_lower)
        else:
            excluded_folders = self.sync.clean_excluded_folder_list(
                folder_list)

        old_excluded_folders = self.sync.excluded_folders

        added_excluded_folders = set(excluded_folders) - set(
            old_excluded_folders)
        added_included_folders = set(old_excluded_folders) - set(
            excluded_folders)

        if not self.pending_first_download():
            # apply changes
            for path in added_excluded_folders:
                self.exclude_folder(path)
            for path in added_included_folders:
                self._include_folder_without_subfolders(path)

        self.sync.excluded_folders = excluded_folders

        return excluded_folders

    def excluded_status(self, dbx_path):
        """
        Returns 'excluded', 'partially excluded' or 'included'. This function will not
        check if the item actually exists on Dropbox.

        :param str dbx_path: Path to item on Dropbox.
        :returns: Excluded status.
        :rtype: str
        """

        dbx_path = dbx_path.lower().rstrip(osp.sep)

        excluded_items = self._conf.get("main",
                                        "excluded_folders") + self._conf.get(
                                            "main", "excluded_files")

        if dbx_path in excluded_items:
            return "excluded"
        elif any(is_child(f, dbx_path) for f in excluded_items):
            return "partially excluded"
        else:
            return "included"

    @with_sync_paused
    def move_dropbox_directory(self, new_path=None):
        """
        Change or set local dropbox directory. This moves all local files to
        the new location. If a file or folder already exists at this location,
        it will be overwritten.

        :param str new_path: Full path to local Dropbox folder. If not given, the
            user will be prompted to input the path.
        """

        # get old and new paths
        old_path = self.sync.dropbox_path
        if new_path is None:
            new_path = self._ask_for_path(self._config_name)

        try:
            if osp.samefile(old_path, new_path):
                return
        except FileNotFoundError:
            pass

        # remove existing items at current location
        try:
            os.unlink(new_path)
        except IsADirectoryError:
            shutil.rmtree(new_path, ignore_errors=True)
        except FileNotFoundError:
            pass

        # move folder from old location or create a new one if no old folder exists
        if osp.isdir(old_path):
            shutil.move(old_path, new_path)
        else:
            os.makedirs(new_path)

        # update config file and client
        self.sync.dropbox_path = new_path

    @with_sync_paused
    def create_dropbox_directory(self, path=None, overwrite=True):
        """
        Set a new local dropbox directory.

        :param str path: Full path to local Dropbox folder. If not given, the user will be
            prompted to input the path.
        :param bool overwrite: If ``True``, any existing file or folder at ``new_path``
            will be replaced.
        """
        # ask for new path
        if path is None:
            path = self._ask_for_path(self._config_name)

        if overwrite:
            # remove any old items at the location
            try:
                shutil.rmtree(path)
            except NotADirectoryError:
                os.unlink(path)
            except FileNotFoundError:
                pass

        # create new folder
        os.makedirs(path, exist_ok=True)

        # update config file and client
        self.sync.dropbox_path = path

    @staticmethod
    def _ask_for_path(config_name):
        """
        Asks for Dropbox path.
        """

        conf = MaestralConfig(config_name)

        default = osp.join(get_home_dir(), conf.get("main",
                                                    "default_dir_name"))

        while True:
            msg = f"Please give Dropbox folder location or press enter for default ['{default}']:"
            res = input(msg).strip("'\" ")

            dropbox_path = osp.expanduser(res or default)
            old_path = osp.expanduser(conf.get("main", "path"))

            same_path = False
            try:
                if osp.samefile(old_path, dropbox_path):
                    same_path = True
            except FileNotFoundError:
                pass

            if osp.exists(dropbox_path) and not same_path:
                msg = f"Directory '{dropbox_path}' already exist. Do you want to overwrite it?"
                yes = click.confirm(msg)
                if yes:
                    return dropbox_path
                else:
                    pass
            else:
                return dropbox_path

    def to_local_path(self, dbx_path):
        return self.sync.to_local_path(dbx_path)

    @staticmethod
    def check_for_updates():
        return check_update_available()

    def _periodic_refresh(self):
        while True:
            # update account info
            self.get_account_info()
            self.get_space_usage()
            self.get_profile_pic()
            # check for maestral updates
            res = self.check_for_updates()
            if not res["error"]:
                self._conf.set("app", "latest_release", res["latest_release"])
            time.sleep(60 * 60)  # 60 min

    def _periodic_watchdog(self):
        while self.monitor._threads_alive():
            system_notifier.notify("WATCHDOG=1")
            time.sleep(int(WATCHDOG_USEC) / (2 * 10**6))

    def shutdown_pyro_daemon(self):
        """Does nothing except for setting the _daemon_running flag to ``False``. This
        will be checked by Pyro periodically to shut down the daemon when requested."""
        self._daemon_running = False
        if NOTIFY_SOCKET and system_notifier:
            # notify systemd that we are shutting down
            system_notifier.notify("STOPPING=1")

    def _loop_condition(self):
        return self._daemon_running

    def __del__(self):
        try:
            self.monitor.stop()
        except:
            pass

    def __repr__(self):
        email = self._conf.get("account", "email")
        account_type = self._conf.get("account", "type")

        return f"<{self.__class__}({email}, {account_type})>"
Example #6
0
class MaestralApiClient(object):
    """Client for Dropbox SDK.

    This client defines basic methods to wrap Dropbox Python SDK calls, such as creating,
    moving, modifying and deleting files and folders on Dropbox and downloading files from
    Dropbox.

    All Dropbox API errors are caught and handled here. ConnectionErrors will
    be caught and handled by :class:`MaestralMonitor` instead.

    :param int timeout: Timeout for individual requests in sec. Defaults to 60 sec.
    """

    SDK_VERSION = "2.0"
    _timeout = 60

    def __init__(self, config_name='maestral', timeout=_timeout):

        self._conf = MaestralConfig(config_name)

        # get Dropbox session
        self.auth = OAuth2Session(config_name)
        if not self.auth.load_token():
            self.auth.link()
        self._timeout = timeout
        self._last_longpoll = None
        self._backoff = 0
        self._retry_count = 0

        # initialize API client
        self.dbx = dropbox.Dropbox(self.auth.access_token,
                                   session=SESSION,
                                   user_agent=USER_AGENT,
                                   timeout=self._timeout)

    @to_maestral_error()
    def get_account_info(self, dbid=None):
        """
        Gets current account information.

        :param str dbid: Dropbox ID of account. If not given, will get the info of our own
            account.
        :returns: :class:`dropbox.users.FullAccount` instance or `None` if failed.
        :rtype: dropbox.users.FullAccount
        """
        if dbid:
            res = self.dbx.users_get_account(dbid)
        else:
            res = self.dbx.users_get_current_account()

        if not dbid:
            # save our own account info to config
            if res.account_type.is_basic():
                account_type = "basic"
            elif res.account_type.is_business():
                account_type = "business"
            elif res.account_type.is_pro():
                account_type = "pro"
            else:
                account_type = ""

            self._conf.set("account", "account_id", res.account_id)
            self._conf.set("account", "email", res.email)
            self._conf.set("account", "display_name", res.name.display_name)
            self._conf.set("account", "abbreviated_name",
                           res.name.abbreviated_name)
            self._conf.set("account", "type", account_type)

        return res

    @to_maestral_error()
    def get_space_usage(self):
        """
        Gets current account space usage.

        :returns: :class:`SpaceUsage` instance or `False` if failed.
        :rtype: SpaceUsage
        """
        res = self.dbx.users_get_space_usage()

        # convert from dropbox.users.SpaceUsage to SpaceUsage
        res.__class__ = SpaceUsage

        # save results to config
        self._conf.set("account", "usage", str(res))
        self._conf.set("account", "usage_type", res.allocation_type())

        return res

    @to_maestral_error()
    def unlink(self):
        """
        Unlinks the Dropbox account and deletes local sync information.
        """
        self.auth.delete_creds()
        self.dbx.auth_token_revoke()  # should only raise auth errors

    @to_maestral_error(dbx_path_arg=1)
    def get_metadata(self, dbx_path, **kwargs):
        """
        Get metadata for Dropbox entry (file or folder). Returns `None` if no
        metadata is available. Keyword arguments are passed on to Dropbox SDK
        files_get_metadata call.

        :param str dbx_path: Path of folder on Dropbox.
        :param kwargs: Keyword arguments for Dropbox SDK files_download_to_file.
        :returns: FileMetadata|FolderMetadata entries or `False` if failed.
        """

        try:
            md = self.dbx.files_get_metadata(dbx_path, **kwargs)
            logger.debug(f"Retrieved metadata for '{md.path_display}'")
        except dropbox.exceptions.ApiError as exc:
            # DropboxAPI error is only raised when the item does not exist on Dropbox
            # this is handled on a DEBUG level since we use call `get_metadata` to check
            # if a file exists
            logger.debug(f"Could not get metadata for '{dbx_path}': {exc}")
            md = False

        return md

    @to_maestral_error(dbx_path_arg=1)
    def list_revisions(self, dbx_path, mode="path", limit=10):
        """
        Lists all file revisions for the given file.

        :param str dbx_path: Path to file on Dropbox.
        :param str mode: Must be "path" or "id". If "id", specify the Dropbox file ID
            instead of the file path to get revisions across move and rename events.
            Defaults to "path".
        :param int limit: Number of revisions to list. Defaults to 10.
        :returns: :class:`dropbox.files.ListRevisionsResult` instance
        """

        mode = dropbox.files.ListRevisionsMode(mode)
        return self.dbx.files_list_revisions(dbx_path, mode=mode, limit=limit)

    @to_maestral_error(dbx_path_arg=1)
    def download(self, dbx_path, dst_path, **kwargs):
        """
        Downloads file from Dropbox to our local folder.

        :param str dbx_path: Path to file on Dropbox.
        :param str dst_path: Path to download destination.
        :param kwargs: Keyword arguments for Dropbox SDK files_download_to_file.
        :returns: :class:`FileMetadata` or
            :class:`FolderMetadata` of downloaded item, `False`
            if request fails or `None` if local copy is already in sync.
        """
        # create local directory if not present
        dst_path_directory = osp.dirname(dst_path)
        try:
            os.makedirs(dst_path_directory)
        except FileExistsError:
            pass

        md = self.dbx.files_download_to_file(dst_path, dbx_path, **kwargs)

        logger.debug(
            f"File '{md.path_display}' (rev {md.rev}) was successfully downloaded as '{dst_path}'"
        )

        return md

    @to_maestral_error(dbx_path_arg=2)
    def upload(self, local_path, dbx_path, chunk_size_mb=5, **kwargs):
        """
        Uploads local file to Dropbox.

        :param str local_path: Path of local file to upload.
        :param str dbx_path: Path to save file on Dropbox.
        :param kwargs: Keyword arguments for Dropbox SDK files_upload.
        :param int chunk_size_mb: Maximum size for individual uploads in MB. Must be
            smaller than 150 MB.
        :returns: Metadata of uploaded file or `False` if upload failed.
        """

        chunk_size_mb = min(chunk_size_mb, 150)
        chunk_size = chunk_size_mb * 10**6  # convert to bytes

        file_size = osp.getsize(local_path)
        file_size_str = bytes_to_str(file_size)
        uploaded = 0

        mtime = osp.getmtime(local_path)
        mtime_dt = datetime.datetime(*time.gmtime(mtime)[:6])

        with open(local_path, "rb") as f:
            if file_size <= chunk_size:
                md = self.dbx.files_upload(f.read(),
                                           dbx_path,
                                           client_modified=mtime_dt,
                                           **kwargs)
            else:
                logger.info(
                    f"Uploading {bytes_to_str(uploaded)}/{file_size_str}...")
                session_start = self.dbx.files_upload_session_start(
                    f.read(chunk_size))
                cursor = dropbox.files.UploadSessionCursor(
                    session_id=session_start.session_id, offset=f.tell())
                commit = dropbox.files.CommitInfo(path=dbx_path,
                                                  client_modified=mtime_dt,
                                                  **kwargs)

                while f.tell() < file_size:
                    if file_size - f.tell() <= chunk_size:
                        md = self.dbx.files_upload_session_finish(
                            f.read(chunk_size), cursor, commit)
                        logger.info(
                            f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..."
                        )
                    else:
                        # Note: we currently do not support resuming interrupted uploads.
                        # However, this can be achieved catching connection errors and
                        # retrying until the upload succeeds. Incorrect offsets due to
                        # a dropped package can be corrected by getting the right
                        # offset from the resulting UploadSessionOffsetError and
                        # resuming the upload from this point.
                        self.dbx.files_upload_session_append_v2(
                            f.read(chunk_size), cursor)
                        cursor.offset = f.tell()
                        uploaded += chunk_size
                        logger.info(
                            f"Uploading {bytes_to_str(uploaded)}/{file_size_str}..."
                        )

        logger.debug(
            f"File '{md.path_display}' (rev {md.rev}) uploaded to Dropbox")

        return md

    @to_maestral_error(dbx_path_arg=1)
    def remove(self, dbx_path, **kwargs):
        """
        Removes file / folder from Dropbox.

        :param str dbx_path: Path to file on Dropbox.
        :param kwargs: Keyword arguments for Dropbox SDK files_delete_v2.
        :returns: Metadata of deleted file or ``False`` if the file does not exist on
            Dropbox.
        :raises: :class:`MaestralApiError`.
        """
        # try to move file (response will be metadata, probably)
        res = self.dbx.files_delete_v2(dbx_path, **kwargs)
        md = res.metadata

        logger.debug(f"Item '{dbx_path}' removed from Dropbox")

        return md

    @to_maestral_error(dbx_path_arg=2)
    def move(self, dbx_path, new_path, **kwargs):
        """
        Moves/renames files or folders on Dropbox.

        :param str dbx_path: Path to file/folder on Dropbox.
        :param str new_path: New path on Dropbox to move to.
        :param kwargs: Keyword arguments for Dropbox SDK files_move_v2.
        :returns: Metadata of moved file/folder.
        :raises: :class:`MaestralApiError`
        """
        res = self.dbx.files_move_v2(dbx_path,
                                     new_path,
                                     allow_shared_folder=True,
                                     allow_ownership_transfer=True,
                                     **kwargs)
        md = res.metadata

        logger.debug(
            f"Item moved from '{dbx_path}' to '{md.path_display}' on Dropbox")

        return md

    @to_maestral_error(dbx_path_arg=1)
    def make_dir(self, dbx_path, **kwargs):
        """
        Creates folder on Dropbox.

        :param str dbx_path: Path o fDropbox folder.
        :param kwargs: Keyword arguments for Dropbox SDK files_create_folder_v2.
        :returns: Metadata of created folder.
        :raises: :class:`MaestralApiError`
        """
        res = self.dbx.files_create_folder_v2(dbx_path, **kwargs)
        md = res.metadata

        logger.debug(f"Created folder '{md.path_display}' on Dropbox")

        return md

    @to_maestral_error(dbx_path_arg=1)
    def get_latest_cursor(self,
                          dbx_path,
                          include_non_downloadable_files=False,
                          **kwargs):
        """
        Gets the latest cursor for the given folder and subfolders.

        :param str dbx_path: Path of folder on Dropbox.
        :param bool include_non_downloadable_files: If ``True``, files that cannot be
            downloaded (at the moment only G-suite files on Dropbox) will be included.
            Defaults to ``False``.
        :param kwargs: Other keyword arguments for Dropbox SDK files_list_folder.
        :returns: The latest cursor representing a state of a folder and its subfolders.
        :rtype: str
        :raises: :class:`MaestralApiError`
        """

        res = self.dbx.files_list_folder_get_latest_cursor(
            dbx_path,
            include_non_downloadable_files=include_non_downloadable_files,
            recursive=True,
            **kwargs,
        )

        return res.cursor

    @to_maestral_error(dbx_path_arg=1)
    def list_folder(self,
                    dbx_path,
                    retry=3,
                    include_non_downloadable_files=False,
                    **kwargs):
        """
        Lists contents of a folder on Dropbox as dictionary mapping unicode
        file names to FileMetadata|FolderMetadata entries.

        :param str dbx_path: Path of folder on Dropbox.
        :param int retry: Number of times to try again call fails because cursor is
            reset. Defaults to 3.
        :param bool include_non_downloadable_files: If ``True``, files that cannot be
            downloaded (at the moment only G-suite files on Dropbox) will be included.
            Defaults to ``False``.
        :param kwargs: Other keyword arguments for Dropbox SDK files_list_folder.
        :returns: :class:`dropbox.files.ListFolderResult` instance.
        :rtype: :class:`dropbox.files.ListFolderResult`
        :raises: :class:`MaestralApiError`
        """

        results = []

        res = self.dbx.files_list_folder(
            dbx_path,
            include_non_downloadable_files=include_non_downloadable_files,
            **kwargs)
        results.append(res)

        idx = 0

        while results[-1].has_more:
            idx += len(results[-1].entries)
            logger.info(f"Indexing {idx}...")
            try:
                more_results = self.dbx.files_list_folder_continue(
                    results[-1].cursor)
                results.append(more_results)
            except dropbox.exceptions.DropboxException as exc:
                new_exc = api_to_maestral_error(exc, dbx_path)
                if isinstance(new_exc,
                              CursorResetError) and self._retry_count < retry:
                    # retry up to three times, then raise
                    self._retry_count += 1
                    self.list_folder(dbx_path, include_non_downloadable_files,
                                     **kwargs)
                else:
                    self._retry_count = 0
                    raise new_exc

        logger.debug(f"Listed contents of folder '{dbx_path}'")

        self._retry_count = 0

        return self.flatten_results(results)

    @staticmethod
    def flatten_results(results):
        """
        Flattens a list of :class:`dropbox.files.ListFolderResult` instances
        and returns their entries only. Only the last cursor will be kept.

        :param list results: List of :class:`dropbox.files.ListFolderResult`
            instances.
        :returns: Single :class:`dropbox.files.ListFolderResult` instance.
        :rtype: :class:`dropbox.files.ListFolderResult`
        """
        entries_all = []
        for result in results:
            entries_all += result.entries

        results_flattened = dropbox.files.ListFolderResult(
            entries=entries_all, cursor=results[-1].cursor, has_more=False)

        return results_flattened

    @to_maestral_error()
    def wait_for_remote_changes(self, last_cursor, timeout=40):
        """
        Waits for remote changes since :param:`last_cursor`. Call this method
        after starting the Dropbox client and periodically to get the latest
        updates.

        :param str last_cursor: Last to cursor to compare for changes.
        :param int timeout: Seconds to wait until timeout. Must be between 30 and 480.
        :returns: ``True`` if changes are available, ``False`` otherwise.
        :rtype: bool
        :raises: :class:`MaestralApiError`
        """

        if not 30 <= timeout <= 480:
            raise ValueError("Timeout must be in range [30, 480]")

        logger.debug(
            f"Waiting for remote changes since cursor:\n{last_cursor}")

        # honour last request to back off
        if self._last_longpoll is not None:
            while time.time() - self._last_longpoll < self._backoff:
                time.sleep(1)

        result = self.dbx.files_list_folder_longpoll(last_cursor,
                                                     timeout=timeout)

        # keep track of last long poll, back off if requested by SDK
        if result.backoff:
            self._backoff = result.backoff + 5
        else:
            self._backoff = 0

        logger.debug(f"Detected remote changes: {result.changes}")

        self._last_longpoll = time.time()

        return result.changes  # will be True or False

    @to_maestral_error()
    def list_remote_changes(self, last_cursor):
        """
        Lists changes to remote Dropbox since :param:`last_cursor`. Call this
        after :method:`wait_for_remote_changes` returns ``True``.

        :param str last_cursor: Last to cursor to compare for changes.
        :returns: :class:`dropbox.files.ListFolderResult` instance.
        :rtype: :class:`dropbox.files.ListFolderResult`
        :raises:
        """

        results = [self.dbx.files_list_folder_continue(last_cursor)]

        while results[-1].has_more:
            more_results = self.dbx.files_list_folder_continue(
                results[-1].cursor)
            results.append(more_results)

        # combine all results into one
        results = self.flatten_results(results)

        logger.debug(f"Listed remote changes: {len(results.entries)} changes")

        return results