def test_unix_permissions(m):
"""
Tests that a newly downloaded file is created with default permissions for our
process and that any locally set permissions are preserved on remote file
modifications.
"""
dbx_path = "/sync_tests/file"
local_path = m.to_local_path(dbx_path)
m.client.upload(resources + "/file.txt", dbx_path)
wait_for_idle(m)
# create a local file and compare its permissions to the new download
reference_file = osp.join(get_home_dir(), "reference")
try:
open(reference_file, "ab").close()
assert os.stat(local_path).st_mode == os.stat(reference_file).st_mode
finally:
delete(reference_file)
# make the local file executable
os.chmod(local_path, 0o744)
new_mode = os.stat(local_path).st_mode # might not be 744...
wait_for_idle(m)
# perform some remote modifications
m.client.upload(resources + "/file1.txt", dbx_path, mode=WriteMode.overwrite)
wait_for_idle(m)
# check that the local permissions have not changed
assert os.stat(local_path).st_mode == new_mode
def test_local_folder_replaced_by_file_and_unsynced_remote_changes(m):
"""
Tests the upload sync when a local folder is replaced by a file and the remote
folder has unsynced changes.
"""
# remote folder is currently not checked for unsynced changes but replaced
os.mkdir(m.test_folder_local + "/folder")
wait_for_idle(m)
with m.sync.sync_lock:
# replace local folder with file
delete(m.test_folder_local + "/folder")
shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder")
# create remote changes
m.client.upload(resources + "/file1.txt", "/sync_tests/folder/file.txt")
wait_for_idle(m)
assert_synced(m)
assert_exists(m, "/sync_tests", "folder")
assert_child_count(m, "/sync_tests", 1)
# check for fatal errors
assert not m.fatal_errors
def test_local_path_error(m):
"""Tests error handling for forbidden file names."""
# paths with backslash are not allowed on Dropbox
# we create such a local folder and assert that it triggers a sync issue
test_path_local = m.test_folder_local + "/folder\\"
test_path_dbx = "/sync_tests/folder\\"
os.mkdir(test_path_local)
wait_for_idle(m)
assert len(m.sync_errors) == 1
assert m.sync_errors[-1]["local_path"] == test_path_local
assert m.sync_errors[-1]["dbx_path"] == test_path_dbx
assert m.sync_errors[-1]["type"] == "PathError"
assert test_path_dbx in m.sync.upload_errors
# remove folder with invalid name and assert that sync issue is cleared
delete(test_path_local)
wait_for_idle(m)
assert len(m.sync_errors) == 0
assert test_path_dbx not in m.sync.upload_errors
# check for fatal errors
assert not m.fatal_errors
def test_folder_tree_local(m):
"""Tests the upload sync of a nested local folder structure."""
# test creating tree
shutil.copytree(resources + "/test_folder", m.test_folder_local + "/test_folder")
snap = DirectorySnapshot(resources + "/test_folder")
num_items = len([p for p in snap.paths if not m.sync.is_excluded(p)])
wait_for_idle(m, 10)
assert_synced(m)
assert_child_count(m, "/sync_tests", num_items)
# test deleting tree
delete(m.test_folder_local + "/test_folder")
wait_for_idle(m)
assert_synced(m)
assert_child_count(m, "/sync_tests", 0)
# check for fatal errors
assert not m.fatal_errors
def test_parallel_deletion_when_paused(m):
"""Tests parallel remote and local deletions of an item."""
# create a local file
shutil.copy(resources + "/file.txt", m.test_folder_local)
wait_for_idle(m)
assert_synced(m)
m.stop_sync()
wait_for_idle(m)
# delete local file
delete(m.test_folder_local + "/file.txt")
# delete remote file
m.client.remove("/sync_tests/file.txt")
m.start_sync()
wait_for_idle(m)
assert_synced(m)
assert_child_count(m, "/sync_tests", 0)
# check for fatal errors
assert not m.fatal_errors
def cleanup_test_config(m: Maestral, test_folder_dbx: Optional[str] = None) -> None:
"""
Shuts down syncing for the given Maestral instance, removes all local files and
folders related to that instance, including the local Dropbox folder, and removes
any '.mignore' files.
:param m: Maestral instance.
:param test_folder_dbx: Optional test folder to clean up.
"""
# stop syncing and clean up remote folder
m.stop_sync()
if test_folder_dbx:
try:
m.client.remove(test_folder_dbx)
except NotFoundError:
pass
try:
m.client.remove("/.mignore")
except NotFoundError:
pass
# remove creds from system keyring
m.client.auth.delete_creds()
# remove local files and folders
delete(m.dropbox_path)
remove_configuration("test-config")
def test_unknown_path_encoding(m, capsys):
"""
Tests the handling of a local path with bytes that cannot be decoded with the
file system encoding reported by the platform.
"""
# create a path with Python surrogate escapes and convert it to bytes
test_path_dbx = "/sync_tests/my_folder_\udce4"
test_path_local = m.sync.to_local_path(test_path_dbx)
test_path_local_bytes = os.fsencode(test_path_local)
# create the local directory while we are syncing
os.mkdir(test_path_local_bytes)
wait_for_idle(m)
# 1) Check that the sync issue is logged
# This requires that our sync logic from the emitted watchdog event all the
# way to `SyncEngine._on_local_created` can handle strings with surrogate escapes.
assert len(m.fatal_errors) == 0
assert len(m.sync_errors) == 1
assert m.sync_errors[-1]["local_path"] == sanitize_string(test_path_local)
assert m.sync_errors[-1]["dbx_path"] == sanitize_string(test_path_dbx)
assert m.sync_errors[-1]["type"] == "PathError"
assert test_path_dbx in m.sync.upload_errors
# 2) Check that the sync is retried after pause / resume
# This requires that our logic to save failed paths in our state file and retry the
# sync on startup can handle strings with surrogate escapes.
m.stop_sync()
m.start_sync()
wait_for_idle(m)
assert len(m.fatal_errors) == 0
assert len(m.sync_errors) == 1
assert m.sync_errors[-1]["local_path"] == sanitize_string(test_path_local)
assert m.sync_errors[-1]["dbx_path"] == sanitize_string(test_path_dbx)
assert m.sync_errors[-1]["type"] == "PathError"
assert test_path_dbx in m.sync.upload_errors
# 3) Check that the error is cleared when the file is deleted
# This requires that `SyncEngine.upload_local_changes_while_inactive` can handle
# strings with surrogate escapes all they way to `SyncEngine._on_local_deleted`.
delete(test_path_local_bytes) # type: ignore
wait_for_idle(m)
assert len(m.fatal_errors) == 0
assert len(m.sync_errors) == 0
assert test_path_dbx not in m.sync.upload_errors
def remove_configuration(config_name):
"""
Removes all config and state files associated with the given configuration.
:param str config_name: The configuration to remove.
"""
MaestralConfig(config_name).cleanup()
MaestralState(config_name).cleanup()
index_file = get_data_path('maestral', f'{config_name}.index')
delete(index_file)
def on_folders_selected(self):
self.mdbx.excluded_items = self.get_excluded_items()
# if any excluded items are currently on the drive, delete them
for item in self.excluded_items:
local_item = self.mdbx.to_local_path(item)
delete(local_item)
# switch to next page
self.stackedWidget.slideInIdx(4)
def reset_sync_state(self):
"""
Resets the sync index and state. Only call this to clean up leftover state
information if a Dropbox was improperly unlinked (e.g., auth token has been
manually deleted). Otherwise leave state management to Maestral.
"""
self.sync.last_cursor = ''
self.sync.last_sync = 0.0
self.sync.clear_rev_index()
delete(self.sync.rev_file_path)
logger.debug("Sync state reset")
def test_delete():
# test deleting file
test_file = tempfile.NamedTemporaryFile()
assert osp.isfile(test_file.name)
delete(test_file.name)
assert not osp.exists(test_file.name)
# test deleting directory
test_dir = tempfile.TemporaryDirectory()
assert osp.isdir(test_dir.name)
delete(test_dir.name)
assert not osp.exists(test_dir.name)
def on_dropbox_location_selected(self):
# start with clean sync state
self.mdbx.reset_sync_state()
# apply dropbox path
try:
if osp.exists(self.dropbox_location):
if is_empty(self.dropbox_location):
delete(self.dropbox_location, raise_error=True)
else:
msg_box = UserDialog(
title="Folder is not empty",
message=(
f'The folder "{osp.basename(self.dropbox_location)}" is '
"not empty. Would you like to merge its content with your "
"Dropbox?"
),
button_names=("Cancel", "Merge"),
parent=self,
)
res = msg_box.exec()
if res == UserDialog.DialogCode.Accepted:
return
elif res == UserDialog.DialogCode.Rejected:
pass
self.mdbx.create_dropbox_directory(self.dropbox_location)
except OSError:
msg_box = UserDialog(
title="Could not set directory",
message=(
"Please check if you have permissions to write to the "
"selected location."
),
parent=self,
)
msg_box.exec()
return
# switch to next page
self.mdbx.set_conf("sync", "excluded_items", [])
self.stackedWidget.slideInIdx(3)
self.treeViewFolders.setFocus()
# populate folder list
if not self.excluded_items: # don't repopulate
self.populate_folders_list()
def on_folders_selected(self):
self.update_selection()
# this won't trigger downloads because we have not yet performed our first sync
self.mdbx.set_excluded_items(self.excluded_items)
# if any excluded items are currently on the drive, delete them
for item in self.excluded_items:
local_item = self.mdbx.to_local_path(item)
delete(local_item)
# switch to next page
self.stackedWidget.slideInIdx(4)
def test_cased_path_candidates():
# test that we can find a unique correctly cased path
# starting from a candidate with scrambled casing
path = "/usr/local/share".upper()
candidates = cased_path_candidates(path)
assert len(candidates) == 1
assert "/usr/local/share" in candidates
candidates = cased_path_candidates("/test", root="/usr/local/share")
assert len(candidates) == 1
assert "/usr/local/share/test" in candidates
home = get_home_dir()
# test that we can get multiple cased path
# candidates on case-sensitive file systems
if is_fs_case_sensitive(home):
parent0 = osp.join(home, "test folder/subfolder")
parent1 = osp.join(home, "Test Folder/subfolder")
os.makedirs(parent0)
os.makedirs(parent1)
path = osp.join(parent0.lower(), "File.txt")
try:
candidates = cased_path_candidates(path)
assert len(candidates) == 2
assert osp.join(parent0, "File.txt") in candidates
assert osp.join(parent1, "File.txt") in candidates
candidates = cased_path_candidates(
"/test folder/subfolder/File.txt", root=home
)
assert len(candidates) == 2
assert osp.join(parent0, "File.txt") in candidates
assert osp.join(parent1, "File.txt") in candidates
finally:
delete(parent0)
delete(parent1)
def _remove_after_excluded(self, dbx_path):
# book keeping
self.sync.clear_sync_error(dbx_path=dbx_path)
self.sync.set_local_rev(dbx_path, None)
# remove folder from local drive
local_path = self.sync.to_local_path(dbx_path)
# dbx_path will be lower-case, we there explicitly run `to_cased_path`
local_path = to_cased_path(local_path)
if local_path:
with self.monitor.fs_event_handler.ignore(
local_path,
recursive=osp.isdir(local_path),
event_types=(EVENT_TYPE_DELETED, )):
delete(local_path)
def test_move_dropbox_folder_to_existing(self):
new_dir_short = "~/New Dropbox"
new_dir = osp.realpath(osp.expanduser(new_dir_short))
os.mkdir(new_dir)
try:
with self.assertRaises(FileExistsError):
self.m.move_dropbox_directory(new_dir)
# assert that sync is still running
self.assertTrue(self.m.syncing)
finally:
# cleanup
delete(new_dir)
def test_move_dropbox_folder_to_existing(m):
new_dir_short = "~/New Dropbox"
new_dir = osp.realpath(osp.expanduser(new_dir_short))
os.mkdir(new_dir)
try:
with pytest.raises(FileExistsError):
m.move_dropbox_directory(new_dir)
# assert that sync is still running
assert m.running
finally:
# cleanup
delete(new_dir)
def on_selected_clicked(self):
# apply dropbox path
try:
if osp.exists(self.dropbox_location):
if is_empty(self.dropbox_location):
delete(self.dropbox_location, raise_error=True)
else:
msg_box = UserDialog(
title="Folder is not empty",
message=(
f'The folder "{osp.basename(self.dropbox_location)}" is '
"not empty. Would you like to merge its content with your "
"Dropbox?"
),
button_names=("Cancel", "Merge"),
parent=self,
)
res = msg_box.exec()
if res == UserDialog.DialogCode.Accepted:
return
elif res == UserDialog.DialogCode.Rejected:
pass
self.mdbx.create_dropbox_directory(self.dropbox_location)
except OSError:
msg_box = UserDialog(
title="Could not set directory",
message=(
"Please check if you have permissions to write to the "
"selected location."
),
parent=self,
)
msg_box.exec()
return
# Resume sync with clean sync state.
self.mdbx.reset_sync_state()
self.mdbx.start_sync()
self.close()
def sync():
local_dir = osp.join(get_home_dir(), "dummy_dir")
os.mkdir(local_dir)
sync = SyncEngine(DropboxClient("test-config"))
sync.fs_events.enable()
sync.dropbox_path = local_dir
observer = Observer()
observer.schedule(sync.fs_events, sync.dropbox_path, recursive=True)
observer.start()
yield sync
observer.stop()
observer.join()
remove_configuration("test-config")
delete(sync.dropbox_path)
def test_local_folder_replaced_by_file(m):
"""Tests the upload sync when a local folder is replaced by a file."""
os.mkdir(m.test_folder_local + "/folder")
wait_for_idle(m)
with m.sync.sync_lock:
# replace local folder with file
delete(m.test_folder_local + "/folder")
shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder")
wait_for_idle(m)
assert_synced(m)
assert osp.isfile(m.test_folder_local + "/folder")
assert_child_count(m, "/sync_tests", 1)
# check for fatal errors
assert not m.fatal_errors
async def on_items_selected(self, btn_name: str) -> None:
self.fs_source.stop_loading()
if btn_name == "Select":
excluded_nodes = self.fs_source.get_nodes_with_state(OFF)
excluded_paths = [node.path_lower for node in excluded_nodes]
self.mdbx.excluded_items = excluded_paths
# if any excluded folders are currently on the drive, delete them
for path in excluded_paths:
local_path = self.mdbx.to_local_path(path)
delete(local_path)
# switch to next page
self.go_forward()
elif btn_name == "Back":
self.go_back()
async def on_dbx_location(self, btn_name: str) -> None:
if btn_name == "Select":
dropbox_path = self.combobox_dbx_location.current_selection
# try to create the directory
# continue to next page if success or alert user if failed
try:
# If a file / folder exists, ask for conflict resolution.
if osp.exists(dropbox_path):
if is_empty(dropbox_path):
delete(dropbox_path, raise_error=True)
else:
should_merge = await self.question_dialog(
title="Folder is not empty",
message=
(f'The folder "{osp.basename(dropbox_path)}" is not '
"empty. Would you like merge its content with "
"your Dropbox?"),
)
if not should_merge:
return
self.mdbx.create_dropbox_directory(dropbox_path)
except OSError:
await self.error_dialog(
title="Could not set folder",
message=("Please make sure that you have permissions "
"to write to the selected location."),
)
else:
self.go_forward()
elif btn_name == "Cancel & Unlink":
self.mdbx.unlink()
self.on_close_button_pressed()
def select_dbx_path_dialog(config_name, allow_merge=False):
"""
A CLI dialog to ask for a local Dropbox folder location.
:param str config_name: The configuration to use for the default folder name.
:param bool allow_merge: If ``True``, allows the selection of an existing folder
without deleting it. Defaults to ``False``.
:returns: Path given by user.
:rtype: str
"""
conf = MaestralConfig(config_name)
default = osp.join(get_home_dir(), conf.get('main', 'default_dir_name'))
while True:
res = click.prompt('Please give Dropbox folder location',
default=default,
type=click.Path(writable=True))
res = res.rstrip(osp.sep)
dropbox_path = osp.expanduser(res or default)
if osp.exists(dropbox_path):
if allow_merge:
choice = click.prompt(text=(
f'Directory "{dropbox_path}" already exists. Do you want to '
f'replace it or merge its content with your Dropbox?'),
type=click.Choice(
['replace', 'merge', 'cancel']))
else:
replace = click.confirm(text=(
f'Directory "{dropbox_path}" already exists. Do you want to '
f'replace it? Its content will be lost!'), )
choice = 'replace' if replace else 'cancel'
if choice == 'replace':
err = delete(dropbox_path)
if err:
click.echo(
f'Could not write to location "{dropbox_path}". Please '
'make sure that you have sufficient permissions.')
else:
return dropbox_path
elif choice == 'merge':
return dropbox_path
else:
return dropbox_path
def unlink(self):
"""
Unlinks the configured Dropbox account but leaves all downloaded files in place.
All syncing metadata will be removed as well. Connection and API errors will be
handled silently but the Dropbox access key will always be removed from the
user's PC.
"""
self.stop_sync()
try:
self.client.unlink()
except (ConnectionError, MaestralApiError):
pass
try:
os.remove(self.sync.rev_file_path)
except OSError:
pass
self.sync.clear_rev_index()
delete(self.sync.rev_file_path)
self._conf.cleanup()
self._state.cleanup()
logger.info('Unlinked Dropbox account.')
def select_dbx_path_dialog(config_name, allow_merge=False):
"""
A CLI dialog to ask for a local dropbox directory path.
:param str config_name: The configuration to use for the default folder name.
:param bool allow_merge: If ``True``, allows for selecting an existing path without
deleting it. Defaults to ``False``.
:returns: Path given by user.
:rtype: str
"""
conf = MaestralConfig(config_name)
default = osp.join(get_home_dir(), conf.get('main', 'default_dir_name'))
while True:
res = click.prompt('Please give Dropbox folder location',
default=default,
type=click.Path(writable=True))
res = res.rstrip(osp.sep)
dropbox_path = osp.expanduser(res or default)
old_path = osp.expanduser(conf.get('main', 'path'))
try:
same_path = osp.samefile(old_path, dropbox_path)
except FileNotFoundError:
same_path = False
if osp.exists(dropbox_path) and not same_path:
msg = (f'Directory "{dropbox_path}" already exist. Do you want to '
'overwrite it? Its content will be lost!')
if click.confirm(msg, prompt_suffix=''):
err = delete(dropbox_path)
if err:
click.echo(
f'Could not write to location "{dropbox_path}". Please '
'make sure that you have sufficient permissions.')
else:
return dropbox_path
elif allow_merge:
msg = 'Would you like to merge its content with your Dropbox?'
if click.confirm(msg, prompt_suffix=''):
return dropbox_path
else:
return dropbox_path
def clean_local(m):
"""Recreates a fresh test folder locally."""
delete(m.dropbox_path + "/.mignore")
delete(m.test_folder_local)
os.mkdir(m.test_folder_local)
def m():
config_name = "test-config"
m = Maestral(config_name)
m.log_level = logging.DEBUG
# link with given token
access_token = os.environ.get("DROPBOX_ACCESS_TOKEN")
refresh_token = os.environ.get("DROPBOX_REFRESH_TOKEN")
if access_token:
m.client._init_sdk_with_token(access_token=access_token)
m.client.auth._access_token = access_token
m.client.auth._token_access_type = "legacy"
elif refresh_token:
m.client._init_sdk_with_token(refresh_token=refresh_token)
m.client.auth._refresh_token = refresh_token
m.client.auth._token_access_type = "offline"
else:
raise RuntimeError(
"Either access token or refresh token must be given as environment "
"variable DROPBOX_ACCESS_TOKEN or DROPBOX_REFRESH_TOKEN."
)
# get corresponding Dropbox ID and store in keyring for other processes
res = m.client.get_account_info()
m.client.auth._account_id = res.account_id
m.client.auth.loaded = True
m.client.auth.save_creds()
# set local Dropbox directory
home = get_home_dir()
local_dropbox_dir = generate_cc_name(home + "/Dropbox", suffix="test runner")
m.create_dropbox_directory(local_dropbox_dir)
# acquire test lock and perform initial sync
lock = DropboxTestLock(m)
if not lock.acquire(timeout=60 * 60):
raise TimeoutError("Could not acquire test lock")
# create / clean our temporary test folder
m.test_folder_dbx = "/sync_tests"
m.test_folder_local = m.to_local_path(m.test_folder_dbx)
try:
m.client.remove(m.test_folder_dbx)
except NotFoundError:
pass
m.client.make_dir(m.test_folder_dbx)
# start syncing
m.start_sync()
wait_for_idle(m)
# return synced and running instance
yield m
# stop syncing and clean up remote folder
m.stop_sync()
try:
m.client.remove(m.test_folder_dbx)
except NotFoundError:
pass
try:
m.client.remove("/.mignore")
except NotFoundError:
pass
# remove all shared links
res = m.client.list_shared_links()
for link in res.links:
m.revoke_shared_link(link.url)
# remove creds from system keyring
m.client.auth.delete_creds()
# remove local files and folders
delete(m.dropbox_path)
remove_configuration(m.config_name)
# release lock
lock.release()
def select_dbx_path_dialog(
config_name: str, default_dir_name: Optional[str] = None, allow_merge: bool = False
) -> str:
"""
A CLI dialog to ask for a local Dropbox folder location.
:param config_name: The configuration to use for the default folder name.
:param default_dir_name: The default directory name. Defaults to
"Dropbox ({config_name})" if not given.
:param allow_merge: If ``True``, allows the selection of an existing folder without
deleting it. Defaults to ``False``.
:returns: Path given by user.
"""
from maestral.utils.appdirs import get_home_dir
from maestral.utils.path import delete
default_dir_name = default_dir_name or f"Dropbox ({config_name.capitalize()})"
default = osp.join(get_home_dir(), default_dir_name)
while True:
res = click.prompt(
"Please give Dropbox folder location",
default=default,
type=click.Path(writable=True),
)
res = res.rstrip(osp.sep)
dropbox_path = osp.expanduser(res or default)
if osp.exists(dropbox_path):
if allow_merge:
choice = click.prompt(
text=(
f'Directory "{dropbox_path}" already exists.\nDo you want to '
f"replace it or merge its content with your Dropbox?"
),
type=click.Choice(["replace", "merge", "cancel"]),
)
else:
replace = click.confirm(
text=(
f'Directory "{dropbox_path}" already exists. Do you want to '
f"replace it? Its content will be lost!"
),
)
choice = "replace" if replace else "cancel"
if choice == "replace":
err = delete(dropbox_path)
if err:
click.echo(
f'Could not write to location "{dropbox_path}". Please '
"make sure that you have sufficient permissions."
)
else:
return dropbox_path
elif choice == "merge":
return dropbox_path
else:
return dropbox_path
def on_dropbox_location_selected(self):
# start with clean sync state
self.mdbx.reset_sync_state()
# apply dropbox path
dropbox_path = osp.join(self.dropbox_location, self.mdbx.get_conf('main', 'default_dir_name'))
if osp.exists(dropbox_path):
if osp.isdir(dropbox_path):
msg_box = UserDialog(
title='Folder already exists',
message=(f'The folder "{dropbox_path}" already exists. Would '
'you like to keep using it?'),
button_names=('Replace', 'Cancel', 'Keep'),
parent=self,
)
msg_box.setAcceptButtonIcon('edit-clear')
res = msg_box.exec_()
else:
dir_name = self.mdbx.get_conf('main', 'default_dir_name')
msg_box = UserDialog(
title='File conflict',
message=(f'There already is a file named "{dir_name}" at this '
'location. Would you like to replace it?'),
button_names=('Replace', 'Cancel'),
parent=self,
)
res = msg_box.exec_()
if res == UserDialog.Rejected:
return
elif res == UserDialog.Accepted:
err = delete(dropbox_path)
if err:
msg_box = UserDialog(
title='Could not write to destination',
message=('Please check if you have permissions to write to the '
'selected location.'),
parent=self
)
msg_box.exec_()
return
elif res == UserDialog.Accepted2:
pass
try:
self.mdbx.create_dropbox_directory(dropbox_path)
except OSError:
msg_box = UserDialog(
title='Could not create directory',
message=('Please check if you have permissions to write to the '
'selected location.'),
parent=self
)
msg_box.exec_()
return
# switch to next page
self.mdbx.set_conf('main', 'excluded_items', [])
self.stackedWidget.slideInIdx(3)
self.treeViewFolders.setFocus()
# populate folder list
if not self.excluded_items: # don't repopulate
self.populate_folders_list()
