def _check_last_sync(self):
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
qm_active = self._queue_manager.active()
qm_size = self._queue_manager.get_overall_size()
empty_polls = self._remote_watcher.get_metrics()["empty_polls"]
if not AbstractOSIntegration.is_windows():
win_info = 'not Windows'
else:
win_info = 'Windows with win queue size = %d and win folder scan size = %d' % (
self._local_watcher.get_win_queue_size(), self._local_watcher.get_win_folder_scan_size())
log.debug('Checking sync completed: queue manager is %s, overall size = %d, empty polls count = %d'
', watchdog queue size = %d, %s',
'active' if qm_active else 'inactive', qm_size, empty_polls,
self._local_watcher.get_watchdog_queue_size(), win_info)
local_metrics = self._local_watcher.get_metrics()
if (qm_size == 0 and not qm_active and empty_polls > 0
and self._local_watcher.empty_events()
and (
not AbstractOSIntegration.is_windows()
or
self._local_watcher.win_queue_empty() and self._local_watcher.win_folder_scan_empty())):
self._dao.update_config("last_sync_date", datetime.datetime.utcnow())
if local_metrics['last_event'] == 0:
log.warn("No watchdog event detected but sync is completed")
if self._sync_started:
self._sync_started = False
log.debug('Emitting syncCompleted for engine %s', self.get_uid())
self.syncCompleted.emit()
def set_folder_icon(self, ref, icon):
if icon is None:
return
if AbstractOSIntegration.is_windows():
self.set_folder_icon_win32(ref, icon)
elif AbstractOSIntegration.is_mac():
self.set_folder_icon_darwin(ref, icon)
def test_untrash_file_with_rename(self):
self.local_client_1.make_file('/', 'File_To_Delete.txt', 'This is a content')
self.wait_sync()
self.assertTrue(self.remote_document_client_1.exists('/File_To_Delete.txt'))
uid = self.local_client_1.get_remote_id('/File_To_Delete.txt')
old_info = self.remote_document_client_1.get_info('/File_To_Delete.txt', use_trash=True)
abs_path = self.local_client_1.abspath('/File_To_Delete.txt')
# Pretend we had trash the file
shutil.move(abs_path, os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'))
self.wait_sync(wait_for_async=True)
self.assertFalse(self.remote_document_client_1.exists('/File_To_Delete.txt'))
self.assertFalse(self.local_client_1.exists('/File_To_Delete.txt'))
with open(os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'), 'w') as f:
f.write('New content')
if AbstractOSIntegration.is_windows():
# Python API overwrite the tag by default
with open(os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt:ndrive'), 'w') as f:
f.write(uid)
# See if it untrash or recreate
shutil.move(os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'), self.local_client_1.abspath('/'))
self.wait_sync(wait_for_async=True)
self.assertTrue(self.remote_document_client_1.exists(old_info.uid))
self.assertTrue(self.local_client_1.exists('/File_To_Delete2.txt'))
self.assertFalse(self.local_client_1.exists('/File_To_Delete.txt'))
self.assertEqual(self.local_client_1.get_content('/File_To_Delete2.txt'), 'New content')
def remove_remote_id(self, ref, name='ndrive'):
# Can be move to another class
path = self._abspath(ref)
log.trace('Removing xattr %s from %s', name, path)
locker = self.unlock_path(path, False)
if AbstractOSIntegration.is_windows():
pathAlt = path + ":" + name
try:
if os.path.exists(pathAlt):
os.remove(pathAlt)
except WindowsError as e:
if e.errno == os.errno.EACCES:
self.unset_path_readonly(path)
os.remove(pathAlt)
self.set_path_readonly(path)
else:
raise e
finally:
self.lock_path(path, locker)
else:
try:
import xattr
if AbstractOSIntegration.is_mac():
xattr.removexattr(path, name)
else:
xattr.removexattr(path, 'user.' + name)
except IOError as e:
# Ignore IOError: [Errno 93] Attribute not found ( Mac )
# IOError: [Errno 61] No data available ( Linux )
if e.errno == 93 or e.errno == 61:
pass
else:
raise
finally:
self.lock_path(path, locker)
def set_folder_icon(self, ref, icon):
if icon is None:
return
if AbstractOSIntegration.is_windows():
self.set_folder_icon_win32(ref, icon)
elif AbstractOSIntegration.is_mac():
self.set_folder_icon_darwin(ref, icon)
def remove_remote_id(self, ref, name='ndrive'):
# Can be move to another class
path = self._abspath(ref)
log.trace('Removing xattr %s from %s', name, path)
locker = self.unlock_path(path, False)
if AbstractOSIntegration.is_windows():
pathAlt = path + ":" + name
try:
os.remove(pathAlt)
except WindowsError as e:
if e.errno == os.errno.EACCES:
self.unset_path_readonly(path)
os.remove(pathAlt)
self.set_path_readonly(path)
else:
raise e
finally:
self.lock_path(path, locker)
else:
try:
import xattr
if AbstractOSIntegration.is_mac():
xattr.removexattr(path, name)
else:
xattr.removexattr(path, 'user.' + name)
finally:
self.lock_path(path, locker)
def remove_remote_id(self, ref, name='ndrive'):
# Can be move to another class
path = self._abspath(ref)
log.trace('Removing xattr %s from %s', name, path)
locker = self.unlock_path(path, False)
if AbstractOSIntegration.is_windows():
pathAlt = path + ":" + name
try:
if os.path.exists(pathAlt):
os.remove(pathAlt)
except WindowsError as e:
if e.errno == os.errno.EACCES:
self.unset_path_readonly(path)
os.remove(pathAlt)
self.set_path_readonly(path)
else:
raise e
finally:
self.lock_path(path, locker)
else:
try:
import xattr
if AbstractOSIntegration.is_mac():
xattr.removexattr(path, name)
else:
xattr.removexattr(path, 'user.' + name)
except IOError as e:
# Ignore IOError: [Errno 93] Attribute not found ( Mac )
# IOError: [Errno 61] No data available ( Linux )
if e.errno == 93 or e.errno == 61:
pass
else:
raise
finally:
self.lock_path(path, locker)
def _check_last_sync(self):
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
empty_events = self._local_watcher.empty_events()
blacklist_size = self._queue_manager.get_errors_count()
qm_size = self._queue_manager.get_overall_size()
qm_active = self._queue_manager.active()
empty_polls = self._remote_watcher.get_metrics()["empty_polls"]
if not AbstractOSIntegration.is_windows():
win_info = 'not Windows'
else:
win_info = 'Windows with win queue size = %d and win folder scan size = %d' % (
self._local_watcher.get_win_queue_size(), self._local_watcher.get_win_folder_scan_size())
log.debug('Checking sync completed: queue manager is %s, overall size = %d, empty polls count = %d'
', local watcher empty events = %d, blacklist = %d, %s',
'active' if qm_active else 'inactive', qm_size, empty_polls,
empty_events, blacklist_size, win_info)
local_metrics = self._local_watcher.get_metrics()
if (qm_size == 0 and not qm_active and empty_polls > 0
and empty_events):
if blacklist_size != 0:
self.syncPartialCompleted.emit()
return
self._dao.update_config("last_sync_date", datetime.datetime.utcnow())
if local_metrics['last_event'] == 0:
log.warn("No watchdog event detected but sync is completed")
if self._sync_started:
self._sync_started = False
log.debug('Emitting syncCompleted for engine %s', self.get_uid())
self.syncCompleted.emit()
def _check_last_sync(self):
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
qm_active = self._queue_manager.active()
qm_size = self._queue_manager.get_overall_size()
empty_polls = self._remote_watcher.get_metrics()["empty_polls"]
log.debug(
"Checking sync completed: queue manager is %s, overall size = %d, empty polls count = %d",
"active" if qm_active else "inactive",
qm_size,
empty_polls,
)
local_metrics = self._local_watcher.get_metrics()
if (
qm_size == 0
and not qm_active
and empty_polls > 0
and self._local_watcher.empty_events()
and (
not AbstractOSIntegration.is_windows()
or self._local_watcher.win_queue_empty()
and self._local_watcher.win_folder_scan_empty()
)
):
self._dao.update_config("last_sync_date", datetime.datetime.utcnow())
if local_metrics["last_event"] == 0:
log.warn("No watchdog event detected but sync is completed")
if self._sync_started:
self._sync_started = False
log.debug("Emitting syncCompleted for engine %s", self.get_uid())
self.syncCompleted.emit()
def _check_last_sync(self):
empty_events = self._local_watcher.empty_events()
blacklist_size = self._queue_manager.get_errors_count()
qm_size = self._queue_manager.get_overall_size()
qm_active = self._queue_manager.active()
empty_polls = self._remote_watcher.get_metrics()["empty_polls"]
if not AbstractOSIntegration.is_windows():
win_info = 'not Windows'
else:
win_info = 'Windows with win queue size = %d and win folder scan size = %d' % (
self._local_watcher.get_win_queue_size(), self._local_watcher.get_win_folder_scan_size())
log.debug('Checking sync completed: queue manager is %s, overall size = %d, empty polls count = %d'
', local watcher empty events = %d, blacklist = %d, %s',
'active' if qm_active else 'inactive', qm_size, empty_polls,
empty_events, blacklist_size, win_info)
local_metrics = self._local_watcher.get_metrics()
if (qm_size == 0 and not qm_active and empty_polls > 0
and empty_events):
if blacklist_size != 0:
self.syncPartialCompleted.emit()
return
self._dao.update_config("last_sync_date", datetime.datetime.utcnow())
if local_metrics['last_event'] == 0:
log.warn("No watchdog event detected but sync is completed")
if self._sync_started:
self._sync_started = False
log.debug('Emitting syncCompleted for engine %s', self.get_uid())
self.syncCompleted.emit()
def is_ignored(self, parent_ref, file_name):
# Add parent_ref to be able to filter on size if needed
ignore = False
# Office temp file
# http://support.microsoft.com/kb/211632
if file_name.startswith("~") and file_name.endswith(".tmp"):
return True
# Emacs auto save file
# http://www.emacswiki.org/emacs/AutoSave
if file_name.startswith("#") and file_name.endswith("#") and len(file_name) > 2:
return True
for suffix in self.ignored_suffixes:
if file_name.endswith(suffix):
ignore = True
break
for prefix in self.ignored_prefixes:
if file_name.startswith(prefix):
ignore = True
break
if ignore:
return True
if AbstractOSIntegration.is_windows():
# NXDRIVE-465
ref = self.get_children_ref(parent_ref, file_name)
path = self._abspath(ref)
if not os.path.exists(path):
return False
import win32con
import win32api
attrs = win32api.GetFileAttributes(path)
if attrs & win32con.FILE_ATTRIBUTE_SYSTEM == win32con.FILE_ATTRIBUTE_SYSTEM:
return True
if attrs & win32con.FILE_ATTRIBUTE_HIDDEN == win32con.FILE_ATTRIBUTE_HIDDEN:
return True
return False
def get_local_client(self, path):
if AbstractOSIntegration.is_windows():
from tests.win_local_client import WindowsLocalClient
return WindowsLocalClient(path)
if AbstractOSIntegration.is_mac():
from tests.mac_local_client import MacLocalClient
return MacLocalClient(path)
return LocalClient(path)
def get_local_client(self, path):
if AbstractOSIntegration.is_windows():
from nxdrive.tests.win_local_client import WindowsLocalClient
return WindowsLocalClient(path)
if AbstractOSIntegration.is_mac():
from nxdrive.tests.mac_local_client import MacLocalClient
return MacLocalClient(path)
return LocalClient(path)
def test_binding_initialization_and_first_sync(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Create some documents in a Nuxeo workspace and bind this server to a
# Nuxeo Drive local folder
self.make_server_tree()
# The root binding operation does not create the local folder yet.
self.assertFalse(local.exists('/'))
# Launch ndrive and check synchronization
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/'))
self.assertTrue(local.exists('/Folder 1'))
self.assertEquals(local.get_content('/Folder 1/File 1.txt'), "aaa")
self.assertTrue(local.exists('/Folder 1/Folder 1.1'))
self.assertEquals(local.get_content('/Folder 1/Folder 1.1/File 2.txt'),
"bbb")
self.assertTrue(local.exists('/Folder 1/Folder 1.2'))
self.assertEquals(local.get_content('/Folder 1/Folder 1.2/File 3.txt'),
"ccc")
self.assertTrue(local.exists('/Folder 2'))
# Cannot predict the resolution in advance
self.assertTrue(remote.get_content(self._duplicate_file_1),
"Some content.")
self.assertTrue(remote.get_content(self._duplicate_file_2),
"Other content.")
if local.get_content(
'/Folder 2/Duplicated File.txt') == "Some content.":
self.assertEquals(
local.get_content('/Folder 2/Duplicated File__1.txt'),
"Other content.")
else:
self.assertEquals(
local.get_content('/Folder 2/Duplicated File.txt'),
"Other content.")
self.assertEquals(
local.get_content('/Folder 2/Duplicated File__1.txt'),
"Some content.")
self.assertEquals(local.get_content('/Folder 2/File 4.txt'), "ddd")
self.assertEquals(local.get_content('/File 5.txt'), "eee")
# Unbind root and resynchronize
remote.unregister_as_root(self.workspace)
# Since errors are generated by the deletion events sent by Watchdog for the workspace children under UNIX,
# don't enforce errors and increase timeout
if not AbstractOSIntegration.is_windows():
timeout = 60
enforce_errors = False
else:
timeout = DEFAULT_WAIT_SYNC_TIMEOUT
enforce_errors = True
self.wait_sync(wait_for_async=True,
timeout=timeout,
enforce_errors=enforce_errors)
self.assertFalse(local.exists('/'))
def test_local_delete_readonly_folder(self):
local_client = self.local_client_1
remote_client = self.remote_document_client_1
# Check local folder
self.assertTrue(local_client.exists(u'/Original Folder 1'))
folder_1_state = self.get_dao_state_from_engine_1(
u'/Original Folder 1')
self.assertTrue(folder_1_state.remote_can_delete)
# Set remote folder as readonly for test user
folder_1_path = TEST_WORKSPACE_PATH + u'/Original Folder 1'
op_input = "doc:" + folder_1_path
self.root_remote_client.execute("Document.SetACE",
op_input=op_input,
user=self.user_1,
permission="Read")
self.root_remote_client.block_inheritance(folder_1_path,
overwrite=False)
self.wait_sync(wait_for_async=True)
# Check can_delete flag in pair state
folder_1_state = self.get_dao_state_from_engine_1(
u'/Original Folder 1')
self.assertFalse(folder_1_state.remote_can_delete)
# Delete local folder
local_client.delete(u'/Original Folder 1')
self.assertFalse(local_client.exists(u'/Original Folder 1'))
self.wait_sync(wait_for_async=True)
self.assertEqual(self.engine_1.get_dao().get_sync_count(), 6)
# Check remote folder and its children have not been deleted
folder_1_remote_info = remote_client.get_info(u'/Original Folder 1')
self.assertEqual(folder_1_remote_info.name, u'Original Folder 1')
file_1_1_remote_info = remote_client.get_info(
u'/Original Folder 1/Original File 1.1.txt')
self.assertEqual(file_1_1_remote_info.name, u'Original File 1.1.txt')
folder_1_1_remote_info = remote_client.get_info(
u'/Original Folder 1/Sub-Folder 1.1')
self.assertEqual(folder_1_1_remote_info.name, u'Sub-Folder 1.1')
folder_1_2_remote_info = remote_client.get_info(
u'/Original Folder 1/Sub-Folder 1.2')
self.assertEqual(folder_1_2_remote_info.name, u'Sub-Folder 1.2')
if not AbstractOSIntegration.is_windows():
# Check filter has been created
self.assertTrue(self.engine_1.get_dao().is_filter(
folder_1_state.remote_parent_path + '/' +
folder_1_state.remote_ref))
# Check local folder haven't been re-created
self.assertFalse(local_client.exists(u'/Original Folder 1'))
def has_folder_icon(self, ref):
target_folder = self._abspath(ref)
if AbstractOSIntegration.is_mac():
meta_file = os.path.join(target_folder, "Icon\r")
return os.path.exists(meta_file)
if AbstractOSIntegration.is_windows():
meta_file = os.path.join(target_folder, "desktop.ini")
return os.path.exists(meta_file)
return False
def testLogs(self):
from nxdrive.osi import AbstractOSIntegration
if AbstractOSIntegration.is_windows():
raise SkipTest("Temporarily skipped, need to investigate")
# NXDRIVE-358
report = Report(self.manager, os.path.join(self.folder, "report.zip"))
log.debug("Strange encoding \xe9")
log.debug(u"Unicode encoding \xe8")
report.generate()
def testLogs(self):
from nxdrive.osi import AbstractOSIntegration
if AbstractOSIntegration.is_windows():
raise SkipTest("Temporarily skipped, need to investigate")
# NXDRIVE-358
report = Report(self.manager, os.path.join(self.folder, "report.zip"))
log.debug("Strange encoding \xe9")
log.debug(u"Unicode encoding \xe8")
report.generate()
def has_folder_icon(self, ref):
target_folder = self._abspath(ref)
if AbstractOSIntegration.is_mac():
meta_file = os.path.join(target_folder, "Icon\r")
return os.path.exists(meta_file)
if AbstractOSIntegration.is_windows():
meta_file = os.path.join(target_folder, "desktop.ini")
return os.path.exists(meta_file)
return False
def rename(self, ref, to_name):
"""Rename a local file or folder
Return the actualized info object.
"""
new_name = safe_filename(to_name)
source_os_path = self._abspath(ref)
parent = ref.rsplit(u'/', 1)[0]
old_name = ref.rsplit(u'/', 1)[1]
parent = u'/' if parent == '' else parent
locker = self.unlock_ref(ref)
try:
# Check if only case renaming
if (old_name != new_name and old_name.lower() == new_name.lower()
and not self.is_case_sensitive()):
# Must use a temp rename as FS is not case sensitive
temp_path = os.tempnam(
self._abspath(parent),
LocalClient.CASE_RENAME_PREFIX + old_name + '_')
if AbstractOSIntegration.is_windows():
import ctypes
ctypes.windll.kernel32.SetFileAttributesW(
unicode(temp_path), 2)
os.rename(source_os_path, temp_path)
source_os_path = temp_path
# Try the os rename part
target_os_path = self._abspath(os.path.join(parent, new_name))
else:
target_os_path, new_name = self._abspath_deduped(
parent, new_name, old_name)
if old_name != new_name:
os.rename(source_os_path, target_os_path)
if AbstractOSIntegration.is_windows():
import ctypes
# See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
ctypes.windll.kernel32.SetFileAttributesW(
unicode(target_os_path), 128)
new_ref = self.get_children_ref(parent, new_name)
return self.get_info(new_ref)
finally:
self.lock_ref(ref, locker & 2)
def unset_folder_icon(self, ref):
'''
Unset the red icon
'''
if AbstractOSIntegration.is_windows():
# TODO Clean version
desktop_ini_file_path = os.path.join(self._abspath(ref), "desktop.ini")
if AbstractOSIntegration.is_mac():
desktop_ini_file_path = os.path.join(self._abspath(ref), "Icon\r")
if os.path.exists(desktop_ini_file_path):
os.remove(desktop_ini_file_path)
def test_synchronize_deletion(self):
local = self.local_client_1
remote = self.remote_document_client_1
self.engine_1.start()
# Create a remote folder with 2 children then synchronize
remote.make_folder('/', 'Remote folder')
remote.make_file('/Remote folder', 'Remote file 1.odt',
'Some content.')
remote.make_file('/Remote folder', 'Remote file 2.odt',
'Other content.')
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/Remote folder'))
self.assertTrue(local.exists('/Remote folder/Remote file 1.odt'))
self.assertTrue(local.exists('/Remote folder/Remote file 2.odt'))
# Delete remote folder then synchronize
remote.delete('/Remote folder')
self.wait_sync(wait_for_async=True)
self.assertFalse(local.exists('/Remote folder'))
self.assertFalse(local.exists('/Remote folder/Remote file 1.odt'))
self.assertFalse(local.exists('/Remote folder/Remote file 2.odt'))
# Create a local folder with 2 children then synchronize
local.make_folder('/', 'Local folder')
local.make_file('/Local folder', 'Local file 1.odt', 'Some content.')
local.make_file('/Local folder', 'Local file 2.odt', 'Other content.')
self.wait_sync()
self.assertTrue(remote.exists('/Local folder'))
self.assertTrue(remote.exists('/Local folder/Local file 1.odt'))
self.assertTrue(remote.exists('/Local folder/Local file 2.odt'))
# Delete local folder then synchronize
time.sleep(OS_STAT_MTIME_RESOLUTION)
local.delete('/Local folder')
# Since errors are generated by the deletion events sent by Watchdog for the folder children under UNIX,
# don't enforce errors and increase timeout
if not AbstractOSIntegration.is_windows():
timeout = 60
enforce_errors = False
else:
timeout = DEFAULT_WAIT_SYNC_TIMEOUT
enforce_errors = True
self.wait_sync(timeout=timeout, enforce_errors=enforce_errors)
self.assertFalse(remote.exists('/Local folder'))
# Wait for async completion as recursive deletion of children is done
# by the BulkLifeCycleChangeListener which is asynchronous
self.wait()
self.assertFalse(remote.exists('/Local folder/Local file 1.odt'))
self.assertFalse(remote.exists('/Local folder/Local file 2.odt'))
def rename(self, ref, to_name):
"""Rename a local file or folder
Return the actualized info object.
"""
new_name = safe_filename(to_name)
source_os_path = self._abspath(ref)
parent = ref.rsplit(u'/', 1)[0]
old_name = ref.rsplit(u'/', 1)[1]
parent = u'/' if parent == '' else parent
locker = self.unlock_ref(ref)
try:
# Check if only case renaming
if (old_name != new_name and old_name.lower() == new_name.lower()
and not self.is_case_sensitive()):
# Must use a temp rename as FS is not case sensitive
temp_path = os.tempnam(self._abspath(parent),
'.ren_' + old_name + '_')
if AbstractOSIntegration.is_windows():
import ctypes
ctypes.windll.kernel32.SetFileAttributesW(
unicode(temp_path), 2)
os.rename(source_os_path, temp_path)
source_os_path = temp_path
# Try the os rename part
target_os_path = self._abspath(os.path.join(parent, new_name))
else:
target_os_path, new_name = self._abspath_deduped(parent,
new_name, old_name)
if old_name != new_name:
os.rename(source_os_path, target_os_path)
if AbstractOSIntegration.is_windows():
import ctypes
# See http://msdn.microsoft.com/en-us/library/aa365535%28v=vs.85%29.aspx
ctypes.windll.kernel32.SetFileAttributesW(
unicode(target_os_path), 128)
new_ref = self.get_children_ref(parent, new_name)
return self.get_info(new_ref)
finally:
self.lock_ref(ref, locker & 2)
def unset_folder_icon(self, ref):
'''
Unset the red icon
'''
if AbstractOSIntegration.is_windows():
# TODO Clean version
desktop_ini_file_path = os.path.join(self._abspath(ref),
"desktop.ini")
if AbstractOSIntegration.is_mac():
desktop_ini_file_path = os.path.join(self._abspath(ref), "Icon\r")
if os.path.exists(desktop_ini_file_path):
os.remove(desktop_ini_file_path)
def test_synchronize_deletion(self):
local = self.local_client_1
remote = self.remote_document_client_1
self.engine_1.start()
# Create a remote folder with 2 children then synchronize
remote.make_folder('/', 'Remote folder',)
remote.make_file('/Remote folder', 'Remote file 1.odt', 'Some content.')
remote.make_file('/Remote folder', 'Remote file 2.odt', 'Other content.')
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/Remote folder'))
self.assertTrue(local.exists('/Remote folder/Remote file 1.odt'))
self.assertTrue(local.exists('/Remote folder/Remote file 2.odt'))
# Delete remote folder then synchronize
remote.delete('/Remote folder')
self.wait_sync(wait_for_async=True)
self.assertFalse(local.exists('/Remote folder'))
self.assertFalse(local.exists('/Remote folder/Remote file 1.odt'))
self.assertFalse(local.exists('/Remote folder/Remote file 2.odt'))
# Create a local folder with 2 children then synchronize
local.make_folder('/', 'Local folder')
local.make_file('/Local folder', 'Local file 1.odt', 'Some content.')
local.make_file('/Local folder', 'Local file 2.odt', 'Other content.')
self.wait_sync()
self.assertTrue(remote.exists('/Local folder'))
self.assertTrue(remote.exists('/Local folder/Local file 1.odt'))
self.assertTrue(remote.exists('/Local folder/Local file 2.odt'))
# Delete local folder then synchronize
time.sleep(OS_STAT_MTIME_RESOLUTION)
local.delete('/Local folder')
# Since errors are generated by the deletion events sent by Watchdog for the folder children under UNIX,
# don't enforce errors and increase timeout
if not AbstractOSIntegration.is_windows():
timeout = 60
enforce_errors = False
else:
timeout = DEFAULT_WAIT_SYNC_TIMEOUT
enforce_errors = True
self.wait_sync(timeout=timeout, enforce_errors=enforce_errors)
self.assertFalse(remote.exists('/Local folder'))
# Wait for async completion as recursive deletion of children is done
# by the BulkLifeCycleChangeListener which is asynchronous
self.wait()
self.assertFalse(remote.exists('/Local folder/Local file 1.odt'))
self.assertFalse(remote.exists('/Local folder/Local file 2.odt'))
def test_local_delete_readonly_folder(self):
local_client = self.local_client_1
remote_client = self.remote_document_client_1
# Check local folder
self.assertTrue(local_client.exists(u'/Original Folder 1'))
folder_1_state = self.get_dao_state_from_engine_1(u'/Original Folder 1')
self.assertTrue(folder_1_state.remote_can_delete)
# Set remote folder as readonly for test user
folder_1_path = TEST_WORKSPACE_PATH + u'/Original Folder 1'
op_input = "doc:" + folder_1_path
self.root_remote_client.execute("Document.SetACE",
op_input=op_input,
user=self.user_1,
permission="Read")
self.root_remote_client.block_inheritance(folder_1_path, overwrite=False)
self.wait_sync(wait_for_async=True)
# Check can_delete flag in pair state
folder_1_state = self.get_dao_state_from_engine_1(u'/Original Folder 1')
self.assertFalse(folder_1_state.remote_can_delete)
# Delete local folder
local_client.delete(u'/Original Folder 1')
self.assertFalse(local_client.exists(u'/Original Folder 1'))
self.wait_sync(wait_for_async=True)
self.assertEqual(self.engine_1.get_dao().get_sync_count(), 6)
# Check remote folder and its children have not been deleted
folder_1_remote_info = remote_client.get_info(u'/Original Folder 1')
self.assertEqual(folder_1_remote_info.name, u'Original Folder 1')
file_1_1_remote_info = remote_client.get_info(u'/Original Folder 1/Original File 1.1.txt')
self.assertEqual(file_1_1_remote_info.name, u'Original File 1.1.txt')
folder_1_1_remote_info = remote_client.get_info(u'/Original Folder 1/Sub-Folder 1.1')
self.assertEqual(folder_1_1_remote_info.name, u'Sub-Folder 1.1')
folder_1_2_remote_info = remote_client.get_info(u'/Original Folder 1/Sub-Folder 1.2')
self.assertEqual(folder_1_2_remote_info.name, u'Sub-Folder 1.2')
if not AbstractOSIntegration.is_windows():
# Check filter has been created
self.assertTrue(
self.engine_1.get_dao().is_filter(
folder_1_state.remote_parent_path + '/' + folder_1_state.remote_ref))
# Check local folder haven't been re-created
self.assertFalse(local_client.exists(u'/Original Folder 1'))
def test_deep_folders(self):
"""
It should fail on Windows:
Explorer cannot deal with very long paths.
"""
if AbstractOSIntegration.is_windows():
# WindowsError: [Error 206] The filename or extension is too long
with self.assertRaises(OSError) as ex:
super(TestLocalClientSimulation, self).test_deep_folders()
self.assertEqual(ex.errno, 206)
else:
super(TestLocalClientSimulation, self).test_deep_folders()
def current_locale(self):
""" Detect the OS default language. """
encoding = locale.getdefaultlocale()[1]
if AbstractOSIntegration.is_windows():
l10n_code = ctypes.windll.kernel32.GetUserDefaultUILanguage()
l10n = locale.windows_locale[l10n_code]
elif AbstractOSIntegration.is_mac():
l10n_code = NSLocale.currentLocale()
l10n = NSLocale.localeIdentifier(l10n_code)
encoding = 'UTF-8'
else:
l10n = locale.getdefaultlocale()[0]
return '.'.join([l10n, encoding])
def __init__(self, engine, dao):
'''
Constructor
'''
super(LocalWatcher, self).__init__(engine, dao)
self.unhandle_fs_event = False
self._event_handler = None
self._windows_queue_threshold = 50
# Delay for the scheduled recursive scans of a created / modified / moved folder under Windows
self._windows_folder_scan_delay = 10000 # 10 seconds
self._windows_watchdog_event_buffer = 8192
self._windows = AbstractOSIntegration.is_windows()
if self._windows:
log.debug('Windows detected so delete event will be delayed by %dms', WIN_MOVE_RESOLUTION_PERIOD)
# TODO Review to delete
self._init()
def test_binding_initialization_and_first_sync(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Create some documents in a Nuxeo workspace and bind this server to a
# Nuxeo Drive local folder
self.make_server_tree()
# The root binding operation does not create the local folder yet.
self.assertFalse(local.exists('/'))
# Launch ndrive and check synchronization
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/'))
self.assertTrue(local.exists('/Folder 1'))
self.assertEqual(local.get_content('/Folder 1/File 1.txt'), "aaa")
self.assertTrue(local.exists('/Folder 1/Folder 1.1'))
self.assertEqual(local.get_content('/Folder 1/Folder 1.1/File 2.txt'), "bbb")
self.assertTrue(local.exists('/Folder 1/Folder 1.2'))
self.assertEqual(local.get_content('/Folder 1/Folder 1.2/File 3.txt'), "ccc")
self.assertTrue(local.exists('/Folder 2'))
# Cannot predict the resolution in advance
self.assertTrue(remote.get_content(self._duplicate_file_1), "Some content.")
self.assertTrue(remote.get_content(self._duplicate_file_2), "Other content.")
if local.duplication_enabled():
if local.get_content('/Folder 2/Duplicated File.txt') == "Some content.":
self.assertEqual(local.get_content('/Folder 2/Duplicated File__1.txt'), "Other content.")
else:
self.assertEqual(local.get_content('/Folder 2/Duplicated File.txt'), "Other content.")
self.assertEqual(local.get_content('/Folder 2/Duplicated File__1.txt'), "Some content.")
self.assertEqual(local.get_content('/Folder 2/File 4.txt'), "ddd")
self.assertEqual(local.get_content('/File 5.txt'), "eee")
# Unbind root and resynchronize
remote.unregister_as_root(self.workspace)
# Since errors are generated by the deletion events sent by Watchdog for the workspace children under UNIX,
# don't enforce errors and increase timeout
if not AbstractOSIntegration.is_windows():
timeout = 60
enforce_errors = False
else:
timeout = DEFAULT_WAIT_SYNC_TIMEOUT
enforce_errors = True
self.wait_sync(wait_for_async=True, timeout=timeout, enforce_errors=enforce_errors)
self.assertFalse(local.exists('/'))
def copy_with_xattr(src, dst):
"""
Custom copy tree method that also copies xattr along with
shutil.copytree functionality.
"""
if AbstractOSIntegration.is_windows():
# Make a copy of file1 using shell (to copy including xattr)
shell.SHFileOperation((0, shellcon.FO_COPY, src, dst,
shellcon.FOF_NOCONFIRMATION, None, None))
elif AbstractOSIntegration.is_mac():
fm = Cocoa.NSFileManager.defaultManager()
fm.copyItemAtPath_toPath_error_(src, dst, None)
else:
shutil.copy2(src, dst)
remote_id = xattr.getxattr(src, 'user.ndrive')
xattr.setxattr(dst, 'user.ndrive', remote_id)
def copy_with_xattr(src, dst):
"""
Custom copy tree method that also copies xattr along with
shutil.copytree functionality.
"""
if AbstractOSIntegration.is_windows():
# Make a copy of file1 using shell (to copy including xattr)
shell.SHFileOperation((0, shellcon.FO_COPY, src, dst,
shellcon.FOF_NOCONFIRMATION, None, None))
elif AbstractOSIntegration.is_mac():
fm = Cocoa.NSFileManager.defaultManager()
fm.copyItemAtPath_toPath_error_(src, dst, None)
else:
shutil.copy2(src, dst)
remote_id = xattr.getxattr(src, 'user.ndrive')
xattr.setxattr(dst, 'user.ndrive', remote_id)
def test_synchronize_windows_foldername_endswith_space(self):
"""
Use nuxeodrive.CreateFolder API to make a folder directly under the workspace "trial ".
Verify if the DS client downloads the folder and trims the space at the end
"""
top_level_children = self.remote_file_system_client_1.get_top_level_children(
)
target = self.remote_file_system_client_1.make_folder(
top_level_children[0]['id'], 'trial ')
self.remote_file_system_client_1.make_file(target.uid, 'aFile.txt',
u'File A Content')
self.remote_file_system_client_1.make_file(target.uid, 'bFile.txt',
u'File B Content')
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.assertTrue(
self.local_root_client_1.exists('/Nuxeo Drive Test Workspace'))
if AbstractOSIntegration.is_windows():
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial/'),
"Folder 'trial ' should be created without trailing space in the name"
)
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial/aFile.txt'),
"trial/aFile.txt should sync")
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial/bFile.txt'),
"trial/bFile.txt should sync")
else:
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial /'),
"Folder 'trial ' should be created with trailing space in the name"
)
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial /aFile.txt'),
"trial/aFile.txt should sync")
self.assertTrue(
self.local_root_client_1.exists(
'/Nuxeo Drive Test Workspace/trial /bFile.txt'),
"trial/bFile.txt should sync")
def test_complex_filenames(self):
"""
It should fail on Windows:
Explorer cannot find the directory as the path is way to long.
"""
if AbstractOSIntegration.is_windows():
try:
# IOError: [Errno 2] No such file or directory
with self.assertRaises(IOError):
super(TestLocalClientSimulation,
self).test_complex_filenames()
except AssertionError:
# Sometimes it does not raise the expected assertion ...
# TODO: More tests to know why.
pass
else:
super(TestLocalClientSimulation, self).test_complex_filenames()
def get_path_remote_id(path, name="ndrive"):
if AbstractOSIntegration.is_windows():
path = path + ":" + name
try:
with open(path, "r") as f:
return unicode(f.read(), 'utf-8')
except:
return None
else:
import xattr
try:
if AbstractOSIntegration.is_mac():
value = xattr.getxattr(path, name)
else:
value = xattr.getxattr(path, 'user.' + name)
return unicode(value, 'utf-8')
except:
return None
def get_path_remote_id(path, name="ndrive"):
if AbstractOSIntegration.is_windows():
path = path + ":" + name
try:
with open(path, "r") as f:
return unicode(f.read(), 'utf-8')
except:
return None
else:
import xattr
try:
if AbstractOSIntegration.is_mac():
value = xattr.getxattr(path, name)
else:
value = xattr.getxattr(path, 'user.' + name)
return unicode(value, 'utf-8')
except:
return None
def is_ignored(self, parent_ref, file_name):
# Add parent_ref to be able to filter on size if needed
ignore = False
# Office temp file
# http://support.microsoft.com/kb/211632
if file_name.startswith("~") and file_name.endswith(".tmp"):
return True
# Emacs auto save file
# http://www.emacswiki.org/emacs/AutoSave
if file_name.startswith("#") and file_name.endswith(
"#") and len(file_name) > 2:
return True
for suffix in self.ignored_suffixes:
if file_name.endswith(suffix):
ignore = True
break
for prefix in self.ignored_prefixes:
if file_name.startswith(prefix):
ignore = True
break
if ignore:
return True
if AbstractOSIntegration.is_windows():
# NXDRIVE-465
ref = self.get_children_ref(parent_ref, file_name)
path = self._abspath(ref)
if not os.path.exists(path):
return False
import win32con
import win32api
attrs = win32api.GetFileAttributes(path)
if attrs & win32con.FILE_ATTRIBUTE_SYSTEM == win32con.FILE_ATTRIBUTE_SYSTEM:
return True
if attrs & win32con.FILE_ATTRIBUTE_HIDDEN == win32con.FILE_ATTRIBUTE_HIDDEN:
return True
# NXDRIVE-655
# Need to check every parent if they are ignored
result = False
path = parent_ref
if path != '/':
file_name = os.path.basename(path)
path = os.path.dirname(path)
result = self.is_ignored(path, file_name)
return result
def test_synchronize_windows_foldername_endswith_space(self):
"""
Use nuxeodrive.CreateFolder API to make a folder directly under the workspace "trial ".
Verify if the DS client downloads the folder and trims the space at the end
"""
top_level_children = self.remote_file_system_client_1.get_top_level_children()
target = self.remote_file_system_client_1.make_folder(top_level_children[0]['id'], 'trial ')
self.remote_file_system_client_1.make_file(target.uid, 'aFile.txt', u'File A Content')
self.remote_file_system_client_1.make_file(target.uid, 'bFile.txt', u'File B Content')
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace'))
if AbstractOSIntegration.is_windows():
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial/'), "Folder 'trial ' should be created without trailing space in the name")
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial/aFile.txt'), "trial/aFile.txt should sync")
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial/bFile.txt'), "trial/bFile.txt should sync")
else:
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial /'), "Folder 'trial ' should be created with trailing space in the name")
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial /aFile.txt'), "trial/aFile.txt should sync")
self.assertTrue(self.local_root_client_1.exists('/Nuxeo Drive Test Workspace/trial /bFile.txt'), "trial/bFile.txt should sync")
def _set_root_icon(self):
local_client = self.get_local_client()
if not local_client.exists('/') or local_client.has_folder_icon('/'):
return
if AbstractOSIntegration.is_mac():
if AbstractOSIntegration.os_version_below("10.10"):
icon = find_icon("NuxeoDrive_Mac_Folder.dat")
else:
icon = find_icon("NuxeoDrive_Mac_Yosemite_Folder.dat")
elif AbstractOSIntegration.is_windows():
icon = find_icon("NuxeoDrive_Windows_Folder.ico")
else:
# No implementation on Linux
return
if icon is None:
return
locker = local_client.unlock_ref('/', unlock_parent=False)
try:
local_client.set_folder_icon('/', icon)
finally:
local_client.lock_ref('/', locker)
def get_path_remote_id(path, name="ndrive"):
if AbstractOSIntegration.is_windows():
path = path + ":" + name
try:
with open(path, "r") as f:
return f.read()
except:
return None
else:
import xattr
try:
if AbstractOSIntegration.is_mac():
value = xattr.getxattr(path, name)
else:
value = xattr.getxattr(path, "user." + name)
if type(value).__name__ == "unicode":
value = unicode(value)
return value
except:
return None
def set_remote_id(self, ref, remote_id, name="ndrive"):
if type(remote_id).__name__ == "unicode":
remote_id = unicodedata.normalize("NFC", remote_id).encode("utf-8")
# Can be move to another class
path = self._abspath(ref)
log.trace("Setting xattr %s with value %r on %r", name, remote_id, path)
locker = self.unlock_path(path, False)
if AbstractOSIntegration.is_windows():
pathAlt = path + ":" + name
try:
if not os.path.exists(path):
raise NotFound()
stat = os.stat(path)
with open(pathAlt, "w") as f:
f.write(remote_id)
# Avoid time modified change
os.utime(path, (stat.st_atime, stat.st_mtime))
except IOError as e:
# Should not happen
if e.errno == os.errno.EACCES:
self.unset_path_readonly(path)
with open(pathAlt, "w") as f:
f.write(remote_id)
self.set_path_readonly(path)
else:
raise e
finally:
self.lock_path(path, locker)
else:
try:
import xattr
stat = os.stat(path)
if AbstractOSIntegration.is_mac():
xattr.setxattr(path, name, remote_id)
else:
xattr.setxattr(path, "user." + name, remote_id)
os.utime(path, (stat.st_atime, stat.st_mtime))
finally:
self.lock_path(path, locker)
def set_remote_id(self, ref, remote_id, name='ndrive'):
if type(remote_id).__name__ == "unicode":
remote_id = unicodedata.normalize('NFC', remote_id).encode('utf-8')
# Can be move to another class
path = self._abspath(ref)
log.trace('Setting xattr %s with value %r on %r', name, remote_id,
path)
locker = self.unlock_path(path, False)
if AbstractOSIntegration.is_windows():
pathAlt = path + ":" + name
try:
if not os.path.exists(path):
raise NotFound()
stat = os.stat(path)
with open(pathAlt, "w") as f:
f.write(remote_id)
# Avoid time modified change
os.utime(path, (stat.st_atime, stat.st_mtime))
except IOError as e:
# Should not happen
if e.errno == os.errno.EACCES:
self.unset_path_readonly(path)
with open(pathAlt, "w") as f:
f.write(remote_id)
self.set_path_readonly(path)
else:
raise e
finally:
self.lock_path(path, locker)
else:
try:
import xattr
stat = os.stat(path)
if AbstractOSIntegration.is_mac():
xattr.setxattr(path, name, remote_id)
else:
xattr.setxattr(path, 'user.' + name, remote_id)
os.utime(path, (stat.st_atime, stat.st_mtime))
finally:
self.lock_path(path, locker)
def test_synchronization_local_watcher_paused_when_offline(self):
""" NXDRIVE-680: fix unwanted local upload when offline. """
local = self.local_client_1
remote = self.remote_document_client_1
engine = self.engine_1
# Create one file locally and wait for sync
engine.start()
self.wait_sync(wait_for_async=True)
local.make_file('/', 'file1.txt', content=b'42')
self.wait_sync()
# Checks
self.assertTrue(remote.exists('/file1.txt'))
self.assertTrue(local.exists('/file1.txt'))
# Simulate offline mode (no more network for instance)
engine.get_queue_manager().suspend()
# Create a bunch of files locally
local.make_folder('/', 'files')
for num in range(60 if AbstractOSIntegration.is_windows() else 20):
local.make_file('/files',
'file-' + str(num) + '.txt',
content=b'Content of file-' + str(num))
self.wait_sync(fail_if_timeout=False)
# Checks
self.assertEqual(len(remote.get_children_info(self.workspace_1)), 1)
self.assertTrue(engine.get_queue_manager().is_paused())
# Restore network connection
engine.get_queue_manager().resume()
# Wait for sync and check synced files
self.wait_sync(wait_for_async=True)
self.assertEqual(len(remote.get_children_info(self.workspace_1)), 2)
self.assertFalse(engine.get_queue_manager().is_paused())
def _set_root_icon(self):
local_client = self.get_local_client()
if local_client.has_folder_icon('/'):
return
if AbstractOSIntegration.is_mac():
if AbstractOSIntegration.os_version_below("10.10"):
icon = find_icon("NuxeoDrive_Mac_Folder.dat")
else:
icon = find_icon("NuxeoDrive_Mac_Yosemite_Folder.dat")
elif AbstractOSIntegration.is_windows():
if AbstractOSIntegration.os_version_below("5.2"):
icon = find_icon("NuxeoDrive_Windows_Xp_Folder.ico")
else:
icon = find_icon("NuxeoDrive_Windows_Folder.ico")
else:
# No implementation on Linux
return
locker = local_client.unlock_ref('/', unlock_parent=False)
try:
local_client.set_folder_icon('/', icon)
finally:
local_client.lock_ref('/', locker)
def test_untrash_file_with_rename(self):
self.local_client_1.make_file('/', 'File_To_Delete.txt',
'This is a content')
self.wait_sync()
self.assertTrue(
self.remote_document_client_1.exists('/File_To_Delete.txt'))
uid = self.local_client_1.get_remote_id('/File_To_Delete.txt')
old_info = self.remote_document_client_1.get_info(
'/File_To_Delete.txt', use_trash=True)
abs_path = self.local_client_1.abspath('/File_To_Delete.txt')
# Pretend we had trash the file
shutil.move(
abs_path,
os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'))
self.wait_sync(wait_for_async=True)
self.assertFalse(
self.remote_document_client_1.exists('/File_To_Delete.txt'))
self.assertFalse(self.local_client_1.exists('/File_To_Delete.txt'))
with open(
os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'),
'w') as f:
f.write('New content')
if AbstractOSIntegration.is_windows():
# Python API overwrite the tag by default
with open(
os.path.join(self.local_test_folder_1,
'File_To_Delete2.txt:ndrive'), 'w') as f:
f.write(uid)
# See if it untrash or recreate
shutil.move(
os.path.join(self.local_test_folder_1, 'File_To_Delete2.txt'),
self.local_client_1.abspath('/'))
self.wait_sync(wait_for_async=True)
self.assertTrue(self.remote_document_client_1.exists(old_info.uid))
self.assertTrue(self.local_client_1.exists('/File_To_Delete2.txt'))
self.assertFalse(self.local_client_1.exists('/File_To_Delete.txt'))
self.assertEqual(
self.local_client_1.get_content('/File_To_Delete2.txt'),
'New content')
def ignore_hidden(self, parent, name, **kwargs):
if AbstractOSIntegration.is_windows():
# 'self' here represents the context where the function is called and is used to facilitate its work; use
# 'duck-typing' to check for the right context.
# skip if called with a context which cannot compute the absolute path, e.g. missing,
# not derived from LocalClient, etc.
if self is None or not hasattr(self, '_abspath') or not hasattr(self, 'get_children_ref'):
return False
# NXDRIVE 465
ref = self.get_children_ref(parent, name)
path = self._abspath(ref)
if not os.path.exists(path):
return False
import win32con
import win32api
attrs = win32api.GetFileAttributes(path)
if attrs & win32con.FILE_ATTRIBUTE_SYSTEM == win32con.FILE_ATTRIBUTE_SYSTEM:
return True
if attrs & win32con.FILE_ATTRIBUTE_HIDDEN == win32con.FILE_ATTRIBUTE_HIDDEN:
return True
return False
class TestWatchers(UnitTestCase):
def get_local_client(self, path):
if self._testMethodName in ('test_local_scan_encoding',
'test_watchdog_encoding'):
return LocalClient(path)
return super(TestWatchers, self).get_local_client(path)
def test_local_scan(self):
files, folders = self.make_local_tree()
self.queue_manager_1.suspend()
self.queue_manager_1._disable = True
self.engine_1.start()
self.wait_remote_scan()
# Workspace should have been reconcile
res = self.engine_1.get_dao().get_states_from_partial_local('/')
# With root
self.assertEqual(len(res), folders + files + 1)
@RandomBug('NXDRIVE-808', target='windows', repeat=2)
def test_reconcile_scan(self):
files, folders = self.make_local_tree()
self.make_server_tree()
# Wait for ES indexing
self.wait()
self.queue_manager_1.suspend()
self.queue_manager_1._disable = True
self.engine_1.start()
self.wait_remote_scan()
# Depending on remote scan results order, the remote duplicated file with the same digest as the local file
# might come first, in which case we get an extra synchronized file,
# or not, in which case we get a conflicted file
self.assertTrue(
self.engine_1.get_dao().get_sync_count() >= folders + files)
# Verify it has been reconciled and all items in queue are synchronized
queue = self.get_full_queue(
self.queue_manager_1.get_local_file_queue())
for item in queue:
if item.remote_name == 'Duplicated File.txt':
self.assertTrue(
item.pair_state in ["synchronized", "conflicted"])
else:
self.assertEqual(item.pair_state, "synchronized")
queue = self.get_full_queue(
self.queue_manager_1.get_local_folder_queue())
for item in queue:
self.assertEqual(item.pair_state, "synchronized")
def test_remote_scan(self):
files, folders = self.make_server_tree()
# Wait for ES indexing
self.wait()
# Add the workspace folder
folders += 1
self.queue_manager_1.suspend()
self.queue_manager_1._disable = True
self.engine_1.start()
self.wait_remote_scan()
res = self.engine_1.get_dao().get_states_from_partial_local('/')
# With root
self.assertEqual(len(res), folders + files + 1)
@RandomBug('NXDRIVE-806', target='linux', mode='BYPASS')
@RandomBug('NXDRIVE-806', target='windows', repeat=2)
def test_local_watchdog_creation(self):
# Test the creation after first local scan
self.queue_manager_1.suspend()
self.queue_manager_1._disable = True
self.engine_1.start()
self.wait_remote_scan()
metrics = self.queue_manager_1.get_metrics()
self.assertEqual(metrics["local_folder_queue"], 0)
self.assertEqual(metrics["local_file_queue"], 0)
files, folders = self.make_local_tree()
self.wait_sync(timeout=3, fail_if_timeout=False)
metrics = self.queue_manager_1.get_metrics()
self.assertNotEquals(metrics["local_folder_queue"], 0)
self.assertNotEquals(metrics["local_file_queue"], 0)
res = self.engine_1.get_dao().get_states_from_partial_local('/')
# With root
self.assertEqual(len(res), folders + files + 1)
def _delete_folder_1(self):
from time import sleep
path = '/Folder 1'
self.local_client_1.delete_final(path)
if sys.platform == 'win32':
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
sleep(WIN_MOVE_RESOLUTION_PERIOD / 1000 + 1)
self.wait_sync(timeout=1, fail_if_timeout=False)
timeout = 5
while (not self.engine_1.get_local_watcher().empty_events()):
sleep(1)
timeout -= 1
if timeout < 0:
break
return '/' + self.workspace_title + path + '/'
def test_local_watchdog_delete_non_synced(self):
# Test the deletion after first local scan
self.test_local_scan()
path = self._delete_folder_1()
children = self.engine_1.get_dao().get_states_from_partial_local(path)
self.assertEqual(len(children), 0)
def test_local_scan_delete_non_synced(self):
# Test the deletion after first local scan
self.test_local_scan()
self.engine_1.stop()
path = self._delete_folder_1()
self.engine_1.start()
self.wait_sync(timeout=5, fail_if_timeout=False)
children = self.engine_1.get_dao().get_states_from_partial_local(path)
self.assertEqual(len(children), 0)
def test_local_watchdog_delete_synced(self):
# Test the deletion after first local scan
self.test_reconcile_scan()
path = self._delete_folder_1()
child = self.engine_1.get_dao().get_state_from_local(path[:-1])
self.assertEqual(child.pair_state, 'locally_deleted')
children = self.engine_1.get_dao().get_states_from_partial_local(path)
self.assertEqual(len(children), 5)
for child in children:
self.assertEqual(child.pair_state, 'parent_locally_deleted')
def test_local_scan_delete_synced(self):
# Test the deletion after first local scan
self.test_reconcile_scan()
self.engine_1.stop()
path = self._delete_folder_1()
self.engine_1.start()
self.wait_sync(timeout=5, fail_if_timeout=False)
child = self.engine_1.get_dao().get_state_from_local(path[:-1])
self.assertEqual(child.pair_state, 'locally_deleted')
children = self.engine_1.get_dao().get_states_from_partial_local(path)
self.assertEqual(len(children), 5)
for child in children:
self.assertEqual(child.pair_state, 'parent_locally_deleted')
def test_local_scan_error(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Synchronize test workspace
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
# Create a local file and use an invalid digest function in local watcher file system client to trigger an error
# during local scan
local.make_file('/', u'Test file.odt', 'Content')
def get_local_client():
return LocalClient(self.local_nxdrive_folder_1,
digest_func='invalid')
original_getter = self.engine_1.get_local_client
self.engine_1.get_local_client = get_local_client
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
self.assertFalse(remote.exists(u'/Test file.odt'))
# Set back original local watcher file system client, launch local scan and check upstream synchronization
self.engine_1.get_local_client = original_getter
self.engine_1.start()
self.wait_sync()
self.assertTrue(remote.exists(u'/Test file.odt'))
def test_local_scan_encoding(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Synchronize test workspace
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
# Create files with Unicode combining accents, Unicode latin characters and no special characters
local.make_file(u'/', u'Accentue\u0301.odt', u'Content')
local.make_folder(u'/', u'P\xf4le applicatif')
local.make_file(u'/P\xf4le applicatif',
u'e\u0302tre ou ne pas \xeatre.odt', u'Content')
local.make_file(u'/', u'No special character.odt', u'Content')
# Launch local scan and check upstream synchronization
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
self.assertTrue(remote.exists(u'/Accentue\u0301.odt'))
self.assertTrue(remote.exists(u'/P\xf4le applicatif'))
self.assertTrue(
remote.exists(
u'/P\xf4le applicatif/e\u0302tre ou ne pas \xeatre.odt'))
self.assertTrue(remote.exists(u'/No special character.odt'))
# Check rename using normalized names as previous local scan has normalized them on the file system
local.rename(u'/Accentu\xe9.odt',
u'Accentue\u0301 avec un e\u0302 et un \xe9.odt')
local.rename(u'/P\xf4le applicatif', u'P\xf4le applique\u0301')
# LocalClient.rename calls LocalClient.get_info then the FileInfo constructor which normalizes names
# on the file system, thus we need to use the normalized name for the parent folder
local.rename(u'/P\xf4le appliqu\xe9/\xeatre ou ne pas \xeatre.odt',
u'avoir et e\u0302tre.odt')
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.engine_1.stop()
self.assertEqual(
remote.get_info(u'/Accentue\u0301.odt').name,
u'Accentu\xe9 avec un \xea et un \xe9.odt')
self.assertEqual(
remote.get_info(u'/P\xf4le applicatif').name,
u'P\xf4le appliqu\xe9')
self.assertEqual(
remote.get_info(
u'/P\xf4le applicatif/e\u0302tre ou ne pas \xeatre.odt').name,
u'avoir et \xeatre.odt')
# Check content update
# NXDRIVE-389: Reload the engine to be sure that the pair are all synchronized
log.debug("Update content of avoir et etre")
local.update_content(u'/Accentu\xe9 avec un \xea et un \xe9.odt',
u'Updated content')
local.update_content(u'/P\xf4le appliqu\xe9/avoir et \xeatre.odt',
u'Updated content')
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
self.assertEqual(remote.get_content(u'/Accentue\u0301.odt'),
u'Updated content')
# NXDRIVE-389: Will be Content and not Updated content
# it is not consider as synced, so conflict is generated
self.assertEqual(
remote.get_content(
u'/P\xf4le applicatif/e\u0302tre ou ne pas \xeatre.odt'),
u'Updated content')
# Check delete
local.delete_final(u'/Accentu\xe9 avec un \xea et un \xe9.odt')
local.delete_final(u'/P\xf4le appliqu\xe9/avoir et \xeatre.odt')
self.engine_1.start()
self.wait_sync()
self.engine_1.stop()
self.assertFalse(remote.exists(u'/Accentue\u0301.odt'))
self.assertFalse(
remote.exists(
u'/P\xf4le applicatif/e\u0302tre ou ne pas \xeatre.odt'))
@skipIf(AbstractOSIntegration.is_windows(),
'Windows cannot have file ending with a space.')
def test_watchdog_space_remover(self):
"""
Test files and folders ending with space.
"""
local = self.local_client_1
remote = self.remote_document_client_1
self.engine_1.start()
self.wait_sync()
local.make_file(u'/', u'Accentue\u0301.odt ', u'Content')
self.wait_sync()
self.assertTrue(remote.exists(u'/Accentue\u0301.odt'))
self.assertFalse(remote.exists(u'/Accentue\u0301.odt '))
local.rename(u'/Accentu\xe9.odt',
u'Accentu\xe9 avec un \xea et un \xe9.odt ')
self.wait_sync()
self.assertEqual(
remote.get_info(u'/Accentue\u0301.odt').name,
u'Accentu\xe9 avec un \xea et un \xe9.odt')
@RandomBug('NXDRIVE-808', target='mac', repeat=5)
def test_watchdog_encoding(self):
local = self.local_client_1
remote = self.remote_document_client_1
self.engine_1.start()
self.wait_sync()
# Create files with Unicode combining accents, Unicode latin characters
# and no special characters
local.make_file(u'/', u'Accentue\u0301.odt', u'Content')
local.make_folder(u'/', u'P\xf4le applicatif')
local.make_folder(u'/', u'Sub folder')
local.make_file(u'/Sub folder', u'e\u0302tre ou ne pas \xeatre.odt',
u'Content')
local.make_file(u'/', u'No special character.odt', u'Content')
self.wait_sync()
self.assertTrue(remote.exists(u'/Accentue\u0301.odt'))
self.assertTrue(remote.exists(u'/P\xf4le applicatif'))
self.assertTrue(remote.exists(u'/Sub folder'))
self.assertTrue(
remote.exists(u'/Sub folder/e\u0302tre ou ne pas \xeatre.odt'))
self.assertTrue(remote.exists(u'/No special character.odt'))
# Check rename using normalized names as previous watchdog handling has
# normalized them on the file system
local.rename(u'/Accentu\xe9.odt',
u'Accentue\u0301 avec un e\u0302 et un \xe9.odt')
local.rename(u'/P\xf4le applicatif', u'P\xf4le applique\u0301')
local.rename(u'/Sub folder/\xeatre ou ne pas \xeatre.odt',
u'avoir et e\u0302tre.odt')
self.wait_sync()
self.assertEqual(
remote.get_info(u'/Accentue\u0301.odt').name,
u'Accentu\xe9 avec un \xea et un \xe9.odt')
self.assertEqual(
remote.get_info(u'/P\xf4le applicatif').name,
u'P\xf4le appliqu\xe9')
info = remote.get_info(u'/Sub folder/e\u0302tre ou ne pas \xeatre.odt')
self.assertEqual(info.name, u'avoir et \xeatre.odt')
# Check content update
local.update_content(u'/Accentu\xe9 avec un \xea et un \xe9.odt',
u'Updated content')
local.update_content(u'/Sub folder/avoir et \xeatre.odt',
u'Updated content')
self.wait_sync()
self.assertEqual(remote.get_content(u'/Accentue\u0301.odt'),
u'Updated content')
content = remote.get_content(
u'/Sub folder/e\u0302tre ou ne pas \xeatre.odt')
self.assertEqual(content, u'Updated content')
# Check delete
local.delete_final(u'/Accentu\xe9 avec un \xea et un \xe9.odt')
local.delete_final(u'/Sub folder/avoir et \xeatre.odt')
self.wait_sync()
self.assertFalse(remote.exists(u'/Accentue\u0301.odt'))
self.assertFalse(
remote.exists(u'/Sub folder/e\u0302tre ou ne pas \xeatre.odt'))
@RandomBug('NXDRIVE-808', target='windows', repeat=2)
def test_watcher_remote_id_setter(self):
local = self.local_client_1
# As some user can rewrite same file for no reason
# Start engine
self.engine_1.start()
# Wait for test workspace synchronization
self.wait_sync()
# Create files with Unicode combining accents, Unicode latin characters and no special characters
file_path = local.abspath('/Test.pdf')
copyfile(self.location + '/resources/testFile.pdf', file_path)
# Wait for test workspace synchronization
self.wait_sync()
remote_id = local.get_remote_id('/Test.pdf')
copyfile(self.location + '/resources/testFile.pdf', file_path)
self.wait_sync()
self.assertEqual(remote_id, local.get_remote_id('/Test.pdf'),
"Should have the remote id")
def test_watcher_remote_id_setter_stopped(self):
local = self.local_client_1
# As some user can rewrite same file for no reason
# Start engine
self.engine_1.start()
# Wait for test workspace synchronization
self.wait_sync()
# Create files with Unicode combining accents, Unicode latin characters and no special characters
file_path = local.abspath('/Test.pdf')
copyfile(self.location + '/resources/testFile.pdf', file_path)
# Wait for test workspace synchronization
self.engine_1.stop()
remote_id = local.get_remote_id('/Test.pdf')
copyfile(self.location + '/resources/testFile.pdf', file_path)
self.engine_1.start()
self.assertEqual(remote_id, local.get_remote_id('/Test.pdf'),
"Should have the remote id")
class TestReadOnly(UnitTestCase):
def setUp(self):
super(TestReadOnly, self).setUp()
self.engine_1.start()
self.wait_sync(wait_for_async=True)
def _set_readonly_permission(self, user, doc_path, grant):
op_input = "doc:" + doc_path
if grant:
self.root_remote_client.execute("Document.SetACE", op_input=op_input, user=user, permission="Read")
self.root_remote_client.block_inheritance(doc_path,
overwrite=False)
else:
self.root_remote_client.execute("Document.SetACE", op_input=op_input, user=user, permission="Write",
grant="true")
def test_rename_readonly_file(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Create documents in the remote root workspace
# then synchronize
remote.make_folder('/', 'Test folder')
remote.make_file('/Test folder', 'joe.odt', 'Some content')
remote.make_file('/Test folder', 'jack.odt', 'Some content')
remote.make_folder('/Test folder', 'Sub folder 1')
remote.make_file('/Test folder/Sub folder 1', 'sub file 1.txt',
'Content')
self._set_readonly_permission(self.user_1, TEST_WORKSPACE_PATH + '/Test folder', True)
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/Test folder'))
self.assertTrue(local.exists('/Test folder/joe.odt'))
self.assertTrue(local.exists('/Test folder/jack.odt'))
self.assertTrue(local.exists('/Test folder/Sub folder 1'))
self.assertTrue(local.exists('/Test folder/Sub folder 1/sub file 1.txt'))
# Local changes
time.sleep(OS_STAT_MTIME_RESOLUTION)
# Create new file
# Fake the readonly forcing
local.unset_readonly('/Test folder')
local.make_file('/Test folder', 'local.odt', 'New local content')
# Create new folder with files
local.make_folder('/Test folder', 'Local sub folder 2')
local.make_file('/Test folder/Local sub folder 2', 'local sub file 2.txt', 'Other local content')
# Update file
local.unset_readonly('/Test folder/joe.odt')
local.update_content('/Test folder/joe.odt', 'Some locally updated content')
local.set_readonly('/Test folder/joe.odt')
local.set_readonly('/Test folder')
# TODO Might rollback if rollback only !
self.wait_sync()
self.assertFalse(remote.exists('/Test folder/local.odt'))
self.assertFalse(remote.exists('/Test folder/Local sub folder 2'))
self.assertFalse(remote.exists('/Test folder/Local sub folder 2/local sub file 2.txt'))
self.assertTrue(local.exists('/Test folder/local.odt'))
self.assertEqual(remote.get_content('/Test folder/joe.odt'), 'Some content')
def touch(self, fname):
try:
with open(fname, 'w') as f:
f.write('Test')
except Exception as e:
log.debug('Exception occurs during touch: %r', e)
return False
return True
@skipIf(AbstractOSIntegration.is_windows(),
'Readonly folder let new file creation')
def test_readonly_user_access(self):
# Should not be able to create content in root folder
fname = os.path.join(self.local_nxdrive_folder_1, 'test.txt')
self.assertFalse(self.touch(fname), "Should not be able to create in ROOT folder")
fname = os.path.join(self.sync_root_folder_1, 'test.txt')
self.assertTrue(self.touch(fname), "Should be able to create in SYNCROOT folder")
fname = os.path.join(self.sync_root_folder_1, 'Test folder', 'test.txt')
self.assertFalse(self.touch(fname), "Should be able to create in SYNCROOT folder")
fname = os.path.join(self.sync_root_folder_1, 'Test folder', 'Sub folder 1', 'test.txt')
self.assertFalse(self.touch(fname), "Should be able to create in SYNCROOT folder")
@skipIf(AbstractOSIntegration.is_windows(),
'Readonly folder let new file creation')
@RandomBug('NXDRIVE-816', target='mac', mode='BYPASS')
def test_file_readonly_change(self):
local = self.local_client_1
remote = self.remote_document_client_1
# Create documents in the remote root workspace
# then synchronize
remote.make_folder('/', 'Test folder')
remote.make_file('/Test folder', 'joe.odt', 'Some content')
remote.make_file('/Test folder', 'jack.odt', 'Some content')
remote.make_folder('/Test folder', 'Sub folder 1')
remote.make_file('/Test folder/Sub folder 1', 'sub file 1.txt',
'Content')
self._set_readonly_permission(self.user_1, TEST_WORKSPACE_PATH + '/Test folder', True)
self.wait_sync(wait_for_async=True)
self.assertTrue(local.exists('/Test folder'))
self.assertTrue(local.exists('/Test folder/joe.odt'))
self.assertTrue(local.exists('/Test folder/jack.odt'))
self.assertTrue(local.exists('/Test folder/Sub folder 1'))
self.assertTrue(local.exists('/Test folder/Sub folder 1/sub file 1.txt'))
# Update the content on the server
self.root_remote_client.update_content(TEST_WORKSPACE_PATH + '/Test folder/joe.odt',
'Some remotely updated content', 'joe.odt')
self.wait_sync(wait_for_async=True)
self.assertTrue(local.get_content('/Test folder/joe.odt'), 'Some remotely updated content')
# Remove the readonly
self._set_readonly_permission(self.user_1, TEST_WORKSPACE_PATH + '/Test folder', False)
self.wait_sync(wait_for_async=True)
fname = os.path.join(self.sync_root_folder_1, 'Test folder', 'test.txt')
fname2 = os.path.join(self.sync_root_folder_1, 'Test folder', 'Sub folder 1', 'test.txt')
# Check it works
self.assertTrue(self.touch(fname))
self.assertTrue(self.touch(fname2))
# First remove the files
os.remove(fname)
os.remove(fname2)
# Put it back readonly
self._set_readonly_permission(self.user_1, TEST_WORKSPACE_PATH + '/Test folder', True)
self.wait_sync(wait_for_async=True)
# Check it works
self.assertFalse(self.touch(fname))
self.assertFalse(self.touch(fname2))
def test_locked_document(self):
remote = self.remote_document_client_1
remote.make_folder('/', 'Test locking')
remote.make_file('/Test locking', 'myDoc.odt', 'Some content')
self.wait_sync(wait_for_async=True)
# Check readonly flag is not set for a document that isn't locked
user1_file_path = os.path.join(self.sync_root_folder_1, 'Test locking', 'myDoc.odt')
self.assertTrue(os.path.exists(user1_file_path))
self.assertTrue(self.touch(user1_file_path))
self.wait_sync()
# Check readonly flag is not set for a document locked by the current user
remote.lock('/Test locking/myDoc.odt')
self.wait_sync(wait_for_async=True)
self.assertTrue(self.touch(user1_file_path))
remote.unlock('/Test locking/myDoc.odt')
self.wait_sync(wait_for_async=True)
# Check readonly flag is set for a document locked by another user
self.remote_document_client_2.lock('/Test locking/myDoc.odt')
self.wait_sync(wait_for_async=True)
self.assertFalse(self.touch(user1_file_path))
# Check readonly flag is unset for a document unlocked by another user
self.remote_document_client_2.unlock('/Test locking/myDoc.odt')
self.wait_sync(wait_for_async=True)
self.assertTrue(self.touch(user1_file_path))
def test_local_readonly_modify(self):
local = self.local_client_1
local.make_folder('/', 'Test')
local.make_file('/Test', 'Test.txt', 'Some content')
self.wait_sync()
self.engine_1.stop()
local.update_content('/Test/Test.txt', 'Another content')
self.engine_1.start()
self.wait_sync()
self.assertEqual(len(self.engine_1.get_dao().get_errors()), 0)
# coding: utf-8
""""
Test if changes made to local file system when Drive is offline sync's back
later when Drive becomes online.
"""
import shutil
from nxdrive.osi import AbstractOSIntegration
from tests.common_unit_test import FILE_CONTENT, UnitTestCase
if AbstractOSIntegration.is_windows():
from win32com.shell import shell, shellcon
else:
import xattr
if AbstractOSIntegration.is_mac():
import Cocoa
class TestOfflineChangesSync(UnitTestCase):
"""
All changes made in local PC while drive is offline should sync later when
drive comes back online.
offline can be one of: suspended, exited or disconnect
See NXDRIVE-686
"""
def setUp(self):
self.engine_1.start()
self.wait_sync(wait_for_async=True)
def test_XLS_conflict_on_locked_document_from_start(self):
if not AbstractOSIntegration.is_windows():
raise SkipTest("Only makes sense under Windows")
self._XLS_local_update_on_locked_document()
def empty_events(self):
return self._watchdog_queue.empty() and ( not AbstractOSIntegration.is_windows() or
self.win_queue_empty() and self.win_folder_scan_empty())
def test_XLS_conflict_on_locked_document_from_start(self):
if not AbstractOSIntegration.is_windows():
raise SkipTest("Only makes sense under Windows")
self._XLS_local_update_on_locked_document()
def test_XLS_conflict_on_locked_document_from_start(self):
if not AbstractOSIntegration.is_windows():
raise SkipTest("Windows Office only test")
self._XLS_local_update_on_locked_document()
# coding: utf-8
""""
Test if changes made to local file system when Drive is offline sync's back
later when Drive becomes online.
"""
import shutil
from nxdrive.osi import AbstractOSIntegration
from tests.common_unit_test import FILE_CONTENT, UnitTestCase
if AbstractOSIntegration.is_windows():
from win32com.shell import shell, shellcon
else:
import xattr
if AbstractOSIntegration.is_mac():
import Cocoa
class TestOfflineChangesSync(UnitTestCase):
"""
All changes made in local PC while drive is offline should sync later when
drive comes back online.
offline can be one of: suspended, exited or disconnect
See NXDRIVE-686
"""
def setUp(self):
self.engine_1.start()
self.wait_sync(wait_for_async=True)
self.local = self.local_client_1
