def clean_folder(self, options):
from nxdrive.client.local_client import LocalClient
if options.local_folder is None:
print "A folder must be specified"
return 0
client = LocalClient(unicode(options.local_folder))
client.clean_xattr_root()
return 0
def clean_folder(self, options):
from nxdrive.client.local_client import LocalClient
if options.local_folder is None:
print('A folder must be specified')
return 0
client = LocalClient(unicode(options.local_folder))
client.clean_xattr_root()
return 0
def test_set_get_xattr_file_not_found():
"""If file is not found, there is no value to retrieve."""
file = pathlib.Path("file-no-found.txt")
# This call should not fail
LocalClient.set_path_remote_id(file, "something")
# And this one should return an empty string
assert LocalClient.get_path_remote_id(file) == ""
def test_get_path(tmp):
folder = tmp()
folder.mkdir()
local = LocalClient(folder)
path = folder / "foo.txt"
path_upper = folder / "FOO.TXT"
# The path does not exist, it returns ROOT
assert local.get_path(pathlib.Path("bar.doc")) == ROOT
# The path exists, it returns
assert local.get_path(path) == pathlib.Path("foo.txt")
assert local.get_path(path_upper) == pathlib.Path("FOO.TXT")
def test_set_get_xattr_invalid_start_byte(tmp):
"""
Ensure this will never happen again:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 8: invalid start byte
"""
folder = tmp()
folder.mkdir()
file = folder / "test-xattr.txt"
file.write_text("bla" * 3)
raw_value, result_needed = b"fdrpMACS\x80", "fdrpMACS"
LocalClient.set_path_remote_id(file, raw_value)
assert LocalClient.get_path_remote_id(file) == result_needed
def test_get_path_permission_error(mocked_resolve, mocked_absolute, tmp):
folder = tmp()
folder.mkdir()
local = LocalClient(folder)
path = folder / "foo.txt"
# Path.resolve() raises a PermissionError, it should fallback on .absolute()
mocked_resolve.side_effect = PermissionError()
path_abs = local.get_path(path)
assert mocked_absolute.called
# Restore the original ehavior and check that .resolved() and .absolute()
# return the same value.
mocked_resolve.reset_mock()
assert local.get_path(path) == path_abs
def __init__(self, manager, folder, url):
super(DirectEdit, self).__init__()
self._test = False
self._manager = manager
self._url = url
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if isinstance(folder, bytes):
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._lock_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
self._manager.get_autolock_service().orphanLocks.connect(
self._autolock_orphans)
self._last_action_timing = -1
def __init__(self, manager, folder):
'''
Constructor
'''
super(DriveEdit, self).__init__()
self._manager = manager
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if type(folder) == str:
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
def __init__(self, manager, folder, url):
super(DirectEdit, self).__init__()
self._test = False
self._manager = manager
self._url = url
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if isinstance(folder, bytes):
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._lock_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
self._manager.get_autolock_service().orphanLocks.connect(self._autolock_orphans)
self._last_action_timing = -1
class DirectEdit(Worker):
localScanFinished = pyqtSignal()
directEditUploadCompleted = pyqtSignal()
openDocument = pyqtSignal(object)
editDocument = pyqtSignal(object)
directEditLockError = pyqtSignal(str, str, str)
directEditConflict = pyqtSignal(str, str, str)
'''
classdocs
'''
def __init__(self, manager, folder, url):
'''
Constructor
'''
super(DirectEdit, self).__init__()
self._manager = manager
self._url = url
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if type(folder) == str:
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._lock_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
self._manager.get_autolock_service().orphanLocks.connect(
self._autolock_orphans)
self._last_action_timing = -1
@pyqtSlot(object)
def _autolock_orphans(self, locks):
log.trace("Orphans lock: %r", locks)
for lock in locks:
if lock.path.startswith(self._folder):
log.debug("Should unlock: %s", lock.path)
ref = self._local_client.get_path(lock.path)
self._lock_queue.put((ref, 'unlock_orphan'))
def autolock_lock(self, src_path):
ref = self._local_client.get_path(src_path)
self._lock_queue.put((ref, 'lock'))
def autolock_unlock(self, src_path):
ref = self._local_client.get_path(src_path)
self._lock_queue.put((ref, 'unlock'))
def start(self):
self._stop = False
super(DirectEdit, self).start()
def stop(self):
super(DirectEdit, self).stop()
self._stop = True
def stop_client(self, reason):
if self._stop:
raise ThreadInterrupt
def handle_url(self, url=None):
if url is None:
url = self._url
if url is None:
return
log.debug("DirectEdit load: '%r'", url)
try:
info = parse_protocol_url(str(url))
except UnicodeEncodeError:
# Firefox seems to be different on the encoding part
info = parse_protocol_url(unicode(url))
if info is None:
return
# Handle backward compatibility
if info.get('item_id') is not None:
self.edit(info['server_url'], info['item_id'])
else:
self.edit(info['server_url'],
info['doc_id'],
filename=info['filename'],
user=info['user'],
download_url=info['download_url'])
def _cleanup(self):
log.debug("Cleanup DirectEdit folder")
# Should unlock any remaining doc that has not been unlocked or ask
if self._local_client.exists('/'):
for child in self._local_client.get_children_info('/'):
if self._local_client.get_remote_id(
child.path, "nxdirecteditlock") is not None:
continue
children = self._local_client.get_children_info(child.path)
if len(children) > 1:
log.warn("Cannot clean this document")
continue
if (len(children) == 0):
# Cleaning the folder it is empty
shutil.rmtree(self._local_client._abspath(child.path),
ignore_errors=True)
continue
ref = children[0].path
uid, engine, remote_client, digest_algorithm, digest = self._extract_edit_info(
ref)
# Don't update if digest are the same
info = self._local_client.get_info(ref)
try:
current_digest = info.get_digest(
digest_func=digest_algorithm)
if (current_digest != digest):
log.warn(
"Document has been modified and not synchronized, readd to upload queue"
)
self._upload_queue.put(ref)
continue
except Exception as e:
log.debug(e)
continue
# Place for handle reopened of interrupted Edit
shutil.rmtree(self._local_client._abspath(child.path),
ignore_errors=True)
if not os.path.exists(self._folder):
os.mkdir(self._folder)
def _get_engine(self, url, user=None):
if url is None:
return None
if url.endswith('/'):
url = url[:-1]
# Simplify port if possible
if url.startswith('http:') and ':80/' in url:
url = url.replace(':80/', '/')
if url.startswith('https:') and ':443/' in url:
url = url.replace(':443/', '/')
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and (user is None or user == bind.username):
return engine
# Some backend are case insensitive
if user is None:
return None
user = user.lower()
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
# Simplify port if possible
if server_url.startswith('http:') and ':80/' in server_url:
server_url = server_url.replace(':80/', '/')
if server_url.startswith('https:') and ':443/' in server_url:
server_url = server_url.replace(':443/', '/')
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and user == bind.username.lower():
return engine
return None
def _download_content(self,
engine,
remote_client,
info,
file_path,
url=None):
file_dir = os.path.dirname(file_path)
file_name = os.path.basename(file_path)
file_out = os.path.join(
file_dir,
DOWNLOAD_TMP_FILE_PREFIX + file_name + DOWNLOAD_TMP_FILE_SUFFIX)
# Close to processor method - should try to refactor ?
pair = engine.get_dao().get_valid_duplicate_file(info.digest)
if pair:
local_client = engine.get_local_client()
existing_file_path = local_client._abspath(pair.local_path)
log.debug(
'Local file matches remote digest %r, copying it from %r',
info.digest, existing_file_path)
shutil.copy(existing_file_path, file_out)
if pair.is_readonly():
log.debug('Unsetting readonly flag on copied file %r',
file_out)
from nxdrive.client.common import BaseClient
BaseClient.unset_path_readonly(file_out)
else:
log.debug('Downloading file %r', info.filename)
if url is not None:
remote_client.do_get(url,
file_out=file_out,
digest=info.digest,
digest_algorithm=info.digest_algorithm)
else:
remote_client.get_blob(info, file_out=file_out)
return file_out
def _display_modal(self, message, values=None):
from nxdrive.wui.application import SimpleApplication
from nxdrive.wui.modal import WebModal
app = SimpleApplication(self._manager, None, {})
dialog = WebModal(app, app.translate(message, values))
dialog.add_button("OK", app.translate("OK"))
dialog.show()
app.exec_()
def _prepare_edit(self, server_url, doc_id, user=None, download_url=None):
start_time = current_milli_time()
engine = self._get_engine(server_url, user=user)
if engine is None:
values = dict()
if user is None:
values['user'] = '******'
else:
values['user'] = user
values['server'] = server_url
log.warn("No engine found for server_url=%s, user=%s, doc_id=%s",
server_url, user, doc_id)
self._display_modal("DIRECT_EDIT_CANT_FIND_ENGINE", values)
return
# Get document info
remote_client = engine.get_remote_doc_client()
# Avoid any link with the engine, remote_doc are not cached so we can do that
remote_client.check_suspended = self.stop_client
info = remote_client.get_info(doc_id)
filename = info.filename
# Create local structure
dir_path = os.path.join(self._folder, doc_id)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
log.debug("Editing %r", filename)
file_path = os.path.join(dir_path, filename)
# Download the file
url = None
if download_url is not None:
url = server_url
if not url.endswith('/'):
url += '/'
url += download_url
tmp_file = self._download_content(engine,
remote_client,
info,
file_path,
url=url)
if tmp_file is None:
log.debug("Download failed")
return
# Set the remote_id
dir_path = self._local_client.get_path(os.path.dirname(file_path))
self._local_client.set_remote_id(dir_path, doc_id)
self._local_client.set_remote_id(dir_path, server_url, "nxdirectedit")
if user is not None:
self._local_client.set_remote_id(dir_path, user,
"nxdirectedituser")
if info.digest is not None:
self._local_client.set_remote_id(dir_path, info.digest,
"nxdirecteditdigest")
# Set digest algorithm if not sent by the server
digest_algorithm = info.digest_algorithm
if digest_algorithm is None:
digest_algorithm = guess_digest_algorithm(info.digest)
self._local_client.set_remote_id(dir_path, digest_algorithm,
"nxdirecteditdigestalgorithm")
self._local_client.set_remote_id(dir_path, filename,
"nxdirecteditname")
# Rename to final filename
# Under Windows first need to delete target file if exists, otherwise will get a 183 WindowsError
if sys.platform == 'win32' and os.path.exists(file_path):
os.unlink(file_path)
os.rename(tmp_file, file_path)
self._last_action_timing = current_milli_time() - start_time
self.openDocument.emit(info)
return file_path
def edit(self,
server_url,
doc_id,
filename=None,
user=None,
download_url=None):
try:
# Handle backward compatibility
if '#' in doc_id:
engine = self._get_engine(server_url)
if engine is None:
log.warn(
"No engine found for %s, cannot edit file with remote ref %s",
server_url, doc_id)
return
self._manager.edit(engine, doc_id)
else:
# Download file
file_path = self._prepare_edit(server_url,
doc_id,
user=user,
download_url=download_url)
# Launch it
if file_path is not None:
self._manager.open_local_file(file_path)
except WindowsError as e:
if e.errno == 13:
# open file anyway
if e.filename is not None:
self._manager.open_local_file(e.filename)
else:
raise e
def _extract_edit_info(self, ref):
dir_path = os.path.dirname(ref)
uid = self._local_client.get_remote_id(dir_path)
server_url = self._local_client.get_remote_id(dir_path, "nxdirectedit")
user = self._local_client.get_remote_id(dir_path, "nxdirectedituser")
engine = self._get_engine(server_url, user=user)
if engine is None:
raise NotFound()
remote_client = engine.get_remote_doc_client()
remote_client.check_suspended = self.stop_client
digest_algorithm = self._local_client.get_remote_id(
dir_path, "nxdirecteditdigestalgorithm")
digest = self._local_client.get_remote_id(dir_path,
"nxdirecteditdigest")
return uid, engine, remote_client, digest_algorithm, digest
def force_update(self, ref, digest):
dir_path = os.path.dirname(ref)
self._local_client.set_remote_id(dir_path, unicode(digest),
"nxdirecteditdigest")
self._upload_queue.put(ref)
def _handle_queues(self):
uploaded = False
# Lock any documents
while (not self._lock_queue.empty()):
try:
item = self._lock_queue.get_nowait()
ref = item[0]
log.trace('Handling DirectEdit lock queue ref: %r', ref)
except Empty:
break
uid = ""
try:
dir_path = os.path.dirname(ref)
uid, engine, remote_client, _, _ = self._extract_edit_info(ref)
if item[1] == 'lock':
remote_client.lock(uid)
self._local_client.set_remote_id(dir_path, "1",
"nxdirecteditlock")
# Emit the lock signal only when the lock is really set
self._manager.get_autolock_service().documentLocked.emit(
os.path.basename(ref))
else:
remote_client.unlock(uid)
if item[1] == 'unlock_orphan':
path = self._local_client._abspath(ref)
log.trace("Remove orphan: %s", path)
self._manager.get_autolock_service().orphan_unlocked(
path)
# Clean the folder
shutil.rmtree(self._local_client._abspath(path),
ignore_errors=True)
self._local_client.remove_remote_id(
dir_path, "nxdirecteditlock")
# Emit the signal only when the unlock is done - might want to avoid the call on orphan
self._manager.get_autolock_service().documentUnlocked.emit(
os.path.basename(ref))
except Exception as e:
# Try again in 30s
log.debug("Can't %s document '%s': %r",
item[1],
ref,
e,
exc_info=True)
self.directEditLockError.emit(item[1], os.path.basename(ref),
uid)
# Unqueue any errors
item = self._error_queue.get()
while (item is not None):
self._upload_queue.put(item.get())
item = self._error_queue.get()
# Handle the upload queue
while (not self._upload_queue.empty()):
try:
ref = self._upload_queue.get_nowait()
log.trace('Handling DirectEdit queue ref: %r', ref)
except Empty:
break
uid, engine, remote_client, digest_algorithm, digest = self._extract_edit_info(
ref)
# Don't update if digest are the same
info = self._local_client.get_info(ref)
try:
current_digest = info.get_digest(digest_func=digest_algorithm)
if current_digest == digest:
continue
start_time = current_milli_time()
log.trace(
"Local digest: %s is different from the recorded one: %s - modification detected for %r",
current_digest, digest, ref)
# TO_REVIEW Should check if server-side blob has changed ?
# Update the document - should verify the remote hash - NXDRIVE-187
remote_info = remote_client.get_info(uid)
if remote_info.digest != digest:
# Conflict detect
log.trace(
"Remote digest: %s is different from the recorded one: %s - conflict detected for %r",
remote_info.digest, digest, ref)
self.directEditConflict.emit(os.path.basename(ref), ref,
remote_info.digest)
continue
log.debug('Uploading file %s',
self._local_client._abspath(ref))
remote_client.stream_update(uid,
self._local_client._abspath(ref),
apply_versioning_policy=True)
# Update hash value
dir_path = os.path.dirname(ref)
self._local_client.set_remote_id(dir_path, current_digest,
'nxdirecteditdigest')
self._last_action_timing = current_milli_time() - start_time
self.editDocument.emit(remote_info)
except ThreadInterrupt:
raise
except Exception as e:
# Try again in 30s
log.trace("Exception on direct edit: %r", e, exc_info=True)
self._error_queue.push(ref, ref)
continue
uploaded = True
if uploaded:
log.debug('Emitting directEditUploadCompleted')
self.directEditUploadCompleted.emit()
def _execute(self):
try:
self._watchdog_queue = Queue()
self._action = Action("Clean up folder")
self._cleanup()
self._action = Action("Setup watchdog")
self._setup_watchdog()
self._end_action()
# Load the target url if Drive was not launched before
self.handle_url()
while (1):
self._interact()
try:
self._handle_queues()
except NotFound:
pass
while (not self._watchdog_queue.empty()):
evt = self._watchdog_queue.get()
self.handle_watchdog_event(evt)
sleep(0.01)
except ThreadInterrupt:
raise
finally:
self._stop_watchdog()
def get_metrics(self):
metrics = super(DirectEdit, self).get_metrics()
if self._event_handler is not None:
metrics['fs_events'] = self._event_handler.counter
metrics['last_action_timing'] = self._last_action_timing
return dict(metrics.items() + self._metrics.items())
def _setup_watchdog(self):
from watchdog.observers import Observer
log.debug("Watching FS modification on : %s", self._folder)
self._event_handler = DriveFSEventHandler(self)
self._observer = Observer()
self._observer.schedule(self._event_handler,
self._folder,
recursive=True)
self._observer.start()
def _stop_watchdog(self, raise_on_error=True):
if self._observer is None:
return
log.info("Stopping FS Observer thread")
try:
self._observer.stop()
except Exception as e:
log.warn("Can't stop FS observer : %r", e)
# Wait for all observers to stop
try:
self._observer.join()
except Exception as e:
log.warn("Can't join FS observer : %r", e)
# Delete all observers
self._observer = None
def is_lock_file(self, name):
return False and ((
name.startswith("~$") # Office lock file
or name.startswith(".~lock."))) # Libre/OpenOffice lock file
def handle_watchdog_event(self, evt):
self._action = Action("Handle watchdog event")
log.debug("Handling watchdog event [%s] on %r", evt.event_type,
evt.src_path)
try:
src_path = normalize_event_filename(evt.src_path)
# Event on the folder by itself
if os.path.isdir(src_path):
return
ref = self._local_client.get_path(src_path)
file_name = os.path.basename(src_path)
if self.is_lock_file(
file_name) and self._manager.get_direct_edit_auto_lock():
if evt.event_type == 'created':
self._lock_queue.put((ref, 'lock'))
elif evt.event_type == 'deleted':
self._lock_queue.put((ref, 'unlock'))
return
if self._local_client.is_temp_file(file_name):
return
queue = False
if evt.event_type == 'modified' or evt.event_type == 'created':
queue = True
if evt.event_type == 'moved':
ref = self._local_client.get_path(evt.dest_path)
file_name = os.path.basename(evt.dest_path)
queue = True
dir_path = self._local_client.get_path(os.path.dirname(src_path))
name = self._local_client.get_remote_id(dir_path,
"nxdirecteditname")
if name is None:
return
if name != file_name:
return
if self._manager.get_direct_edit_auto_lock(
) and self._local_client.get_remote_id(dir_path,
"nxdirecteditlock") != "1":
self._manager.get_autolock_service().set_autolock(
src_path, self)
if queue:
# ADD TO UPLOAD QUEUE
self._upload_queue.put(ref)
return
except Exception as e:
log.warn("Watchdog exception : %r", e, exc_info=True)
finally:
self._end_action()
class DriveEdit(Worker):
localScanFinished = pyqtSignal()
driveEditUploadCompleted = pyqtSignal()
openDocument = pyqtSignal(object)
editDocument = pyqtSignal(object)
driveEditLockError = pyqtSignal(str, str, str)
driveEditConflict = pyqtSignal(str, str, str)
'''
classdocs
'''
def __init__(self, manager, folder, url):
'''
Constructor
'''
super(DriveEdit, self).__init__()
self._manager = manager
self._url = url
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if type(folder) == str:
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._lock_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
self._manager.get_autolock_service().orphanLocks.connect(self._autolock_orphans)
self._last_action_timing = -1
@pyqtSlot(object)
def _autolock_orphans(self, locks):
log.trace("Orphans lock: %r", locks)
for lock in locks:
if lock.path.startswith(self._folder):
log.debug("Should unlock: %s", lock.path)
ref = self._local_client.get_path(lock.path)
self._lock_queue.put((ref, 'unlock_orphan'))
def autolock_lock(self, src_path):
ref = self._local_client.get_path(src_path)
self._lock_queue.put((ref, 'lock'))
def autolock_unlock(self, src_path):
ref = self._local_client.get_path(src_path)
self._lock_queue.put((ref, 'unlock'))
def stop(self):
super(DriveEdit, self).stop()
self._stop = True
def stop_client(self, reason):
if self._stop:
raise ThreadInterrupt
def handle_url(self, url=None):
if url is None:
url = self._url
if url is None:
return
log.debug("DriveEdit load: '%r'", url)
try:
info = parse_protocol_url(str(url))
except UnicodeEncodeError:
# Firefox seems to be different on the encoding part
info = parse_protocol_url(unicode(url))
if info is None:
return
# Handle backward compatibility
if info.get('item_id') is not None:
self.edit(info['server_url'], info['item_id'])
else:
self.edit(info['server_url'], info['doc_id'], filename=info['filename'],
user=info['user'], download_url=info['download_url'])
def _cleanup(self):
log.debug("Cleanup DriveEdit folder")
# Should unlock any remaining doc that has not been unlocked or ask
if self._local_client.exists('/'):
for child in self._local_client.get_children_info('/'):
if self._local_client.get_remote_id(child.path, "nxdriveeditlock") is not None:
continue
# Place for handle reopened of interrupted Edit
shutil.rmtree(self._local_client._abspath(child.path), ignore_errors=True)
if not os.path.exists(self._folder):
os.mkdir(self._folder)
def _get_engine(self, url, user=None):
if url is None:
return None
if url.endswith('/'):
url = url[:-1]
# Simplify port if possible
if url.startswith('http:') and ':80/' in url:
url = url.replace(':80/', '/')
if url.startswith('https:') and ':443/' in url:
url = url.replace(':443/', '/')
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and (user is None or user == bind.username):
return engine
# Some backend are case insensitive
if user is None:
return None
user = user.lower()
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
# Simplify port if possible
if server_url.startswith('http:') and ':80/' in server_url:
server_url = server_url.replace(':80/', '/')
if server_url.startswith('https:') and ':443/' in server_url:
server_url = server_url.replace(':443/', '/')
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and user == bind.username.lower():
return engine
return None
def _download_content(self, engine, remote_client, info, file_path, url=None):
file_dir = os.path.dirname(file_path)
file_name = os.path.basename(file_path)
file_out = os.path.join(file_dir, DOWNLOAD_TMP_FILE_PREFIX + file_name
+ DOWNLOAD_TMP_FILE_SUFFIX)
# Close to processor method - should try to refactor ?
pair = engine.get_dao().get_valid_duplicate_file(info.digest)
if pair:
local_client = engine.get_local_client()
existing_file_path = local_client._abspath(pair.local_path)
log.debug('Local file matches remote digest %r, copying it from %r', info.digest, existing_file_path)
shutil.copy(existing_file_path, file_out)
if pair.is_readonly():
log.debug('Unsetting readonly flag on copied file %r', file_out)
from nxdrive.client.common import BaseClient
BaseClient.unset_path_readonly(file_out)
else:
log.debug('Downloading file %r', info.filename)
if url is not None:
remote_client.do_get(url, file_out=file_out, digest=info.digest, digest_algorithm=info.digest_algorithm)
else:
remote_client.get_blob(info, file_out=file_out)
return file_out
def _display_modal(self, message, values=None):
from nxdrive.wui.application import SimpleApplication
from nxdrive.wui.modal import WebModal
app = SimpleApplication(self._manager, None, {})
dialog = WebModal(app, app.translate(message, values))
dialog.add_button("OK", app.translate("OK"))
dialog.show()
app.exec_()
def _prepare_edit(self, server_url, doc_id, user=None, download_url=None):
start_time = current_milli_time()
engine = self._get_engine(server_url, user=user)
if engine is None:
values = dict()
if user is None:
values['user'] = '******'
else:
values['user'] = user
values['server'] = server_url
log.warn("No engine found for server_url=%s, user=%s, doc_id=%s", server_url, user, doc_id)
self._display_modal("DIRECT_EDIT_CANT_FIND_ENGINE", values)
return
# Get document info
remote_client = engine.get_remote_doc_client()
# Avoid any link with the engine, remote_doc are not cached so we can do that
remote_client.check_suspended = self.stop_client
info = remote_client.get_info(doc_id)
filename = info.filename
# Create local structure
dir_path = os.path.join(self._folder, doc_id)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
log.debug("Editing %r", filename)
file_path = os.path.join(dir_path, filename)
# Download the file
url = None
if download_url is not None:
url = server_url
if not url.endswith('/'):
url += '/'
url += download_url
tmp_file = self._download_content(engine, remote_client, info, file_path, url=url)
if tmp_file is None:
log.debug("Download failed")
return
# Set the remote_id
dir_path = self._local_client.get_path(os.path.dirname(file_path))
self._local_client.set_remote_id(dir_path, doc_id)
self._local_client.set_remote_id(dir_path, server_url, "nxdriveedit")
if user is not None:
self._local_client.set_remote_id(dir_path, user, "nxdriveedituser")
if info.digest is not None:
self._local_client.set_remote_id(dir_path, info.digest, "nxdriveeditdigest")
# Set digest algorithm if not sent by the server
digest_algorithm = info.digest_algorithm
if digest_algorithm is None:
digest_algorithm = guess_digest_algorithm(info.digest)
self._local_client.set_remote_id(dir_path, digest_algorithm, "nxdriveeditdigestalgorithm")
self._local_client.set_remote_id(dir_path, filename, "nxdriveeditname")
# Rename to final filename
# Under Windows first need to delete target file if exists, otherwise will get a 183 WindowsError
if sys.platform == 'win32' and os.path.exists(file_path):
os.unlink(file_path)
os.rename(tmp_file, file_path)
self._last_action_timing = current_milli_time() - start_time
self.openDocument.emit(info)
return file_path
def edit(self, server_url, doc_id, filename=None, user=None, download_url=None):
try:
# Handle backward compatibility
if '#' in doc_id:
engine = self._get_engine(server_url)
if engine is None:
log.warn("No engine found for %s, cannot edit file with remote ref %s", server_url, doc_id)
return
self._manager.edit(engine, doc_id)
else:
# Download file
file_path = self._prepare_edit(server_url, doc_id, user=user, download_url=download_url)
# Launch it
if file_path is not None:
self._manager.open_local_file(file_path)
except WindowsError as e:
if e.errno == 13:
# open file anyway
if e.filename is not None:
self._manager.open_local_file(e.filename)
else:
raise e
def _extract_edit_info(self, ref):
dir_path = os.path.dirname(ref)
uid = self._local_client.get_remote_id(dir_path)
server_url = self._local_client.get_remote_id(dir_path, "nxdriveedit")
user = self._local_client.get_remote_id(dir_path, "nxdriveedituser")
engine = self._get_engine(server_url, user=user)
if engine is None:
raise NotFound()
remote_client = engine.get_remote_doc_client()
remote_client.check_suspended = self.stop_client
digest_algorithm = self._local_client.get_remote_id(dir_path, "nxdriveeditdigestalgorithm")
digest = self._local_client.get_remote_id(dir_path, "nxdriveeditdigest")
return uid, engine, remote_client, digest_algorithm, digest
def force_update(self, ref, digest):
dir_path = os.path.dirname(ref)
self._local_client.set_remote_id(dir_path, unicode(digest), "nxdriveeditdigest")
self._upload_queue.put(ref)
def _handle_queues(self):
uploaded = False
# Lock any documents
while (not self._lock_queue.empty()):
try:
item = self._lock_queue.get_nowait()
ref = item[0]
log.trace('Handling DriveEdit lock queue ref: %r', ref)
except Empty:
break
uid = ""
try:
dir_path = os.path.dirname(ref)
uid, engine, remote_client, _, _ = self._extract_edit_info(ref)
if item[1] == 'lock':
remote_client.lock(uid)
self._local_client.set_remote_id(dir_path, "1", "nxdriveeditlock")
# Emit the lock signal only when the lock is really set
self._manager.get_autolock_service().documentLocked.emit(os.path.basename(ref))
else:
remote_client.unlock(uid)
if item[1] == 'unlock_orphan':
path = self._local_client._abspath(ref)
log.trace("Remove orphan: %s", path)
self._manager.get_autolock_service().orphan_unlocked(path)
# Clean the folder
shutil.rmtree(self._local_client._abspath(path), ignore_errors=True)
self._local_client.remove_remote_id(dir_path, "nxdriveeditlock")
# Emit the signal only when the unlock is done - might want to avoid the call on orphan
self._manager.get_autolock_service().documentUnlocked.emit(os.path.basename(ref))
except Exception as e:
# Try again in 30s
log.debug("Can't %s document '%s': %r", item[1], ref, e, exc_info=True)
self.driveEditLockError.emit(item[1], os.path.basename(ref), uid)
# Unqueue any errors
item = self._error_queue.get()
while (item is not None):
self._upload_queue.put(item.get())
item = self._error_queue.get()
# Handle the upload queue
while (not self._upload_queue.empty()):
try:
ref = self._upload_queue.get_nowait()
log.trace('Handling DriveEdit queue ref: %r', ref)
except Empty:
break
uid, engine, remote_client, digest_algorithm, digest = self._extract_edit_info(ref)
# Don't update if digest are the same
info = self._local_client.get_info(ref)
try:
current_digest = info.get_digest(digest_func=digest_algorithm)
if current_digest == digest:
continue
start_time = current_milli_time()
log.trace("Local digest: %s is different from the recorded one: %s - modification detected for %r",
current_digest, digest, ref)
# TO_REVIEW Should check if server-side blob has changed ?
# Update the document - should verify the remote hash - NXDRIVE-187
remote_info = remote_client.get_info(uid)
if remote_info.digest != digest:
# Conflict detect
log.trace("Remote digest: %s is different from the recorded one: %s - conflict detected for %r",
remote_info.digest, digest, ref)
self.driveEditConflict.emit(os.path.basename(ref), ref, remote_info.digest)
continue
log.debug('Uploading file %s', self._local_client._abspath(ref))
remote_client.stream_update(uid, self._local_client._abspath(ref), apply_versioning_policy=True)
# Update hash value
dir_path = os.path.dirname(ref)
self._local_client.set_remote_id(dir_path, current_digest, 'nxdriveeditdigest')
self._last_action_timing = current_milli_time() - start_time
self.editDocument.emit(remote_info)
except ThreadInterrupt:
raise
except Exception as e:
# Try again in 30s
log.trace("Exception on drive edit: %r", e, exc_info=True)
self._error_queue.push(ref, ref)
continue
uploaded = True
if uploaded:
log.debug('Emitting driveEditUploadCompleted')
self.driveEditUploadCompleted.emit()
def _execute(self):
try:
self._watchdog_queue = Queue()
self._action = Action("Clean up folder")
self._cleanup()
self._action = Action("Setup watchdog")
self._setup_watchdog()
self._end_action()
# Load the target url if Drive was not launched before
self.handle_url()
while (1):
self._interact()
try:
self._handle_queues()
except NotFound:
pass
while (not self._watchdog_queue.empty()):
evt = self._watchdog_queue.get()
self.handle_watchdog_event(evt)
sleep(0.01)
except ThreadInterrupt:
raise
finally:
self._stop_watchdog()
def get_metrics(self):
metrics = super(DriveEdit, self).get_metrics()
if self._event_handler is not None:
metrics['fs_events'] = self._event_handler.counter
metrics['last_action_timing'] = self._last_action_timing
return dict(metrics.items() + self._metrics.items())
def _setup_watchdog(self):
from watchdog.observers import Observer
log.debug("Watching FS modification on : %s", self._folder)
self._event_handler = DriveFSEventHandler(self)
self._observer = Observer()
self._observer.schedule(self._event_handler, self._folder, recursive=True)
self._observer.start()
def _stop_watchdog(self, raise_on_error=True):
if self._observer is None:
return
log.info("Stopping FS Observer thread")
try:
self._observer.stop()
except Exception as e:
log.warn("Can't stop FS observer : %r", e)
# Wait for all observers to stop
try:
self._observer.join()
except Exception as e:
log.warn("Can't join FS observer : %r", e)
# Delete all observers
self._observer = None
def is_lock_file(self, name):
return False and ((name.startswith("~$") # Office lock file
or name.startswith(".~lock."))) # Libre/OpenOffice lock file
def handle_watchdog_event(self, evt):
self._action = Action("Handle watchdog event")
log.debug("Handling watchdog event [%s] on %r", evt.event_type, evt.src_path)
try:
src_path = normalize_event_filename(evt.src_path)
# Event on the folder by itself
if os.path.isdir(src_path):
return
ref = self._local_client.get_path(src_path)
file_name = os.path.basename(src_path)
if self.is_lock_file(file_name) and self._manager.get_drive_edit_auto_lock():
if evt.event_type == 'created':
self._lock_queue.put((ref, 'lock'))
elif evt.event_type == 'deleted':
self._lock_queue.put((ref, 'unlock'))
return
if self._local_client.is_temp_file(file_name):
return
queue = False
if evt.event_type == 'modified' or evt.event_type == 'created':
queue = True
if evt.event_type == 'moved':
ref = self._local_client.get_path(evt.dest_path)
file_name = os.path.basename(evt.dest_path)
queue = True
dir_path = self._local_client.get_path(os.path.dirname(src_path))
name = self._local_client.get_remote_id(dir_path, "nxdriveeditname")
if name is None:
return
if name != file_name:
return
if self._manager.get_drive_edit_auto_lock() and self._local_client.get_remote_id(dir_path, "nxdriveeditlock") != "1":
self._manager.get_autolock_service().set_autolock(src_path, self)
if queue:
# ADD TO UPLOAD QUEUE
self._upload_queue.put(ref)
return
except Exception as e:
log.warn("Watchdog exception : %r", e, exc_info=True)
finally:
self._end_action()
class DriveEdit(Worker):
localScanFinished = pyqtSignal()
driveEditUploadCompleted = pyqtSignal()
'''
classdocs
'''
def __init__(self, manager, folder):
'''
Constructor
'''
super(DriveEdit, self).__init__()
self._manager = manager
self._thread.started.connect(self.run)
self._event_handler = None
self._metrics = dict()
self._metrics['edit_files'] = 0
self._observer = None
if type(folder) == str:
folder = unicode(folder)
self._folder = folder
self._local_client = LocalClient(self._folder)
self._upload_queue = Queue()
self._error_queue = BlacklistQueue()
self._stop = False
def stop(self):
super(DriveEdit, self).stop()
self._stop = True
def stop_client(self, reason):
if self._stop:
raise ThreadInterrupt
def _cleanup(self):
log.debug("Cleanup DriveEdit folder")
shutil.rmtree(self._folder, ignore_errors=True)
if not os.path.exists(self._folder):
os.mkdir(self._folder)
def _get_engine(self, url, user=None):
if url.endswith('/'):
url = url[:-1]
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and (user is None or user == bind.username):
return engine
# Some backend are case insensitive
if user is None:
return None
user = user.lower()
for engine in self._manager.get_engines().values():
bind = engine.get_binder()
server_url = bind.server_url
if server_url.endswith('/'):
server_url = server_url[:-1]
if server_url == url and user == bind.username.lower():
return engine
return None
def _download_content(self, engine, remote_client, info, file_path, url=None):
file_dir = os.path.dirname(file_path)
file_name = os.path.basename(file_path)
file_out = os.path.join(file_dir, DOWNLOAD_TMP_FILE_PREFIX + file_name
+ DOWNLOAD_TMP_FILE_SUFFIX)
# Close to processor method - should try to refactor ?
pair = engine.get_dao().get_valid_duplicate_file(info.digest)
if pair:
local_client = engine.get_local_client()
shutil.copy(local_client._abspath(pair.local_path), file_out)
else:
if url is not None:
remote_client.do_get(url, file_out=file_out, digest=info.digest, digest_algorithm=info.digest_algorithm)
else:
remote_client.get_blob(info.uid, file_out=file_out)
return file_out
def _prepare_edit(self, server_url, doc_id, filename, user=None, download_url=None):
engine = self._get_engine(server_url, user=user)
if engine is None:
# TO_REVIEW Display an error message
log.debug("No engine found for %s(%s)", server_url, doc_id)
return
# Get document info
remote_client = engine.get_remote_doc_client()
# Avoid any link with the engine, remote_doc are not cached so we can do that
remote_client.check_suspended = self.stop_client
info = remote_client.get_info(doc_id)
# Create local structure
dir_path = os.path.join(self._folder, doc_id)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
log.trace('Raw filename: %r', filename)
filename = safe_filename(urllib2.unquote(filename))
log.trace('Unquoted filename = %r', filename)
decoded_filename = force_decode(filename)
if decoded_filename is None:
decoded_filename = filename
else:
# Always use utf-8 encoding for xattr
filename = decoded_filename.encode('utf-8')
log.debug("Editing %r ('nxdriveeditname' xattr: %r)", decoded_filename, filename)
file_path = os.path.join(dir_path, decoded_filename)
# Download the file
url = None
if download_url is not None:
url = server_url
if not url.endswith('/'):
url += '/'
url += download_url
tmp_file = self._download_content(engine, remote_client, info, file_path, url=url)
if tmp_file is None:
log.debug("Download failed")
return
# Set the remote_id
dir_path = self._local_client.get_path(os.path.dirname(file_path))
self._local_client.set_remote_id(dir_path, doc_id)
self._local_client.set_remote_id(dir_path, server_url, "nxdriveedit")
if user is not None:
self._local_client.set_remote_id(dir_path, user, "nxdriveedituser")
if info.digest is not None:
self._local_client.set_remote_id(dir_path, info.digest, "nxdriveeditdigest")
# Set digest algorithm if not sent by the server
digest_algorithm = info.digest_algorithm
if digest_algorithm is None:
digest_algorithm = guess_digest_algorithm(info.digest)
self._local_client.set_remote_id(dir_path, digest_algorithm, "nxdriveeditdigestalgorithm")
self._local_client.set_remote_id(dir_path, filename, "nxdriveeditname")
# Rename to final filename
# Under Windows first need to delete target file if exists, otherwise will get a 183 WindowsError
if sys.platform == 'win32' and os.path.exists(file_path):
os.unlink(file_path)
os.rename(tmp_file, file_path)
return file_path
def edit(self, server_url, doc_id, filename, user=None, download_url=None):
# Download file
file_path = self._prepare_edit(server_url, doc_id, filename, user=user, download_url=download_url)
# Launch it
if file_path is not None:
self._manager.open_local_file(file_path)
def _handle_queue(self):
uploaded = False
# Unqueue any errors
item = self._error_queue.get()
while (item is not None):
self._upload_queue.put(item.get())
item = self._error_queue.get()
# Handle the upload queue
while (not self._upload_queue.empty()):
try:
ref = self._upload_queue.get_nowait()
log.trace('Handling DriveEdit queue ref: %r', ref)
except Empty:
break
dir_path = os.path.dirname(ref)
uid = self._local_client.get_remote_id(dir_path)
server_url = self._local_client.get_remote_id(dir_path, "nxdriveedit")
user = self._local_client.get_remote_id(dir_path, "nxdriveedituser")
engine = self._get_engine(server_url, user=user)
remote_client = engine.get_remote_doc_client()
remote_client.check_suspended = self.stop_client
digest_algorithm = self._local_client.get_remote_id(dir_path, "nxdriveeditdigestalgorithm")
digest = self._local_client.get_remote_id(dir_path, "nxdriveeditdigest")
# Don't update if digest are the same
info = self._local_client.get_info(ref)
try:
if info.get_digest(digest_func=digest_algorithm) == digest:
continue
# TO_REVIEW Should check if server-side blob has changed ?
# Update the document - should verify the remote hash - NXDRIVE-187
log.debug('Uploading file %s for user %s', self._local_client._abspath(ref),
user)
remote_client.stream_update(uid, self._local_client._abspath(ref), apply_versioning_policy=True)
except ThreadInterrupt:
raise
except Exception as e:
# Try again in 30s
log.trace("Exception on drive edit: %r", e)
self._error_queue.push(ref, ref)
continue
uploaded = True
if uploaded:
log.debug('Emitting driveEditUploadCompleted')
self.driveEditUploadCompleted.emit()
def _execute(self):
try:
self._watchdog_queue = Queue()
self._action = Action("Clean up folder")
self._cleanup()
self._action = Action("Setup watchdog")
self._setup_watchdog()
self._end_action()
while (1):
self._interact()
try:
self._handle_queue()
except NotFound:
pass
while (not self._watchdog_queue.empty()):
evt = self._watchdog_queue.get()
self.handle_watchdog_event(evt)
sleep(0.01)
except ThreadInterrupt:
raise
finally:
self._stop_watchdog()
def get_metrics(self):
metrics = super(DriveEdit, self).get_metrics()
if self._event_handler is not None:
metrics['fs_events'] = self._event_handler.counter
return dict(metrics.items() + self._metrics.items())
def _setup_watchdog(self):
from watchdog.observers import Observer
log.debug("Watching FS modification on : %s", self._folder)
self._event_handler = DriveFSEventHandler(self)
self._observer = Observer()
self._observer.schedule(self._event_handler, self._folder, recursive=True)
self._observer.start()
def _stop_watchdog(self, raise_on_error=True):
if self._observer is None:
return
log.info("Stopping FS Observer thread")
try:
self._observer.stop()
except Exception as e:
log.warn("Can't stop FS observer : %r", e)
# Wait for all observers to stop
try:
self._observer.join()
except Exception as e:
log.warn("Can't join FS observer : %r", e)
# Delete all observers
self._observer = None
def handle_watchdog_event(self, evt):
self._action = Action("Handle watchdog event")
log.debug("Handling watchdog event [%s] on %r", evt.event_type, evt.src_path)
try:
src_path = normalize_event_filename(evt.src_path)
# Event on the folder by itself
if os.path.isdir(src_path):
return
ref = self._local_client.get_path(src_path)
file_name = os.path.basename(src_path)
if self._local_client.is_temp_file(file_name):
return
queue = False
if evt.event_type == 'modified' or evt.event_type == 'created':
queue = True
if evt.event_type == 'moved':
ref = self._local_client.get_path(evt.dest_path)
file_name = os.path.basename(evt.dest_path)
queue = True
dir_path = self._local_client.get_path(os.path.dirname(src_path))
name = self._local_client.get_remote_id(dir_path, "nxdriveeditname")
if name is None:
return
decoded_name = force_decode(name)
if decoded_name is not None:
name = decoded_name
if name != file_name:
return
if queue:
# ADD TO UPLOAD QUEUE
self._upload_queue.put(ref)
return
except Exception as e:
log.warn("Watchdog exception : %r" % e)
log.exception(e)
finally:
self._end_action()