def __init__(self, config_folder, echo=None, poolclass=None, timeout=20): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.debug("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.debug("nxdrive configured in '%s'", self.config_folder) if echo is None: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.timeout = timeout # Handle connection to the local Nuxeo Drive configuration and # metadata sqlite database. self._engine, self._session_maker = init_db( self.config_folder, echo=echo, poolclass=poolclass) self._local = local() self._remote_error = None self.device_id = self.get_device_config().device_id self.synchronizer = Synchronizer(self) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar()
def __init__(self, config_folder, echo=False, echo_pool=False, poolclass=None, handshake_timeout=60, timeout=20, page_size=None, max_errors=3): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.info("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.info("nxdrive configured in '%s'", self.config_folder) if not echo: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.handshake_timeout = handshake_timeout self.timeout = timeout self.max_errors = max_errors # Handle connection to the local Nuxeo Drive configuration and # metadata SQLite database. self._engine, self._session_maker = init_db( self.config_folder, echo=echo, echo_pool=echo_pool, poolclass=poolclass) # Migrate SQLite database if needed migrate_db(self._engine) # Thread-local storage for the remote client cache self._local = local() self._client_cache_timestamps = dict() self._remote_error = None self._local_error = None device_config = self.get_device_config() self.device_id = device_config.device_id self.version = nxdrive.__version__ self.updated = self.update_version(device_config) # HTTP proxy settings self.proxies = None self.proxy_exceptions = None self.refresh_proxies(device_config=device_config) # Recently modified items for each server binding self.recently_modified = {} self.synchronizer = Synchronizer(self, page_size=page_size) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar()
class Controller(object): """Manage configuration and perform Nuxeo Drive Operations This class is thread safe: instance can be shared by multiple threads as DB sessions and Nuxeo clients are thread locals. """ # Used for binding server / roots and managing tokens remote_doc_client_factory = RemoteDocumentClient # Used for FS synchronization operations remote_fs_client_factory = RemoteFileSystemClient def __init__(self, config_folder, echo=None, poolclass=None, timeout=20): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.debug("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.debug("nxdrive configured in '%s'", self.config_folder) if echo is None: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.timeout = timeout # Handle connection to the local Nuxeo Drive configuration and # metadata sqlite database. self._engine, self._session_maker = init_db( self.config_folder, echo=echo, poolclass=poolclass) self._local = local() self._remote_error = None self.device_id = self.get_device_config().device_id self.synchronizer = Synchronizer(self) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar() def get_session(self): """Reuse the thread local session for this controller Using the controller in several thread should be thread safe as long as this method is always called to fetch the session instance. """ return self._session_maker() def get_device_config(self, session=None): """Fetch the singleton configuration object for this device""" if session is None: session = self.get_session() try: return session.query(DeviceConfig).one() except NoResultFound: device_config = DeviceConfig() # generate a unique device id session.add(device_config) session.commit() return device_config def stop(self): """Stop the Nuxeo Drive synchronization thread As the process asking the synchronization to stop might not be the same as the process running the synchronization (especially when used from the commandline without the graphical user interface and its tray icon menu) we use a simple empty marker file a cross platform way to pass the stop message between the two. """ pid = self.synchronizer.check_running(process_name="sync") if pid is not None: # Create a stop file marker for the running synchronization # process log.info("Telling synchronization process %d to stop." % pid) stop_file = os.path.join(self.config_folder, "stop_%d" % pid) open(safe_long_path(stop_file), 'wb').close() else: log.info("No running synchronization process to stop.") def children_states(self, folder_path): """List the status of the children of a folder The state of the folder is a summary of their descendant rather than their own instric synchronization step which is of little use for the end user. """ session = self.get_session() # Find the server binding for this absolute path try: binding, path = self._binding_path(folder_path, session=session) except NotFound: return [] try: folder_state = session.query(LastKnownState).filter_by( local_folder=binding.local_folder, local_path=path, ).one() except NoResultFound: return [] states = self._pair_states_recursive(session, folder_state) return [(os.path.basename(s.local_path), pair_state) for s, pair_state in states if s.local_parent_path == path] def _pair_states_recursive(self, session, doc_pair): """Recursive call to collect pair state under a given location.""" if not doc_pair.folderish: return [(doc_pair, doc_pair.pair_state)] if doc_pair.local_path is not None and doc_pair.remote_ref is not None: f = or_( LastKnownState.local_parent_path == doc_pair.local_path, LastKnownState.remote_parent_ref == doc_pair.remote_ref, ) elif doc_pair.local_path is not None: f = LastKnownState.local_parent_path == doc_pair.local_path elif doc_pair.remote_ref is not None: f = LastKnownState.remote_parent_ref == doc_pair.remote_ref else: raise ValueError("Illegal state %r: at least path or remote_ref" " should be not None." % doc_pair) children_states = session.query(LastKnownState).filter_by( local_folder=doc_pair.local_folder).filter(f).order_by( asc(LastKnownState.local_name), asc(LastKnownState.remote_name), ).all() results = [] for child_state in children_states: sub_results = self._pair_states_recursive(session, child_state) results.extend(sub_results) # A folder stays synchronized (or unknown) only if all the descendants # are themselfves synchronized. pair_state = doc_pair.pair_state for _, sub_pair_state in results: if sub_pair_state != 'synchronized': pair_state = 'children_modified' break # Pre-pend the folder state to the descendants return [(doc_pair, pair_state)] + results def _binding_path(self, local_path, session=None): """Find a server binding and relative path for a given FS path""" local_path = normalized_path(local_path) # Check exact binding match binding = self.get_server_binding(local_path, session=session, raise_if_missing=False) if binding is not None: return binding, u'/' # Check for bindings that are prefix of local_path session = self.get_session() all_bindings = session.query(ServerBinding).all() matching_bindings = [sb for sb in all_bindings if local_path.startswith( sb.local_folder + os.path.sep)] if len(matching_bindings) == 0: raise NotFound("Could not find any server binding for " + local_path) elif len(matching_bindings) > 1: raise RuntimeError("Found more than one binding for %s: %r" % ( local_path, matching_bindings)) binding = matching_bindings[0] path = local_path[len(binding.local_folder):] path = path.replace(os.path.sep, u'/') return binding, path def get_server_binding(self, local_folder, raise_if_missing=False, session=None): """Find the ServerBinding instance for a given local_folder""" local_folder = normalized_path(local_folder) if session is None: session = self.get_session() try: return session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() except NoResultFound: if raise_if_missing: raise RuntimeError( "Folder '%s' is not bound to any Nuxeo server" % local_folder) return None def list_server_bindings(self, session=None): if session is None: session = self.get_session() return session.query(ServerBinding).all() def bind_server(self, local_folder, server_url, username, password): """Bind a local folder to a remote nuxeo server""" session = self.get_session() local_folder = normalized_path(local_folder) if not os.path.exists(local_folder): os.makedirs(local_folder) self.register_folder_link(local_folder) # check the connection to the server by issuing an authentication # request server_url = self._normalize_url(server_url) nxclient = self.remote_doc_client_factory( server_url, username, self.device_id, password) token = nxclient.request_token() if token is not None: # The server supports token based identification: do not store the # password in the DB password = None try: server_binding = session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() if (server_binding.remote_user != username or server_binding.server_url != server_url): raise RuntimeError( "%s is already bound to '%s' with user '%s'" % ( local_folder, server_binding.server_url, server_binding.remote_user)) if token is None and server_binding.remote_password != password: # Update password info if required server_binding.remote_password = password log.info("Updating password for user '%s' on server '%s'", username, server_url) if token is not None and server_binding.remote_token != token: log.info("Updating token for user '%s' on server '%s'", username, server_url) # Update the token info if required server_binding.remote_token = token # Ensure that the password is not stored in the DB if server_binding.remote_password is not None: server_binding.remote_password = None except NoResultFound: log.info("Binding '%s' to '%s' with account '%s'", local_folder, server_url, username) server_binding = ServerBinding(local_folder, server_url, username, remote_password=password, remote_token=token) session.add(server_binding) # Creating the toplevel state for the server binding local_client = LocalClient(server_binding.local_folder) local_info = local_client.get_info(u'/') remote_client = self.get_remote_fs_client(server_binding) remote_info = remote_client.get_filesystem_root_info() state = LastKnownState(server_binding.local_folder, local_info=local_info, local_state='synchronized', remote_info=remote_info, remote_state='synchronized') session.add(state) session.commit() return server_binding def unbind_server(self, local_folder): """Remove the binding to a Nuxeo server Local files are not deleted""" session = self.get_session() local_folder = normalized_path(local_folder) binding = self.get_server_binding(local_folder, raise_if_missing=True, session=session) # Revoke token if necessary if binding.remote_token is not None: try: nxclient = self.remote_doc_client_factory( binding.server_url, binding.remote_user, self.device_id, token=binding.remote_token) log.info("Revoking token for '%s' with account '%s'", binding.server_url, binding.remote_user) nxclient.revoke_token() except POSSIBLE_NETWORK_ERROR_TYPES: log.warning("Could not connect to server '%s' to revoke token", binding.server_url) except Unauthorized: # Token is already revoked pass # Invalidate client cache self.invalidate_client_cache(binding.server_url) # Delete binding info in local DB log.info("Unbinding '%s' from '%s' with account '%s'", local_folder, binding.server_url, binding.remote_user) session.delete(binding) session.commit() def unbind_all(self): """Unbind all server and revoke all tokens This is useful for cleanup in integration test code. """ session = self.get_session() for sb in session.query(ServerBinding).all(): self.unbind_server(sb.local_folder) def bind_root(self, local_folder, remote_ref, repository='default', session=None): """Bind local root to a remote root (folderish document in Nuxeo). local_folder must be already bound to an existing Nuxeo server. remote_ref must be the IdRef or PathRef of an existing folderish document on the remote server bound to the local folder. """ session = self.get_session() if session is None else session local_folder = normalized_path(local_folder) server_binding = self.get_server_binding( local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Register the root on the server nxclient.register_as_root(remote_ref) def unbind_root(self, local_folder, remote_ref, repository='default', session=None): """Remove binding to remote folder""" session = self.get_session() if session is None else session server_binding = self.get_server_binding( local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Unregister the root on the server nxclient.unregister_as_root(remote_ref) def list_pending(self, limit=100, local_folder=None, ignore_in_error=None, session=None): """List pending files to synchronize, ordered by path Ordering by path makes it possible to synchronize sub folders content only once the parent folders have already been synchronized. If ingore_in_error is not None and is a duration in second, skip pair states states that have recently triggered a synchronization error. """ if session is None: session = self.get_session() predicates = [LastKnownState.pair_state != 'synchronized'] if local_folder is not None: predicates.append(LastKnownState.local_folder == local_folder) if ignore_in_error is not None and ignore_in_error > 0: max_date = datetime.utcnow() - timedelta(seconds=ignore_in_error) predicates.append(or_( LastKnownState.last_sync_error_date == None, LastKnownState.last_sync_error_date < max_date)) return session.query(LastKnownState).filter( *predicates ).order_by( # Ensure that newly created remote folders will be synchronized # before their children while keeping a fixed named based # deterministic ordering to make the tests readable asc(LastKnownState.remote_parent_path), asc(LastKnownState.remote_name), asc(LastKnownState.remote_ref), # Ensure that newly created local folders will be synchronized # before their children asc(LastKnownState.local_path) ).limit(limit).all() def next_pending(self, local_folder=None, session=None): """Return the next pending file to synchronize or None""" pending = self.list_pending(limit=1, local_folder=local_folder, session=session) return pending[0] if len(pending) > 0 else None def _get_client_cache(self): if not hasattr(self._local, 'remote_clients'): self._local.remote_clients = dict() return self._local.remote_clients def get_remote_fs_client(self, server_binding): """Return a client for the FileSystem abstraction.""" cache = self._get_client_cache() sb = server_binding cache_key = (sb.server_url, sb.remote_user, self.device_id) remote_client = cache.get(cache_key) if remote_client is None: remote_client = self.remote_fs_client_factory( sb.server_url, sb.remote_user, self.device_id, token=sb.remote_token, password=sb.remote_password, timeout=self.timeout, cookie_jar=self.cookie_jar) cache[cache_key] = remote_client # Make it possible to have the remote client simulate any kind of # failure: this is useful for ensuring that cookies used for load # balancer affinity (e.g. AWSELB) are shared by all the automation # clients managed by a given controller remote_client.make_raise(self._remote_error) return remote_client def get_remote_doc_client(self, server_binding, repository='default', base_folder=None): """Return an instance of Nuxeo Document Client""" sb = server_binding return self.remote_doc_client_factory( sb.server_url, sb.remote_user, self.device_id, token=sb.remote_token, password=sb.remote_password, repository=repository, base_folder=base_folder, timeout=self.timeout, cookie_jar=self.cookie_jar) def get_remote_client(self, server_binding, repository='default', base_folder=None): # Backward compat return self.get_remote_doc_client(server_binding, repository=repository, base_folder=base_folder) def invalidate_client_cache(self, server_url): cache = self._get_client_cache() for key, client in cache.items(): if client.server_url == server_url: del cache[key] def get_state(self, server_url, remote_ref): """Find a pair state for the provided remote document identifiers.""" server_url = self._normalize_url(server_url) session = self.get_session() try: states = session.query(LastKnownState).filter_by( remote_ref=remote_ref, ).all() for state in states: if (state.server_binding.server_url == server_url): return state except NoResultFound: return None def get_state_for_local_path(self, local_os_path): """Find a DB state from a local filesystem path""" session = self.get_session() sb, local_path = self._binding_path(local_os_path, session=session) return session.query(LastKnownState).filter_by( local_folder=sb.local_folder, local_path=local_path).one() def launch_file_editor(self, server_url, remote_ref): """Find the local file if any and start OS editor on it.""" state = self.get_state(server_url, remote_ref) if state is None: # TODO: synchronize to a dedicated special root for one time edit log.warning('Could not find local file for server_url=%s ' 'and remote_ref=%s', server_url, remote_ref) return # TODO: check synchronization of this state first # Find the best editor for the file according to the OS configuration file_path = state.get_local_abspath() self.open_local_file(file_path) def open_local_file(self, file_path): """Launch the local OS program on the given file / folder.""" log.debug('Launching editor on %s', file_path) if sys.platform == 'win32': os.startfile(file_path) elif sys.platform == 'darwin': subprocess.Popen(['open', file_path]) else: try: subprocess.Popen(['xdg-open', file_path]) except OSError: # xdg-open should be supported by recent Gnome, KDE, Xfce log.error("Failed to find and editor for: '%s'", file_path) def make_remote_raise(self, error): """Helper method to simulate network failure for testing""" self._remote_error = error def dispose(self): """Release all database resources""" self.get_session().close_all() self._engine.pool.dispose() def _normalize_url(self, url): """Ensure that user provided url always has a trailing '/'""" if url is None or not url: raise ValueError("Invalid url: %r" % url) if not url.endswith(u'/'): return url + u'/' return url def register_folder_link(self, folder_path): if sys.platform == 'darwin': self.register_folder_link_darwin(folder_path) # TODO: implement Windows and Linux support here def register_folder_link_darwin(self, folder_path): try: from LaunchServices import LSSharedFileListCreate from LaunchServices import kLSSharedFileListFavoriteItems from LaunchServices import LSSharedFileListInsertItemURL from LaunchServices import kLSSharedFileListItemBeforeFirst from LaunchServices import CFURLCreateWithString except ImportError: log.warning("PyObjC package is not installed:" " skipping favorite link creation") return folder_path = normalized_path(folder_path) folder_name = os.path.basename(folder_path) lst = LSSharedFileListCreate(None, kLSSharedFileListFavoriteItems, None) if lst is None: log.warning("Could not fetch the Finder favorite list.") return url = CFURLCreateWithString(None, "file://" + quote(folder_path), None) if url is None: log.warning("Could not generate valid favorite URL for: %s", folder_path) return # Register the folder as favorite if not already there item = LSSharedFileListInsertItemURL( lst, kLSSharedFileListItemBeforeFirst, folder_name, None, url, {}, []) if item is not None: log.debug("Registered new favorite in Finder for: %s", folder_path)
class Controller(object): """Manage configuration and perform Nuxeo Drive Operations This class is thread safe: instance can be shared by multiple threads as DB sessions and Nuxeo clients are thread locals. """ # Used for binding server / roots and managing tokens remote_doc_client_factory = RemoteDocumentClient # Used for FS synchronization operations remote_fs_client_factory = RemoteFileSystemClient # Used for FS synchronization operations remote_filtered_fs_client_factory = RemoteFilteredFileSystemClient def __init__(self, config_folder, echo=False, echo_pool=False, poolclass=None, handshake_timeout=60, timeout=20, page_size=None, max_errors=3): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.info("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.info("nxdrive configured in '%s'", self.config_folder) if not echo: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.handshake_timeout = handshake_timeout self.timeout = timeout self.max_errors = max_errors # Handle connection to the local Nuxeo Drive configuration and # metadata SQLite database. self._engine, self._session_maker = init_db( self.config_folder, echo=echo, echo_pool=echo_pool, poolclass=poolclass) # Migrate SQLite database if needed migrate_db(self._engine) # Thread-local storage for the remote client cache self._local = local() self._client_cache_timestamps = dict() self._remote_error = None self._local_error = None device_config = self.get_device_config() self.device_id = device_config.device_id self.version = nxdrive.__version__ self.updated = self.update_version(device_config) # HTTP proxy settings self.proxies = None self.proxy_exceptions = None self.refresh_proxies(device_config=device_config) # Recently modified items for each server binding self.recently_modified = {} self.synchronizer = Synchronizer(self, page_size=page_size) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar() def trash_modified_file(self): return False def get_session(self): """Reuse the thread local session for this controller Using the controller in several thread should be thread safe as long as this method is always called to fetch the session instance. """ return self._session_maker() def get_device_config(self, session=None): """Fetch the singleton configuration object for this device""" if session is None: session = self.get_session() try: return session.query(DeviceConfig).one() except NoResultFound: device_config = DeviceConfig() # generate a unique device id session.add(device_config) session.commit() return device_config def get_version(self): return self.version def update_version(self, device_config): if self.version != device_config.client_version: log.info("Detected version upgrade: current version = %s," " new version = %s => upgrading current version," " yet DB upgrade might be needed.", device_config.client_version, self.version) device_config.client_version = self.version self.get_session().commit() return True return False def is_updated(self): return self.updated def refresh_update_info(self, local_folder): session = self.get_session() sb = self.get_server_binding(local_folder, session=session) self._set_update_info(sb) session.commit() def _set_update_info(self, server_binding, remote_client=None): try: remote_client = (remote_client if remote_client is not None else self.get_remote_doc_client(server_binding)) update_info = remote_client.get_update_info() log.info("Fetched update info from server: %r", update_info) server_binding.server_version = update_info['serverVersion'] server_binding.update_url = update_info['updateSiteURL'] except Exception as e: log.warning("Cannot get update info from server because of: %s", e) # Fall back on default server version if needed if server_binding.server_version is None: server_binding.server_version = self.get_default_server_version() log.debug("Server version is null or not available, falling back" " on default one: %s", server_binding.server_version) # Fall back on default update site URL if needed if server_binding.update_url is None: server_binding.update_url = self.get_default_update_site_url() log.debug("Update site URL is null or not available, falling back" " on default one: %s", server_binding.update_url) @deprecated def get_default_server_version(self): return None def get_default_update_site_url(self): return DEFAULT_UPDATE_SITE_URL def get_proxy_settings(self, device_config=None): """Fetch proxy settings from database""" dc = (self.get_device_config() if device_config is None else device_config) # Decrypt password with token as the secret token = self.get_first_token() if dc.proxy_password is not None and token is not None: password = decrypt(dc.proxy_password, token) else: # If no server binding or no token available # (possibly after token revocation) reset password password = '' return ProxySettings(config=dc.proxy_config, proxy_type=dc.proxy_type, server=dc.proxy_server, port=dc.proxy_port, authenticated=dc.proxy_authenticated, username=dc.proxy_username, password=password, exceptions=dc.proxy_exceptions) def set_proxy_settings(self, proxy_settings): session = self.get_session() device_config = self.get_device_config(session) device_config.proxy_config = proxy_settings.config device_config.proxy_type = proxy_settings.proxy_type device_config.proxy_server = proxy_settings.server device_config.proxy_port = proxy_settings.port device_config.proxy_exceptions = proxy_settings.exceptions device_config.proxy_authenticated = proxy_settings.authenticated device_config.proxy_username = proxy_settings.username # Encrypt password with token as the secret token = self.get_first_token(session) if token is None: raise MissingToken("Your token has been revoked," " please update your password to acquire a new one.") password = encrypt(proxy_settings.password, token) device_config.proxy_password = password session.commit() log.info("Proxy settings successfully updated: %r", proxy_settings) self.invalidate_client_cache() def get_general_settings(self, device_config=None): """Fetch general settings from database""" dc = (self.get_device_config() if device_config is None else device_config) return GeneralSettings(auto_update=dc.auto_update) def set_general_settings(self, general_settings): session = self.get_session() device_config = self.get_device_config(session) device_config.auto_update = general_settings.auto_update session.commit() log.info("General settings successfully updated: %r", general_settings) def is_auto_update(self, device_config=None): return self.get_general_settings( device_config=device_config).auto_update def set_auto_update(self, auto_update): session = self.get_session() device_config = self.get_device_config(session) device_config.auto_update = auto_update session.commit() log.info("Auto update setting successfully updated: %r", auto_update) def refresh_proxies(self, proxy_settings=None, device_config=None): """Refresh current proxies with the given settings""" # If no proxy settings passed fetch them from database proxy_settings = (proxy_settings if proxy_settings is not None else self.get_proxy_settings( device_config=device_config)) self.proxies, self.proxy_exceptions = get_proxies_for_handler( proxy_settings) def get_server_binding(self, local_folder, raise_if_missing=False, session=None): """Find the ServerBinding instance for a given local_folder""" local_folder = normalized_path(local_folder) if session is None: session = self.get_session() try: return session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() except NoResultFound: if raise_if_missing: raise RuntimeError( "Folder '%s' is not bound to any Nuxeo server" % local_folder) return None def list_server_bindings(self, session=None): if session is None: session = self.get_session() return session.query(ServerBinding).all() def get_first_token(self, session=None): """Get the token from the first server binding""" if session is None: session = self.get_session() server_bindings = self.list_server_bindings(session) if not server_bindings: return None sb = server_bindings[0] return sb.remote_token def get_server_binding_settings(self): """Fetch server binding settings from database""" server_bindings = self.list_server_bindings() if not server_bindings: return ServerBindingSettings( local_folder=default_nuxeo_drive_folder()) else: # TODO: handle multiple server bindings, for now take the first one # See https://jira.nuxeo.com/browse/NXP-12716 sb = server_bindings[0] return ServerBindingSettings(server_url=sb.server_url, server_version=sb.server_version, username=sb.remote_user, local_folder=sb.local_folder, initialized=True, pwd_update_required=sb.has_invalid_credentials()) def is_credentials_update_required(self): server_bindings = self.list_server_bindings() if not server_bindings: return True else: # TODO: handle multiple server bindings, for now consider that # credentials update is required if at least one binding has # invalid credentials # See https://jira.nuxeo.com/browse/NXP-12716 for server_binding in server_bindings: if server_binding.has_invalid_credentials(): return True return False def stop(self): """Stop the Nuxeo Drive synchronization thread As the process asking the synchronization to stop might not be the same as the process running the synchronization (especially when used from the commandline without the graphical user interface and its tray icon menu) we use a simple empty marker file a cross platform way to pass the stop message between the two. """ pid = self.synchronizer.check_running(process_name="sync") if pid is not None: # Create a stop file marker for the running synchronization # process log.info("Telling synchronization process %d to stop." % pid) stop_file = os.path.join(self.config_folder, "stop_%d" % pid) open(safe_long_path(stop_file), 'wb').close() else: log.info("No running synchronization process to stop.") def children_states(self, folder_path): """List the status of the children of a folder The state of the folder is a summary of their descendant rather than their own instric synchronization step which is of little use for the end user. """ session = self.get_session() # Find the server binding for this absolute path try: binding, path = self._binding_path(folder_path, session=session) except NotFound: return [] try: folder_state = session.query(LastKnownState).filter_by( local_folder=binding.local_folder, local_path=path, ).one() except NoResultFound: return [] states = self._pair_states_recursive(session, folder_state) return [(os.path.basename(s.local_path), pair_state) for s, pair_state in states if s.local_parent_path == path] def _pair_states_recursive(self, session, doc_pair): """Recursive call to collect pair state under a given location.""" if not doc_pair.folderish: return [(doc_pair, doc_pair.pair_state)] if doc_pair.local_path is not None and doc_pair.remote_ref is not None: f = or_( LastKnownState.local_parent_path == doc_pair.local_path, LastKnownState.remote_parent_ref == doc_pair.remote_ref, ) elif doc_pair.local_path is not None: f = LastKnownState.local_parent_path == doc_pair.local_path elif doc_pair.remote_ref is not None: f = LastKnownState.remote_parent_ref == doc_pair.remote_ref else: raise ValueError("Illegal state %r: at least path or remote_ref" " should be not None." % doc_pair) children_states = session.query(LastKnownState).filter_by( local_folder=doc_pair.local_folder).filter(f).order_by( asc(LastKnownState.local_name), asc(LastKnownState.remote_name), ).all() results = [] for child_state in children_states: sub_results = self._pair_states_recursive(session, child_state) results.extend(sub_results) # A folder stays synchronized (or unknown) only if all the descendants # are themselfves synchronized. pair_state = doc_pair.pair_state for _, sub_pair_state in results: if sub_pair_state != 'synchronized': pair_state = 'children_modified' break # Pre-pend the folder state to the descendants return [(doc_pair, pair_state)] + results def _binding_path(self, local_path, session=None): """Find a server binding and relative path for a given FS path""" local_path = normalized_path(local_path) # Check exact binding match binding = self.get_server_binding(local_path, session=session, raise_if_missing=False) if binding is not None: return binding, u'/' # Check for bindings that are prefix of local_path session = self.get_session() all_bindings = session.query(ServerBinding).all() matching_bindings = [sb for sb in all_bindings if local_path.startswith( sb.local_folder + os.path.sep)] if len(matching_bindings) == 0: raise NotFound("Could not find any server binding for " + local_path) elif len(matching_bindings) > 1: raise RuntimeError("Found more than one binding for %s: %r" % ( local_path, matching_bindings)) binding = matching_bindings[0] path = local_path[len(binding.local_folder):] path = path.replace(os.path.sep, u'/') return binding, path def bind_server(self, local_folder, server_url, username, password): """Bind a local folder to a remote nuxeo server""" session = self.get_session() local_folder = normalized_path(local_folder) # check the connection to the server by issuing an authentication # request server_url = self._normalize_url(server_url) nxclient = self.remote_doc_client_factory( server_url, username, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=password, timeout=self.handshake_timeout) token = nxclient.request_token() if token is not None: # The server supports token based identification: do not store the # password in the DB password = None try: try: # Look for an existing server binding for the given local # folder server_binding = session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() if server_binding.server_url != server_url: raise RuntimeError( "%s is already bound to '%s'" % ( local_folder, server_binding.server_url)) if server_binding.remote_user != username: # Update username info if required server_binding.remote_user = username log.info("Updating username to '%s' on server '%s'", username, server_url) if (token is None and server_binding.remote_password != password): # Update password info if required server_binding.remote_password = password log.info("Updating password for user '%s' on server '%s'", username, server_url) if token is not None and server_binding.remote_token != token: log.info("Updating token for user '%s' on server '%s'", username, server_url) # Update the token info if required server_binding.remote_token = token # Ensure that the password is not stored in the DB if server_binding.remote_password is not None: server_binding.remote_password = None # If the top level state for the server binding doesn't exist, # create the local folder and the top level state. This can be # the case when initializing the DB manually with a SQL script. try: session.query(LastKnownState).filter_by(local_path='/', local_folder=local_folder).one() except NoResultFound: self._make_local_folder(local_folder) self._add_top_level_state(server_binding, session) except NoResultFound: # No server binding found for the given local folder # First create local folder in the file system self._make_local_folder(local_folder) # Create ServerBinding instance in DB log.info("Binding '%s' to '%s' with account '%s'", local_folder, server_url, username) server_binding = ServerBinding(local_folder, server_url, username, remote_password=password, remote_token=token) session.add(server_binding) # Create the top level state for the server binding self._add_top_level_state(server_binding, session) # Set update info self._set_update_info(server_binding, remote_client=nxclient) except: # In case an AddonNotInstalled exception is raised, need to # invalidate the remote client cache for it to be aware of the new # operations when the addon gets installed if server_binding is not None: self.invalidate_client_cache(server_binding.server_url) session.rollback() raise session.commit() return server_binding def _add_top_level_state(self, server_binding, session): local_client = LocalClient(server_binding.local_folder) local_info = local_client.get_info(u'/') remote_client = self.get_remote_fs_client(server_binding) remote_info = remote_client.get_filesystem_root_info() state = LastKnownState(server_binding.local_folder, local_info=local_info, local_state='synchronized', remote_info=remote_info, remote_state='synchronized') session.add(state) def _make_local_folder(self, local_folder): if not os.path.exists(local_folder): os.makedirs(local_folder) self.register_folder_link(local_folder) def unbind_server(self, local_folder): """Remove the binding to a Nuxeo server Local files are not deleted""" session = self.get_session() local_folder = normalized_path(local_folder) binding = self.get_server_binding(local_folder, raise_if_missing=True, session=session) # Revoke token if necessary if binding.remote_token is not None: try: nxclient = self.remote_doc_client_factory( binding.server_url, binding.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, token=binding.remote_token, timeout=self.timeout) log.info("Revoking token for '%s' with account '%s'", binding.server_url, binding.remote_user) nxclient.revoke_token() except POSSIBLE_NETWORK_ERROR_TYPES: log.warning("Could not connect to server '%s' to revoke token", binding.server_url) except Unauthorized: # Token is already revoked pass # Invalidate client cache self.invalidate_client_cache(binding.server_url) # Delete binding info in local DB log.info("Unbinding '%s' from '%s' with account '%s'", local_folder, binding.server_url, binding.remote_user) session.delete(binding) session.commit() def unbind_all(self): """Unbind all server and revoke all tokens This is useful for cleanup in integration test code. """ session = self.get_session() for sb in session.query(ServerBinding).all(): self.unbind_server(sb.local_folder) def bind_root(self, local_folder, remote_ref, repository='default', session=None): """Bind local root to a remote root (folderish document in Nuxeo). local_folder must be already bound to an existing Nuxeo server. remote_ref must be the IdRef or PathRef of an existing folderish document on the remote server bound to the local folder. """ session = self.get_session() if session is None else session local_folder = normalized_path(local_folder) server_binding = self.get_server_binding( local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Register the root on the server nxclient.register_as_root(remote_ref) def unbind_root(self, local_folder, remote_ref, repository='default', session=None): """Remove binding to remote folder""" session = self.get_session() if session is None else session server_binding = self.get_server_binding( local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Unregister the root on the server nxclient.unregister_as_root(remote_ref) def get_max_errors(self): return self.max_errors def list_on_errors(self, limit=100, session=None): if session is None: session = self.get_session() # Only consider pair states that are not synchronized # and ignore unsynchronized ones predicates = [LastKnownState.pair_state != 'synchronized', LastKnownState.pair_state != 'unsynchronized'] # Don't try to sync file that have too many error predicates.append(LastKnownState.error_count >= self.get_max_errors()) return session.query(LastKnownState).filter( *predicates ).order_by( # Ensure that newly created remote folders will be synchronized # before their children while keeping a fixed named based # deterministic ordering to make the tests readable asc(LastKnownState.remote_parent_path), asc(LastKnownState.remote_name), asc(LastKnownState.remote_ref), # Ensure that newly created local folders will be synchronized # before their children asc(LastKnownState.local_path) ).limit(limit).all() def list_pending(self, limit=100, local_folder=None, ignore_in_error=None, session=None): """List pending files to synchronize, ordered by path Ordering by path makes it possible to synchronize sub folders content only once the parent folders have already been synchronized. If ingore_in_error is not None and is a duration in second, skip pair states that have recently triggered a synchronization error. """ if session is None: session = self.get_session() # Only consider pair states that are not synchronized # and ignore unsynchronized ones predicates = [LastKnownState.pair_state != 'synchronized', LastKnownState.pair_state != 'unsynchronized'] # Don't try to sync file that have too many error predicates.append(LastKnownState.error_count < self.get_max_errors()) if local_folder is not None: predicates.append(LastKnownState.local_folder == local_folder) if ignore_in_error is not None and ignore_in_error > 0: max_date = datetime.utcnow() - timedelta(seconds=ignore_in_error) predicates.append(or_( LastKnownState.last_sync_error_date == None, LastKnownState.last_sync_error_date < max_date)) return session.query(LastKnownState).filter( *predicates ).order_by( # Ensure that newly created remote folders will be synchronized # before their children while keeping a fixed named based # deterministic ordering to make the tests readable asc(LastKnownState.remote_parent_path), asc(LastKnownState.remote_name), asc(LastKnownState.remote_ref), # Ensure that newly created local folders will be synchronized # before their children asc(LastKnownState.local_path) ).limit(limit).all() def next_pending(self, local_folder=None, session=None): """Return the next pending file to synchronize or None""" pending = self.list_pending(limit=1, local_folder=local_folder, session=session) return pending[0] if len(pending) > 0 else None def init_recently_modified(self): server_bindings = self.list_server_bindings() if server_bindings: for sb in server_bindings: self.recently_modified[sb.local_folder] = self.list_recently_modified(sb.local_folder) log.info("Initialized list of recently modified items" " in %s: %r", sb.local_folder, [item.local_name for item in self.get_recently_modified(sb.local_folder)]) def get_recently_modified(self, local_folder): return self.recently_modified[local_folder] def update_recently_modified(self, doc_pair): local_folder = doc_pair.local_folder self.recently_modified[local_folder] = self.list_recently_modified(local_folder) session = self.get_session() log.info("Updated list of recently modified items in %s: %r", local_folder, [item.local_name for item in self.get_recently_modified(local_folder)]) def list_recently_modified(self, local_folder): """List recently modified pairs ordered by last local modification. """ session = self.get_session() predicates = [LastKnownState.local_folder == local_folder] # Only consider pair states that are synchronized predicates.append(LastKnownState.pair_state == 'synchronized') # Don't consider folders predicates.append(LastKnownState.folderish == False) items = session.query(LastKnownState).filter( *predicates ).order_by( desc(LastKnownState.last_sync_date), ).options().limit(self.get_number_recently_modified()).all() # Remove objects from session result = [] for item in items: session.expunge(item) result.append(item) return result def get_number_recently_modified(self): return DEFAULT_NUMBER_RECENTLY_MODIFIED def _get_client_cache(self): if not hasattr(self._local, 'remote_clients'): self._local.remote_clients = dict() return self._local.remote_clients def get_remote_fs_client(self, server_binding, filtered=True): """Return a client for the FileSystem abstraction.""" cache = self._get_client_cache() sb = server_binding cache_key = (sb.server_url, sb.remote_user, self.device_id, filtered) remote_client_cache = cache.get(cache_key) if remote_client_cache is not None: remote_client = remote_client_cache[0] timestamp = remote_client_cache[1] client_cache_timestamp = self._client_cache_timestamps.get(cache_key) if remote_client_cache is None or timestamp < client_cache_timestamp: if filtered: remote_client = self.remote_filtered_fs_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, self.get_session(), proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, timeout=self.timeout, cookie_jar=self.cookie_jar, check_suspended=self.synchronizer.check_suspended) else: remote_client = self.remote_fs_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, timeout=self.timeout, cookie_jar=self.cookie_jar, check_suspended=self.synchronizer.check_suspended) if client_cache_timestamp is None: client_cache_timestamp = 0 self._client_cache_timestamps[cache_key] = 0 cache[cache_key] = remote_client, client_cache_timestamp # Make it possible to have the remote client simulate any kind of # network or server failure: this is useful for example to ensure that # cookies used for load balancer affinity (e.g. AWSELB) are shared by # all the Automation clients managed by a given controller. remote_client.make_remote_raise(self._remote_error) # Make it possible to have the remote client simulate any kind of # local device failure: this is useful for example to test a "No space # left on device" issue when downloading a file. remote_client.make_local_raise(self._local_error) return remote_client def get_remote_doc_client(self, server_binding, repository='default', base_folder=None): """Return an instance of Nuxeo Document Client""" sb = server_binding return self.remote_doc_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, repository=repository, base_folder=base_folder, timeout=self.timeout, cookie_jar=self.cookie_jar) def get_local_client(self, local_folder): """Return a file system client for the given local folder""" return LocalClient(local_folder) def invalidate_client_cache(self, server_url=None): for key in self._client_cache_timestamps: if server_url is None or key[0] == server_url: now = datetime.utcnow().utctimetuple() self._client_cache_timestamps[key] = calendar.timegm(now) # Re-fetch HTTP proxy settings self.refresh_proxies() def get_state(self, server_url, remote_ref): """Find a pair state for the provided remote document identifiers.""" server_url = self._normalize_url(server_url) session = self.get_session() try: states = session.query(LastKnownState).filter_by( remote_ref=remote_ref, ).all() for state in states: if (state.server_binding.server_url == server_url): return state except NoResultFound: return None def get_state_for_local_path(self, local_os_path): """Find a DB state from a local filesystem path""" session = self.get_session() sb, local_path = self._binding_path(local_os_path, session=session) return session.query(LastKnownState).filter_by( local_folder=sb.local_folder, local_path=local_path).one() def launch_file_editor(self, server_url, remote_ref): """Find the local file if any and start OS editor on it.""" state = self.get_state(server_url, remote_ref) if state is None: # TODO: synchronize to a dedicated special root for one time edit log.warning('Could not find local file for server_url=%s ' 'and remote_ref=%s', server_url, remote_ref) return # TODO: check synchronization of this state first # Find the best editor for the file according to the OS configuration file_path = state.get_local_abspath() self.open_local_file(file_path) def open_local_file(self, file_path): """Launch the local OS program on the given file / folder.""" log.debug('Launching editor on %s', file_path) if sys.platform == 'win32': os.startfile(file_path) elif sys.platform == 'darwin': subprocess.Popen(['open', file_path]) else: try: subprocess.Popen(['xdg-open', file_path]) except OSError: # xdg-open should be supported by recent Gnome, KDE, Xfce log.error("Failed to find and editor for: '%s'", file_path) def make_remote_raise(self, error): """Helper method to simulate network failure for testing""" self._remote_error = error def make_local_raise(self, error): """Helper method to simulate local device failure for testing""" self._local_error = error def dispose(self): """Release database resources. Close thread-local Session, ending any transaction in progress and releasing underlying connections from the pool. Note that releasing all connections from the pool using Session.close_all() or SingletonThreadPool.dispose() is not an option here as the Python SQLite driver pysqlite used by SQLAlchemy doesn't let you close a connection from a thread that didn't create it (except under Windows, see below). In our case at least two threads are involved, the GUI and the synchronization one, so each one needs to close its own Session by calling this function. Beware that starting more threads than the pool size (default is 5) might lead to a ProgrammingError when SingletonThreadPool._cleanup() gets called, for the reason just mentioned. Also note that calling Session.close() never seems to remove the connection object from the pool, even if the thread owning it is dead. Under Windows we need to release all connections from the pool calling SingletonThreadPool.dispose(), which strangely doesn't raise a ProgrammingError - probably due to a different implementation of the pysqlite driver -, otherwise we might get a WindowsError at tear down in tests using multiple threads when trying to remove the temporary test folder because of it being used by another Python process than the main one... """ session = self.get_session() log.debug("Closing thread-local Session %r, ending any transaction" " in progress and releasing underlying connections from" " the pool", session) session.close() if sys.platform == 'win32': log.debug("As we are under Windows, dispose connection pool to" " make sure all connections are closed, avoiding any" " WindowsError due to a Python process using the" " database file") self._engine.pool.dispose() def _normalize_url(self, url): """Ensure that user provided url always has a trailing '/'""" if url is None or not url: raise ValueError("Invalid url: %r" % url) if not url.endswith(u'/'): return url + u'/' return url def register_folder_link(self, folder_path): if sys.platform == 'darwin': self.register_folder_link_darwin(folder_path) # TODO: implement Windows and Linux support here def register_folder_link_darwin(self, folder_path): try: from LaunchServices import LSSharedFileListCreate from LaunchServices import kLSSharedFileListFavoriteItems from LaunchServices import LSSharedFileListInsertItemURL from LaunchServices import kLSSharedFileListItemBeforeFirst from LaunchServices import CFURLCreateWithString except ImportError: log.warning("PyObjC package is not installed:" " skipping favorite link creation") return folder_path = normalized_path(folder_path) folder_name = os.path.basename(folder_path) lst = LSSharedFileListCreate(None, kLSSharedFileListFavoriteItems, None) if lst is None: log.warning("Could not fetch the Finder favorite list.") return url = CFURLCreateWithString(None, "file://" + quote(folder_path), None) if url is None: log.warning("Could not generate valid favorite URL for: %s", folder_path) return # Register the folder as favorite if not already there item = LSSharedFileListInsertItemURL( lst, kLSSharedFileListItemBeforeFirst, folder_name, None, url, {}, []) if item is not None: log.debug("Registered new favorite in Finder for: %s", folder_path)
def __init__(self, config_folder, echo=False, echo_pool=False, poolclass=None, handshake_timeout=60, timeout=20, page_size=None, max_errors=3): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.info("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.info("nxdrive configured in '%s'", self.config_folder) if not echo: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.handshake_timeout = handshake_timeout self.timeout = timeout self.max_errors = max_errors # Handle connection to the local Nuxeo Drive configuration and # metadata SQLite database. self._engine, self._session_maker = init_db(self.config_folder, echo=echo, echo_pool=echo_pool, poolclass=poolclass) # Migrate SQLite database if needed migrate_db(self._engine) # Thread-local storage for the remote client cache self._local = local() self._client_cache_timestamps = dict() self._remote_error = None self._local_error = None device_config = self.get_device_config() self.device_id = device_config.device_id self.version = nxdrive.__version__ self.updated = self.update_version(device_config) # HTTP proxy settings self.proxies = None self.proxy_exceptions = None self.refresh_proxies(device_config=device_config) # Recently modified items for each server binding self.recently_modified = {} self.synchronizer = Synchronizer(self, page_size=page_size) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar()
class Controller(object): """Manage configuration and perform Nuxeo Drive Operations This class is thread safe: instance can be shared by multiple threads as DB sessions and Nuxeo clients are thread locals. """ # Used for binding server / roots and managing tokens remote_doc_client_factory = RemoteDocumentClient # Used for FS synchronization operations remote_fs_client_factory = RemoteFileSystemClient # Used for FS synchronization operations remote_filtered_fs_client_factory = RemoteFilteredFileSystemClient def __init__(self, config_folder, echo=False, echo_pool=False, poolclass=None, handshake_timeout=60, timeout=20, page_size=None, max_errors=3): # Log the installation location for debug nxdrive_install_folder = os.path.dirname(nxdrive.__file__) nxdrive_install_folder = os.path.realpath(nxdrive_install_folder) log.info("nxdrive installed in '%s'", nxdrive_install_folder) # Log the configuration location for debug config_folder = os.path.expanduser(config_folder) self.config_folder = os.path.realpath(config_folder) if not os.path.exists(self.config_folder): os.makedirs(self.config_folder) log.info("nxdrive configured in '%s'", self.config_folder) if not echo: echo = os.environ.get('NX_DRIVE_LOG_SQL', None) is not None self.handshake_timeout = handshake_timeout self.timeout = timeout self.max_errors = max_errors # Handle connection to the local Nuxeo Drive configuration and # metadata SQLite database. self._engine, self._session_maker = init_db(self.config_folder, echo=echo, echo_pool=echo_pool, poolclass=poolclass) # Migrate SQLite database if needed migrate_db(self._engine) # Thread-local storage for the remote client cache self._local = local() self._client_cache_timestamps = dict() self._remote_error = None self._local_error = None device_config = self.get_device_config() self.device_id = device_config.device_id self.version = nxdrive.__version__ self.updated = self.update_version(device_config) # HTTP proxy settings self.proxies = None self.proxy_exceptions = None self.refresh_proxies(device_config=device_config) # Recently modified items for each server binding self.recently_modified = {} self.synchronizer = Synchronizer(self, page_size=page_size) # Make all the automation client related to this controller # share cookies using threadsafe jar self.cookie_jar = CookieJar() def trash_modified_file(self): return False def get_session(self): """Reuse the thread local session for this controller Using the controller in several thread should be thread safe as long as this method is always called to fetch the session instance. """ return self._session_maker() def get_device_config(self, session=None): """Fetch the singleton configuration object for this device""" if session is None: session = self.get_session() try: return session.query(DeviceConfig).one() except NoResultFound: device_config = DeviceConfig() # generate a unique device id session.add(device_config) session.commit() return device_config def get_version(self): return self.version def update_version(self, device_config): if self.version != device_config.client_version: log.info( "Detected version upgrade: current version = %s," " new version = %s => upgrading current version," " yet DB upgrade might be needed.", device_config.client_version, self.version) device_config.client_version = self.version self.get_session().commit() return True return False def is_updated(self): return self.updated def refresh_update_info(self, local_folder): session = self.get_session() sb = self.get_server_binding(local_folder, session=session) self._set_update_info(sb) session.commit() def _set_update_info(self, server_binding, remote_client=None): try: remote_client = (remote_client if remote_client is not None else self.get_remote_doc_client(server_binding)) update_info = remote_client.get_update_info() log.info("Fetched update info from server: %r", update_info) server_binding.server_version = update_info['serverVersion'] server_binding.update_url = update_info['updateSiteURL'] except Exception as e: log.warning("Cannot get update info from server because of: %s", e) # Fall back on default server version if needed if server_binding.server_version is None: server_binding.server_version = self.get_default_server_version() log.debug( "Server version is null or not available, falling back" " on default one: %s", server_binding.server_version) # Fall back on default update site URL if needed if server_binding.update_url is None: server_binding.update_url = self.get_default_update_site_url() log.debug( "Update site URL is null or not available, falling back" " on default one: %s", server_binding.update_url) @deprecated def get_default_server_version(self): return None def get_default_update_site_url(self): return DEFAULT_UPDATE_SITE_URL def get_proxy_settings(self, device_config=None): """Fetch proxy settings from database""" dc = (self.get_device_config() if device_config is None else device_config) # Decrypt password with token as the secret token = self.get_first_token() if dc.proxy_password is not None and token is not None: password = decrypt(dc.proxy_password, token) else: # If no server binding or no token available # (possibly after token revocation) reset password password = '' return ProxySettings(config=dc.proxy_config, proxy_type=dc.proxy_type, server=dc.proxy_server, port=dc.proxy_port, authenticated=dc.proxy_authenticated, username=dc.proxy_username, password=password, exceptions=dc.proxy_exceptions) def set_proxy_settings(self, proxy_settings): session = self.get_session() device_config = self.get_device_config(session) device_config.proxy_config = proxy_settings.config device_config.proxy_type = proxy_settings.proxy_type device_config.proxy_server = proxy_settings.server device_config.proxy_port = proxy_settings.port device_config.proxy_exceptions = proxy_settings.exceptions device_config.proxy_authenticated = proxy_settings.authenticated device_config.proxy_username = proxy_settings.username # Encrypt password with token as the secret token = self.get_first_token(session) if token is None: raise MissingToken( "Your token has been revoked," " please update your password to acquire a new one.") password = encrypt(proxy_settings.password, token) device_config.proxy_password = password session.commit() log.info("Proxy settings successfully updated: %r", proxy_settings) self.invalidate_client_cache() def get_general_settings(self, device_config=None): """Fetch general settings from database""" dc = (self.get_device_config() if device_config is None else device_config) return GeneralSettings(auto_update=dc.auto_update) def set_general_settings(self, general_settings): session = self.get_session() device_config = self.get_device_config(session) device_config.auto_update = general_settings.auto_update session.commit() log.info("General settings successfully updated: %r", general_settings) def is_auto_update(self, device_config=None): return self.get_general_settings( device_config=device_config).auto_update def set_auto_update(self, auto_update): session = self.get_session() device_config = self.get_device_config(session) device_config.auto_update = auto_update session.commit() log.info("Auto update setting successfully updated: %r", auto_update) def refresh_proxies(self, proxy_settings=None, device_config=None): """Refresh current proxies with the given settings""" # If no proxy settings passed fetch them from database proxy_settings = (proxy_settings if proxy_settings is not None else self.get_proxy_settings(device_config=device_config)) self.proxies, self.proxy_exceptions = get_proxies_for_handler( proxy_settings) def get_server_binding(self, local_folder, raise_if_missing=False, session=None): """Find the ServerBinding instance for a given local_folder""" local_folder = normalized_path(local_folder) if session is None: session = self.get_session() try: return session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() except NoResultFound: if raise_if_missing: raise RuntimeError( "Folder '%s' is not bound to any Nuxeo server" % local_folder) return None def list_server_bindings(self, session=None): if session is None: session = self.get_session() return session.query(ServerBinding).all() def get_first_token(self, session=None): """Get the token from the first server binding""" if session is None: session = self.get_session() server_bindings = self.list_server_bindings(session) if not server_bindings: return None sb = server_bindings[0] return sb.remote_token def get_server_binding_settings(self): """Fetch server binding settings from database""" server_bindings = self.list_server_bindings() if not server_bindings: return ServerBindingSettings( local_folder=default_nuxeo_drive_folder()) else: # TODO: handle multiple server bindings, for now take the first one # See https://jira.nuxeo.com/browse/NXP-12716 sb = server_bindings[0] return ServerBindingSettings( server_url=sb.server_url, server_version=sb.server_version, username=sb.remote_user, local_folder=sb.local_folder, initialized=True, pwd_update_required=sb.has_invalid_credentials()) def is_credentials_update_required(self): server_bindings = self.list_server_bindings() if not server_bindings: return True else: # TODO: handle multiple server bindings, for now consider that # credentials update is required if at least one binding has # invalid credentials # See https://jira.nuxeo.com/browse/NXP-12716 for server_binding in server_bindings: if server_binding.has_invalid_credentials(): return True return False def stop(self): """Stop the Nuxeo Drive synchronization thread As the process asking the synchronization to stop might not be the same as the process running the synchronization (especially when used from the commandline without the graphical user interface and its tray icon menu) we use a simple empty marker file a cross platform way to pass the stop message between the two. """ pid = self.synchronizer.check_running(process_name="sync") if pid is not None: # Create a stop file marker for the running synchronization # process log.info("Telling synchronization process %d to stop." % pid) stop_file = os.path.join(self.config_folder, "stop_%d" % pid) open(safe_long_path(stop_file), 'wb').close() else: log.info("No running synchronization process to stop.") def children_states(self, folder_path): """List the status of the children of a folder The state of the folder is a summary of their descendant rather than their own instric synchronization step which is of little use for the end user. """ session = self.get_session() # Find the server binding for this absolute path try: binding, path = self._binding_path(folder_path, session=session) except NotFound: return [] try: folder_state = session.query(LastKnownState).filter_by( local_folder=binding.local_folder, local_path=path, ).one() except NoResultFound: return [] states = self._pair_states_recursive(session, folder_state) return [(os.path.basename(s.local_path), pair_state) for s, pair_state in states if s.local_parent_path == path] def _pair_states_recursive(self, session, doc_pair): """Recursive call to collect pair state under a given location.""" if not doc_pair.folderish: return [(doc_pair, doc_pair.pair_state)] if doc_pair.local_path is not None and doc_pair.remote_ref is not None: f = or_( LastKnownState.local_parent_path == doc_pair.local_path, LastKnownState.remote_parent_ref == doc_pair.remote_ref, ) elif doc_pair.local_path is not None: f = LastKnownState.local_parent_path == doc_pair.local_path elif doc_pair.remote_ref is not None: f = LastKnownState.remote_parent_ref == doc_pair.remote_ref else: raise ValueError("Illegal state %r: at least path or remote_ref" " should be not None." % doc_pair) children_states = session.query(LastKnownState).filter_by( local_folder=doc_pair.local_folder).filter(f).order_by( asc(LastKnownState.local_name), asc(LastKnownState.remote_name), ).all() results = [] for child_state in children_states: sub_results = self._pair_states_recursive(session, child_state) results.extend(sub_results) # A folder stays synchronized (or unknown) only if all the descendants # are themselfves synchronized. pair_state = doc_pair.pair_state for _, sub_pair_state in results: if sub_pair_state != 'synchronized': pair_state = 'children_modified' break # Pre-pend the folder state to the descendants return [(doc_pair, pair_state)] + results def _binding_path(self, local_path, session=None): """Find a server binding and relative path for a given FS path""" local_path = normalized_path(local_path) # Check exact binding match binding = self.get_server_binding(local_path, session=session, raise_if_missing=False) if binding is not None: return binding, u'/' # Check for bindings that are prefix of local_path session = self.get_session() all_bindings = session.query(ServerBinding).all() matching_bindings = [ sb for sb in all_bindings if local_path.startswith(sb.local_folder + os.path.sep) ] if len(matching_bindings) == 0: raise NotFound("Could not find any server binding for " + local_path) elif len(matching_bindings) > 1: raise RuntimeError("Found more than one binding for %s: %r" % (local_path, matching_bindings)) binding = matching_bindings[0] path = local_path[len(binding.local_folder):] path = path.replace(os.path.sep, u'/') return binding, path def bind_server(self, local_folder, server_url, username, password): """Bind a local folder to a remote nuxeo server""" session = self.get_session() local_folder = normalized_path(local_folder) # check the connection to the server by issuing an authentication # request server_url = self._normalize_url(server_url) nxclient = self.remote_doc_client_factory( server_url, username, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=password, timeout=self.handshake_timeout) token = nxclient.request_token() if token is not None: # The server supports token based identification: do not store the # password in the DB password = None try: try: # Look for an existing server binding for the given local # folder server_binding = session.query(ServerBinding).filter( ServerBinding.local_folder == local_folder).one() if server_binding.server_url != server_url: raise RuntimeError( "%s is already bound to '%s'" % (local_folder, server_binding.server_url)) if server_binding.remote_user != username: # Update username info if required server_binding.remote_user = username log.info("Updating username to '%s' on server '%s'", username, server_url) if (token is None and server_binding.remote_password != password): # Update password info if required server_binding.remote_password = password log.info("Updating password for user '%s' on server '%s'", username, server_url) if token is not None and server_binding.remote_token != token: log.info("Updating token for user '%s' on server '%s'", username, server_url) # Update the token info if required server_binding.remote_token = token # Ensure that the password is not stored in the DB if server_binding.remote_password is not None: server_binding.remote_password = None # If the top level state for the server binding doesn't exist, # create the local folder and the top level state. This can be # the case when initializing the DB manually with a SQL script. try: session.query(LastKnownState).filter_by( local_path='/', local_folder=local_folder).one() except NoResultFound: self._make_local_folder(local_folder) self._add_top_level_state(server_binding, session) except NoResultFound: # No server binding found for the given local folder # First create local folder in the file system self._make_local_folder(local_folder) # Create ServerBinding instance in DB log.info("Binding '%s' to '%s' with account '%s'", local_folder, server_url, username) server_binding = ServerBinding(local_folder, server_url, username, remote_password=password, remote_token=token) session.add(server_binding) # Create the top level state for the server binding self._add_top_level_state(server_binding, session) # Set update info self._set_update_info(server_binding, remote_client=nxclient) except: # In case an AddonNotInstalled exception is raised, need to # invalidate the remote client cache for it to be aware of the new # operations when the addon gets installed if server_binding is not None: self.invalidate_client_cache(server_binding.server_url) session.rollback() raise session.commit() return server_binding def _add_top_level_state(self, server_binding, session): local_client = LocalClient(server_binding.local_folder) local_info = local_client.get_info(u'/') remote_client = self.get_remote_fs_client(server_binding) remote_info = remote_client.get_filesystem_root_info() state = LastKnownState(server_binding.local_folder, local_info=local_info, local_state='synchronized', remote_info=remote_info, remote_state='synchronized') session.add(state) def _make_local_folder(self, local_folder): if not os.path.exists(local_folder): os.makedirs(local_folder) self.register_folder_link(local_folder) def unbind_server(self, local_folder): """Remove the binding to a Nuxeo server Local files are not deleted""" session = self.get_session() local_folder = normalized_path(local_folder) binding = self.get_server_binding(local_folder, raise_if_missing=True, session=session) # Revoke token if necessary if binding.remote_token is not None: try: nxclient = self.remote_doc_client_factory( binding.server_url, binding.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, token=binding.remote_token, timeout=self.timeout) log.info("Revoking token for '%s' with account '%s'", binding.server_url, binding.remote_user) nxclient.revoke_token() except POSSIBLE_NETWORK_ERROR_TYPES: log.warning("Could not connect to server '%s' to revoke token", binding.server_url) except Unauthorized: # Token is already revoked pass # Invalidate client cache self.invalidate_client_cache(binding.server_url) # Delete binding info in local DB log.info("Unbinding '%s' from '%s' with account '%s'", local_folder, binding.server_url, binding.remote_user) session.delete(binding) session.commit() def unbind_all(self): """Unbind all server and revoke all tokens This is useful for cleanup in integration test code. """ session = self.get_session() for sb in session.query(ServerBinding).all(): self.unbind_server(sb.local_folder) def bind_root(self, local_folder, remote_ref, repository='default', session=None): """Bind local root to a remote root (folderish document in Nuxeo). local_folder must be already bound to an existing Nuxeo server. remote_ref must be the IdRef or PathRef of an existing folderish document on the remote server bound to the local folder. """ session = self.get_session() if session is None else session local_folder = normalized_path(local_folder) server_binding = self.get_server_binding(local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Register the root on the server nxclient.register_as_root(remote_ref) def unbind_root(self, local_folder, remote_ref, repository='default', session=None): """Remove binding to remote folder""" session = self.get_session() if session is None else session server_binding = self.get_server_binding(local_folder, raise_if_missing=True, session=session) nxclient = self.get_remote_doc_client(server_binding, repository=repository) # Unregister the root on the server nxclient.unregister_as_root(remote_ref) def get_max_errors(self): return self.max_errors def list_on_errors(self, limit=100, session=None): if session is None: session = self.get_session() # Only consider pair states that are not synchronized # and ignore unsynchronized ones predicates = [ LastKnownState.pair_state != 'synchronized', LastKnownState.pair_state != 'unsynchronized' ] # Don't try to sync file that have too many error predicates.append(LastKnownState.error_count >= self.get_max_errors()) return session.query(LastKnownState).filter(*predicates).order_by( # Ensure that newly created remote folders will be synchronized # before their children while keeping a fixed named based # deterministic ordering to make the tests readable asc(LastKnownState.remote_parent_path), asc(LastKnownState.remote_name), asc(LastKnownState.remote_ref), # Ensure that newly created local folders will be synchronized # before their children asc(LastKnownState.local_path)).limit(limit).all() def list_pending(self, limit=100, local_folder=None, ignore_in_error=None, session=None): """List pending files to synchronize, ordered by path Ordering by path makes it possible to synchronize sub folders content only once the parent folders have already been synchronized. If ingore_in_error is not None and is a duration in second, skip pair states that have recently triggered a synchronization error. """ if session is None: session = self.get_session() # Only consider pair states that are not synchronized # and ignore unsynchronized ones predicates = [ LastKnownState.pair_state != 'synchronized', LastKnownState.pair_state != 'unsynchronized' ] # Don't try to sync file that have too many error predicates.append(LastKnownState.error_count < self.get_max_errors()) if local_folder is not None: predicates.append(LastKnownState.local_folder == local_folder) if ignore_in_error is not None and ignore_in_error > 0: max_date = datetime.utcnow() - timedelta(seconds=ignore_in_error) predicates.append( or_(LastKnownState.last_sync_error_date == None, LastKnownState.last_sync_error_date < max_date)) return session.query(LastKnownState).filter(*predicates).order_by( # Ensure that newly created remote folders will be synchronized # before their children while keeping a fixed named based # deterministic ordering to make the tests readable asc(LastKnownState.remote_parent_path), asc(LastKnownState.remote_name), asc(LastKnownState.remote_ref), # Ensure that newly created local folders will be synchronized # before their children asc(LastKnownState.local_path)).limit(limit).all() def next_pending(self, local_folder=None, session=None): """Return the next pending file to synchronize or None""" pending = self.list_pending(limit=1, local_folder=local_folder, session=session) return pending[0] if len(pending) > 0 else None def init_recently_modified(self): server_bindings = self.list_server_bindings() if server_bindings: for sb in server_bindings: self.recently_modified[ sb.local_folder] = self.list_recently_modified( sb.local_folder) log.info( "Initialized list of recently modified items" " in %s: %r", sb.local_folder, [ item.local_name for item in self.get_recently_modified(sb.local_folder) ]) def get_recently_modified(self, local_folder): return self.recently_modified[local_folder] def update_recently_modified(self, doc_pair): local_folder = doc_pair.local_folder self.recently_modified[local_folder] = self.list_recently_modified( local_folder) session = self.get_session() log.info("Updated list of recently modified items in %s: %r", local_folder, [ item.local_name for item in self.get_recently_modified(local_folder) ]) def list_recently_modified(self, local_folder): """List recently modified pairs ordered by last local modification. """ session = self.get_session() predicates = [LastKnownState.local_folder == local_folder] # Only consider pair states that are synchronized predicates.append(LastKnownState.pair_state == 'synchronized') # Don't consider folders predicates.append(LastKnownState.folderish == False) items = session.query(LastKnownState).filter(*predicates).order_by( desc(LastKnownState.last_sync_date), ).options().limit( self.get_number_recently_modified()).all() # Remove objects from session result = [] for item in items: session.expunge(item) result.append(item) return result def get_number_recently_modified(self): return DEFAULT_NUMBER_RECENTLY_MODIFIED def _get_client_cache(self): if not hasattr(self._local, 'remote_clients'): self._local.remote_clients = dict() return self._local.remote_clients def get_remote_fs_client(self, server_binding, filtered=True): """Return a client for the FileSystem abstraction.""" cache = self._get_client_cache() sb = server_binding cache_key = (sb.server_url, sb.remote_user, self.device_id, filtered) remote_client_cache = cache.get(cache_key) if remote_client_cache is not None: remote_client = remote_client_cache[0] timestamp = remote_client_cache[1] client_cache_timestamp = self._client_cache_timestamps.get(cache_key) if remote_client_cache is None or timestamp < client_cache_timestamp: if filtered: remote_client = self.remote_filtered_fs_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, self.get_session(), proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, timeout=self.timeout, cookie_jar=self.cookie_jar, check_suspended=self.synchronizer.check_suspended) else: remote_client = self.remote_fs_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, timeout=self.timeout, cookie_jar=self.cookie_jar, check_suspended=self.synchronizer.check_suspended) if client_cache_timestamp is None: client_cache_timestamp = 0 self._client_cache_timestamps[cache_key] = 0 cache[cache_key] = remote_client, client_cache_timestamp # Make it possible to have the remote client simulate any kind of # network or server failure: this is useful for example to ensure that # cookies used for load balancer affinity (e.g. AWSELB) are shared by # all the Automation clients managed by a given controller. remote_client.make_remote_raise(self._remote_error) # Make it possible to have the remote client simulate any kind of # local device failure: this is useful for example to test a "No space # left on device" issue when downloading a file. remote_client.make_local_raise(self._local_error) return remote_client def get_remote_doc_client(self, server_binding, repository='default', base_folder=None): """Return an instance of Nuxeo Document Client""" sb = server_binding return self.remote_doc_client_factory( sb.server_url, sb.remote_user, self.device_id, self.version, proxies=self.proxies, proxy_exceptions=self.proxy_exceptions, password=sb.remote_password, token=sb.remote_token, repository=repository, base_folder=base_folder, timeout=self.timeout, cookie_jar=self.cookie_jar) def get_local_client(self, local_folder): """Return a file system client for the given local folder""" return LocalClient(local_folder) def invalidate_client_cache(self, server_url=None): for key in self._client_cache_timestamps: if server_url is None or key[0] == server_url: now = datetime.utcnow().utctimetuple() self._client_cache_timestamps[key] = calendar.timegm(now) # Re-fetch HTTP proxy settings self.refresh_proxies() def get_state(self, server_url, remote_ref): """Find a pair state for the provided remote document identifiers.""" server_url = self._normalize_url(server_url) session = self.get_session() try: states = session.query(LastKnownState).filter_by( remote_ref=remote_ref, ).all() for state in states: if (state.server_binding.server_url == server_url): return state except NoResultFound: return None def get_state_for_local_path(self, local_os_path): """Find a DB state from a local filesystem path""" session = self.get_session() sb, local_path = self._binding_path(local_os_path, session=session) return session.query(LastKnownState).filter_by( local_folder=sb.local_folder, local_path=local_path).one() def launch_file_editor(self, server_url, remote_ref): """Find the local file if any and start OS editor on it.""" state = self.get_state(server_url, remote_ref) if state is None: # TODO: synchronize to a dedicated special root for one time edit log.warning( 'Could not find local file for server_url=%s ' 'and remote_ref=%s', server_url, remote_ref) return # TODO: check synchronization of this state first # Find the best editor for the file according to the OS configuration file_path = state.get_local_abspath() self.open_local_file(file_path) def open_local_file(self, file_path): """Launch the local OS program on the given file / folder.""" log.debug('Launching editor on %s', file_path) if sys.platform == 'win32': os.startfile(file_path) elif sys.platform == 'darwin': subprocess.Popen(['open', file_path]) else: try: subprocess.Popen(['xdg-open', file_path]) except OSError: # xdg-open should be supported by recent Gnome, KDE, Xfce log.error("Failed to find and editor for: '%s'", file_path) def make_remote_raise(self, error): """Helper method to simulate network failure for testing""" self._remote_error = error def make_local_raise(self, error): """Helper method to simulate local device failure for testing""" self._local_error = error def dispose(self): """Release database resources. Close thread-local Session, ending any transaction in progress and releasing underlying connections from the pool. Note that releasing all connections from the pool using Session.close_all() or SingletonThreadPool.dispose() is not an option here as the Python SQLite driver pysqlite used by SQLAlchemy doesn't let you close a connection from a thread that didn't create it (except under Windows, see below). In our case at least two threads are involved, the GUI and the synchronization one, so each one needs to close its own Session by calling this function. Beware that starting more threads than the pool size (default is 5) might lead to a ProgrammingError when SingletonThreadPool._cleanup() gets called, for the reason just mentioned. Also note that calling Session.close() never seems to remove the connection object from the pool, even if the thread owning it is dead. Under Windows we need to release all connections from the pool calling SingletonThreadPool.dispose(), which strangely doesn't raise a ProgrammingError - probably due to a different implementation of the pysqlite driver -, otherwise we might get a WindowsError at tear down in tests using multiple threads when trying to remove the temporary test folder because of it being used by another Python process than the main one... """ session = self.get_session() log.debug( "Closing thread-local Session %r, ending any transaction" " in progress and releasing underlying connections from" " the pool", session) session.close() if sys.platform == 'win32': log.debug("As we are under Windows, dispose connection pool to" " make sure all connections are closed, avoiding any" " WindowsError due to a Python process using the" " database file") self._engine.pool.dispose() def _normalize_url(self, url): """Ensure that user provided url always has a trailing '/'""" if url is None or not url: raise ValueError("Invalid url: %r" % url) if not url.endswith(u'/'): return url + u'/' return url def register_folder_link(self, folder_path): if sys.platform == 'darwin': self.register_folder_link_darwin(folder_path) # TODO: implement Windows and Linux support here def register_folder_link_darwin(self, folder_path): try: from LaunchServices import LSSharedFileListCreate from LaunchServices import kLSSharedFileListFavoriteItems from LaunchServices import LSSharedFileListInsertItemURL from LaunchServices import kLSSharedFileListItemBeforeFirst from LaunchServices import CFURLCreateWithString except ImportError: log.warning("PyObjC package is not installed:" " skipping favorite link creation") return folder_path = normalized_path(folder_path) folder_name = os.path.basename(folder_path) lst = LSSharedFileListCreate(None, kLSSharedFileListFavoriteItems, None) if lst is None: log.warning("Could not fetch the Finder favorite list.") return url = CFURLCreateWithString(None, "file://" + quote(folder_path), None) if url is None: log.warning("Could not generate valid favorite URL for: %s", folder_path) return # Register the folder as favorite if not already there item = LSSharedFileListInsertItemURL(lst, kLSSharedFileListItemBeforeFirst, folder_name, None, url, {}, []) if item is not None: log.debug("Registered new favorite in Finder for: %s", folder_path)