def _abspath(self, ref):
"""Absolute path on the operating system"""
if not ref.startswith(u'/'):
raise ValueError("LocalClient expects ref starting with '/'")
path_suffix = ref[1:].replace('/', os.path.sep)
path = normalized_path(os.path.join(self.base_folder, path_suffix))
return safe_long_path(path)
def init_db(nxdrive_home, echo=False, echo_pool=False, scoped_sessions=True,
poolclass=None):
"""Return an engine and session maker configured for using nxdrive_home
The database is created in nxdrive_home if missing and the tables
are initialized based on the model classes from this module (they
all inherit the same abstract base class.
If scoped_sessions is True, sessions built with this maker are reusable
thread local singletons.
"""
# We store the DB as SQLite files in the nxdrive_home folder
dbfile = os.path.join(normalized_path(nxdrive_home), 'nxdrive.db')
# SQLite cannot share connections across threads hence it's safer to
# enforce this at the connection pool level
poolclass = SingletonThreadPool if poolclass is None else poolclass
engine = create_engine('sqlite:///' + dbfile, poolclass=poolclass)
# Configure SQLAlchemy logging
if echo:
logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
if echo_pool:
logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)
# Ensure that the tables are properly initialized
Base.metadata.create_all(engine)
maker = sessionmaker(bind=engine)
if scoped_sessions:
maker = scoped_session(maker)
return engine, maker
def __init__(self, manager, options):
super(MetadataApplication, self).__init__(manager, options, [])
self.manager = manager
self.options = options
self.file_path = normalized_path(options.file)
self.dialog = CreateMetadataWebDialog(manager, self.file_path, self)
self.dialog.show()
def _binding_path(self, local_path, session=None):
"""Find a server binding and relative path for a given FS path"""
local_path = normalized_path(local_path)
# Check exact binding match
binding = self.get_server_binding(local_path,
session=session,
raise_if_missing=False)
if binding is not None:
return binding, u'/'
# Check for bindings that are prefix of local_path
session = self.get_session()
all_bindings = session.query(ServerBinding).all()
matching_bindings = [
sb for sb in all_bindings
if local_path.startswith(sb.local_folder + os.path.sep)
]
if len(matching_bindings) == 0:
raise NotFound("Could not find any server binding for " +
local_path)
elif len(matching_bindings) > 1:
raise RuntimeError("Found more than one binding for %s: %r" %
(local_path, matching_bindings))
binding = matching_bindings[0]
path = local_path[len(binding.local_folder):]
path = path.replace(os.path.sep, u'/')
return binding, path
def setup_method(
self, test_method, register_roots=True, user_2=True, server_profile=None
):
""" Setup method that will be invoked for every test method of a class."""
log.info("TEST master setup start")
# This will be the name of the workspace and a tag on Sentry
node = os.getenv("NODE_NAME", sys.platform).lower() # Only set from Jenkins
self.current_test = f"{test_method.__name__}-{node}"
# To be replaced with fixtures when migrating to 100% pytest
self.nuxeo_url = nuxeo_url() # fixture name: nuxeo_url
self.version = __version__ # fixture name: version
self.root_remote = root_remote()
self.fake = FAKER
self.location = LOCATION
self.server_profile = server_profile
if server_profile:
self.root_remote.activate_profile(server_profile)
self.users = [self._create_user(1)]
if user_2:
self.users.append(self._create_user(2))
self._create_workspace(self.current_test)
# Add proper rights for all users on the root workspace
users = [user.uid for user in self.users]
try:
self.ws.add_permission({"permission": "ReadWrite", "users": users})
except BadQuery:
# *users* is a valid parameter starting with Nuxeo 10.3.
# Keep that compatibility code for test_volume.py to work on old customers server.
for user in self.users:
self.ws.add_permission(
{"permission": "ReadWrite", "username": user.uid}
)
Options.delay = TEST_DEFAULT_DELAY
self.connected = False
self.app = StubQApplication([], self)
self._wait_sync = {}
self._wait_remote_scan = {}
self._remote_changes_count = {}
self._no_remote_changes = {}
self.report_path = os.getenv("REPORT_PATH")
self.tmpdir = (
normalized_path(tempfile.gettempdir()) / str(uuid4()).split("-")[0]
)
self.upload_tmp_dir = self.tmpdir / "uploads"
self.upload_tmp_dir.mkdir(parents=True)
self._append_user_attrs(1, register_roots)
if user_2:
self._append_user_attrs(2, register_roots)
def register_folder_link_darwin(self, folder_path):
try:
from LaunchServices import LSSharedFileListCreate
from LaunchServices import kLSSharedFileListFavoriteItems
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
except ImportError:
log.warning("PyObjC package is not installed:"
" skipping favorite link creation")
return
folder_path = normalized_path(folder_path)
folder_name = os.path.basename(folder_path)
lst = LSSharedFileListCreate(None, kLSSharedFileListFavoriteItems,
None)
if lst is None:
log.warning("Could not fetch the Finder favorite list.")
return
url = CFURLCreateWithString(None, "file://" + quote(folder_path), None)
if url is None:
log.warning("Could not generate valid favorite URL for: %s",
folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(lst,
kLSSharedFileListItemBeforeFirst,
folder_name, None, url, {}, [])
if item is not None:
log.debug("Registered new favorite in Finder for: %s", folder_path)
def register_folder_link_darwin(self, folder_path):
try:
from LaunchServices import LSSharedFileListCreate
from LaunchServices import kLSSharedFileListFavoriteItems
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
except ImportError:
log.warning("PyObjC package is not installed:"
" skipping favorite link creation")
return
folder_path = normalized_path(folder_path)
folder_name = os.path.basename(folder_path)
lst = LSSharedFileListCreate(None, kLSSharedFileListFavoriteItems,
None)
if lst is None:
log.warning("Could not fetch the Finder favorite list.")
return
url = CFURLCreateWithString(None, "file://" + quote(folder_path), None)
if url is None:
log.warning("Could not generate valid favorite URL for: %s",
folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(
lst, kLSSharedFileListItemBeforeFirst, folder_name, None, url,
{}, [])
if item is not None:
log.debug("Registered new favorite in Finder for: %s", folder_path)
def bind_engine(self, engine_type, local_folder, name, binder, starts=True):
"""Bind a local folder to a remote nuxeo server"""
if name is None and hasattr(binder, 'url'):
name = self._get_engine_name(binder.url)
if not self.check_local_folder_available(local_folder):
raise FolderAlreadyUsed()
if not engine_type in self._engine_types:
raise EngineTypeMissing()
if self._engines is None:
self.load()
local_folder = normalized_path(local_folder)
if local_folder == self.get_configuration_folder():
# Prevent from binding in the configuration folder
raise FolderAlreadyUsed()
uid = uuid.uuid1().hex
# TODO Check that engine is not inside another or same position
engine_def = self._dao.add_engine(engine_type, local_folder, uid, name)
try:
self._engines[uid] = self._engine_types[engine_type](self, engine_def, binder=binder,
remote_watcher_delay=self.remote_watcher_delay)
self._engine_definitions.append(engine_def)
except Exception as e:
log.exception(e)
if uid in self._engines:
del self._engines[uid]
self._dao.delete_engine(uid)
# TODO Remove the db ?
raise e
# As new engine was just bound, refresh application update status
self.refresh_update_status()
if starts:
self._engines[uid].start()
self.newEngine.emit(self._engines[uid])
return self._engines[uid]
def _create_drive_edit(self, url):
from nxdrive.drive_edit import DriveEdit
self._drive_edit = DriveEdit(
self, os.path.join(normalized_path(self.nxdrive_home), "edit"),
url)
self.started.connect(self._drive_edit._thread.start)
return self._drive_edit
def unbind_server(self, local_folder):
"""Remove the binding to a Nuxeo server
Local files are not deleted"""
session = self.get_session()
local_folder = normalized_path(local_folder)
binding = self.get_server_binding(local_folder, raise_if_missing=True,
session=session)
# Revoke token if necessary
if binding.remote_token is not None:
try:
nxclient = self.remote_doc_client_factory(
binding.server_url,
binding.remote_user,
self.device_id,
token=binding.remote_token)
log.info("Revoking token for '%s' with account '%s'",
binding.server_url, binding.remote_user)
nxclient.revoke_token()
except POSSIBLE_NETWORK_ERROR_TYPES:
log.warning("Could not connect to server '%s' to revoke token",
binding.server_url)
except Unauthorized:
# Token is already revoked
pass
# Invalidate client cache
self.invalidate_client_cache(binding.server_url)
# Delete binding info in local DB
log.info("Unbinding '%s' from '%s' with account '%s'",
local_folder, binding.server_url, binding.remote_user)
session.delete(binding)
session.commit()
def init_db(nxdrive_home,
echo=False,
echo_pool=False,
scoped_sessions=True,
poolclass=None):
"""Return an engine and session maker configured for using nxdrive_home
The database is created in nxdrive_home if missing and the tables
are initialized based on the model classes from this module (they
all inherit the same abstract base class.
If scoped_sessions is True, sessions built with this maker are reusable
thread local singletons.
"""
# We store the DB as SQLite files in the nxdrive_home folder
dbfile = os.path.join(normalized_path(nxdrive_home), 'nxdrive.db')
# SQLite cannot share connections across threads hence it's safer to
# enforce this at the connection pool level
poolclass = SingletonThreadPool if poolclass is None else poolclass
engine = create_engine('sqlite:///' + dbfile, poolclass=poolclass)
# Configure SQLAlchemy logging
if echo:
logging.getLogger('sqlalchemy.engine').setLevel(logging.DEBUG)
if echo_pool:
logging.getLogger('sqlalchemy.pool').setLevel(logging.DEBUG)
# Ensure that the tables are properly initialized
Base.metadata.create_all(engine)
maker = sessionmaker(bind=engine)
if scoped_sessions:
maker = scoped_session(maker)
return engine, maker
def setUp(self):
"""
1. create folder 'temp/a1' with more than 20 files in it
2. create folder 'temp/a2', empty
3. copy 'a1' and 'a2', in this order to the test sync root
4. repeat step 3, but copy 'a2' and 'a1', in this order
(to the test sync root)
5. Verify that both folders and their content is sync to DM,
in both steps 3 and 4
"""
self.engine_1.start()
self.wait_sync(wait_for_async=True)
local = self.local_1
assert local.exists("/")
self.workspace_abspath = local.abspath("/")
# Create folder a1 and a2 under a temp folder
self.local_temp = normalized_path(tempfile.mkdtemp(self.TEMP_FOLDER))
self.folder1 = self.local_temp / self.FOLDER_A1
self.folder1.mkdir(parents=True)
self.folder2 = self.local_temp / self.FOLDER_A2
self.folder2.mkdir(parents=True)
# Add files in folder 'temp/a1'
for file_num in range(1, self.NUMBER_OF_LOCAL_FILES + 1):
filename = self.FILENAME_PATTERN % file_num
(self.folder1 / filename).write_bytes(FILE_CONTENT)
def setUp(self):
"""
1. create folder 'temp/a1' with more than 20 files in it
2. create folder 'temp/a2', empty
3. copy 'a1' and 'a2', in this order to the test sync root
4. repeat step 3, but copy 'a2' and 'a1', in this order
(to the test sync root)
5. Verify that both folders and their content is sync to DM,
in both steps 3 and 4
"""
self.engine_1.start()
self.wait_sync(wait_for_async=True)
local = self.local_1
assert local.exists("/")
self.workspace_abspath = local.abspath("/")
# Create folder a1 and a2 under a temp folder
self.local_temp = normalized_path(tempfile.mkdtemp(self.TEMP_FOLDER))
self.folder1 = self.local_temp / self.FOLDER_A1
self.folder1.mkdir(parents=True)
self.folder2 = self.local_temp / self.FOLDER_A2
self.folder2.mkdir(parents=True)
# Add files in folder 'temp/a1'
for file_num in range(1, self.NUMBER_OF_LOCAL_FILES + 1):
filename = self.FILENAME_PATTERN % file_num
(self.folder1 / filename).write_bytes(FILE_CONTENT)
def register_folder_link(self, folder_path, name=None):
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
favorites = self._get_favorite_list() or []
if not favorites:
log.warning('Could not fetch the Finder favorite list.')
return
folder_path = normalized_path(folder_path)
if name is None:
name = self._manager.app_name
else:
name = os.path.basename(name)
if self._find_item_in_list(favorites, name):
return
url = CFURLCreateWithString(
None, 'file://' + urllib2.quote(folder_path), None)
if url is None:
log.warning(
'Could not generate valid favorite URL for: %r', folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(
favorites, kLSSharedFileListItemBeforeFirst, name, None, url, {}, [])
if item is not None:
log.debug('Registered new favorite in Finder for: %r', folder_path)
def register_folder_link(self, folder_path, name=None):
try:
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
except ImportError:
log.warning("PyObjC package is not installed:"
" skipping favorite link creation")
return
folder_path = normalized_path(folder_path)
if name is None:
name = self._manager.get_appname()
lst = self._get_favorite_list()
if lst is None:
log.warning("Could not fetch the Finder favorite list.")
return
url = CFURLCreateWithString(None,
"file://" + urllib2.quote(folder_path),
None)
if url is None:
log.warning("Could not generate valid favorite URL for: %s",
folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(lst,
kLSSharedFileListItemBeforeFirst,
name, None, url, {}, [])
if item is not None:
log.debug("Registered new favorite in Finder for: %s", folder_path)
def register_folder_link(self, folder_path, name=None):
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
favorites = self._get_favorite_list() or []
if not favorites:
log.warning('Could not fetch the Finder favorite list.')
return
folder_path = normalized_path(folder_path)
if name is None:
name = self._manager.app_name
else:
name = os.path.basename(name)
if self._find_item_in_list(favorites, name):
return
url = CFURLCreateWithString(None,
'file://' + urllib2.quote(folder_path),
None)
if url is None:
log.warning('Could not generate valid favorite URL for: %r',
folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(favorites,
kLSSharedFileListItemBeforeFirst,
name, None, url, {}, [])
if item is not None:
log.debug('Registered new favorite in Finder for: %r', folder_path)
def __init__(self, manager, options):
super(MetadataApplication, self).__init__([])
self.manager = manager
self.options = options
self.file_path = normalized_path(options.file)
self.dialog = CreateMetadataWebDialog(manager, self.file_path, self)
self.dialog.show()
def __init__(self, fname):
root = normalized_path(__file__).parent.parent
src = root / "resources" / "databases" / fname
dst = src.with_name(f"{uuid4()}.db")
shutil.copy(src, dst)
time.sleep(1)
super().__init__(dst)
def register_folder_link(self, folder_path, name=None):
try:
from LaunchServices import LSSharedFileListInsertItemURL
from LaunchServices import kLSSharedFileListItemBeforeFirst
from LaunchServices import CFURLCreateWithString
except ImportError:
log.warning("PyObjC package is not installed:"
" skipping favorite link creation")
return
folder_path = normalized_path(folder_path)
if name is None:
name = self._manager.get_appname()
lst = self._get_favorite_list()
if lst is None:
log.warning("Could not fetch the Finder favorite list.")
return
url = CFURLCreateWithString(None, "file://"
+ urllib2.quote(folder_path), None)
if url is None:
log.warning("Could not generate valid favorite URL for: %s",
folder_path)
return
# Register the folder as favorite if not already there
item = LSSharedFileListInsertItemURL(
lst, kLSSharedFileListItemBeforeFirst, name, None, url,
{}, [])
if item is not None:
log.debug("Registered new favorite in Finder for: %s", folder_path)
def _abspath(self, ref):
"""Absolute path on the operating system"""
if not ref.startswith(u'/'):
raise ValueError("LocalClient expects ref starting with '/'")
path_suffix = ref[1:].replace('/', os.path.sep)
path = normalized_path(os.path.join(self.base_folder, path_suffix))
return safe_long_path(path)
def _binding_path(self, local_path, session=None):
"""Find a server binding and relative path for a given FS path"""
local_path = normalized_path(local_path)
# Check exact binding match
binding = self.get_server_binding(local_path, session=session,
raise_if_missing=False)
if binding is not None:
return binding, u'/'
# Check for bindings that are prefix of local_path
session = self.get_session()
all_bindings = session.query(ServerBinding).all()
matching_bindings = [sb for sb in all_bindings
if local_path.startswith(
sb.local_folder + os.path.sep)]
if len(matching_bindings) == 0:
raise NotFound("Could not find any server binding for "
+ local_path)
elif len(matching_bindings) > 1:
raise RuntimeError("Found more than one binding for %s: %r" % (
local_path, matching_bindings))
binding = matching_bindings[0]
path = local_path[len(binding.local_folder):]
path = path.replace(os.path.sep, u'/')
return binding, path
def cmd(tmp):
path = tmp() / "config"
path.mkdir(parents=True, exist_ok=True)
Options.nxdrive_home = normalized_path(path)
yield CliHandler()
with suppress(FileNotFoundError):
(Options.nxdrive_home / "config.ini").unlink()
def cmd(tmp):
path = tmp() / "config"
path.mkdir(parents=True, exist_ok=True)
Options.nxdrive_home = normalized_path(path)
yield CliHandler()
with suppress(FileNotFoundError):
(Options.nxdrive_home / "config.ini").unlink()
def _migrate(self):
from nxdrive.engine.dao.sqlite import ManagerDAO
self._dao = ManagerDAO(self._get_db())
old_db = os.path.join(normalized_path(self.nxdrive_home), "nxdrive.db")
if os.path.exists(old_db):
import sqlite3
from nxdrive.engine.dao.sqlite import CustomRow
conn = sqlite3.connect(old_db)
conn.row_factory = CustomRow
c = conn.cursor()
cfg = c.execute("SELECT * FROM device_config LIMIT 1").fetchone()
if cfg is not None:
self.device_id = cfg.device_id
self._dao.update_config("device_id", cfg.device_id)
self._dao.update_config("proxy_config", cfg.proxy_config)
self._dao.update_config("proxy_type", cfg.proxy_type)
self._dao.update_config("proxy_server", cfg.proxy_server)
self._dao.update_config("proxy_port", cfg.proxy_port)
self._dao.update_config("proxy_authenticated", cfg.proxy_authenticated)
self._dao.update_config("proxy_username", cfg.proxy_username)
self._dao.update_config("auto_update", cfg.auto_update)
# Copy first server binding
rows = c.execute("SELECT * FROM server_bindings").fetchall()
if not rows:
return
first_row = True
for row in rows:
row.url = row.server_url
log.debug("Binding server from Nuxeo Drive V1: [%s, %s]", row.url, row.remote_user)
row.username = row.remote_user
row.password = None
row.token = row.remote_token
row.no_fscheck = True
engine = self.bind_engine(
self._get_default_server_type(),
row["local_folder"],
self._get_engine_name(row.url),
row,
starts=False,
)
log.trace("Resulting server binding remote_token %r", row.remote_token)
if first_row:
first_engine_def = row
first_engine = engine
first_row = False
else:
engine.dispose_db()
# Copy filters for first engine as V1 only supports filtering for the first server binding
filters = c.execute("SELECT * FROM filters")
for filter_obj in filters:
if first_engine_def.local_folder != filter_obj.local_folder:
continue
log.trace("Filter Row from DS1 %r", filter_obj)
first_engine.add_filter(filter_obj["path"])
first_engine.dispose_db()
def _migrate(self):
from nxdrive.engine.dao.sqlite import ManagerDAO
self._dao = ManagerDAO(self._get_db())
old_db = os.path.join(normalized_path(self.nxdrive_home), "nxdrive.db")
if os.path.exists(old_db):
import sqlite3
from nxdrive.engine.dao.sqlite import CustomRow
conn = sqlite3.connect(old_db)
conn.row_factory = CustomRow
c = conn.cursor()
cfg = c.execute("SELECT * FROM device_config LIMIT 1").fetchone()
if cfg is not None:
self.device_id = cfg.device_id
self._dao.update_config("device_id", cfg.device_id)
self._dao.update_config("proxy_config", cfg.proxy_config)
self._dao.update_config("proxy_type", cfg.proxy_type)
self._dao.update_config("proxy_server", cfg.proxy_server)
self._dao.update_config("proxy_port", cfg.proxy_port)
self._dao.update_config("proxy_authenticated",
cfg.proxy_authenticated)
self._dao.update_config("proxy_username", cfg.proxy_username)
self._dao.update_config("auto_update", cfg.auto_update)
# Copy first server binding
rows = c.execute("SELECT * FROM server_bindings").fetchall()
if not rows:
return
first_row = True
for row in rows:
row.url = row.server_url
log.debug("Binding server from Nuxeo Drive V1: [%s, %s]",
row.url, row.remote_user)
row.username = row.remote_user
row.password = None
row.token = row.remote_token
row.no_fscheck = True
engine = self.bind_engine(self._get_default_server_type(),
row["local_folder"],
self._get_engine_name(row.url),
row,
starts=False)
log.trace("Resulting server binding remote_token %r",
row.remote_token)
if first_row:
first_engine_def = row
first_engine = engine
first_row = False
else:
engine.dispose_db()
# Copy filters for first engine as V1 only supports filtering for the first server binding
filters = c.execute("SELECT * FROM filters")
for filter_obj in filters:
if first_engine_def.local_folder != filter_obj.local_folder:
continue
log.trace("Filter Row from DS1 %r", filter_obj)
first_engine.add_filter(filter_obj["path"])
first_engine.dispose_db()
def abspath(self, ref):
# type: (Text) -> Text
""" Absolute path on the operating system. """
if not ref.startswith(u'/'):
raise ValueError('LocalClient expects ref starting with "/"',
locals())
path_suffix = ref[1:].replace('/', os.path.sep)
path = normalized_path(os.path.join(self.base_folder, path_suffix))
return safe_long_path(path)
def bind_engine(self,
engine_type,
local_folder,
name,
binder,
starts=True):
"""Bind a local folder to a remote nuxeo server"""
if name is None and hasattr(binder, 'url'):
name = self._get_engine_name(binder.url)
if hasattr(binder, 'url'):
url = binder.url
if '#' in url:
# Last part of the url is the engine type
engine_type = url.split('#')[1]
binder.url = url.split('#')[0]
log.debug(
"Engine type has been specified in the url: %s will be used",
engine_type)
if not self.check_local_folder_available(local_folder):
raise FolderAlreadyUsed()
if not engine_type in self._engine_types:
raise EngineTypeMissing()
if self._engines is None:
self.load()
local_folder = normalized_path(local_folder)
if local_folder == self.get_configuration_folder():
# Prevent from binding in the configuration folder
raise FolderAlreadyUsed()
uid = uuid.uuid1().hex
# TODO Check that engine is not inside another or same position
engine_def = self._dao.add_engine(engine_type, local_folder, uid, name)
try:
self._engines[uid] = self._engine_types[engine_type](
self,
engine_def,
binder=binder,
remote_watcher_delay=self.remote_watcher_delay)
self._engine_definitions.append(engine_def)
except Exception as e:
log.exception(e)
if uid in self._engines:
del self._engines[uid]
self._dao.delete_engine(uid)
# TODO Remove the db ?
raise e
# As new engine was just bound, refresh application update status
self.refresh_update_status()
if starts:
self._engines[uid].start()
self.newEngine.emit(self._engines[uid])
return self._engines[uid]
def setup_method(
self, test_method, register_roots=True, user_2=True, server_profile=None
):
""" Setup method that will be invoked for every test method of a class."""
log.info("TEST master setup start")
self.current_test = test_method.__name__
# To be replaced with fixtures when migrating to 100% pytest
self.nuxeo_url = nuxeo_url() # fixture name: nuxeo_url
self.version = __version__ # fixture name: version
self.root_remote = root_remote()
self.fake = FAKER
self.location = LOCATION
self.server_profile = server_profile
if server_profile:
self.root_remote.activate_profile(server_profile)
self.users = [self._create_user(1)]
if user_2:
self.users.append(self._create_user(2))
self._create_workspace(self.current_test)
# Add proper rights for all users on the root workspace
users = [user.uid for user in self.users]
self.ws.add_permission({"permission": "ReadWrite", "users": users})
Options.delay = TEST_DEFAULT_DELAY
self.connected = False
self.app = StubQApplication([], self)
self._wait_sync = {}
self._wait_remote_scan = {}
self._remote_changes_count = {}
self._no_remote_changes = {}
self.report_path = os.getenv("REPORT_PATH")
self.tmpdir = (
normalized_path(tempfile.gettempdir()) / str(uuid4()).split("-")[0]
)
self.upload_tmp_dir = self.tmpdir / "uploads"
self.upload_tmp_dir.mkdir(parents=True)
self._append_user_attrs(1, register_roots)
if user_2:
self._append_user_attrs(2, register_roots)
def setup_method(
self, test_method, register_roots=True, user_2=True, server_profile=None
):
""" Setup method that will be invoked for every test method of a class."""
log.info("TEST master setup start")
self.current_test = test_method.__name__
# To be replaced with fixtures when migrating to 100% pytest
self.nuxeo_url = nuxeo_url() # fixture name: nuxeo_url
self.version = __version__ # fixture name: version
self.root_remote = root_remote()
self.fake = FAKER
self.location = LOCATION
self.server_profile = server_profile
if server_profile:
self.root_remote.activate_profile(server_profile)
self.users = [self._create_user(1)]
if user_2:
self.users.append(self._create_user(2))
self._create_workspace(self.current_test)
# Add proper rights for all users on the root workspace
users = [user.uid for user in self.users]
self.ws.add_permission({"permission": "ReadWrite", "users": users})
Options.delay = TEST_DEFAULT_DELAY
self.connected = False
self.app = StubQApplication([], self)
self._wait_sync = {}
self._wait_remote_scan = {}
self._remote_changes_count = {}
self._no_remote_changes = {}
self.report_path = os.getenv("REPORT_PATH")
self.tmpdir = (
normalized_path(tempfile.gettempdir()) / str(uuid4()).split("-")[0]
)
self.upload_tmp_dir = self.tmpdir / "uploads"
self.upload_tmp_dir.mkdir(parents=True)
self._append_user_attrs(1, register_roots)
if user_2:
self._append_user_attrs(2, register_roots)
def get_server_binding(self, local_folder, raise_if_missing=False,
session=None):
"""Find the ServerBinding instance for a given local_folder"""
local_folder = normalized_path(local_folder)
if session is None:
session = self.get_session()
try:
return session.query(ServerBinding).filter(
ServerBinding.local_folder == local_folder).one()
except NoResultFound:
if raise_if_missing:
raise RuntimeError(
"Folder '%s' is not bound to any Nuxeo server"
% local_folder)
return None
def handle(self, argv):
""" Parse options, setup logs and manager and dispatch execution. """
options = self.parse_cli(argv)
if hasattr(options, 'local_folder'):
options.local_folder = normalized_path(options.local_folder)
# 'launch' is the default command if None is provided
command = getattr(options, 'command', 'launch')
if command != 'uninstall':
# Configure the logging framework, except for the tests as they
# configure their own.
# Don't need uninstall logs either for now.
self._configure_logger(command, options)
self.log = get_logger(__name__)
self.log.debug("Command line: argv=%r, options=%r", ' '.join(argv),
options)
# Update default options
Options.update(options, setter='cli')
if command != 'uninstall':
# Install utility to help debugging segmentation faults
self._install_faulthandler()
# Initialize a manager for this process
self.manager = self.get_manager()
# Find the command to execute based on the
handler = getattr(self, command, None)
if not handler:
raise NotImplementedError('No handler implemented for command ' +
command)
try:
return handler(options)
except Exception as e:
if Options.debug:
# Make it possible to use the postmortem debugger
raise
msg = e.msg if hasattr(e, 'msg') else e
self.log.error("Error executing '%s': %s",
command,
msg,
exc_info=True)
def get_server_binding(self,
local_folder,
raise_if_missing=False,
session=None):
"""Find the ServerBinding instance for a given local_folder"""
local_folder = normalized_path(local_folder)
if session is None:
session = self.get_session()
try:
return session.query(ServerBinding).filter(
ServerBinding.local_folder == local_folder).one()
except NoResultFound:
if raise_if_missing:
raise RuntimeError(
"Folder '%s' is not bound to any Nuxeo server" %
local_folder)
return None
def handle(self, argv):
""" Parse options, setup logs and manager and dispatch execution. """
options = self.parse_cli(argv)
if hasattr(options, 'local_folder'):
options.local_folder = normalized_path(options.local_folder)
# 'launch' is the default command if None is provided
command = getattr(options, 'command', 'launch')
if command != 'uninstall':
# Configure the logging framework, except for the tests as they
# configure their own.
# Don't need uninstall logs either for now.
self._configure_logger(command, options)
self.log = get_logger(__name__)
self.log.debug("Command line: argv=%r, options=%r",
' '.join(argv), options)
# Update default options
Options.update(options, setter='cli')
if command != 'uninstall':
# Install utility to help debugging segmentation faults
self._install_faulthandler()
# Initialize a manager for this process
self.manager = self.get_manager()
# Find the command to execute based on the
handler = getattr(self, command, None)
if not handler:
raise NotImplementedError(
'No handler implemented for command ' + command)
try:
return handler(options)
except Exception as e:
if Options.debug:
# Make it possible to use the postmortem debugger
raise
msg = e.msg if hasattr(e, 'msg') else e
self.log.error("Error executing '%s': %s", command, msg,
exc_info=True)
def bind_engine(self, engine_type, local_folder, name, binder, starts=True):
"""Bind a local folder to a remote nuxeo server"""
if name is None and hasattr(binder, 'url'):
name = self._get_engine_name(binder.url)
if hasattr(binder, 'url'):
url = binder.url
if '#' in url:
# Last part of the url is the engine type
engine_type = url.split('#')[1]
binder.url = url.split('#')[0]
log.debug("Engine type has been specified in the url: %s will be used", engine_type)
if not self.check_local_folder_available(local_folder):
raise FolderAlreadyUsed()
if not engine_type in self._engine_types:
raise EngineTypeMissing()
if self._engines is None:
self.load()
local_folder = normalized_path(local_folder)
if local_folder == self.get_configuration_folder():
# Prevent from binding in the configuration folder
raise FolderAlreadyUsed()
uid = uuid.uuid1().hex
# TODO Check that engine is not inside another or same position
engine_def = self._dao.add_engine(engine_type, local_folder, uid, name)
try:
self._engines[uid] = self._engine_types[engine_type](
self, engine_def, binder=binder,
remote_watcher_delay=self.remote_watcher_delay,
processors=get_number_of_processors(BaseAutomationClient.get_upload_rate_limit(),
BaseAutomationClient.get_download_rate_limit()))
self._engine_definitions.append(engine_def)
except Exception as e:
log.exception(e)
if uid in self._engines:
del self._engines[uid]
self._dao.delete_engine(uid)
# TODO Remove the db ?
raise e
# As new engine was just bound, refresh application update status
self.refresh_update_status()
if starts:
self._engines[uid].start()
self.newEngine.emit(self._engines[uid])
return self._engines[uid]
def bind_root(self, local_folder, remote_ref, repository='default',
session=None):
"""Bind local root to a remote root (folderish document in Nuxeo).
local_folder must be already bound to an existing Nuxeo server.
remote_ref must be the IdRef or PathRef of an existing folderish
document on the remote server bound to the local folder.
"""
session = self.get_session() if session is None else session
local_folder = normalized_path(local_folder)
server_binding = self.get_server_binding(
local_folder, raise_if_missing=True, session=session)
nxclient = self.get_remote_doc_client(server_binding,
repository=repository)
# Register the root on the server
nxclient.register_as_root(remote_ref)
def unbind_server(self, local_folder):
"""Remove the binding to a Nuxeo server
Local files are not deleted"""
session = self.get_session()
local_folder = normalized_path(local_folder)
binding = self.get_server_binding(local_folder,
raise_if_missing=True,
session=session)
# Revoke token if necessary
if binding.remote_token is not None:
try:
nxclient = self.remote_doc_client_factory(
binding.server_url,
binding.remote_user,
self.device_id,
self.version,
proxies=self.proxies,
proxy_exceptions=self.proxy_exceptions,
token=binding.remote_token,
timeout=self.timeout)
log.info("Revoking token for '%s' with account '%s'",
binding.server_url, binding.remote_user)
nxclient.revoke_token()
except POSSIBLE_NETWORK_ERROR_TYPES:
log.warning("Could not connect to server '%s' to revoke token",
binding.server_url)
except Unauthorized:
# Token is already revoked
pass
# Invalidate client cache
self.invalidate_client_cache(binding.server_url)
# Delete binding info in local DB
log.info("Unbinding '%s' from '%s' with account '%s'", local_folder,
binding.server_url, binding.remote_user)
session.delete(binding)
session.commit()
def clean_dir(_dir: Path, retry: int = 1, max_retries: int = 5) -> None:
_dir = safe_long_path(_dir)
if not _dir.exists():
return
test_data = os.environ.get("TEST_SAVE_DATA")
if test_data:
shutil.move(_dir, test_data)
return
try:
for path, folders, filenames in os.walk(_dir):
dirpath = normalized_path(path)
for folder in folders:
unset_path_readonly(dirpath / folder)
for filename in filenames:
unset_path_readonly(dirpath / filename)
shutil.rmtree(_dir)
except Exception:
if retry < max_retries:
sleep(2)
clean_dir(_dir, retry=retry + 1)
def clean_dir(_dir: Path, retry: int = 1, max_retries: int = 5) -> None:
_dir = safe_long_path(_dir)
if not _dir.exists():
return
test_data = os.environ.get("TEST_SAVE_DATA")
if test_data:
shutil.move(_dir, test_data)
return
try:
for path, folders, filenames in os.walk(_dir):
dirpath = normalized_path(path)
for folder in folders:
unset_path_readonly(dirpath / folder)
for filename in filenames:
unset_path_readonly(dirpath / filename)
shutil.rmtree(_dir)
except Exception:
if retry < max_retries:
sleep(2)
clean_dir(_dir, retry=retry + 1)
def bind_root(self,
local_folder,
remote_ref,
repository='default',
session=None):
"""Bind local root to a remote root (folderish document in Nuxeo).
local_folder must be already bound to an existing Nuxeo server.
remote_ref must be the IdRef or PathRef of an existing folderish
document on the remote server bound to the local folder.
"""
session = self.get_session() if session is None else session
local_folder = normalized_path(local_folder)
server_binding = self.get_server_binding(local_folder,
raise_if_missing=True,
session=session)
nxclient = self.get_remote_doc_client(server_binding,
repository=repository)
# Register the root on the server
nxclient.register_as_root(remote_ref)
# 1s time resolution as we truncate remote last modification time to the
# seconds in RemoteFileInfo.from_dict() because of the datetime
# resolution of some databases (MySQL...)
REMOTE_MODIFICATION_TIME_RESOLUTION = 1.0
# 1s resolution on HFS+ on OSX
# ~0.01 sec for NTFS
# 0.001 sec for EXT4FS
OS_STAT_MTIME_RESOLUTION = 1.0
log = getLogger(__name__)
DEFAULT_WAIT_SYNC_TIMEOUT = 10
FILE_CONTENT = b"Lorem ipsum dolor sit amet ..."
FAKER = Faker("en_US")
LOCATION = normalized_path(__file__).parent.parent
Translator(LOCATION / "resources" / "i18n")
def nuxeo_url() -> str:
"""Retrieve the Nuxeo URL."""
return env.NXDRIVE_TEST_NUXEO_URL.split("#")[0]
def root_remote(base_folder: str = "/") -> DocRemote:
return DocRemote(
nuxeo_url(),
env.NXDRIVE_TEST_USERNAME,
"nxdrive-test-administrator-device",
__version__,
def _get_db_file(self):
return os.path.join(normalized_path(self._manager.nxdrive_home),
'ndrive_' + self.uid + '.db')
def _create_drive_edit(self, url):
from nxdrive.drive_edit import DriveEdit
self._drive_edit = DriveEdit(self, os.path.join(normalized_path(self.nxdrive_home), "edit"), url)
self.started.connect(self._drive_edit._thread.start)
return self._drive_edit
def _get_db(self):
return os.path.join(normalized_path(self.nxdrive_home), "manager.db")
def _get_db(self):
return os.path.join(normalized_path(self.nxdrive_home), "manager.db")
def _get_db_file(self):
return os.path.join(normalized_path(self._manager.get_configuration_folder()),
"ndrive_" + self._uid + ".db")
def bind_server(self, local_folder, server_url, username, password):
"""Bind a local folder to a remote nuxeo server"""
session = self.get_session()
local_folder = normalized_path(local_folder)
# check the connection to the server by issuing an authentication
# request
server_url = self._normalize_url(server_url)
nxclient = self.remote_doc_client_factory(
server_url,
username,
self.device_id,
self.version,
proxies=self.proxies,
proxy_exceptions=self.proxy_exceptions,
password=password,
timeout=self.handshake_timeout)
token = nxclient.request_token()
if token is not None:
# The server supports token based identification: do not store the
# password in the DB
password = None
try:
try:
# Look for an existing server binding for the given local
# folder
server_binding = session.query(ServerBinding).filter(
ServerBinding.local_folder == local_folder).one()
if server_binding.server_url != server_url:
raise RuntimeError(
"%s is already bound to '%s'" %
(local_folder, server_binding.server_url))
if server_binding.remote_user != username:
# Update username info if required
server_binding.remote_user = username
log.info("Updating username to '%s' on server '%s'",
username, server_url)
if (token is None
and server_binding.remote_password != password):
# Update password info if required
server_binding.remote_password = password
log.info("Updating password for user '%s' on server '%s'",
username, server_url)
if token is not None and server_binding.remote_token != token:
log.info("Updating token for user '%s' on server '%s'",
username, server_url)
# Update the token info if required
server_binding.remote_token = token
# Ensure that the password is not stored in the DB
if server_binding.remote_password is not None:
server_binding.remote_password = None
# If the top level state for the server binding doesn't exist,
# create the local folder and the top level state. This can be
# the case when initializing the DB manually with a SQL script.
try:
session.query(LastKnownState).filter_by(
local_path='/', local_folder=local_folder).one()
except NoResultFound:
self._make_local_folder(local_folder)
self._add_top_level_state(server_binding, session)
except NoResultFound:
# No server binding found for the given local folder
# First create local folder in the file system
self._make_local_folder(local_folder)
# Create ServerBinding instance in DB
log.info("Binding '%s' to '%s' with account '%s'",
local_folder, server_url, username)
server_binding = ServerBinding(local_folder,
server_url,
username,
remote_password=password,
remote_token=token)
session.add(server_binding)
# Create the top level state for the server binding
self._add_top_level_state(server_binding, session)
# Set update info
self._set_update_info(server_binding, remote_client=nxclient)
except:
# In case an AddonNotInstalled exception is raised, need to
# invalidate the remote client cache for it to be aware of the new
# operations when the addon gets installed
if server_binding is not None:
self.invalidate_client_cache(server_binding.server_url)
session.rollback()
raise
session.commit()
return server_binding
def metadata(self, options):
file_path = normalized_path(options.file)
self.manager.open_metadata_window(file_path)
def cmd(tmp):
path = tmp() / "config"
path.mkdir(parents=True, exist_ok=True)
Options.nxdrive_home = normalized_path(path)
yield CliHandler()
def __init__(self, fname):
db = normalized_path(__file__).parent.parent / "resources" / fname
tmp = db.with_name(f"{uuid4()}.db")
shutil.copy(db, tmp)
time.sleep(1)
super().__init__(tmp)
def metadata(self, options):
file_path = normalized_path(options.file)
self.manager.open_metadata_window(file_path)
def bind_server(self, local_folder, server_url, username, password):
"""Bind a local folder to a remote nuxeo server"""
session = self.get_session()
local_folder = normalized_path(local_folder)
# check the connection to the server by issuing an authentication
# request
server_url = self._normalize_url(server_url)
nxclient = self.remote_doc_client_factory(
server_url, username, self.device_id, self.version,
proxies=self.proxies, proxy_exceptions=self.proxy_exceptions,
password=password, timeout=self.handshake_timeout)
token = nxclient.request_token()
if token is not None:
# The server supports token based identification: do not store the
# password in the DB
password = None
try:
try:
# Look for an existing server binding for the given local
# folder
server_binding = session.query(ServerBinding).filter(
ServerBinding.local_folder == local_folder).one()
if server_binding.server_url != server_url:
raise RuntimeError(
"%s is already bound to '%s'" % (
local_folder, server_binding.server_url))
if server_binding.remote_user != username:
# Update username info if required
server_binding.remote_user = username
log.info("Updating username to '%s' on server '%s'",
username, server_url)
if (token is None
and server_binding.remote_password != password):
# Update password info if required
server_binding.remote_password = password
log.info("Updating password for user '%s' on server '%s'",
username, server_url)
if token is not None and server_binding.remote_token != token:
log.info("Updating token for user '%s' on server '%s'",
username, server_url)
# Update the token info if required
server_binding.remote_token = token
# Ensure that the password is not stored in the DB
if server_binding.remote_password is not None:
server_binding.remote_password = None
# If the top level state for the server binding doesn't exist,
# create the local folder and the top level state. This can be
# the case when initializing the DB manually with a SQL script.
try:
session.query(LastKnownState).filter_by(local_path='/',
local_folder=local_folder).one()
except NoResultFound:
self._make_local_folder(local_folder)
self._add_top_level_state(server_binding, session)
except NoResultFound:
# No server binding found for the given local folder
# First create local folder in the file system
self._make_local_folder(local_folder)
# Create ServerBinding instance in DB
log.info("Binding '%s' to '%s' with account '%s'",
local_folder, server_url, username)
server_binding = ServerBinding(local_folder, server_url,
username,
remote_password=password,
remote_token=token)
session.add(server_binding)
# Create the top level state for the server binding
self._add_top_level_state(server_binding, session)
# Set update info
self._set_update_info(server_binding, remote_client=nxclient)
except:
# In case an AddonNotInstalled exception is raised, need to
# invalidate the remote client cache for it to be aware of the new
# operations when the addon gets installed
if server_binding is not None:
self.invalidate_client_cache(server_binding.server_url)
session.rollback()
raise
session.commit()
return server_binding
def bind_server(self, local_folder, server_url, username, password):
"""Bind a local folder to a remote nuxeo server"""
session = self.get_session()
local_folder = normalized_path(local_folder)
if not os.path.exists(local_folder):
os.makedirs(local_folder)
self.register_folder_link(local_folder)
# check the connection to the server by issuing an authentication
# request
server_url = self._normalize_url(server_url)
nxclient = self.remote_doc_client_factory(
server_url, username, self.device_id, password)
token = nxclient.request_token()
if token is not None:
# The server supports token based identification: do not store the
# password in the DB
password = None
try:
server_binding = session.query(ServerBinding).filter(
ServerBinding.local_folder == local_folder).one()
if (server_binding.remote_user != username
or server_binding.server_url != server_url):
raise RuntimeError(
"%s is already bound to '%s' with user '%s'" % (
local_folder, server_binding.server_url,
server_binding.remote_user))
if token is None and server_binding.remote_password != password:
# Update password info if required
server_binding.remote_password = password
log.info("Updating password for user '%s' on server '%s'",
username, server_url)
if token is not None and server_binding.remote_token != token:
log.info("Updating token for user '%s' on server '%s'",
username, server_url)
# Update the token info if required
server_binding.remote_token = token
# Ensure that the password is not stored in the DB
if server_binding.remote_password is not None:
server_binding.remote_password = None
except NoResultFound:
log.info("Binding '%s' to '%s' with account '%s'",
local_folder, server_url, username)
server_binding = ServerBinding(local_folder, server_url, username,
remote_password=password,
remote_token=token)
session.add(server_binding)
# Creating the toplevel state for the server binding
local_client = LocalClient(server_binding.local_folder)
local_info = local_client.get_info(u'/')
remote_client = self.get_remote_fs_client(server_binding)
remote_info = remote_client.get_filesystem_root_info()
state = LastKnownState(server_binding.local_folder,
local_info=local_info,
local_state='synchronized',
remote_info=remote_info,
remote_state='synchronized')
session.add(state)
session.commit()
return server_binding
def __init__(self, fname):
db = normalized_path(__file__).parent.parent / "resources" / fname
tmp = db.with_name(f"{uuid4()}.db")
shutil.copy(db, tmp)
time.sleep(1)
super().__init__(tmp)
def orphan_unlocked(path: Path) -> None:
"""
Mocked autolock.orphan_unlocked method.
Path is normalized before because safe_long_path() is not used yet in the database.
"""
self.direct_edit._manager.dao.unlock_path(normalized_path(path))
# seconds in RemoteFileInfo.from_dict() because of the datetime
# resolution of some databases (MySQL...)
REMOTE_MODIFICATION_TIME_RESOLUTION = 1.0
# 1s resolution on HFS+ on OSX
# ~0.01 sec for NTFS
# 0.001 sec for EXT4FS
OS_STAT_MTIME_RESOLUTION = 1.0
log = getLogger(__name__)
DEFAULT_NUXEO_URL = "http://localhost:8080/nuxeo"
DEFAULT_WAIT_SYNC_TIMEOUT = 10
FILE_CONTENT = b"Lorem ipsum dolor sit amet ..."
FAKER = Faker("en_US")
LOCATION = normalized_path(__file__).parent.parent
Translator(LOCATION / "resources" / "i18n")
def nuxeo_url() -> str:
"""Retrieve the Nuxeo URL."""
url = os.getenv("NXDRIVE_TEST_NUXEO_URL", DEFAULT_NUXEO_URL)
url = url.split("#")[0]
return url
def root_remote(base_folder: str = "/") -> DocRemote:
return DocRemote(
nuxeo_url(),