class UnitTestCase(unittest.TestCase):
def setUpServer(self, server_profile=None):
# Long timeout for the root client that is responsible for the test
# environment set: this client is doing the first query on the Nuxeo
# server and might need to wait for a long time without failing for
# Nuxeo to finish initialize the repo on the first request after
# startup
self.root_remote_client = RemoteDocumentClient(
self.nuxeo_url, self.admin_user,
u'nxdrive-test-administrator-device', self.version,
password=self.password, base_folder=u'/', timeout=60)
# Activate given profile if needed, eg. permission hierarchy
if server_profile is not None:
self.root_remote_client.activate_profile(server_profile)
# Call the Nuxeo operation to setup the integration test environment
credentials = self.root_remote_client.execute(
"NuxeoDrive.SetupIntegrationTests",
userNames="user_1, user_2", permission='ReadWrite')
credentials = [c.strip().split(u":") for c in credentials.split(u",")]
self.user_1, self.password_1 = credentials[0]
self.user_2, self.password_2 = credentials[1]
ws_info = self.root_remote_client.fetch(TEST_WORKSPACE_PATH)
self.workspace = ws_info[u'uid']
self.workspace_title = ws_info[u'title']
self.workspace_1 = self.workspace
self.workspace_2 = self.workspace
self.workspace_title_1 = self.workspace_title
self.workspace_title_2 = self.workspace_title
def tearDownServer(self, server_profile=None):
# Don't need to revoke tokens for the file system remote clients
# since they use the same users as the remote document clients
self.root_remote_client.execute("NuxeoDrive.TearDownIntegrationTests")
# Deactivate given profile if needed, eg. permission hierarchy
if server_profile is not None:
self.root_remote_client.deactivate_profile(server_profile)
def setUpApp(self, server_profile=None):
# Check the Nuxeo server test environment
self.nuxeo_url = os.environ.get('NXDRIVE_TEST_NUXEO_URL')
self.admin_user = os.environ.get('NXDRIVE_TEST_USER')
self.password = os.environ.get('NXDRIVE_TEST_PASSWORD')
self.build_workspace = os.environ.get('WORKSPACE')
self.result = None
self.tearedDown = False
# Take default parameter if none has been set
if self.nuxeo_url is None:
self.nuxeo_url = "http://localhost:8080/nuxeo"
if self.admin_user is None:
self.admin_user = "******"
if self.password is None:
self.password = "******"
self.tmpdir = None
if self.build_workspace is not None:
self.tmpdir = os.path.join(self.build_workspace, "tmp")
if not os.path.isdir(self.tmpdir):
os.makedirs(self.tmpdir)
self.upload_tmp_dir = tempfile.mkdtemp(u'-nxdrive-uploads', dir=self.tmpdir)
if None in (self.nuxeo_url, self.admin_user, self.password):
raise unittest.SkipTest(
"No integration server configuration found in environment.")
# Check the local filesystem test environment
self.local_test_folder_1 = tempfile.mkdtemp(u'-nxdrive-tests-user-1', dir=self.tmpdir)
self.local_test_folder_2 = tempfile.mkdtemp(u'-nxdrive-tests-user-2', dir=self.tmpdir)
self.local_nxdrive_folder_1 = os.path.join(
self.local_test_folder_1, u'Nuxeo Drive')
os.mkdir(self.local_nxdrive_folder_1)
self.local_nxdrive_folder_2 = os.path.join(
self.local_test_folder_2, u'Nuxeo Drive')
os.mkdir(self.local_nxdrive_folder_2)
self.nxdrive_conf_folder_1 = os.path.join(
self.local_test_folder_1, u'nuxeo-drive-conf')
os.mkdir(self.nxdrive_conf_folder_1)
self.nxdrive_conf_folder_2 = os.path.join(
self.local_test_folder_2, u'nuxeo-drive-conf')
os.mkdir(self.nxdrive_conf_folder_2)
from mock import Mock
options = Mock()
options.debug = False
options.delay = TEST_DEFAULT_DELAY
options.force_locale = None
options.proxy_server = None
options.log_level_file = None
options.update_site_url = None
options.beta_update_site_url = None
options.autolock_interval = 30
options.nxdrive_home = self.nxdrive_conf_folder_1
self.manager_1 = Manager(options)
self.connected = False
import nxdrive
nxdrive_path = os.path.dirname(nxdrive.__file__)
i18n_path = os.path.join(nxdrive_path, 'tests', 'resources', "i18n.js")
Translator(self.manager_1, i18n_path)
options.nxdrive_home = self.nxdrive_conf_folder_2
Manager._singleton = None
self.manager_2 = Manager(options)
self.version = __version__
url = self.nuxeo_url
log.debug("Will use %s as url", url)
if '#' in url:
# Remove the engine type for the rest of the test
self.nuxeo_url = url.split('#')[0]
self.setUpServer(server_profile)
self.engine_1 = self.manager_1.bind_server(self.local_nxdrive_folder_1, url, self.user_1,
self.password_1, start_engine=False)
self.engine_2 = self.manager_2.bind_server(self.local_nxdrive_folder_2, url, self.user_2,
self.password_2, start_engine=False)
self.engine_1.syncCompleted.connect(self.app.sync_completed)
self.engine_1.get_remote_watcher().remoteScanFinished.connect(self.app.remote_scan_completed)
self.engine_1.get_remote_watcher().changesFound.connect(self.app.remote_changes_found)
self.engine_1.get_remote_watcher().noChangesFound.connect(self.app.no_remote_changes_found)
self.engine_2.syncCompleted.connect(self.app.sync_completed)
self.engine_2.get_remote_watcher().remoteScanFinished.connect(self.app.remote_scan_completed)
self.engine_2.get_remote_watcher().changesFound.connect(self.app.remote_changes_found)
self.engine_2.get_remote_watcher().noChangesFound.connect(self.app.no_remote_changes_found)
self.queue_manager_1 = self.engine_1.get_queue_manager()
self.queue_manager_2 = self.engine_2.get_queue_manager()
self.sync_root_folder_1 = os.path.join(self.local_nxdrive_folder_1, self.workspace_title_1)
self.sync_root_folder_2 = os.path.join(self.local_nxdrive_folder_2, self.workspace_title_2)
self.local_root_client_1 = self.engine_1.get_local_client()
self.local_root_client_2 = self.engine_2.get_local_client()
self.local_client_1 = LocalClient(os.path.join(self.local_nxdrive_folder_1, self.workspace_title_1))
self.local_client_2 = LocalClient(os.path.join(self.local_nxdrive_folder_2, self.workspace_title_2))
# Document client to be used to create remote test documents
# and folders
remote_document_client_1 = RemoteDocumentClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1, base_folder=self.workspace_1,
upload_tmp_dir=self.upload_tmp_dir)
remote_document_client_2 = RemoteDocumentClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2, base_folder=self.workspace_2,
upload_tmp_dir=self.upload_tmp_dir)
# File system client to be used to create remote test documents
# and folders
remote_file_system_client_1 = RemoteFileSystemClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1, upload_tmp_dir=self.upload_tmp_dir)
remote_file_system_client_2 = RemoteFileSystemClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2, upload_tmp_dir=self.upload_tmp_dir)
self.remote_restapi_client_1 = RestAPIClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1
)
self.remote_restapi_client_2 = RestAPIClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2
)
# Register root
remote_document_client_1.register_as_root(self.workspace_1)
remote_document_client_2.register_as_root(self.workspace_2)
self.remote_document_client_1 = remote_document_client_1
self.remote_document_client_2 = remote_document_client_2
self.remote_file_system_client_1 = remote_file_system_client_1
self.remote_file_system_client_2 = remote_file_system_client_2
self._wait_sync = {self.engine_1.get_uid(): True, self.engine_2.get_uid(): True}
self._wait_remote_scan = {self.engine_1.get_uid(): True, self.engine_2.get_uid(): True}
self._remote_changes_count = {self.engine_1.get_uid(): 0, self.engine_2.get_uid(): 0}
self._no_remote_changes = {self.engine_1.get_uid(): False, self.engine_2.get_uid(): False}
def wait_sync(self, wait_for_async=False, timeout=DEFAULT_WAIT_SYNC_TIMEOUT, fail_if_timeout=True,
wait_for_engine_1=True, wait_for_engine_2=False, wait_win=False, enforce_errors=True):
log.debug("Wait for sync")
# First wait for server if needed
if wait_for_async:
self.wait()
if sys.platform == "win32" and wait_win:
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
log.trace("Need to wait for Windows delete resolution")
sleep(WIN_MOVE_RESOLUTION_PERIOD/1000)
self._wait_sync = {
self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2
}
self._no_remote_changes = {self.engine_1.get_uid(): not wait_for_engine_1,
self.engine_2.get_uid(): not wait_for_engine_2}
if enforce_errors:
if not self.connected:
self.engine_1.syncPartialCompleted.connect(self.engine_1.get_queue_manager().requeue_errors)
self.engine_2.syncPartialCompleted.connect(self.engine_1.get_queue_manager().requeue_errors)
self.connected = True
elif self.connected:
self.engine_1.syncPartialCompleted.disconnect(self.engine_1.get_queue_manager().requeue_errors)
self.engine_2.syncPartialCompleted.disconnect(self.engine_1.get_queue_manager().requeue_errors)
self.connected = False
while timeout > 0:
sleep(1)
timeout = timeout - 1
if sum(self._wait_sync.values()) == 0:
if wait_for_async:
log.debug('Sync completed, _wait_remote_scan = %r, remote changes count = %r,'
' no remote changes = %r',
self._wait_remote_scan, self._remote_changes_count, self._no_remote_changes)
wait_remote_scan = False
if wait_for_engine_1:
wait_remote_scan = self._wait_remote_scan[self.engine_1.get_uid()]
if wait_for_engine_2:
wait_remote_scan = wait_remote_scan or self._wait_remote_scan[self.engine_2.get_uid()]
is_remote_changes = True
is_change_summary_over = True
if wait_for_engine_1:
is_remote_changes = self._remote_changes_count[self.engine_1.get_uid()] > 0
is_change_summary_over = self._no_remote_changes[self.engine_1.get_uid()]
if wait_for_engine_2:
is_remote_changes = (is_remote_changes
and self._remote_changes_count[self.engine_2.get_uid()] > 0)
is_change_summary_over = (is_change_summary_over
and self._no_remote_changes[self.engine_2.get_uid()])
if (not wait_remote_scan or is_remote_changes and is_change_summary_over):
self._wait_remote_scan = {self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2}
self._remote_changes_count = {self.engine_1.get_uid(): 0, self.engine_2.get_uid(): 0}
self._no_remote_changes = {self.engine_1.get_uid(): False, self.engine_2.get_uid(): False}
log.debug('Ended wait for sync, setting _wait_remote_scan values to True,'
' _remote_changes_count values to 0 and _no_remote_changes values to False')
return
else:
log.debug("Sync completed, ended wait for sync")
return
if fail_if_timeout:
log.warn("Wait for sync timeout has expired")
if wait_for_engine_1 and self.engine_1.get_dao().get_syncing_count() != 0:
self.fail("Wait for sync timeout expired")
if wait_for_engine_2 and self.engine_2.get_dao().get_syncing_count() != 0:
self.fail("Wait for sync timeout expired")
else:
log.debug("Wait for sync timeout")
def wait_remote_scan(self, timeout=DEFAULT_WAIT_REMOTE_SCAN_TIMEOUT, wait_for_engine_1=True,
wait_for_engine_2=False):
log.debug("Wait for remote scan")
self._wait_remote_scan = {self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2}
while timeout > 0:
sleep(1)
if sum(self._wait_remote_scan.values()) == 0:
log.debug("Ended wait for remote scan")
return
timeout = timeout - 1
self.fail("Wait for remote scan timeout expired")
def is_profiling(self):
return 'DRIVE_YAPPI' in os.environ and yappi is not None
def setup_profiler(self):
if not self.is_profiling():
return
yappi.start()
def teardown_profiler(self):
if not self.is_profiling():
return
path = os.environ["DRIVE_YAPPI"]
if not os.path.exists(path):
os.mkdir(path)
report_path = os.path.join(path, self.id() + '-yappi-threads')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("tid", 15), 2: ("ttot", 8), 3: ("scnt", 10)}
yappi.get_thread_stats().print_all(out=fd, columns=columns)
report_path = os.path.join(path, self.id() + '-yappi-fcts')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
stats = yappi.get_func_stats()
stats.strip_dirs()
stats.print_all(out=fd, columns=columns)
log.debug("Profiler Report generated in '%s'", report_path)
def run(self, result=None):
self.app = TestQApplication([], self)
self.setUpApp()
self.result = result
# TODO Should use a specific application
def launch_test():
log.debug("UnitTest thread started")
sleep(1)
self.setup_profiler()
super(UnitTestCase, self).run(result)
self.teardown_profiler()
self.app.quit()
log.debug("UnitTest thread finished")
sync_thread = Thread(target=launch_test)
sync_thread.start()
self.app.exec_()
sync_thread.join(30)
self.tearDownApp()
del self.app
log.debug("UnitTest run finished")
def tearDown(self):
unittest.TestCase.tearDown(self)
if not self.tearedDown:
self.tearDownApp()
def tearDownApp(self, server_profile=None):
if self.tearedDown:
return
if sys.exc_info() != (None, None, None):
self.generate_report()
elif self.result is not None:
if hasattr(self.result, "wasSuccessful") and not self.result.wasSuccessful():
self.generate_report()
log.debug("TearDown unit test")
# Unbind all
self.manager_1.unbind_all()
self.manager_1.dispose_db()
self.manager_2.unbind_all()
self.manager_2.dispose_db()
Manager._singleton = None
self.tearDownServer(server_profile)
clean_dir(self.upload_tmp_dir)
clean_dir(self.local_test_folder_1)
clean_dir(self.local_test_folder_2)
del self.engine_1
self.engine_1 = None
del self.engine_2
self.engine_2 = None
del self.local_client_1
self.local_client_1 = None
del self.local_client_2
self.local_client_2 = None
del self.remote_document_client_1
self.remote_document_client_1 = None
del self.remote_document_client_2
self.remote_document_client_2 = None
del self.remote_file_system_client_1
self.remote_file_system_client_1 = None
del self.remote_file_system_client_2
self.remote_file_system_client_2 = None
self.tearedDown = True
def _interact(self, pause=0):
self.app.processEvents()
if pause > 0:
sleep(pause)
while (self.app.hasPendingEvents()):
self.app.processEvents()
def make_local_tree(self, root=None, local_client=None):
if local_client is None:
local_client = self.local_root_client_1
if root is None:
root = u"/" + self.workspace_title
if not local_client.exists(root):
local_client.make_folder(u"/", self.workspace_title)
# create some folders
folder_1 = local_client.make_folder(root, u'Folder 1')
folder_1_1 = local_client.make_folder(folder_1, u'Folder 1.1')
folder_1_2 = local_client.make_folder(folder_1, u'Folder 1.2')
folder_2 = local_client.make_folder(root, u'Folder 2')
# create some files
local_client.make_file(folder_2, u'Duplicated File.txt', content=b"Some content.")
local_client.make_file(folder_1, u'File 1.txt', content=b"aaa")
local_client.make_file(folder_1_1, u'File 2.txt', content=b"bbb")
local_client.make_file(folder_1_2, u'File 3.txt', content=b"ccc")
local_client.make_file(folder_2, u'File 4.txt', content=b"ddd")
local_client.make_file(root, u'File 5.txt', content=b"eee")
return (6, 5)
def make_server_tree(self, deep=True):
remote_client = self.remote_document_client_1
# create some folders on the server
folder_1 = remote_client.make_folder(self.workspace, u'Folder 1')
folder_2 = remote_client.make_folder(self.workspace, u'Folder 2')
if deep:
folder_1_1 = remote_client.make_folder(folder_1, u'Folder 1.1')
folder_1_2 = remote_client.make_folder(folder_1, u'Folder 1.2')
# create some files on the server
if deep:
self._duplicate_file_1 = remote_client.make_file(folder_2, u'Duplicated File.txt',
content=b"Some content.")
self._duplicate_file_2 = remote_client.make_file(folder_2, u'Duplicated File.txt',
content=b"Other content.")
if deep:
remote_client.make_file(folder_1, u'File 1.txt', content=b"aaa")
remote_client.make_file(folder_1_1, u'File 2.txt', content=b"bbb")
remote_client.make_file(folder_1_2, u'File 3.txt', content=b"ccc")
remote_client.make_file(folder_2, u'File 4.txt', content=b"ddd")
remote_client.make_file(self.workspace, u'File 5.txt', content=b"eee")
return (7, 4) if deep else (1, 2)
def get_local_child_count(self, path):
dir_count = 0
file_count = 0
for _, dirnames, filenames in os.walk(path):
dir_count += len(dirnames)
file_count += len(filenames)
if os.path.exists(os.path.join(path, '.partials')):
dir_count -= 1
return (dir_count, file_count)
def get_full_queue(self, queue, dao=None):
if dao is None:
dao = self.engine_1.get_dao()
result = []
while (len(queue) > 0):
result.append(dao.get_state_from_id(queue.pop().id))
return result
def generate_report(self):
if "REPORT_PATH" not in os.environ:
return
report_path = os.path.join(os.environ["REPORT_PATH"], self.id())
self.manager_1.generate_report(report_path)
log.debug("Report generated in '%s'", report_path)
def wait(self, retry=3):
try:
self.root_remote_client.wait()
except Exception as e:
log.debug("Exception while waiting for server : %r", e)
# Not the nicest
if retry > 0:
log.debug("Retry to wait")
self.wait(retry - 1)
def generate_random_jpg(self, filename, size):
try:
import numpy
from PIL import Image
except:
# Create random file
with open(filename, 'wb') as f:
f.write(os.urandom(1024 * size))
return
a = numpy.random.rand(size, size, 3) * 255
im_out = Image.fromarray(a.astype('uint8')).convert('RGBA')
im_out.save(filename)
def assertNxPart(self, path, name=None, present=True):
os_path = self.local_client_1._abspath(path)
children = os.listdir(os_path)
for child in children:
if len(child) < 8:
continue
if name is not None and len(child) < len(name) + 8:
continue
if child[0] == "." and child[-7:] == ".nxpart":
if name is None or child[1:len(name)+1] == name:
if present:
return
else:
self.fail("nxpart found in : '%s'" % (path))
if present:
self.fail("nxpart not found in : '%s'" % (path))
def get_dao_state_from_engine_1(self, path):
"""
Returns the pair from dao of engine 1 according to the path.
:param path: The path to document (from workspace, ex: /Folder is converted to /{{workspace_title_1}}/Folder).
:return: The pair from dao of engine 1 according to the path.
"""
abs_path = '/' + self.workspace_title_1 + path
return self.engine_1.get_dao().get_state_from_local(abs_path)
class UnitTestCase(unittest.TestCase):
def setUpServer(self, server_profile=None):
# Long timeout for the root client that is responsible for the test
# environment set: this client is doing the first query on the Nuxeo
# server and might need to wait for a long time without failing for
# Nuxeo to finish initialize the repo on the first request after
# startup
self.root_remote_client = RemoteDocumentClient(
self.nuxeo_url, self.admin_user,
u'nxdrive-test-administrator-device', self.version,
password=self.password, base_folder=u'/', timeout=60)
# Activate given profile if needed, eg. permission hierarchy
if server_profile is not None:
self.root_remote_client.activate_profile(server_profile)
# Call the Nuxeo operation to setup the integration test environment
credentials = self.root_remote_client.execute(
"NuxeoDrive.SetupIntegrationTests",
userNames="user_1, user_2", permission='ReadWrite')
credentials = [c.strip().split(u":") for c in credentials.split(u",")]
self.user_1, self.password_1 = credentials[0]
self.user_2, self.password_2 = credentials[1]
ws_info = self.root_remote_client.fetch(TEST_WORKSPACE_PATH)
self.workspace = ws_info[u'uid']
self.workspace_title = ws_info[u'title']
self.workspace_1 = self.workspace
self.workspace_2 = self.workspace
self.workspace_title_1 = self.workspace_title
self.workspace_title_2 = self.workspace_title
def tearDownServer(self, server_profile=None):
# Don't need to revoke tokens for the file system remote clients
# since they use the same users as the remote document clients
self.root_remote_client.execute("NuxeoDrive.TearDownIntegrationTests")
# Deactivate given profile if needed, eg. permission hierarchy
if server_profile is not None:
self.root_remote_client.deactivate_profile(server_profile)
def setUpApp(self, server_profile=None):
# Check the Nuxeo server test environment
self.nuxeo_url = os.environ.get('NXDRIVE_TEST_NUXEO_URL')
self.admin_user = os.environ.get('NXDRIVE_TEST_USER')
self.password = os.environ.get('NXDRIVE_TEST_PASSWORD')
self.build_workspace = os.environ.get('WORKSPACE')
self.result = None
self.tearedDown = False
# Take default parameter if none has been set
if self.nuxeo_url is None:
self.nuxeo_url = "http://localhost:8080/nuxeo"
if self.admin_user is None:
self.admin_user = "******"
if self.password is None:
self.password = "******"
self.tmpdir = None
if self.build_workspace is not None:
self.tmpdir = os.path.join(self.build_workspace, "tmp")
if not os.path.isdir(self.tmpdir):
os.makedirs(self.tmpdir)
self.upload_tmp_dir = tempfile.mkdtemp(u'-nxdrive-uploads', dir=self.tmpdir)
if None in (self.nuxeo_url, self.admin_user, self.password):
raise unittest.SkipTest(
"No integration server configuration found in environment.")
# Check the local filesystem test environment
self.local_test_folder_1 = tempfile.mkdtemp(u'-nxdrive-tests-user-1', dir=self.tmpdir)
self.local_test_folder_2 = tempfile.mkdtemp(u'-nxdrive-tests-user-2', dir=self.tmpdir)
self.local_nxdrive_folder_1 = os.path.join(
self.local_test_folder_1, u'Nuxeo Drive')
os.mkdir(self.local_nxdrive_folder_1)
self.local_nxdrive_folder_2 = os.path.join(
self.local_test_folder_2, u'Nuxeo Drive')
os.mkdir(self.local_nxdrive_folder_2)
self.nxdrive_conf_folder_1 = os.path.join(
self.local_test_folder_1, u'nuxeo-drive-conf')
os.mkdir(self.nxdrive_conf_folder_1)
self.nxdrive_conf_folder_2 = os.path.join(
self.local_test_folder_2, u'nuxeo-drive-conf')
os.mkdir(self.nxdrive_conf_folder_2)
from mock import Mock
options = Mock()
options.debug = False
options.delay = TEST_DEFAULT_DELAY
options.force_locale = None
options.proxy_server = None
options.log_level_file = None
options.update_site_url = None
options.beta_update_site_url = None
options.autolock_interval = 30
options.nxdrive_home = self.nxdrive_conf_folder_1
options.version == __version__
self.manager_1 = Manager(options)
self.connected = False
self.version = self.manager_1.get_version()
import nxdrive
nxdrive_path = os.path.dirname(nxdrive.__file__)
i18n_path = os.path.join(nxdrive_path, 'tests', 'resources', "i18n.js")
Translator(self.manager_1, i18n_path)
options.nxdrive_home = self.nxdrive_conf_folder_2
Manager._singleton = None
self.manager_2 = Manager(options)
url = self.nuxeo_url
log.debug("Will use %s as url", url)
if '#' in url:
# Remove the engine type for the rest of the test
self.nuxeo_url = url.split('#')[0]
self.setUpServer(server_profile)
self.engine_1 = self.manager_1.bind_server(self.local_nxdrive_folder_1, url, self.user_1,
self.password_1, start_engine=False)
self.engine_2 = self.manager_2.bind_server(self.local_nxdrive_folder_2, url, self.user_2,
self.password_2, start_engine=False)
self.engine_1.syncCompleted.connect(self.app.sync_completed)
self.engine_1.get_remote_watcher().remoteScanFinished.connect(self.app.remote_scan_completed)
self.engine_1.get_remote_watcher().changesFound.connect(self.app.remote_changes_found)
self.engine_1.get_remote_watcher().noChangesFound.connect(self.app.no_remote_changes_found)
self.engine_2.syncCompleted.connect(self.app.sync_completed)
self.engine_2.get_remote_watcher().remoteScanFinished.connect(self.app.remote_scan_completed)
self.engine_2.get_remote_watcher().changesFound.connect(self.app.remote_changes_found)
self.engine_2.get_remote_watcher().noChangesFound.connect(self.app.no_remote_changes_found)
self.queue_manager_1 = self.engine_1.get_queue_manager()
self.queue_manager_2 = self.engine_2.get_queue_manager()
self.sync_root_folder_1 = os.path.join(self.local_nxdrive_folder_1, self.workspace_title_1)
self.sync_root_folder_2 = os.path.join(self.local_nxdrive_folder_2, self.workspace_title_2)
self.local_root_client_1 = self.engine_1.get_local_client()
self.local_root_client_2 = self.engine_2.get_local_client()
self.local_client_1 = LocalClient(os.path.join(self.local_nxdrive_folder_1, self.workspace_title_1))
self.local_client_2 = LocalClient(os.path.join(self.local_nxdrive_folder_2, self.workspace_title_2))
# Document client to be used to create remote test documents
# and folders
remote_document_client_1 = RemoteDocumentClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1, base_folder=self.workspace_1,
upload_tmp_dir=self.upload_tmp_dir)
remote_document_client_2 = RemoteDocumentClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2, base_folder=self.workspace_2,
upload_tmp_dir=self.upload_tmp_dir)
# File system client to be used to create remote test documents
# and folders
remote_file_system_client_1 = RemoteFileSystemClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1, upload_tmp_dir=self.upload_tmp_dir)
remote_file_system_client_2 = RemoteFileSystemClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2, upload_tmp_dir=self.upload_tmp_dir)
self.remote_restapi_client_1 = RestAPIClient(
self.nuxeo_url, self.user_1, u'nxdrive-test-device-1',
self.version,
password=self.password_1
)
self.remote_restapi_client_2 = RestAPIClient(
self.nuxeo_url, self.user_2, u'nxdrive-test-device-2',
self.version,
password=self.password_2
)
# Register root
remote_document_client_1.register_as_root(self.workspace_1)
remote_document_client_2.register_as_root(self.workspace_2)
self.remote_document_client_1 = remote_document_client_1
self.remote_document_client_2 = remote_document_client_2
self.remote_file_system_client_1 = remote_file_system_client_1
self.remote_file_system_client_2 = remote_file_system_client_2
self._wait_sync = {self.engine_1.get_uid(): True, self.engine_2.get_uid(): True}
self._wait_remote_scan = {self.engine_1.get_uid(): True, self.engine_2.get_uid(): True}
self._remote_changes_count = {self.engine_1.get_uid(): 0, self.engine_2.get_uid(): 0}
self._no_remote_changes = {self.engine_1.get_uid(): False, self.engine_2.get_uid(): False}
def wait_sync(self, wait_for_async=False, timeout=DEFAULT_WAIT_SYNC_TIMEOUT, fail_if_timeout=True,
wait_for_engine_1=True, wait_for_engine_2=False, wait_win=False, enforce_errors=True):
log.debug("Wait for sync")
# First wait for server if needed
if wait_for_async:
self.wait()
if sys.platform == "win32" and wait_win:
from nxdrive.engine.watcher.local_watcher import WIN_MOVE_RESOLUTION_PERIOD
log.trace("Need to wait for Windows delete resolution")
sleep(WIN_MOVE_RESOLUTION_PERIOD/1000)
self._wait_sync = {
self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2
}
self._no_remote_changes = {self.engine_1.get_uid(): not wait_for_engine_1,
self.engine_2.get_uid(): not wait_for_engine_2}
if enforce_errors:
if not self.connected:
self.engine_1.syncPartialCompleted.connect(self.engine_1.get_queue_manager().requeue_errors)
self.engine_2.syncPartialCompleted.connect(self.engine_1.get_queue_manager().requeue_errors)
self.connected = True
elif self.connected:
self.engine_1.syncPartialCompleted.disconnect(self.engine_1.get_queue_manager().requeue_errors)
self.engine_2.syncPartialCompleted.disconnect(self.engine_1.get_queue_manager().requeue_errors)
self.connected = False
while timeout > 0:
sleep(1)
timeout = timeout - 1
if sum(self._wait_sync.values()) == 0:
if wait_for_async:
log.debug('Sync completed, _wait_remote_scan = %r, remote changes count = %r,'
' no remote changes = %r',
self._wait_remote_scan, self._remote_changes_count, self._no_remote_changes)
wait_remote_scan = False
if wait_for_engine_1:
wait_remote_scan = self._wait_remote_scan[self.engine_1.get_uid()]
if wait_for_engine_2:
wait_remote_scan = wait_remote_scan or self._wait_remote_scan[self.engine_2.get_uid()]
is_remote_changes = True
is_change_summary_over = True
if wait_for_engine_1:
is_remote_changes = self._remote_changes_count[self.engine_1.get_uid()] > 0
is_change_summary_over = self._no_remote_changes[self.engine_1.get_uid()]
if wait_for_engine_2:
is_remote_changes = (is_remote_changes
and self._remote_changes_count[self.engine_2.get_uid()] > 0)
is_change_summary_over = (is_change_summary_over
and self._no_remote_changes[self.engine_2.get_uid()])
if (not wait_remote_scan or is_remote_changes and is_change_summary_over):
self._wait_remote_scan = {self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2}
self._remote_changes_count = {self.engine_1.get_uid(): 0, self.engine_2.get_uid(): 0}
self._no_remote_changes = {self.engine_1.get_uid(): False, self.engine_2.get_uid(): False}
log.debug('Ended wait for sync, setting _wait_remote_scan values to True,'
' _remote_changes_count values to 0 and _no_remote_changes values to False')
return
else:
log.debug("Sync completed, ended wait for sync")
return
if fail_if_timeout:
log.warn("Wait for sync timeout has expired")
if wait_for_engine_1 and self.engine_1.get_dao().get_syncing_count() != 0:
self.fail("Wait for sync timeout expired")
if wait_for_engine_2 and self.engine_2.get_dao().get_syncing_count() != 0:
self.fail("Wait for sync timeout expired")
else:
log.debug("Wait for sync timeout")
def wait_remote_scan(self, timeout=DEFAULT_WAIT_REMOTE_SCAN_TIMEOUT, wait_for_engine_1=True,
wait_for_engine_2=False):
log.debug("Wait for remote scan")
self._wait_remote_scan = {self.engine_1.get_uid(): wait_for_engine_1,
self.engine_2.get_uid(): wait_for_engine_2}
while timeout > 0:
sleep(1)
if sum(self._wait_remote_scan.values()) == 0:
log.debug("Ended wait for remote scan")
return
timeout = timeout - 1
self.fail("Wait for remote scan timeout expired")
def is_profiling(self):
return 'DRIVE_YAPPI' in os.environ and yappi is not None
def setup_profiler(self):
if not self.is_profiling():
return
yappi.start()
def teardown_profiler(self):
if not self.is_profiling():
return
path = os.environ["DRIVE_YAPPI"]
if not os.path.exists(path):
os.mkdir(path)
report_path = os.path.join(path, self.id() + '-yappi-threads')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("tid", 15), 2: ("ttot", 8), 3: ("scnt", 10)}
yappi.get_thread_stats().print_all(out=fd, columns=columns)
report_path = os.path.join(path, self.id() + '-yappi-fcts')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
stats = yappi.get_func_stats()
stats.strip_dirs()
stats.print_all(out=fd, columns=columns)
log.debug("Profiler Report generated in '%s'", report_path)
def run(self, result=None):
repeat = 1
testMethod = getattr(self, self._testMethodName)
if hasattr(testMethod, '_repeat'):
repeat = testMethod._repeat
while repeat > 0:
self.app = TestQApplication([], self)
self.setUpApp()
self.result = result
# TODO Should use a specific application
def launch_test():
log.debug("UnitTest thread started")
sleep(1)
self.setup_profiler()
super(UnitTestCase, self).run(result)
self.teardown_profiler()
self.app.quit()
log.debug("UnitTest thread finished")
sync_thread = Thread(target=launch_test)
sync_thread.start()
self.app.exec_()
sync_thread.join(30)
self.tearDownApp()
del self.app
repeat -= 1
log.debug("UnitTest run finished")
def tearDown(self):
unittest.TestCase.tearDown(self)
if not self.tearedDown:
self.tearDownApp()
def tearDownApp(self, server_profile=None):
if self.tearedDown:
return
if sys.exc_info() != (None, None, None):
self.generate_report()
elif self.result is not None:
if hasattr(self.result, "wasSuccessful") and not self.result.wasSuccessful():
self.generate_report()
log.debug("TearDown unit test")
# Unbind all
self.manager_1.unbind_all()
self.manager_1.dispose_db()
self.manager_2.unbind_all()
self.manager_2.dispose_db()
Manager._singleton = None
self.tearDownServer(server_profile)
clean_dir(self.upload_tmp_dir)
clean_dir(self.local_test_folder_1)
clean_dir(self.local_test_folder_2)
del self.engine_1
self.engine_1 = None
del self.engine_2
self.engine_2 = None
del self.local_client_1
self.local_client_1 = None
del self.local_client_2
self.local_client_2 = None
del self.remote_document_client_1
self.remote_document_client_1 = None
del self.remote_document_client_2
self.remote_document_client_2 = None
del self.remote_file_system_client_1
self.remote_file_system_client_1 = None
del self.remote_file_system_client_2
self.remote_file_system_client_2 = None
self.tearedDown = True
def _interact(self, pause=0):
self.app.processEvents()
if pause > 0:
sleep(pause)
while (self.app.hasPendingEvents()):
self.app.processEvents()
def make_local_tree(self, root=None, local_client=None):
if local_client is None:
local_client = self.local_root_client_1
if root is None:
root = u"/" + self.workspace_title
if not local_client.exists(root):
local_client.make_folder(u"/", self.workspace_title)
# create some folders
folder_1 = local_client.make_folder(root, u'Folder 1')
folder_1_1 = local_client.make_folder(folder_1, u'Folder 1.1')
folder_1_2 = local_client.make_folder(folder_1, u'Folder 1.2')
folder_2 = local_client.make_folder(root, u'Folder 2')
# create some files
local_client.make_file(folder_2, u'Duplicated File.txt', content=b"Some content.")
local_client.make_file(folder_1, u'File 1.txt', content=b"aaa")
local_client.make_file(folder_1_1, u'File 2.txt', content=b"bbb")
local_client.make_file(folder_1_2, u'File 3.txt', content=b"ccc")
local_client.make_file(folder_2, u'File 4.txt', content=b"ddd")
local_client.make_file(root, u'File 5.txt', content=b"eee")
return (6, 5)
def make_server_tree(self, deep=True):
remote_client = self.remote_document_client_1
# create some folders on the server
folder_1 = remote_client.make_folder(self.workspace, u'Folder 1')
folder_2 = remote_client.make_folder(self.workspace, u'Folder 2')
if deep:
folder_1_1 = remote_client.make_folder(folder_1, u'Folder 1.1')
folder_1_2 = remote_client.make_folder(folder_1, u'Folder 1.2')
# create some files on the server
if deep:
self._duplicate_file_1 = remote_client.make_file(folder_2, u'Duplicated File.txt',
content=b"Some content.")
self._duplicate_file_2 = remote_client.make_file(folder_2, u'Duplicated File.txt',
content=b"Other content.")
if deep:
remote_client.make_file(folder_1, u'File 1.txt', content=b"aaa")
remote_client.make_file(folder_1_1, u'File 2.txt', content=b"bbb")
remote_client.make_file(folder_1_2, u'File 3.txt', content=b"ccc")
remote_client.make_file(folder_2, u'File 4.txt', content=b"ddd")
remote_client.make_file(self.workspace, u'File 5.txt', content=b"eee")
return (7, 4) if deep else (1, 2)
def get_local_child_count(self, path):
dir_count = 0
file_count = 0
for _, dirnames, filenames in os.walk(path):
dir_count += len(dirnames)
file_count += len(filenames)
if os.path.exists(os.path.join(path, '.partials')):
dir_count -= 1
return (dir_count, file_count)
def get_full_queue(self, queue, dao=None):
if dao is None:
dao = self.engine_1.get_dao()
result = []
while (len(queue) > 0):
result.append(dao.get_state_from_id(queue.pop().id))
return result
def generate_report(self):
if "REPORT_PATH" not in os.environ:
return
report_path = os.path.join(os.environ["REPORT_PATH"], self.id())
self.manager_1.generate_report(report_path)
log.debug("Report generated in '%s'", report_path)
def wait(self, retry=3):
try:
self.root_remote_client.wait()
except Exception as e:
log.debug("Exception while waiting for server : %r", e)
# Not the nicest
if retry > 0:
log.debug("Retry to wait")
self.wait(retry - 1)
def _set_read_permission(self, user, doc_path, grant):
op_input = "doc:" + doc_path
if grant:
self.root_remote_client.execute("Document.SetACE",
op_input=op_input,
user=user,
permission="Read",
grant="true")
else:
self.root_remote_client.block_inheritance(doc_path)
def generate_random_jpg(self, filename, size):
try:
import numpy
from PIL import Image
except:
# Create random file
with open(filename, 'wb') as f:
f.write(os.urandom(1024 * size))
return
a = numpy.random.rand(size, size, 3) * 255
im_out = Image.fromarray(a.astype('uint8')).convert('RGBA')
im_out.save(filename)
def assertNxPart(self, path, name=None, present=True):
os_path = self.local_client_1._abspath(path)
children = os.listdir(os_path)
for child in children:
if len(child) < 8:
continue
if name is not None and len(child) < len(name) + 8:
continue
if child[0] == "." and child[-7:] == ".nxpart":
if name is None or child[1:len(name)+1] == name:
if present:
return
else:
self.fail("nxpart found in : '%s'" % (path))
if present:
self.fail("nxpart not found in : '%s'" % (path))
def get_dao_state_from_engine_1(self, path):
"""
Returns the pair from dao of engine 1 according to the path.
:param path: The path to document (from workspace, ex: /Folder is converted to /{{workspace_title_1}}/Folder).
:return: The pair from dao of engine 1 according to the path.
"""
abs_path = '/' + self.workspace_title_1 + path
return self.engine_1.get_dao().get_state_from_local(abs_path)