def __init__(self, *args, **kwargs):
super(MetaManager, self).__init__(*args, **kwargs)
self.app = kwargs['parent']
self.managers = {}
server_home = FileContentsManager()
server_home.root_dir = self.root_dir
self.managers['server-home'] = server_home
if self.enable_custom_handlers:
enable_custom_handlers()
for alias, path in self.file_dirs.items():
fb = FileContentsManager()
fb.root_dir = path
self.managers[alias] = fb
for alias, path in self.bundle_dirs.items():
fb = BundleNotebookManager()
fb.root_dir = path
self.managers[alias] = fb
for user, pw in self.github_accounts:
gh = notebook_gisthub(user, pw)
gbm = GistNotebookManager(gisthub=gh)
self.managers['gist:' + user] = gbm
self.middleware = {}
for name, middleware in self.manager_middleware.items():
cls = import_item(middleware)
self.middleware[name] = cls(parent=self, log=self.log)
self.root = RootManager(meta_manager=self)
def test_model_to_files(self):
with TemporaryDirectory() as td:
fm = FileContentsManager(root_dir=td)
model = fm.create_file()
# need content
model = fm.get_model(model['name'], model['path'])
files = model_to_files(model)
name = model['name']
# files should only contain one file
nt.assert_items_equal(files, [name])
# add a file
model['__files'] = {'file1.txt': 'file1txt content'}
files = model_to_files(model)
nt.assert_items_equal(files, [name, 'file1.txt'])
nt.assert_equal(files['file1.txt'], 'file1txt content')
def test_model_to_files(self):
with TemporaryDirectory() as td:
fm = FileContentsManager(root_dir=td)
model = fm.create_file()
# need content
model = fm.get_model(model['name'], model['path'])
files = model_to_files(model)
name = model['name']
# files should only contain one file
nt.assert_items_equal(files, [name])
# add a file
model['__files'] = {'file1.txt': 'file1txt content'}
files = model_to_files(model)
nt.assert_items_equal(files, [name, 'file1.txt'])
nt.assert_equal(files['file1.txt'],
'file1txt content')
def test_download_checkpoints(self):
"""
Create two checkpoints for two notebooks, then call
download_checkpoints.
Assert that we get the correct version of both notebooks.
"""
self.contents.new({'type': 'directory'}, 'subdir')
paths = ('a.ipynb', 'subdir/a.ipynb')
expected_content = {}
for path in paths:
# Create and checkpoint.
self.contents.new(path=path)
self.contents.create_checkpoint(path)
model = self.add_markdown_cell(path)
self.contents.create_checkpoint(path)
# Assert greater because FileContentsManager creates a checkpoint
# on creation, but this isn't part of the spec.
self.assertGreater(len(self.contents.list_checkpoints(path)), 2)
# Store the content to verify correctness after download.
expected_content[path] = model['content']
with TemporaryDirectory() as td:
download_checkpoints(
self.checkpoints.db_url,
td,
user='******',
)
fm = FileContentsManager(root_dir=td)
root_entries = sorted(m['path'] for m in fm.get('')['content'])
self.assertEqual(root_entries, ['a.ipynb', 'subdir'])
subdir_entries = sorted(
m['path'] for m in fm.get('subdir')['content']
)
self.assertEqual(subdir_entries, ['subdir/a.ipynb'])
for path in paths:
content = fm.get(path)['content']
self.assertEqual(expected_content[path], content)
def checkpoint_all(db_url, directory, user):
"""
Upload the current state of a directory for each user.
"""
print("Checkpointing directory {directory} for user {user}".format(
directory=directory, user=user,
))
cp_mgr = PostgresCheckpoints(
db_url=db_url,
user_id=user,
create_user_on_startup=False,
)
contents_mgr = FileContentsManager(
root_dir=directory,
checkpoints=cp_mgr,
)
cps = {}
for dirname, subdirs, files in walk(contents_mgr):
for fname in files:
if fname.endswith('.ipynb'):
cps[fname] = contents_mgr.create_checkpoint(fname)
return cps
def setUp(self):
drop_testing_db_tables()
migrate_testing_db()
self.td = TemporaryDirectory()
self.checkpoints = PostgresCheckpoints(
user_id='test',
db_url=TEST_DB_URL,
)
self.contents = FileContentsManager(
root_dir=self.td.name,
checkpoints=self.checkpoints,
)
self.checkpoints.ensure_user()
def __init__(self, **kwargs):
self.file_contents_manager = FileContentsManager()
self.client_side_contents_manager = ClientSideContentsManager()
class MixedContentsManager(ContentsManager):
DRIVE_PATH_SENTINEL = 'gdrive'
def __init__(self, **kwargs):
self.file_contents_manager = FileContentsManager()
self.client_side_contents_manager = ClientSideContentsManager()
def is_drive_path(self, path):
components = path.split('/')
return components and components[0] == self.DRIVE_PATH_SENTINEL
# ContentsManager API part 1: methods that must be
# implemented in subclasses.
def dir_exists(self, path):
if self.is_drive_path(path):
return self.client_side_contents_manager.dir_exists(path)
return self.file_contents_manager.dir_exists(path)
def is_hidden(self, path):
if self.is_drive_path(path):
return self.client_side_contents_manager.is_hidden(path)
return self.file_contents_manager.is_hidden(path)
def file_exists(self, path=''):
if self.is_drive_path(path):
return self.client_side_contents_manager.file_exists(path)
return self.file_contents_manager.file_exists(path)
def exists(self, path):
return self.file_contents_manager.exists(path)
def get(self, path, **kwargs):
if self.is_drive_path(path):
return self.client_side_contents_manager.get(path, **kwargs)
return self.file_contents_manager.get(path, **kwargs)
def save(self, model, path):
return self.file_contents_manager.save(model, path)
def update(self, model, path):
return self.file_contents_manager.update(model, path)
def delete(self, path):
return self.file_contents_manager.delete(path)
def create_checkpoint(self, path):
return self.file_contents_manager.create_checkpoint(path)
def list_checkpoints(self, path):
return self.file_contents_manager.list_checkpoints(path)
def restore_checkpoint(self, checkpoint_id, path):
return self.file_contents_manager.restore_checkpoint(
checkpoint_id, path)
def delete_checkpoint(self, checkpoint_id, path):
return self.file_contents_manager.delete_checkpoint(
checkpoint_id, path)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.filemanager = FileContentsManager(*args, **kwargs)
class BackwardsFileContentsManager(BackwardsCompatMixin, NBXContentsManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.filemanager = FileContentsManager(*args, **kwargs)
def get_notebook(self, name, path='', content=True, **kwargs):
model = self.get(name, path, content=content, **kwargs)
# note, since this is calling IPython's .get, the model
# path will be the newer full path. reset it since we're
# translating back in MetaManager.
model['path'] = path
return model
def get_model_dir(self, name, path='', content=True, **kwargs):
model = self.get(name, path, content=content, **kwargs)
for nb in model['content']:
# reset the model back to pre v4 name/path
nb['name'] = nb['path'].rsplit('/')[-1]
nb['path'] = path
return model
def notebook_exists(self, name, path=''):
return self.file_exists(name, path) and name.endswith('.ipynb')
# this is dumb, quite literally just translating back
def is_hidden(self, path):
return self.filemanager.is_hidden(path)
def get(self, name, path, **kwargs):
path = _fullpath(name, path)
model = self.filemanager.get(path, **kwargs)
return model
def save(self, model, name, path=''):
path = _fullpath(name, path)
return self.filemanager.save(model, path)
def update(self, model, name, path=''):
path = _fullpath(name, path)
return self.filemanager.update(model, path)
def delete(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.delete(path)
def rename(self, old_name, old_path, new_name, new_path):
path = _fullpath(name, path)
return self.filemanager.rename(old_path, new_path)
def file_exists(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.file_exists(path)
def path_exists(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.exists(path)
def exists(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.exists(path)
# checkpoint stuff
def get_checkpoint_path(self, checkpoint_id, name, path=''):
path = _fullpath(name, path)
return self.filemanager.get_checkpoint_path(checkpoint_id, path)
def get_checkpoint_model(self, checkpoint_id, name, path=''):
path = _fullpath(name, path)
return self.filemanager.get_checkpoint_model(checkpoint_id, path)
def create_checkpoint(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.create_checkpoint(path)
def list_checkpoints(self, name, path=''):
path = _fullpath(name, path)
return self.filemanager.list_checkpoints(path)
def restore_checkpoint(self, checkpoint_id, name, path=''):
path = _fullpath(name, path)
return self.filemanager.restore_checkpoint(checkpoint_id, path)
def delete_checkpoint(self, checkpoint_id, name, path=''):
path = _fullpath(name, path)
return self.filemanager.delete_checkpoint(checkpoint_id, path)
def get_kernel_path(self, name, path='', model=None):
path = _fullpath(name, path)
return self.filemanager.get_kernel_path(path, model=model)
class TestUploadDownload(TestCase):
def setUp(self):
drop_testing_db_tables()
migrate_testing_db()
self.td = TemporaryDirectory()
self.checkpoints = PostgresCheckpoints(
user_id='test',
db_url=TEST_DB_URL,
)
self.contents = FileContentsManager(
root_dir=self.td.name,
checkpoints=self.checkpoints,
)
self.checkpoints.ensure_user()
def tearDown(self):
self.td.cleanup()
def add_markdown_cell(self, path):
# Load and update
model = self.contents.get(path=path)
model['content'].cells.append(
new_markdown_cell('Created by test: ' + path)
)
# Save and checkpoint again.
self.contents.save(model, path=path)
return model
def test_download_checkpoints(self):
"""
Create two checkpoints for two notebooks, then call
download_checkpoints.
Assert that we get the correct version of both notebooks.
"""
self.contents.new({'type': 'directory'}, 'subdir')
paths = ('a.ipynb', 'subdir/a.ipynb')
expected_content = {}
for path in paths:
# Create and checkpoint.
self.contents.new(path=path)
self.contents.create_checkpoint(path)
model = self.add_markdown_cell(path)
self.contents.create_checkpoint(path)
# Assert greater because FileContentsManager creates a checkpoint
# on creation, but this isn't part of the spec.
self.assertGreater(len(self.contents.list_checkpoints(path)), 2)
# Store the content to verify correctness after download.
expected_content[path] = model['content']
with TemporaryDirectory() as td:
download_checkpoints(
self.checkpoints.db_url,
td,
user='******',
)
fm = FileContentsManager(root_dir=td)
root_entries = sorted(m['path'] for m in fm.get('')['content'])
self.assertEqual(root_entries, ['a.ipynb', 'subdir'])
subdir_entries = sorted(
m['path'] for m in fm.get('subdir')['content']
)
self.assertEqual(subdir_entries, ['subdir/a.ipynb'])
for path in paths:
content = fm.get(path)['content']
self.assertEqual(expected_content[path], content)
def test_checkpoint_all(self):
"""
Test that checkpoint_all correctly makes a checkpoint for all files.
"""
paths = populate(self.contents)
original_content_minus_trust = {
# Remove metadata that we expect to have dropped
path: strip_transient(self.contents.get(path)['content'])
for path in paths
}
original_cps = {}
for path in paths:
# Create a checkpoint, then update the file.
original_cps[path] = self.contents.create_checkpoint(path)
self.add_markdown_cell(path)
# Verify that we still have the old version checkpointed.
cp_content = {
path: self.checkpoints.get_notebook_checkpoint(
cp['id'],
path,
)['content']
for path, cp in iteritems(original_cps)
}
self.assertEqual(original_content_minus_trust, cp_content)
new_cps = checkpoint_all(
self.checkpoints.db_url,
self.td.name,
self.checkpoints.user_id,
)
new_cp_content = {
path: self.checkpoints.get_notebook_checkpoint(
cp['id'],
path,
)['content']
for path, cp in iteritems(new_cps)
}
for path, new_content in iteritems(new_cp_content):
old_content = original_content_minus_trust[_norm_unicode(path)]
self.assertEqual(
new_content['cells'][:-1],
old_content['cells'],
)
self.assertEqual(
new_content['cells'][-1],
new_markdown_cell('Created by test: ' + _norm_unicode(path)),
)
