def __init__(self, *args, **kwargs): super(MetaManager, self).__init__(*args, **kwargs) self.app = kwargs['parent'] self.managers = {} server_home = FileContentsManager() server_home.root_dir = self.root_dir self.managers['server-home'] = server_home if self.enable_custom_handlers: enable_custom_handlers() for alias, path in self.file_dirs.items(): fb = FileContentsManager() fb.root_dir = path self.managers[alias] = fb for alias, path in self.bundle_dirs.items(): fb = BundleNotebookManager() fb.root_dir = path self.managers[alias] = fb for user, pw in self.github_accounts: gh = notebook_gisthub(user, pw) gbm = GistNotebookManager(gisthub=gh) self.managers['gist:' + user] = gbm self.middleware = {} for name, middleware in self.manager_middleware.items(): cls = import_item(middleware) self.middleware[name] = cls(parent=self, log=self.log) self.root = RootManager(meta_manager=self)
def test_model_to_files(self): with TemporaryDirectory() as td: fm = FileContentsManager(root_dir=td) model = fm.create_file() # need content model = fm.get_model(model['name'], model['path']) files = model_to_files(model) name = model['name'] # files should only contain one file nt.assert_items_equal(files, [name]) # add a file model['__files'] = {'file1.txt': 'file1txt content'} files = model_to_files(model) nt.assert_items_equal(files, [name, 'file1.txt']) nt.assert_equal(files['file1.txt'], 'file1txt content')
def test_download_checkpoints(self): """ Create two checkpoints for two notebooks, then call download_checkpoints. Assert that we get the correct version of both notebooks. """ self.contents.new({'type': 'directory'}, 'subdir') paths = ('a.ipynb', 'subdir/a.ipynb') expected_content = {} for path in paths: # Create and checkpoint. self.contents.new(path=path) self.contents.create_checkpoint(path) model = self.add_markdown_cell(path) self.contents.create_checkpoint(path) # Assert greater because FileContentsManager creates a checkpoint # on creation, but this isn't part of the spec. self.assertGreater(len(self.contents.list_checkpoints(path)), 2) # Store the content to verify correctness after download. expected_content[path] = model['content'] with TemporaryDirectory() as td: download_checkpoints( self.checkpoints.db_url, td, user='******', ) fm = FileContentsManager(root_dir=td) root_entries = sorted(m['path'] for m in fm.get('')['content']) self.assertEqual(root_entries, ['a.ipynb', 'subdir']) subdir_entries = sorted( m['path'] for m in fm.get('subdir')['content'] ) self.assertEqual(subdir_entries, ['subdir/a.ipynb']) for path in paths: content = fm.get(path)['content'] self.assertEqual(expected_content[path], content)
def checkpoint_all(db_url, directory, user): """ Upload the current state of a directory for each user. """ print("Checkpointing directory {directory} for user {user}".format( directory=directory, user=user, )) cp_mgr = PostgresCheckpoints( db_url=db_url, user_id=user, create_user_on_startup=False, ) contents_mgr = FileContentsManager( root_dir=directory, checkpoints=cp_mgr, ) cps = {} for dirname, subdirs, files in walk(contents_mgr): for fname in files: if fname.endswith('.ipynb'): cps[fname] = contents_mgr.create_checkpoint(fname) return cps
def setUp(self): drop_testing_db_tables() migrate_testing_db() self.td = TemporaryDirectory() self.checkpoints = PostgresCheckpoints( user_id='test', db_url=TEST_DB_URL, ) self.contents = FileContentsManager( root_dir=self.td.name, checkpoints=self.checkpoints, ) self.checkpoints.ensure_user()
def __init__(self, **kwargs): self.file_contents_manager = FileContentsManager() self.client_side_contents_manager = ClientSideContentsManager()
class MixedContentsManager(ContentsManager): DRIVE_PATH_SENTINEL = 'gdrive' def __init__(self, **kwargs): self.file_contents_manager = FileContentsManager() self.client_side_contents_manager = ClientSideContentsManager() def is_drive_path(self, path): components = path.split('/') return components and components[0] == self.DRIVE_PATH_SENTINEL # ContentsManager API part 1: methods that must be # implemented in subclasses. def dir_exists(self, path): if self.is_drive_path(path): return self.client_side_contents_manager.dir_exists(path) return self.file_contents_manager.dir_exists(path) def is_hidden(self, path): if self.is_drive_path(path): return self.client_side_contents_manager.is_hidden(path) return self.file_contents_manager.is_hidden(path) def file_exists(self, path=''): if self.is_drive_path(path): return self.client_side_contents_manager.file_exists(path) return self.file_contents_manager.file_exists(path) def exists(self, path): return self.file_contents_manager.exists(path) def get(self, path, **kwargs): if self.is_drive_path(path): return self.client_side_contents_manager.get(path, **kwargs) return self.file_contents_manager.get(path, **kwargs) def save(self, model, path): return self.file_contents_manager.save(model, path) def update(self, model, path): return self.file_contents_manager.update(model, path) def delete(self, path): return self.file_contents_manager.delete(path) def create_checkpoint(self, path): return self.file_contents_manager.create_checkpoint(path) def list_checkpoints(self, path): return self.file_contents_manager.list_checkpoints(path) def restore_checkpoint(self, checkpoint_id, path): return self.file_contents_manager.restore_checkpoint( checkpoint_id, path) def delete_checkpoint(self, checkpoint_id, path): return self.file_contents_manager.delete_checkpoint( checkpoint_id, path)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.filemanager = FileContentsManager(*args, **kwargs)
class BackwardsFileContentsManager(BackwardsCompatMixin, NBXContentsManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.filemanager = FileContentsManager(*args, **kwargs) def get_notebook(self, name, path='', content=True, **kwargs): model = self.get(name, path, content=content, **kwargs) # note, since this is calling IPython's .get, the model # path will be the newer full path. reset it since we're # translating back in MetaManager. model['path'] = path return model def get_model_dir(self, name, path='', content=True, **kwargs): model = self.get(name, path, content=content, **kwargs) for nb in model['content']: # reset the model back to pre v4 name/path nb['name'] = nb['path'].rsplit('/')[-1] nb['path'] = path return model def notebook_exists(self, name, path=''): return self.file_exists(name, path) and name.endswith('.ipynb') # this is dumb, quite literally just translating back def is_hidden(self, path): return self.filemanager.is_hidden(path) def get(self, name, path, **kwargs): path = _fullpath(name, path) model = self.filemanager.get(path, **kwargs) return model def save(self, model, name, path=''): path = _fullpath(name, path) return self.filemanager.save(model, path) def update(self, model, name, path=''): path = _fullpath(name, path) return self.filemanager.update(model, path) def delete(self, name, path=''): path = _fullpath(name, path) return self.filemanager.delete(path) def rename(self, old_name, old_path, new_name, new_path): path = _fullpath(name, path) return self.filemanager.rename(old_path, new_path) def file_exists(self, name, path=''): path = _fullpath(name, path) return self.filemanager.file_exists(path) def path_exists(self, name, path=''): path = _fullpath(name, path) return self.filemanager.exists(path) def exists(self, name, path=''): path = _fullpath(name, path) return self.filemanager.exists(path) # checkpoint stuff def get_checkpoint_path(self, checkpoint_id, name, path=''): path = _fullpath(name, path) return self.filemanager.get_checkpoint_path(checkpoint_id, path) def get_checkpoint_model(self, checkpoint_id, name, path=''): path = _fullpath(name, path) return self.filemanager.get_checkpoint_model(checkpoint_id, path) def create_checkpoint(self, name, path=''): path = _fullpath(name, path) return self.filemanager.create_checkpoint(path) def list_checkpoints(self, name, path=''): path = _fullpath(name, path) return self.filemanager.list_checkpoints(path) def restore_checkpoint(self, checkpoint_id, name, path=''): path = _fullpath(name, path) return self.filemanager.restore_checkpoint(checkpoint_id, path) def delete_checkpoint(self, checkpoint_id, name, path=''): path = _fullpath(name, path) return self.filemanager.delete_checkpoint(checkpoint_id, path) def get_kernel_path(self, name, path='', model=None): path = _fullpath(name, path) return self.filemanager.get_kernel_path(path, model=model)
class TestUploadDownload(TestCase): def setUp(self): drop_testing_db_tables() migrate_testing_db() self.td = TemporaryDirectory() self.checkpoints = PostgresCheckpoints( user_id='test', db_url=TEST_DB_URL, ) self.contents = FileContentsManager( root_dir=self.td.name, checkpoints=self.checkpoints, ) self.checkpoints.ensure_user() def tearDown(self): self.td.cleanup() def add_markdown_cell(self, path): # Load and update model = self.contents.get(path=path) model['content'].cells.append( new_markdown_cell('Created by test: ' + path) ) # Save and checkpoint again. self.contents.save(model, path=path) return model def test_download_checkpoints(self): """ Create two checkpoints for two notebooks, then call download_checkpoints. Assert that we get the correct version of both notebooks. """ self.contents.new({'type': 'directory'}, 'subdir') paths = ('a.ipynb', 'subdir/a.ipynb') expected_content = {} for path in paths: # Create and checkpoint. self.contents.new(path=path) self.contents.create_checkpoint(path) model = self.add_markdown_cell(path) self.contents.create_checkpoint(path) # Assert greater because FileContentsManager creates a checkpoint # on creation, but this isn't part of the spec. self.assertGreater(len(self.contents.list_checkpoints(path)), 2) # Store the content to verify correctness after download. expected_content[path] = model['content'] with TemporaryDirectory() as td: download_checkpoints( self.checkpoints.db_url, td, user='******', ) fm = FileContentsManager(root_dir=td) root_entries = sorted(m['path'] for m in fm.get('')['content']) self.assertEqual(root_entries, ['a.ipynb', 'subdir']) subdir_entries = sorted( m['path'] for m in fm.get('subdir')['content'] ) self.assertEqual(subdir_entries, ['subdir/a.ipynb']) for path in paths: content = fm.get(path)['content'] self.assertEqual(expected_content[path], content) def test_checkpoint_all(self): """ Test that checkpoint_all correctly makes a checkpoint for all files. """ paths = populate(self.contents) original_content_minus_trust = { # Remove metadata that we expect to have dropped path: strip_transient(self.contents.get(path)['content']) for path in paths } original_cps = {} for path in paths: # Create a checkpoint, then update the file. original_cps[path] = self.contents.create_checkpoint(path) self.add_markdown_cell(path) # Verify that we still have the old version checkpointed. cp_content = { path: self.checkpoints.get_notebook_checkpoint( cp['id'], path, )['content'] for path, cp in iteritems(original_cps) } self.assertEqual(original_content_minus_trust, cp_content) new_cps = checkpoint_all( self.checkpoints.db_url, self.td.name, self.checkpoints.user_id, ) new_cp_content = { path: self.checkpoints.get_notebook_checkpoint( cp['id'], path, )['content'] for path, cp in iteritems(new_cps) } for path, new_content in iteritems(new_cp_content): old_content = original_content_minus_trust[_norm_unicode(path)] self.assertEqual( new_content['cells'][:-1], old_content['cells'], ) self.assertEqual( new_content['cells'][-1], new_markdown_cell('Created by test: ' + _norm_unicode(path)), )