class PickleShareDBTestCase(TestCase):
def setUp(self):
self.tempdir = TemporaryDirectory()
def tearDown(self):
self.tempdir.cleanup()
def test_picklesharedb(self):
db = PickleShareDB(self.tempdir.name)
db.clear()
print("Should be empty:", db.items())
db['hello'] = 15
db['aku ankka'] = [1, 2, 313]
db['paths/nest/ok/keyname'] = [1, (5, 46)]
db.hset('hash', 'aku', 12)
db.hset('hash', 'ankka', 313)
self.assertEqual(db.hget('hash', 'aku'), 12)
self.assertEqual(db.hget('hash', 'ankka'), 313)
print("all hashed", db.hdict('hash'))
print(db.keys())
print(db.keys('paths/nest/ok/k*'))
print(dict(db)) # snapsot of whole db
db.uncache() # frees memory, causes re-reads later
# shorthand for accessing deeply nested files
lnk = db.getlink('myobjects/test')
lnk.foo = 2
lnk.bar = lnk.foo + 5
self.assertEqual(lnk.bar, 7)
@skip("Too slow for regular running.")
def test_stress(self):
db = PickleShareDB('~/fsdbtest')
import time
import sys
for i in range(1000):
for j in range(1000):
if i % 15 == 0 and i < 200:
if str(j) in db:
del db[str(j)]
continue
if j % 33 == 0:
time.sleep(0.02)
db[str(j)] = db.get(str(j), []) + \
[(i, j, "proc %d" % os.getpid())]
db.hset('hash', j, db.hget('hash', j, 15) + 1)
print(i, end=' ')
sys.stdout.flush()
if i % 10 == 0:
db.uncache()
def setUp(self):
self.package = package = "tmp{0}".format(repr(random.random())[2:])
"""Temporary valid python package name."""
self.value = int(random.random() * 10000)
self.tempdir = TemporaryDirectory()
self.__orig_cwd = os.getcwdu()
sys.path.insert(0, self.tempdir.name)
self.writefile(os.path.join(package, "__init__.py"), "")
self.writefile(
os.path.join(package, "sub.py"),
"""
x = {0!r}
""".format(
self.value
),
)
self.writefile(
os.path.join(package, "relative.py"),
"""
from .sub import x
""",
)
self.writefile(
os.path.join(package, "absolute.py"),
"""
from {0}.sub import x
""".format(
package
),
)
def setUp(self):
self._temp_dir = TemporaryDirectory()
self.td = self._temp_dir.name
self.contents_manager = FileContentsManager(
root_dir=self.td,
log=logging.getLogger()
)
def setUp(self):
self._temp_dir = TemporaryDirectory()
self.td = self._temp_dir.name
self.notebook_manager = FileNotebookManager(
notebook_dir=self.td,
log=logging.getLogger()
)
def setUp(self):
self._temp_dir = TemporaryDirectory()
self.td = self._temp_dir.name
self._file_manager = FileContentsManager(root_dir=self.td)
self.contents_manager = HybridContentsManager(
managers={'': self._file_manager}
)
def install_my_kernel_spec(user=True, prefix=None):
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
# copy kernel.js to the right place
import shutil
dir_path = os.path.dirname(os.path.realpath(__file__))
jsPath = os.path.join(dir_path, os.path.join('js', 'kernel.js'))
shutil.copy2(jsPath, td)
# copy gf.css to the right place
css = os.path.join(dir_path, os.path.join('css', 'gf.css'))
shutil.copy2(css, td)
print('Installing Jupyter kernel spec GF')
KernelSpecManager().install_kernel_spec(td, 'GF', user=user, prefix=prefix)
def run(self):
# Regular installation
install.run(self)
# Now write the kernelspec
from IPython.kernel.kernelspec import install_kernel_spec
from IPython.utils.tempdir import TemporaryDirectory
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
# TODO: Copy resources once they're specified
log.info('Installing IPython kernel spec')
install_kernel_spec(td,
'mysql_kernel',
user=self.user,
replace=True)
def install_my_kernel_spec(user=True):
ldflags, ld_lib_path = find_boost_ldflags()
env = {
"CXX": find_compiler(),
"CPPFLAGS": quote_list(find_boost_cppflags()),
"LDFLAGS": quote_list(ldflags),
"RUNTIME_LIB_PATH": ''
}
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(make_kernel_json(env), f, sort_keys=True)
# TODO: Copy resources once they're specified
print('Installing IPython kernel spec')
install_kernel_spec(td, 'fakerepl', user=user, replace=True)
for name in sorted(env):
print("%s=%s" % (name, env[name]))
def test_run__name__():
with TemporaryDirectory() as td:
path = pjoin(td, "foo.py")
with open(path, "w", encoding="utf-8") as f:
f.write("q = __name__")
_ip.user_ns.pop("q", None)
_ip.magic("run {}".format(path))
assert _ip.user_ns.pop("q") == "__main__"
_ip.magic("run -n {}".format(path))
assert _ip.user_ns.pop("q") == "foo"
try:
_ip.magic("run -i -n {}".format(path))
assert _ip.user_ns.pop("q") == "foo"
finally:
_ip.magic('reset -f')
def install(self):
with TemporaryDirectory() as td:
allow_all_rx = 0o755
os.chmod(td, allow_all_rx)
for kernel_name in self._kernel_spec_manager.find_kernel_specs():
if not self._should_containerize_kernel(kernel_name):
continue
kernel_display_name = 'Containerized ' + kernel_name
with open(os.path.join(td, 'kernel.json'), 'w') as f:
kernel_json = self._get_kernel_json(
kernel_name, kernel_display_name)
logger.debug("Installing kernel with definition: " +
str(kernel_json))
json.dump(kernel_json, f, sort_keys=True)
safe_name = kernel_display_name.replace(' ', '_')
self._kernel_spec_manager.install_kernel_spec(
td, safe_name, prefix=sys.prefix, replace=True)
logger.info('Installed kernel: ' + safe_name)
def test_abspath_file_completions():
ip = get_ipython()
with TemporaryDirectory() as tmpdir:
prefix = os.path.join(tmpdir, 'foo')
suffixes = ['1', '2']
names = [prefix + s for s in suffixes]
for n in names:
open(n, 'w').close()
# Check simple completion
c = ip.complete(prefix)[1]
nt.assert_equal(c, names)
# Now check with a function call
cmd = 'a = f("%s' % prefix
c = ip.complete(prefix, cmd)[1]
comp = [prefix + s for s in suffixes]
nt.assert_equal(c, comp)
def test_profile_create_ipython_dir():
"""ipython profile create respects --ipython-dir"""
with TemporaryDirectory() as td:
getoutput(
[
sys.executable,
"-m",
"IPython",
"profile",
"create",
"foo",
"--ipython-dir=%s" % td,
]
)
profile_dir = os.path.join(td, "profile_foo")
assert os.path.exists(profile_dir)
ipython_config = os.path.join(profile_dir, "ipython_config.py")
assert os.path.exists(ipython_config)
def check_handler_with_file(self, inpath, handler):
shell = self.shell
configname = '{0}_image_handler'.format(handler)
funcname = 'handle_image_{0}'.format(handler)
assert hasattr(shell, configname)
assert hasattr(shell, funcname)
with TemporaryDirectory() as tmpdir:
outpath = os.path.join(tmpdir, 'data')
cmd = [sys.executable, SCRIPT_PATH, inpath, outpath]
setattr(shell, configname, cmd)
getattr(shell, funcname)(self.data, self.mime)
# cmd is called and file is closed. So it's safe to open now.
with open(outpath, 'rb') as file:
transferred = file.read()
self.assertEqual(transferred, self.raw)
def test_good_symlink(self):
with TemporaryDirectory() as td:
cm = FileContentsManager(root_dir=td)
parent = 'test good symlink'
name = 'good symlink'
path = '{0}/{1}'.format(parent, name)
_make_dir(cm, parent)
file_model = cm.new(path=parent + '/zfoo.txt')
# create a good symlink
self.symlink(cm, file_model['path'], path)
symlink_model = cm.get(path, content=False)
dir_model = cm.get(parent)
self.assertEqual(
sorted(dir_model['content'], key=lambda x: x['name']),
[symlink_model, file_model],
)
def test_run_tb():
"""Test traceback offset in %run"""
with TemporaryDirectory() as td:
path = pjoin(td, 'foo.py')
with open(path, 'w') as f:
f.write('\n'.join([
"def foo():",
" return bar()",
"def bar():",
" raise RuntimeError('hello!')",
"foo()",
]))
with capture_output() as io:
_ip.magic('run {}'.format(path))
out = io.stdout
nt.assert_not_in("execfile", out)
nt.assert_in("RuntimeError", out)
nt.assert_equal(out.count("---->"), 3)
def test_run__name__():
with TemporaryDirectory() as td:
path = pjoin(td, 'foo.py')
with open(path, 'w') as f:
f.write("q = __name__")
_ip.user_ns.pop('q', None)
_ip.magic('run {}'.format(path))
nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
_ip.magic('run -n {}'.format(path))
nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
try:
_ip.magic('run -i -n {}'.format(path))
nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
finally:
_ip.magic('reset -f')
def install_my_kernel_spec(user=True, prefix=None):
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
# TODO: Copy resources once they're specified
print('Installing IPython kernel spec')
KernelSpecManager().install_kernel_spec(td,
'bash',
user=user,
replace=True,
prefix=prefix)
# Also install as shell for consistency, since it's the name DC images use
KernelSpecManager().install_kernel_spec(td,
'shell',
user=user,
replace=True,
prefix=prefix)
def install_my_kernel_spec(user=True):
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
path_of_file = dirname(
abspath(__file__)) + "/jupyter_kernel_polymake/resources/"
filenames = [
"three.js", "Detector.js", "controls/TrackballControls.js",
"renderers/SVGRenderer.js", "renderers/CanvasRenderer.js",
"renderers/Projector.js", "menu.svg", "close.svg"
]
for i in filenames:
file_copy(path_of_file + i, td)
file_copy(path_of_file + "kernel.js", td)
file_copy(path_of_file + "logo-32x32.png", td)
file_copy(path_of_file + "logo-64x64.png", td)
print('Installing jupyter kernel spec for polymake')
install_kernel_spec(td, 'polymake', user=user, replace=True)
def start(self):
kernel_spec = self.kernel_class.kernel_json
with TemporaryDirectory() as td:
dirname = os.path.join(td, kernel_spec['name'])
os.mkdir(dirname)
with open(os.path.join(dirname, 'kernel.json'), 'w') as f:
json.dump(kernel_spec, f, sort_keys=True)
filenames = ['logo-64x64.png', 'logo-32x32.png']
for filename in filenames:
data = pkgutil.get_data(__name__.split('.')[0],
'images/' + filename)
with open(os.path.join(dirname, filename), 'wb') as f:
f.write(data)
try:
subprocess.check_call(
[sys.executable, '-m', 'jupyter',
'kernelspec', 'install'] + self.argv + [dirname])
except CalledProcessError as exc:
sys.exit(exc.returncode)
def test_save_with_no_args():
ip = get_ipython()
ip.history_manager.reset() # Clear any existing history.
cmds = ["a=1", "def b():\n return a**2", "print(a, b())", "%save"]
for i, cmd in enumerate(cmds, start=1):
ip.history_manager.store_inputs(i, cmd)
with TemporaryDirectory() as tmpdir:
path = os.path.join(tmpdir, "testsave.py")
ip.run_line_magic("save", path)
content = Path(path).read_text()
expected_content = dedent("""\
# coding: utf-8
a=1
def b():
return a**2
print(a, b())
""")
assert content == expected_content
def run(self):
install.run(self)
from IPython.kernel.kernelspec import install_kernel_spec
from IPython.utils.tempdir import TemporaryDirectory
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
log.info('Installing kernel spec')
try:
install_kernel_spec(td,
'metakernel_bash',
user=self.user,
replace=True)
except:
install_kernel_spec(td,
'metakernel_bash',
user=not self.user,
replace=True)
def setup_class(cls):
cls.td = TemporaryDirectory()
cls.config = Config()
cls.config.NotebookApp.contents_manager_class = HybridContentsManager
cls.config.HybridContentsManager.manager_classes = {
'': PostgresContentsManager,
cls.files_prefix: FileContentsManager,
}
cls.config.HybridContentsManager.manager_kwargs = {
'': {
'user_id': 'test',
'db_url': TEST_DB_URL
},
cls.files_prefix: {
'root_dir': cls.td.name
},
}
super(HybridContentsPGRootAPITest, cls).setup_class()
def run(self):
install.run(self)
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
log.info('Installing kernel spec')
install_kernel_resources(td, files=['logo-64x64.png'])
kernel_name = kernel_json['name']
try:
install_kernel_spec(td,
kernel_name,
user=self.user,
replace=True)
except:
install_kernel_spec(td,
kernel_name,
user=not self.user,
replace=True)
def test_save():
"""Test %save."""
ip = get_ipython()
ip.history_manager.reset() # Clear any existing history.
cmds = [u"a=1", u"def b():\n return a**2", u"print(a, b())"]
for i, cmd in enumerate(cmds, start=1):
ip.history_manager.store_inputs(i, cmd)
with TemporaryDirectory() as tmpdir:
file = os.path.join(tmpdir, "testsave.py")
ip.run_line_magic("save", "%s 1-10" % file)
with open(file) as f:
content = f.read()
nt.assert_equal(content.count(cmds[0]), 1)
nt.assert_in('coding: utf-8', content)
ip.run_line_magic("save", "-a %s 1-10" % file)
with open(file) as f:
content = f.read()
nt.assert_equal(content.count(cmds[0]), 2)
nt.assert_in('coding: utf-8', content)
def install_my_kernel_spec(user=True, prefix=None):
user = '******' in sys.argv or not _is_root()
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
kernel_name = kernel_json['name']
try:
install_kernel_spec(td,
kernel_name,
user=user,
replace=True,
prefix=prefix)
except:
install_kernel_spec(td,
kernel_name,
user=not user,
replace=True,
prefix=prefix)
def _run_cmd(self):
with TemporaryDirectory() as IPYTHONDIR:
env = os.environ.copy()
env['IPYTHONDIR'] = IPYTHONDIR
# print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg
output = subprocess.PIPE if self.buffer_output else None
subp = subprocess.Popen(self.call_args,
stdout=output,
stderr=output,
env=env)
self.processes.append(subp)
# If this fails, the process will be left in self.processes and
# cleaned up later, but if the wait call succeeds, then we can
# clear the stored process.
retcode = subp.wait()
self.processes.pop()
self.stdout = subp.stdout
self.stderr = subp.stderr
return retcode
def install_my_kernel_spec(user=True, prefix=None):
if "python2" in sys.executable:
print("I think this needs python3")
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
# TODO: Copy resources once they're specified
print('Installing IPython kernel spec of micropython')
k = KernelSpecManager()
k.install_kernel_spec(td,
'Micropython',
user=user,
replace=True,
prefix=prefix)
h = k.get_kernel_spec("micropython")
print("...into", h.resource_dir)
def __init__(self):
self.td = TemporaryDirectory()
self.to_clean_up = ['.dot', '.png', '.gfo']
self.pipe = os.pipe()
self.gf_shell = Popen((find_executable('gf'), '--run'),
stdin=PIPE,
stderr=self.pipe[1],
stdout=self.pipe[1],
text=True)
self.commandcounter = 0
self.infile = os.fdopen(self.pipe[0])
# catch any initial messages
sep = self.write_separator()
self.gf_shell.stdin.flush()
self.initialOutput = self.get_output(sep)
self.out_count = 0
def test_download_checkpoints(self):
"""
Create two checkpoints for two notebooks, then call
download_checkpoints.
Assert that we get the correct version of both notebooks.
"""
self.contents.new({'type': 'directory'}, 'subdir')
paths = ('a.ipynb', 'subdir/a.ipynb')
expected_content = {}
for path in paths:
# Create and checkpoint.
self.contents.new(path=path)
self.contents.create_checkpoint(path)
model = self.add_markdown_cell(path)
self.contents.create_checkpoint(path)
# Assert greater because FileContentsManager creates a checkpoint
# on creation, but this isn't part of the spec.
self.assertGreater(len(self.contents.list_checkpoints(path)), 2)
# Store the content to verify correctness after download.
expected_content[path] = model['content']
with TemporaryDirectory() as td:
download_checkpoints(
self.checkpoints.db_url,
td,
user='******',
)
fm = FileContentsManager(root_dir=td)
root_entries = sorted(m['path'] for m in fm.get('')['content'])
self.assertEqual(root_entries, ['a.ipynb', 'subdir'])
subdir_entries = sorted(m['path']
for m in fm.get('subdir')['content'])
self.assertEqual(subdir_entries, ['subdir/a.ipynb'])
for path in paths:
content = fm.get(path)['content']
self.assertEqual(expected_content[path], content)
def start(self):
if self.user and self.prefix:
self.exit("Can't specify both user and prefix. Please choose one or\
the other.")
self.log.info('Installing SPARQL kernel')
with TemporaryDirectory() as td:
os.chmod(td, 0o755) # Starts off as 700, not user readable
# Add kernel spec
if len(self.logdir):
kernel_json['env'] = { 'LOGDIR_DEFAULT' : self.logdir }
with open(os.path.join(td, 'kernel.json'), 'w') as f:
json.dump(kernel_json, f, sort_keys=True)
# Add resources
install_kernel_resources(td, resource=PKGNAME)
# Install JSON kernel specification + resources
self.log.info('Installing kernel spec')
self.sourcedir = td
install_dir = self.kernel_spec_manager.install_kernel_spec( td,
kernel_name=self.kernel_name,
user=self.user,
prefix=self.prefix,
replace=self.replace,
)
self.log.info( "Installed into %s", install_dir )
#install_kernel( self.kernel_spec_manager )
#self.create_kernel_json( install_dir )
# Install the custom css
self.log.info('Installing CSS')
if self.user:
# Use the ~/.jupyter/custom dir
import jupyter_core
destd = os.path.join( jupyter_core.paths.jupyter_config_dir(),'custom')
else:
# Use the system custom dir
import notebook
destd = os.path.join( notebook.DEFAULT_STATIC_FILES_PATH, 'custom' )
self.log.info('Installing CSS into %s', destd)
install_custom_css( destd, PKGNAME )
def test_get_os_path(self):
# full filesystem path should be returned with correct operating system
# separators.
with TemporaryDirectory() as td:
root = td
fm = FileContentsManager(root_dir=root)
path = fm._get_os_path('/path/to/notebook/test.ipynb')
rel_path_list = '/path/to/notebook/test.ipynb'.split('/')
fs_path = os.path.join(fm.root_dir, *rel_path_list)
self.assertEqual(path, fs_path)
fm = FileContentsManager(root_dir=root)
path = fm._get_os_path('test.ipynb')
fs_path = os.path.join(fm.root_dir, 'test.ipynb')
self.assertEqual(path, fs_path)
fm = FileContentsManager(root_dir=root)
path = fm._get_os_path('////test.ipynb')
fs_path = os.path.join(fm.root_dir, 'test.ipynb')
self.assertEqual(path, fs_path)
def test_deepreload():
"Test that dreload does deep reloads and skips excluded modules."
with TemporaryDirectory() as tmpdir:
with prepended_to_syspath(tmpdir):
with open(os.path.join(tmpdir, "A.py"), "w") as f:
f.write("class Object(object):\n pass\n")
with open(os.path.join(tmpdir, "B.py"), "w") as f:
f.write("import A\n")
import A
import B
# Test that A is not reloaded.
obj = A.Object()
dreload(B, exclude=["A"])
nt.assert_true(isinstance(obj, A.Object))
# Test that A is reloaded.
obj = A.Object()
dreload(B)
nt.assert_false(isinstance(obj, A.Object))
def test_create_notebook(self):
with TemporaryDirectory() as td:
# Test in root directory
nm = FileNotebookManager(notebook_dir=td)
model = nm.create_notebook()
assert isinstance(model, dict)
self.assertIn('name', model)
self.assertIn('path', model)
self.assertEqual(model['name'], 'Untitled0.ipynb')
self.assertEqual(model['path'], '')
# Test in sub-directory
sub_dir = '/foo/'
self.make_dir(nm.notebook_dir, 'foo')
model = nm.create_notebook(None, sub_dir)
assert isinstance(model, dict)
self.assertIn('name', model)
self.assertIn('path', model)
self.assertEqual(model['name'], 'Untitled0.ipynb')
self.assertEqual(model['path'], sub_dir.strip('/'))
def test_403(self):
if hasattr(os, 'getuid'):
if os.getuid() == 0:
raise SkipTest("Can't test permissions as root")
if sys.platform.startswith('win'):
raise SkipTest("Can't test permissions on Windows")
with TemporaryDirectory() as td:
cm = FileContentsManager(root_dir=td)
model = cm.new_untitled(type='file')
os_path = cm._get_os_path(model['path'])
os.chmod(os_path, 0o400)
try:
with cm.open(os_path, 'w') as f:
f.write(u"don't care")
except HTTPError as e:
self.assertEqual(e.status_code, 403)
else:
self.fail("Should have raised HTTPError(403)")
def test_get_os_path(self):
# full filesystem path should be returned with correct operating system
# separators.
with TemporaryDirectory() as td:
nbdir = os.path.join(td, 'notebooks')
fm = FileNotebookManager(notebook_dir=nbdir)
path = fm.get_os_path('test.ipynb', '/path/to/notebook/')
rel_path_list = '/path/to/notebook/test.ipynb'.split('/')
fs_path = os.path.join(fm.notebook_dir, *rel_path_list)
self.assertEqual(path, fs_path)
fm = FileNotebookManager(notebook_dir=nbdir)
path = fm.get_os_path('test.ipynb')
fs_path = os.path.join(fm.notebook_dir, 'test.ipynb')
self.assertEqual(path, fs_path)
fm = FileNotebookManager(notebook_dir=nbdir)
path = fm.get_os_path('test.ipynb', '////')
fs_path = os.path.join(fm.notebook_dir, 'test.ipynb')
self.assertEqual(path, fs_path)
