def test_distributed_n5_datasink():
tmp_path = create_tmp_dir(prefix="test_distributed_n5_datasink")
filesystem = OsFs(tmp_path.as_posix())
outer_path = PurePosixPath("test_distributed_n5_datasink.n5")
inner_path = PurePosixPath("/data")
full_path = PurePosixPath("test_distributed_n5_datasink.n5/data")
attributes = N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=datasource.tile_shape,
c_axiskeys=data.axiskeys, #FIXME: double check this
dataType=datasource.dtype,
compression=RawCompressor())
sink = N5DatasetSink(outer_path=outer_path,
inner_path=inner_path,
filesystem=filesystem,
attributes=attributes)
sink_writer = sink.create()
assert not isinstance(sink_writer, Exception)
sink_writers = [sink_writer] * 4
for idx, piece in enumerate(DataRoi(datasource).default_split()):
sink = sink_writers[idx % len(sink_writers)]
sink.write(piece.retrieve())
n5ds = N5DataSource(filesystem=filesystem, path=full_path)
assert n5ds.retrieve() == data
def testResolveFile(self):
'''Test notebook.resolve_file()'''
dir = Dir(tests.create_tmp_dir('notebook_testResolveFile'))
path = Path('Foo:Bar')
self.notebook.dir = dir
self.notebook.get_store(path).dir = dir
self.notebook.config['Notebook']['document_root'] = './notebook_document_root'
doc_root = self.notebook.get_document_root()
for link, wanted, cleaned in (
('~/test.txt', File('~/test.txt'), '~/test.txt'),
(r'~\test.txt', File('~/test.txt'), '~/test.txt'),
('file:///test.txt', File('file:///test.txt'), None),
('file:/test.txt', File('file:///test.txt'), None),
('file://localhost/test.txt', File('file:///test.txt'), None),
('/test.txt', doc_root.file('test.txt'), '/test.txt'),
('./test.txt', dir.file('Foo/Bar/test.txt'), './test.txt'),
(r'.\test.txt', dir.file('Foo/Bar/test.txt'), './test.txt'),
('../test.txt', dir.file('Foo/test.txt'), '../test.txt'),
(r'..\test.txt', dir.file('Foo/test.txt'), '../test.txt'),
('../Bar/Baz/test.txt', dir.file('Foo/Bar/Baz/test.txt'), './Baz/test.txt'),
(r'C:\foo\bar', File('file:///C:/foo/bar'), None),
(r'Z:\foo\bar', File('file:///Z:/foo/bar'), None),
):
#~ print link, '>>', self.notebook.resolve_file(link, path)
self.assertEqual(
self.notebook.resolve_file(link, path), wanted)
self.assertEqual(
self.notebook.relative_filepath(wanted, path), cleaned)
# check relative path without Path
self.assertEqual(
self.notebook.relative_filepath(doc_root.file('foo.txt')), '/foo.txt')
def test_writing_to_precomputed_chunks():
tmp_path = create_tmp_dir(prefix="test_writing_to_precomputed_chunks")
datasource = ArrayDataSource(data=data, tile_shape=Shape5D(x=10, y=10))
scale = PrecomputedChunksScale.from_datasource(
datasource=datasource,
key=PurePosixPath("my_test_data"),
encoding=RawEncoder())
sink_path = PurePosixPath("mytest.precomputed")
filesystem = OsFs(tmp_path.as_posix())
datasink = PrecomputedChunksScaleSink(
filesystem=filesystem,
info_dir=sink_path,
scale=scale,
dtype=datasource.dtype,
num_channels=datasource.shape.c,
)
creation_result = datasink.create()
if isinstance(creation_result, Exception):
raise creation_result
for tile in datasource.roi.get_datasource_tiles():
datasink.write(tile.retrieve())
precomp_datasource = PrecomputedChunksDataSource(
path=sink_path, filesystem=filesystem, resolution=scale.resolution)
reloaded_data = precomp_datasource.retrieve()
assert reloaded_data == data
def test_httpfs():
httpd = start_test_server(create_tmp_dir(prefix="http_fs_test"), port=8123)
try:
fs = HttpFs(read_url=Url.parse_or_raise("http://localhost:8123/"))
eprint(" --> Opening some file...")
with fs.openbin("/dir1/file1.txt", "r") as f:
assert f.read() == "file1_contents".encode("ascii")
eprint(" --> Verifying that opendir works nested dirs...")
dir1 = fs.opendir("dir2")
assert dir1.openbin("file2.txt",
"r").read() == "file2_contents".encode("ascii")
fs2 = HttpFs(read_url=Url.parse_or_raise("http://localhost:8123/dir1"))
assert fs2.desc("file2.txt") == "http://localhost:8123/dir1/file2.txt"
# check that "/" maps to the base url that was used to create the filesystem
assert fs2.desc("/file2.txt") == "http://localhost:8123/dir1/file2.txt"
#check that .. works even when creating the fs
fs_updir = HttpFs(
read_url=Url.parse_or_raise("http://localhost:8123/dir1/.."))
with fs_updir.openbin("dir1/file1.txt", "r") as f:
assert f.read() == "file1_contents".encode("ascii")
finally:
httpd.shutdown()
def runTest(self):
'''Test GUI daemon interaction'''
dir = Dir(tests.create_tmp_dir('daemon'))
socket = dir.file('test-socket')
pidfile = dir.file('daemon.pid')
assert Daemon == UnixSocketDaemon
Daemon.socket_address = socket.path
Daemon.pidfile = pidfile.path
pid = None
klass = 'tests.daemon.MockDaemonClient'
try:
# Start daemon
proxy = SocketDaemonProxy()
pid = int(pidfile.read())
self.assertTrue(pid > 0)
# Check child object interaction
object1 = proxy.get_object(klass, 'Obj1', 'ID1')
file = dir.file('test1.txt')
self.assertTrue(object1.write(file.path, msg='foobar\n'))
waitfile(file)
self.assertEqual(file.read(), 'ID1: foobar\n')
object2 = proxy.get_object(klass, 'Obj2', 'ID2')
file = dir.file('test2.txt')
self.assertTrue(object2.write(file.path, msg='baz\n'))
waitfile(file)
self.assertEqual(file.read(), 'ID2: baz\n')
self.assertEqual(proxy.list_objects(),
[(klass, 'Obj1'), (klass, 'Obj2')]
)
# See daemon quit after last child
object1.exit()
object2.exit()
waitfile(pidfile, exists=False)
try:
os.kill(pid, signal.SIGUSR1)
except OSError:
pass # no such process
else:
assert False, 'Process still running'
except Exception, error:
# clean up process by pidfile
if pid and pid > 0:
try:
print 'Kill process %i' % pid
os.kill(pid, signal.SIGKILL)
except OSError:
pass
raise error
def testFileHandle(self):
'''Test FileHandle object'''
self.on_close_called = False
tmpdir = tests.create_tmp_dir('fs_testFile')
fh = FileHandle(
tmpdir+'/foo.txt', mode='w', on_close=self.on_close)
fh.write('duss')
fh.close()
self.assertTrue(self.on_close_called)
def export(self):
dir = Dir(create_tmp_dir('export_SourceFiles'))
notebook = Notebook(dir=dir)
for name, text in get_test_data('wiki'):
page = notebook.get_page(Path(name))
page.parse('wiki', text)
notebook.store_page(page)
file = dir.file('Test/foo.txt')
self.assertTrue(file.exists())
check_call(['python', './zim.py', '--export', '--template=Default', dir.path, '--output', self.dir.path])
def setUp(self):
TestStoresMemory.setUp(self)
tmpdir = tests.create_tmp_dir('stores_TestFiles')
self.dir = Dir([tmpdir, 'store-files'])
self.mem = self.store
store = zim.stores.get_store('files')
self.store = store.Store(
path=Path(':'), notebook=Notebook(), dir=self.dir )
for parent, page in walk(self.mem):
if page.hascontent:
mypage = self.store.get_page(page)
mypage.set_parsetree(page.get_parsetree())
self.store.store_page(mypage)
def testFile(self):
'''Test File object'''
tmpdir = tests.create_tmp_dir('fs_testFile')
file = File(tmpdir+'/foo/bar/baz.txt')
assert not file.exists()
file.touch()
self.assertTrue(os.path.isfile(tmpdir+'/foo/bar/baz.txt'))
File(tmpdir+'/anotherfile.txt').touch()
file.cleanup()
self.assertTrue(os.path.isfile(tmpdir+'/anotherfile.txt'))
self.assertTrue(os.path.isdir(tmpdir))
self.assertFalse(os.path.isfile(tmpdir+'/foo/bar/baz.txt'))
self.assertFalse(os.path.isdir(tmpdir+'/foo'))
# without codecs
file = File(tmpdir+'/foo.txt')
fh = file.open('w', encoding=None)
self.assertTrue(isinstance(fh, FileHandle))
fh.write('tja')
fh.close()
# with codecs
file = File(tmpdir+'/bar.txt')
file.writelines(['c\n', 'd\n'])
self.assertEqual(file.readlines(), ['c\n', 'd\n'])
# with error
try:
fh = file.open('w')
fh.write('foo')
raise IOError
except IOError:
del fh
self.assertEqual(file.readlines(), ['c\n', 'd\n'])
self.assertTrue(os.path.isfile(file.path+'.zim.new~'))
# test read-only
path = tmpdir+'/read-only-file.txt'
open(path, 'w').write('test 123')
os.chmod(path, 0444)
file = File(path)
self.assertRaises(OverWriteError, file.write, 'Overwritten!')
os.chmod(path, 0644) # make it removable again
# with windows line-ends
file = open(tmpdir+'/newlines.txt', 'wb')
# binary mode means no automatic newline conversions
file.write('Some lines\r\nWith win32 newlines\r\n')
file = File(tmpdir+'/newlines.txt')
self.assertEqual(file.read(), 'Some lines\nWith win32 newlines\n')
def get_files_notebook(key):
# We fill the notebook using the store interface, as this test comes before
# the notebook test, but after the store test.
dir = Dir(tests.create_tmp_dir('index_'+key))
notebook = Notebook(dir=dir)
store = notebook.get_store(':')
manifest = []
for name, text in tests.get_test_data('wiki'):
manifest.append(name)
page = store.get_page(Path(name))
page.parse('wiki', text)
store.store_page(page)
notebook.testdata_manifest = tests.expand_manifest(manifest)
return notebook
def testDir(self):
'''Test Dir object'''
tmpdir = tests.create_tmp_dir('fs_testDir')
dir = Dir(tmpdir+'/foo/bar')
assert not dir.exists()
# TODO test list()
# TODO - test file(FILE), + test exception
# TODO - test subdir(DIR), + test excepion
file1 = dir.file('unique.txt')
file1.touch()
file2 = dir.new_file('unique.txt')
file2.touch()
file3 = dir.new_file('unique.txt')
self.assertEqual(file1.basename, 'unique.txt')
self.assertEqual(file2.basename, 'unique001.txt')
self.assertEqual(file3.basename, 'unique002.txt')
def test_n5_datasink():
tmp_path = create_tmp_dir(prefix="test_n5_datasink")
sink = N5DatasetSink(
filesystem=OsFs(tmp_path.as_posix()),
outer_path=PurePosixPath("test_n5_datasink.n5"),
inner_path=PurePosixPath("/data"),
attributes=N5DatasetAttributes(
dimensions=datasource.shape,
blockSize=Shape5D(x=10, y=10),
c_axiskeys=data.axiskeys, #FIXME: double check this
dataType=datasource.dtype,
compression=RawCompressor(),
location=Point5D.zero(x=7, y=13)))
sink_writer = sink.create()
assert not isinstance(sink_writer, Exception)
for tile in DataRoi(datasource).split(sink.tile_shape):
sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13)))
n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path)
saved_data = n5ds.retrieve()
assert saved_data.location == Point5D.zero(x=7, y=13)
assert saved_data == data
def test_writing_to_offset_precomputed_chunks():
tmp_path = create_tmp_dir(
prefix="test_writing_to_offset_precomputed_chunks")
data_at_1000_1000 = data.translated(
Point5D(x=1000, y=1000) - data.location)
datasource = ArrayDataSource(data=data_at_1000_1000,
tile_shape=Shape5D(x=10, y=10))
scale = PrecomputedChunksScale.from_datasource(
datasource=datasource,
key=PurePosixPath("my_test_data"),
encoding=RawEncoder())
sink_path = PurePosixPath("mytest.precomputed")
filesystem = OsFs(tmp_path.as_posix())
print(f"\n\n will write to '{filesystem.geturl(sink_path.as_posix())}' ")
datasink = PrecomputedChunksScaleSink(
filesystem=filesystem,
info_dir=sink_path,
scale=scale,
num_channels=datasource.shape.c,
dtype=datasource.dtype,
)
creation_result = datasink.create()
if isinstance(creation_result, Exception):
raise creation_result
for tile in datasource.roi.get_datasource_tiles():
datasink.write(tile.retrieve())
precomp_datasource = PrecomputedChunksDataSource(
path=sink_path, filesystem=filesystem, resolution=scale.resolution)
reloaded_data = precomp_datasource.retrieve(
interval=data_at_1000_1000.interval)
assert (reloaded_data.raw("xyz") == data.raw("xyz")).all()
def setUp(self):
self.dir = Dir(create_tmp_dir('export_ExportedFiles'))
def runTest(self):
root = Dir(tests.create_tmp_dir('notebook_TestGetNotebook'))
# Start empty - see this is no issue
list = get_notebook_list()
self.assertTrue(isinstance(list, NotebookList))
self.assertFalse(list)
nb, page = resolve_notebook('foo')
self.assertTrue(nb is None)
nb = resolve_default_notebook()
self.assertTrue(nb is None)
# Non-existing dir
dir = root.subdir('/notebook')
nb, page = resolve_notebook(dir.path)
self.assertEqual(nb, dir)
# Now create it
init_notebook(dir, name='foo')
file = dir.file('notebook.zim')
nb, page = resolve_notebook(dir.path)
self.assertEqual(nb, dir)
nb, page = resolve_notebook(file.uri)
self.assertEqual(nb, dir)
file = dir.file('foo/bar/baz.txt')
file.touch()
nb, page = resolve_notebook(file.path)
self.assertEqual(nb, dir)
self.assertEqual(page, Path('foo:bar:baz'))
# And put it in the list and resolve it by name
list.append(dir.uri)
list.write()
list = get_notebook_list()
self.assertTrue(len(list) == 1)
nb, page = resolve_notebook('foo')
self.assertEqual(nb, dir)
# Single notebook is automatically the default
nb = resolve_default_notebook()
self.assertEqual(nb, dir)
# But not anymore after adding second notebook
list.append('file:///foo/bar')
list.write()
list = get_notebook_list()
self.assertTrue(len(list) == 2)
self.assertEqual(list[:], [dir.uri, 'file:///foo/bar'])
nb, page = resolve_notebook('foo')
self.assertEqual(nb, dir)
self.assertTrue(isinstance(get_notebook(nb), Notebook))
nb = resolve_default_notebook()
self.assertTrue(nb is None)
list.default = 'file:///default/foo'
list.write()
list = get_notebook_list()
nb = resolve_default_notebook()
self.assertEqual(nb, Dir('/default/foo'))
self.assertEqual(get_notebook(nb), None)
# Check interwiki parsing
self.assertEqual(interwiki_link('wp?Foo'), 'http://en.wikipedia.org/wiki/Foo')
self.assertEqual(interwiki_link('foo?Foo'), dir.uri+'?Foo')
nb, page = resolve_notebook(dir.uri+'?Foo')
self.assertEqual(nb, dir)
self.assertEqual(page, 'Foo')
# Check backward compatibility
file = File('tests/data/notebook-list-old-format.list')
wanted = [Dir('~/Notes').uri, Dir('/home/user/code/zim.debug').uri, Dir('/home/user/Foo Bar').uri]
list = NotebookList(file)
self.assertEqual(list[:], wanted)
self.assertEqual(list.default, Dir('/home/user/code/zim.debug').uri)
def setUp(self):
self.path = tests.create_tmp_dir('fs_testOverwrite')+'/file.txt'
