def test_distributed_n5_datasink(): tmp_path = create_tmp_dir(prefix="test_distributed_n5_datasink") filesystem = OsFs(tmp_path.as_posix()) outer_path = PurePosixPath("test_distributed_n5_datasink.n5") inner_path = PurePosixPath("/data") full_path = PurePosixPath("test_distributed_n5_datasink.n5/data") attributes = N5DatasetAttributes( dimensions=datasource.shape, blockSize=datasource.tile_shape, c_axiskeys=data.axiskeys, #FIXME: double check this dataType=datasource.dtype, compression=RawCompressor()) sink = N5DatasetSink(outer_path=outer_path, inner_path=inner_path, filesystem=filesystem, attributes=attributes) sink_writer = sink.create() assert not isinstance(sink_writer, Exception) sink_writers = [sink_writer] * 4 for idx, piece in enumerate(DataRoi(datasource).default_split()): sink = sink_writers[idx % len(sink_writers)] sink.write(piece.retrieve()) n5ds = N5DataSource(filesystem=filesystem, path=full_path) assert n5ds.retrieve() == data
def testResolveFile(self): '''Test notebook.resolve_file()''' dir = Dir(tests.create_tmp_dir('notebook_testResolveFile')) path = Path('Foo:Bar') self.notebook.dir = dir self.notebook.get_store(path).dir = dir self.notebook.config['Notebook']['document_root'] = './notebook_document_root' doc_root = self.notebook.get_document_root() for link, wanted, cleaned in ( ('~/test.txt', File('~/test.txt'), '~/test.txt'), (r'~\test.txt', File('~/test.txt'), '~/test.txt'), ('file:///test.txt', File('file:///test.txt'), None), ('file:/test.txt', File('file:///test.txt'), None), ('file://localhost/test.txt', File('file:///test.txt'), None), ('/test.txt', doc_root.file('test.txt'), '/test.txt'), ('./test.txt', dir.file('Foo/Bar/test.txt'), './test.txt'), (r'.\test.txt', dir.file('Foo/Bar/test.txt'), './test.txt'), ('../test.txt', dir.file('Foo/test.txt'), '../test.txt'), (r'..\test.txt', dir.file('Foo/test.txt'), '../test.txt'), ('../Bar/Baz/test.txt', dir.file('Foo/Bar/Baz/test.txt'), './Baz/test.txt'), (r'C:\foo\bar', File('file:///C:/foo/bar'), None), (r'Z:\foo\bar', File('file:///Z:/foo/bar'), None), ): #~ print link, '>>', self.notebook.resolve_file(link, path) self.assertEqual( self.notebook.resolve_file(link, path), wanted) self.assertEqual( self.notebook.relative_filepath(wanted, path), cleaned) # check relative path without Path self.assertEqual( self.notebook.relative_filepath(doc_root.file('foo.txt')), '/foo.txt')
def test_writing_to_precomputed_chunks(): tmp_path = create_tmp_dir(prefix="test_writing_to_precomputed_chunks") datasource = ArrayDataSource(data=data, tile_shape=Shape5D(x=10, y=10)) scale = PrecomputedChunksScale.from_datasource( datasource=datasource, key=PurePosixPath("my_test_data"), encoding=RawEncoder()) sink_path = PurePosixPath("mytest.precomputed") filesystem = OsFs(tmp_path.as_posix()) datasink = PrecomputedChunksScaleSink( filesystem=filesystem, info_dir=sink_path, scale=scale, dtype=datasource.dtype, num_channels=datasource.shape.c, ) creation_result = datasink.create() if isinstance(creation_result, Exception): raise creation_result for tile in datasource.roi.get_datasource_tiles(): datasink.write(tile.retrieve()) precomp_datasource = PrecomputedChunksDataSource( path=sink_path, filesystem=filesystem, resolution=scale.resolution) reloaded_data = precomp_datasource.retrieve() assert reloaded_data == data
def test_httpfs(): httpd = start_test_server(create_tmp_dir(prefix="http_fs_test"), port=8123) try: fs = HttpFs(read_url=Url.parse_or_raise("http://localhost:8123/")) eprint(" --> Opening some file...") with fs.openbin("/dir1/file1.txt", "r") as f: assert f.read() == "file1_contents".encode("ascii") eprint(" --> Verifying that opendir works nested dirs...") dir1 = fs.opendir("dir2") assert dir1.openbin("file2.txt", "r").read() == "file2_contents".encode("ascii") fs2 = HttpFs(read_url=Url.parse_or_raise("http://localhost:8123/dir1")) assert fs2.desc("file2.txt") == "http://localhost:8123/dir1/file2.txt" # check that "/" maps to the base url that was used to create the filesystem assert fs2.desc("/file2.txt") == "http://localhost:8123/dir1/file2.txt" #check that .. works even when creating the fs fs_updir = HttpFs( read_url=Url.parse_or_raise("http://localhost:8123/dir1/..")) with fs_updir.openbin("dir1/file1.txt", "r") as f: assert f.read() == "file1_contents".encode("ascii") finally: httpd.shutdown()
def runTest(self): '''Test GUI daemon interaction''' dir = Dir(tests.create_tmp_dir('daemon')) socket = dir.file('test-socket') pidfile = dir.file('daemon.pid') assert Daemon == UnixSocketDaemon Daemon.socket_address = socket.path Daemon.pidfile = pidfile.path pid = None klass = 'tests.daemon.MockDaemonClient' try: # Start daemon proxy = SocketDaemonProxy() pid = int(pidfile.read()) self.assertTrue(pid > 0) # Check child object interaction object1 = proxy.get_object(klass, 'Obj1', 'ID1') file = dir.file('test1.txt') self.assertTrue(object1.write(file.path, msg='foobar\n')) waitfile(file) self.assertEqual(file.read(), 'ID1: foobar\n') object2 = proxy.get_object(klass, 'Obj2', 'ID2') file = dir.file('test2.txt') self.assertTrue(object2.write(file.path, msg='baz\n')) waitfile(file) self.assertEqual(file.read(), 'ID2: baz\n') self.assertEqual(proxy.list_objects(), [(klass, 'Obj1'), (klass, 'Obj2')] ) # See daemon quit after last child object1.exit() object2.exit() waitfile(pidfile, exists=False) try: os.kill(pid, signal.SIGUSR1) except OSError: pass # no such process else: assert False, 'Process still running' except Exception, error: # clean up process by pidfile if pid and pid > 0: try: print 'Kill process %i' % pid os.kill(pid, signal.SIGKILL) except OSError: pass raise error
def testFileHandle(self): '''Test FileHandle object''' self.on_close_called = False tmpdir = tests.create_tmp_dir('fs_testFile') fh = FileHandle( tmpdir+'/foo.txt', mode='w', on_close=self.on_close) fh.write('duss') fh.close() self.assertTrue(self.on_close_called)
def export(self): dir = Dir(create_tmp_dir('export_SourceFiles')) notebook = Notebook(dir=dir) for name, text in get_test_data('wiki'): page = notebook.get_page(Path(name)) page.parse('wiki', text) notebook.store_page(page) file = dir.file('Test/foo.txt') self.assertTrue(file.exists()) check_call(['python', './zim.py', '--export', '--template=Default', dir.path, '--output', self.dir.path])
def setUp(self): TestStoresMemory.setUp(self) tmpdir = tests.create_tmp_dir('stores_TestFiles') self.dir = Dir([tmpdir, 'store-files']) self.mem = self.store store = zim.stores.get_store('files') self.store = store.Store( path=Path(':'), notebook=Notebook(), dir=self.dir ) for parent, page in walk(self.mem): if page.hascontent: mypage = self.store.get_page(page) mypage.set_parsetree(page.get_parsetree()) self.store.store_page(mypage)
def testFile(self): '''Test File object''' tmpdir = tests.create_tmp_dir('fs_testFile') file = File(tmpdir+'/foo/bar/baz.txt') assert not file.exists() file.touch() self.assertTrue(os.path.isfile(tmpdir+'/foo/bar/baz.txt')) File(tmpdir+'/anotherfile.txt').touch() file.cleanup() self.assertTrue(os.path.isfile(tmpdir+'/anotherfile.txt')) self.assertTrue(os.path.isdir(tmpdir)) self.assertFalse(os.path.isfile(tmpdir+'/foo/bar/baz.txt')) self.assertFalse(os.path.isdir(tmpdir+'/foo')) # without codecs file = File(tmpdir+'/foo.txt') fh = file.open('w', encoding=None) self.assertTrue(isinstance(fh, FileHandle)) fh.write('tja') fh.close() # with codecs file = File(tmpdir+'/bar.txt') file.writelines(['c\n', 'd\n']) self.assertEqual(file.readlines(), ['c\n', 'd\n']) # with error try: fh = file.open('w') fh.write('foo') raise IOError except IOError: del fh self.assertEqual(file.readlines(), ['c\n', 'd\n']) self.assertTrue(os.path.isfile(file.path+'.zim.new~')) # test read-only path = tmpdir+'/read-only-file.txt' open(path, 'w').write('test 123') os.chmod(path, 0444) file = File(path) self.assertRaises(OverWriteError, file.write, 'Overwritten!') os.chmod(path, 0644) # make it removable again # with windows line-ends file = open(tmpdir+'/newlines.txt', 'wb') # binary mode means no automatic newline conversions file.write('Some lines\r\nWith win32 newlines\r\n') file = File(tmpdir+'/newlines.txt') self.assertEqual(file.read(), 'Some lines\nWith win32 newlines\n')
def get_files_notebook(key): # We fill the notebook using the store interface, as this test comes before # the notebook test, but after the store test. dir = Dir(tests.create_tmp_dir('index_'+key)) notebook = Notebook(dir=dir) store = notebook.get_store(':') manifest = [] for name, text in tests.get_test_data('wiki'): manifest.append(name) page = store.get_page(Path(name)) page.parse('wiki', text) store.store_page(page) notebook.testdata_manifest = tests.expand_manifest(manifest) return notebook
def testDir(self): '''Test Dir object''' tmpdir = tests.create_tmp_dir('fs_testDir') dir = Dir(tmpdir+'/foo/bar') assert not dir.exists() # TODO test list() # TODO - test file(FILE), + test exception # TODO - test subdir(DIR), + test excepion file1 = dir.file('unique.txt') file1.touch() file2 = dir.new_file('unique.txt') file2.touch() file3 = dir.new_file('unique.txt') self.assertEqual(file1.basename, 'unique.txt') self.assertEqual(file2.basename, 'unique001.txt') self.assertEqual(file3.basename, 'unique002.txt')
def test_n5_datasink(): tmp_path = create_tmp_dir(prefix="test_n5_datasink") sink = N5DatasetSink( filesystem=OsFs(tmp_path.as_posix()), outer_path=PurePosixPath("test_n5_datasink.n5"), inner_path=PurePosixPath("/data"), attributes=N5DatasetAttributes( dimensions=datasource.shape, blockSize=Shape5D(x=10, y=10), c_axiskeys=data.axiskeys, #FIXME: double check this dataType=datasource.dtype, compression=RawCompressor(), location=Point5D.zero(x=7, y=13))) sink_writer = sink.create() assert not isinstance(sink_writer, Exception) for tile in DataRoi(datasource).split(sink.tile_shape): sink_writer.write(tile.retrieve().translated(Point5D.zero(x=7, y=13))) n5ds = N5DataSource(filesystem=sink.filesystem, path=sink.full_path) saved_data = n5ds.retrieve() assert saved_data.location == Point5D.zero(x=7, y=13) assert saved_data == data
def test_writing_to_offset_precomputed_chunks(): tmp_path = create_tmp_dir( prefix="test_writing_to_offset_precomputed_chunks") data_at_1000_1000 = data.translated( Point5D(x=1000, y=1000) - data.location) datasource = ArrayDataSource(data=data_at_1000_1000, tile_shape=Shape5D(x=10, y=10)) scale = PrecomputedChunksScale.from_datasource( datasource=datasource, key=PurePosixPath("my_test_data"), encoding=RawEncoder()) sink_path = PurePosixPath("mytest.precomputed") filesystem = OsFs(tmp_path.as_posix()) print(f"\n\n will write to '{filesystem.geturl(sink_path.as_posix())}' ") datasink = PrecomputedChunksScaleSink( filesystem=filesystem, info_dir=sink_path, scale=scale, num_channels=datasource.shape.c, dtype=datasource.dtype, ) creation_result = datasink.create() if isinstance(creation_result, Exception): raise creation_result for tile in datasource.roi.get_datasource_tiles(): datasink.write(tile.retrieve()) precomp_datasource = PrecomputedChunksDataSource( path=sink_path, filesystem=filesystem, resolution=scale.resolution) reloaded_data = precomp_datasource.retrieve( interval=data_at_1000_1000.interval) assert (reloaded_data.raw("xyz") == data.raw("xyz")).all()
def setUp(self): self.dir = Dir(create_tmp_dir('export_ExportedFiles'))
def runTest(self): root = Dir(tests.create_tmp_dir('notebook_TestGetNotebook')) # Start empty - see this is no issue list = get_notebook_list() self.assertTrue(isinstance(list, NotebookList)) self.assertFalse(list) nb, page = resolve_notebook('foo') self.assertTrue(nb is None) nb = resolve_default_notebook() self.assertTrue(nb is None) # Non-existing dir dir = root.subdir('/notebook') nb, page = resolve_notebook(dir.path) self.assertEqual(nb, dir) # Now create it init_notebook(dir, name='foo') file = dir.file('notebook.zim') nb, page = resolve_notebook(dir.path) self.assertEqual(nb, dir) nb, page = resolve_notebook(file.uri) self.assertEqual(nb, dir) file = dir.file('foo/bar/baz.txt') file.touch() nb, page = resolve_notebook(file.path) self.assertEqual(nb, dir) self.assertEqual(page, Path('foo:bar:baz')) # And put it in the list and resolve it by name list.append(dir.uri) list.write() list = get_notebook_list() self.assertTrue(len(list) == 1) nb, page = resolve_notebook('foo') self.assertEqual(nb, dir) # Single notebook is automatically the default nb = resolve_default_notebook() self.assertEqual(nb, dir) # But not anymore after adding second notebook list.append('file:///foo/bar') list.write() list = get_notebook_list() self.assertTrue(len(list) == 2) self.assertEqual(list[:], [dir.uri, 'file:///foo/bar']) nb, page = resolve_notebook('foo') self.assertEqual(nb, dir) self.assertTrue(isinstance(get_notebook(nb), Notebook)) nb = resolve_default_notebook() self.assertTrue(nb is None) list.default = 'file:///default/foo' list.write() list = get_notebook_list() nb = resolve_default_notebook() self.assertEqual(nb, Dir('/default/foo')) self.assertEqual(get_notebook(nb), None) # Check interwiki parsing self.assertEqual(interwiki_link('wp?Foo'), 'http://en.wikipedia.org/wiki/Foo') self.assertEqual(interwiki_link('foo?Foo'), dir.uri+'?Foo') nb, page = resolve_notebook(dir.uri+'?Foo') self.assertEqual(nb, dir) self.assertEqual(page, 'Foo') # Check backward compatibility file = File('tests/data/notebook-list-old-format.list') wanted = [Dir('~/Notes').uri, Dir('/home/user/code/zim.debug').uri, Dir('/home/user/Foo Bar').uri] list = NotebookList(file) self.assertEqual(list[:], wanted) self.assertEqual(list.default, Dir('/home/user/code/zim.debug').uri)
def setUp(self): self.path = tests.create_tmp_dir('fs_testOverwrite')+'/file.txt'