def test_create_handler(self):
posix_handler = chainerio.create_handler("posix")
self.assertIsInstance(posix_handler,
chainerio.filesystems.posix.PosixFileSystem)
hdfs_handler = chainerio.create_handler("hdfs")
self.assertIsInstance(hdfs_handler,
chainerio.filesystems.hdfs.HdfsFileSystem)
another_posix_handler = chainerio.create_handler("posix")
self.assertNotEqual(posix_handler, another_posix_handler)
with self.assertRaises(ValueError):
chainerio.create_handler("unsupported_scheme")
def test_read_bytes(self):
with chainerio.create_handler(self.fs) as handler:
with handler.open(self.tmpfile_name, "wb") as tmpfile:
tmpfile.write(self.test_string_b)
with handler.open(self.tmpfile_name, "rb") as f:
self.assertEqual(self.test_string_b, f.read())
def load_snapshot(target,
directory,
filename=None,
fs=None,
fail_on_no_file=False):
assert directory is not None or filename is not None
if fs is None:
fs = chainerio
elif isinstance(fs, str):
fs = chainerio.create_handler(fs)
else:
fs = fs
if filename is None and directory is not None:
filename = _scan_directory(fs, directory)
if filename is None:
if fail_on_no_file:
raise RuntimeError('No snapshot found from %s' % directory)
return
if directory is not None:
filename = os.path.join(directory, filename)
with fs.open(filename, 'rb') as fp:
load_npz(fp, target)
def test_mkdir(self):
test_dir_name = "testmkdir"
with chainerio.create_handler(self.fs) as handler:
handler.mkdir(test_dir_name)
self.assertTrue(handler.isdir(test_dir_name))
handler.remove(test_dir_name)
def test_exists(self):
non_exist_file = "non_exist_file.txt"
with chainerio.create_handler(self.fs) as handler:
self.assertTrue(handler.exists(self.tmpfile_name))
self.assertTrue(handler.exists("/"))
self.assertFalse(handler.exists(non_exist_file))
def setUp(self):
# The following zip layout is created for all the tests
# outside.zip
# | - testfile1
n = 1 << 20
self.test_string = make_random_str(n)
self.fs_handler = chainerio.create_handler("posix")
# the most outside zip
self.zip_file_name = "outside"
# nested zip and nested file
self.tmpdir = tempfile.TemporaryDirectory()
# test file
self.testfile_name = "testfile1"
# paths used in making outside.zip
testfile_path = os.path.join(self.tmpdir.name, self.testfile_name)
# paths used in tests
self.zip_file_path = self.zip_file_name + ".zip"
with open(testfile_path, "w") as tmpfile:
tmpfile.write(self.test_string)
# this will include outside.zip itself into the zip
make_zip(self.zip_file_path, root_dir=self.tmpdir.name, base_dir=".")
def test_remove(self):
test_file = "test_remove.txt"
test_dir = "test_dir/"
nested_dir = os.path.join(test_dir, "nested_file/")
nested_file = os.path.join(nested_dir, test_file)
with chainerio.create_handler(self.fs) as handler:
with handler.open(test_file, 'w') as fp:
fp.write('foobar')
# test remove on one file
self.assertTrue(handler.exists(test_file))
handler.remove(test_file)
self.assertFalse(handler.exists(test_file))
# test remove on directory
handler.makedirs(nested_dir)
with handler.open(nested_file, 'w') as fp:
fp.write('foobar')
self.assertTrue(handler.exists(test_dir))
self.assertTrue(handler.exists(nested_dir))
self.assertTrue(handler.exists(nested_file))
handler.remove(test_dir, True)
self.assertFalse(handler.exists(test_dir))
self.assertFalse(handler.exists(nested_dir))
self.assertFalse(handler.exists(nested_file))
def setUp(self):
self.test_string = "this is a test string\n"
self.fs = "hdfs"
self.tmpfile_name = "tmpfile.txt"
with chainerio.create_handler(self.fs) as handler:
with handler.open(self.tmpfile_name, "w") as tmpfile:
tmpfile.write(self.test_string)
def test_open_non_exist(self):
non_exist_file = "non_exist_file.txt"
if os.path.exists(non_exist_file):
os.remove(non_exist_file)
with chainerio.create_handler(self.fs) as handler:
self.assertRaises(IOError, handler.open, non_exist_file)
def setUp(self):
test_string = "this is a test string\n"
self.test_string_b = test_string.encode("utf-8")
self.fs = "hdfs"
self.tmpfile_name = "tmpfile.txt"
with chainerio.create_handler(self.fs) as handler:
with handler.open(self.tmpfile_name, "wb") as tmpfile:
tmpfile.write(self.test_string_b)
def test_makedirs(self):
test_dir_name = "testmkdir/"
nested_dir_name = test_dir_name + "nested_dir"
with chainerio.create_handler(self.fs) as handler:
handler.makedirs(nested_dir_name)
self.assertTrue(handler.isdir(nested_dir_name))
handler.remove(test_dir_name, True)
def test_read_string(self):
with chainerio.create_handler(self.fs) as handler:
with handler.open(self.tmpfile_name, "w") as tmpfile:
tmpfile.write(self.test_string)
with handler.open(self.tmpfile_name, "r") as f:
self.assertEqual(self.test_string, f.read())
with handler.open(self.tmpfile_name, "r") as f:
self.assertEqual(self.test_string, f.readline())
def test_read_bytes(self):
with chainerio.create_handler(self.fs) as handler:
with tempfile.NamedTemporaryFile("w+b", delete=False) as tmpfile:
tmpfile_path = tmpfile.name
tmpfile.write(self.test_string_bytes)
with handler.open(tmpfile_path, mode="rb") as loaded_file:
self.assertEqual(self.test_string_bytes, loaded_file.read())
handler.remove(tmpfile_path)
def test_picle(self):
pickle_file_name = "test_pickle.pickle"
test_data = {'test_elem1': b'balabala', 'test_elem2': 'balabala'}
with chainerio.create_handler(self.fs) as handler:
with handler.open(pickle_file_name, 'wb') as f:
pickle.dump(test_data, f)
with handler.open(pickle_file_name, 'rb') as f:
loaded_obj = pickle.load(f)
self.assertEqual(test_data, loaded_obj)
handler.remove(pickle_file_name, True)
def __init__(self, directory: str, savefun=None, fs=None):
assert directory is not None
self.directory = directory
self.savefun = save_npz if savefun is None else savefun
if fs is None:
self.fs = chainerio
elif isinstance(fs, str):
self.fs = chainerio.create_handler(fs)
else:
self.fs = fs
if not self.fs.exists(self.directory):
self.fs.makedirs(self.directory)
def setUp(self):
self.test_string = "this is a test string\n"
self.test_string_b = self.test_string.encode("utf-8")
self.zip_file_name = "test"
self.zip_file_path = self.zip_file_name + ".zip"
self.dir_name = "testdir/"
self.zipped_file_name = "testfile"
self.zipped_file_path = os.path.join(self.dir_name,
self.zipped_file_name)
self.fs_handler = chainerio.create_handler("posix")
os.mkdir(self.dir_name)
with open(self.zipped_file_path, "w") as tmpfile:
tmpfile.write(self.test_string)
shutil.make_archive(self.zip_file_name, "zip", base_dir=self.dir_name)
def test_rename(self):
with chainerio.create_handler(self.fs) as handler:
with handler.open('src', 'w') as fp:
fp.write('foobar')
self.assertTrue(handler.exists('src'))
self.assertFalse(handler.exists('dst'))
handler.rename('src', 'dst')
self.assertFalse(handler.exists('src'))
self.assertTrue(handler.exists('dst'))
with handler.open('dst', 'r') as fp:
data = fp.read()
assert data == 'foobar'
handler.remove('dst', True)
def test_list(self):
with chainerio.create_handler(self.fs) as handler:
file_generator = handler.list()
self.assertIsInstance(file_generator, Iterable)
file_list = list(file_generator)
self.assertIn(self.tmpfile_name, file_list, self.tmpfile_name)
# An exception is raised when the given path is not a directory
self.assertRaises(NotADirectoryError, list,
handler.list(self.tmpfile_name))
for test_dir_name in ["testmkdir", "testmkdir/"]:
nested_dir_name1 = "nested_dir1"
nested_dir_name2 = "nested_dir2"
nested_file_name = "file"
nested_dir1 = os.path.join(test_dir_name, nested_dir_name1)
nested_dir2 = os.path.join(test_dir_name, nested_dir_name2)
nested_file = os.path.join(nested_dir2, nested_file_name)
nested_file_relative = os.path.join(nested_dir_name2,
nested_file_name)
try:
handler.makedirs(nested_dir1)
handler.makedirs(nested_dir2)
with handler.open(nested_file, "w") as f:
f.write(self.test_string)
recursive_file_generator = handler.list(test_dir_name,
recursive=True)
self.assertIsInstance(recursive_file_generator, Iterable)
file_list = list(recursive_file_generator)
self.assertIn(nested_dir_name1, file_list)
self.assertIn(nested_dir_name2, file_list)
self.assertIn(nested_file_relative, file_list)
normal_file_generator = handler.list(test_dir_name)
self.assertIsInstance(recursive_file_generator, Iterable)
file_list = list(normal_file_generator)
self.assertIn(nested_dir_name1, file_list)
self.assertIn(nested_dir_name2, file_list)
self.assertNotIn(nested_file_relative, file_list)
finally:
handler.remove(test_dir_name, True)
def test_list(self):
# directory layout
# testlsdir
# | - nested_dir1
# | | - nested_dir3
# | _ nested_dir2
for test_dir_name in ["testlsdir", "testlsdir/"]:
try:
tmpdir = tempfile.TemporaryDirectory()
nested_dir_name1 = "nested_dir1"
nested_dir_name2 = "nested_dir2"
nested_dir_name3 = "nested_dir3"
test_dir_path = os.path.join(tmpdir.name, test_dir_name)
nested_dir_path1 = os.path.join(test_dir_path,
nested_dir_name1)
nested_dir_path2 = os.path.join(test_dir_path,
nested_dir_name2)
nested_dir_path3 = os.path.join(nested_dir_path1,
nested_dir_name3)
nested_dir_relative_path3 = os.path.join(
nested_dir_name1, nested_dir_name3)
with chainerio.create_handler(self.fs) as handler:
handler.makedirs(nested_dir_path1)
handler.makedirs(nested_dir_path2)
handler.makedirs(nested_dir_path3)
self.assertIsInstance(handler.list(), Iterable)
full_list_of_file = list(
handler.list(test_dir_path, recursive=True))
self.assertIn(nested_dir_name1, full_list_of_file)
self.assertIn(nested_dir_name2, full_list_of_file)
self.assertIn(nested_dir_relative_path3, full_list_of_file)
first_level_list_of_file = list(
handler.list(test_dir_path))
self.assertIn(nested_dir_name1, first_level_list_of_file)
self.assertIn(nested_dir_name2, first_level_list_of_file)
self.assertNotIn(nested_dir_relative_path3,
first_level_list_of_file)
finally:
tmpdir.cleanup()
def test_list(self):
with chainerio.create_handler(self.fs) as handler:
self.assertIsInstance(handler.list(), Iterable)
def setUp(self):
# The following zip layout is created for all the tests
# outside.zip
# | - testdir1
# | | - nested1.zip
# | | - nested_dir
# | | - nested
# | - testdir2
# | | - testfile1
# | - testfile2
self.test_string = "this is a test string\n"
self.nested_test_string = \
"this is a test string for nested zip\n"
self.test_string_b = self.test_string.encode("utf-8")
self.nested_test_string_b = \
self.nested_test_string.encode("utf-8")
self.fs_handler = chainerio.create_handler("posix")
# the most outside zip
self.zip_file_name = "outside"
# nested zip and nested file
self.tmpdir = tempfile.TemporaryDirectory()
self.nested_zipped_file_name = "nested"
self.nested_dir_name = "nested_dir/"
self.nested_dir_path = os.path.join(self.tmpdir.name,
self.nested_dir_name)
self.nested_zip_file_name = "nested.zip"
# directory and file
self.dir_name1 = "testdir1/"
self.dir_name2 = "testdir2/"
self.zipped_file_name = "testfile1"
self.testfile_name = "testfile2"
# paths used in making outside.zip
dir_path1 = os.path.join(self.tmpdir.name, self.dir_name1)
dir_path2 = os.path.join(self.tmpdir.name, self.dir_name2)
testfile_path = os.path.join(self.tmpdir.name, self.testfile_name)
nested_dir_path = os.path.join(self.tmpdir.name, self.nested_dir_name)
zipped_file_path = os.path.join(dir_path2, self.zipped_file_name)
nested_zipped_file_path = os.path.join(nested_dir_path,
self.nested_zipped_file_name)
nested_zip_file_path = os.path.join(dir_path1,
self.nested_zip_file_name)
# paths used in tests
self.zip_file_path = self.zip_file_name + ".zip"
self.zipped_file_path = os.path.join(self.dir_name2,
self.zipped_file_name)
self.nested_zip_path = os.path.join(self.dir_name1,
self.nested_zip_file_name)
self.nested_zipped_file_path = os.path.join(
self.nested_dir_name, self.nested_zipped_file_name)
self.non_exists_list = [
"does_not_exist", "does_not_exist/", "does/not/exist"
]
os.mkdir(dir_path1)
os.mkdir(dir_path2)
os.mkdir(nested_dir_path)
with open(zipped_file_path, "w") as tmpfile:
tmpfile.write(self.test_string)
with open(nested_zipped_file_path, "w") as tmpfile:
tmpfile.write(self.nested_test_string)
with open(testfile_path, "w") as tmpfile:
tmpfile.write(self.test_string)
make_zip(nested_zip_file_path,
root_dir=self.tmpdir.name,
base_dir=self.nested_dir_name)
shutil.rmtree(nested_dir_path)
# this will include outside.zip itself into the zip
make_zip(self.zip_file_path, root_dir=self.tmpdir.name, base_dir=".")
def test_isdir(self):
with chainerio.create_handler(self.fs) as handler:
self.assertTrue(handler.isdir("/"))
self.assertFalse(handler.isdir("test_posix_handler.py"))
def tearDown(self):
with chainerio.create_handler(self.fs) as handler:
try:
handler.remove(self.tmpfile_name)
except IOError:
pass
def test_isdir(self):
with chainerio.create_handler(self.fs) as handler:
self.assertTrue(handler.isdir("/"))
self.assertFalse(handler.isdir(self.tmpfile_name))
def test_info(self):
with chainerio.create_handler(self.fs) as handler:
self.assertIsInstance(handler.info(), str)
def test_list(self):
with chainerio.create_handler(self.fs) as handler:
file_generator = handler.list()
self.assertIsInstance(file_generator, Iterable)
file_list = list(file_generator)
self.assertIn(self.tmpfile_name, file_list)
def test_read_non_exist(self):
non_exist_file = "non_exist_file.txt"
with chainerio.create_handler(self.fs) as handler:
self.assertRaises(IOError, handler.open, non_exist_file)
