def append(self):
replication = 1 # see https://issues.apache.org/jira/browse/HDFS-3091
content, update = utils.make_random_data(), utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w", replication=replication) as fo:
fo.write(content)
try:
with utils.silent_call(self.fs.open_file, path, "a") as fo:
fo.write(update)
except IOError:
sys.stderr.write("NOT SUPPORTED ... ")
return
else:
with self.fs.open_file(path) as fi:
self.assertEqual(fi.read(), content + update)
def append(self):
replication = 1 # see https://issues.apache.org/jira/browse/HDFS-3091
content, update = utils.make_random_data(), utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w", replication=replication) as fo:
fo.write(content)
try:
with utils.silent_call(self.fs.open_file, path, "a") as fo:
fo.write(update)
except IOError:
sys.stderr.write("NOT SUPPORTED ... ")
return
else:
with self.fs.open_file(path) as fi:
self.assertEqual(fi.read(), content + update)
def write(self):
content = utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(content)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(bytearray(content))
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
chunk = create_string_buffer(len(content))
chunk[:] = content
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(chunk)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(u"some unicode data")
# try to write a unicode object
with self.fs.open_file(path, "w") as fo:
u = u"a string" + utils.UNI_CHR
data = u.encode("utf-8")
bytes_written = fo.write(u)
self.assertEqual(bytes_written, len(data))
def copy_on_self(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
path1 = self._make_random_path()
self.fs.copy(path, self.fs, path1)
with self.fs.open_file(path1) as f:
self.assertEqual(f.read(), content)
def copy_on_self(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
path1 = self._make_random_path()
self.fs.copy(path, self.fs, path1)
with self.fs.open_file(path1) as f:
self.assertEqual(f.read(), content)
def write(self):
content = utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(content)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(bytearray(content))
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
chunk = create_string_buffer(len(content))
chunk[:] = content
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(chunk)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(u'some unicode data')
# try to write a unicode object
with self.fs.open_file(path, "w") as fo:
u = u'a string' + utils.UNI_CHR
data = u.encode('utf-8')
bytes_written = fo.write(u)
self.assertEqual(bytes_written, len(data))
def write_chunk(self):
content = utils.make_random_data()
chunk = create_string_buffer(len(content))
chunk[:] = content
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write_chunk(chunk)
self.assertEqual(bytes_written, len(content))
def write_chunk(self):
content = utils.make_random_data()
chunk = create_string_buffer(len(content))
chunk[:] = content
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write_chunk(chunk)
self.assertEqual(bytes_written, len(content))
def move(self):
content = utils.make_random_data(printable=True)
from_path = self._make_random_file(content=content)
to_path = self._make_random_path()
self.fs.move(from_path, self.fs, to_path)
self.assertFalse(self.fs.exists(from_path))
with self.fs.open_file(to_path) as f:
self.assertEqual(f.read(), content)
self.assertRaises(ValueError, self.fs.move, "", self.fs, "")
def move(self):
content = utils.make_random_data(printable=True)
from_path = self._make_random_file(content=content)
to_path = self._make_random_path()
self.fs.move(from_path, self.fs, to_path)
self.assertFalse(self.fs.exists(from_path))
with self.fs.open_file(to_path) as f:
self.assertEqual(f.read(), content)
self.assertRaises(ValueError, self.fs.move, "", self.fs, "")
def pread_chunk(self):
content = utils.make_random_data()
offset, length = 2, 3
chunk = create_string_buffer(length)
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
bytes_read = f.pread_chunk(offset, chunk)
self.assertEqual(bytes_read, length)
self.assertEqual(chunk.value, content[offset : offset + length])
self.assertEqual(f.tell(), 0)
def __read_chunk(self, chunk_factory):
content = utils.make_random_data()
path = self._make_random_file(content=content)
size = len(content)
for chunk_size in size - 1, size, size + 1:
with self.fs.open_file(path) as f:
chunk = chunk_factory(chunk_size)
bytes_read = f.read_chunk(chunk)
self.assertEqual(bytes_read, min(size, chunk_size))
self.assertEqual(_get_value(chunk), content[:bytes_read])
def __read_chunk(self, chunk_factory):
content = utils.make_random_data()
path = self._make_random_file(content=content)
size = len(content)
for chunk_size in size - 1, size, size + 1:
with self.fs.open_file(path) as f:
chunk = chunk_factory(chunk_size)
bytes_read = f.read_chunk(chunk)
self.assertEqual(bytes_read, min(size, chunk_size))
self.assertEqual(_get_value(chunk), content[:bytes_read])
def pread_chunk(self):
content = utils.make_random_data()
offset, length = 2, 3
chunk = create_string_buffer(length)
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
bytes_read = f.pread_chunk(offset, chunk)
self.assertEqual(bytes_read, length)
self.assertEqual(chunk.value, content[offset:offset + length])
self.assertEqual(f.tell(), 0)
def read(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(-1), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(3), content[:3])
self.assertEqual(f.read(3), content[3:6])
self.assertRaisesExternal(IOError, f.write, content)
def read(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(-1), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(3), content[:3])
self.assertEqual(f.read(3), content[3:6])
self.assertRaisesExternal(IOError, f.write, content)
def get_path_info(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
info = self.fs.get_path_info(path)
self.__check_path_info(info, kind="file", size=len(content))
self.assertTrue(info["name"].endswith(path))
path = self._make_random_dir()
info = self.fs.get_path_info(path)
self.__check_path_info(info, kind="directory")
self.assertTrue(info["name"].endswith(path))
self.assertRaises(IOError, self.fs.get_path_info, self._make_random_path())
self.assertRaises(ValueError, self.fs.get_path_info, "")
def file_attrs(self):
path = self._make_random_path()
with self.fs.open_file(path, os.O_WRONLY) as f:
self.assertTrue(f.name.endswith(path))
self.assertEqual(f.size, 0)
self.assertEqual(f.mode, "w")
content = utils.make_random_data()
f.write(content)
self.assertEqual(f.size, len(content))
with self.fs.open_file(path) as f:
self.assertTrue(f.name.endswith(path))
self.assertEqual(f.size, len(content))
self.assertEqual(f.mode, "r")
def file_attrs(self):
path = self._make_random_path()
with self.fs.open_file(path, os.O_WRONLY) as f:
self.assertTrue(f.name.endswith(path))
self.assertEqual(f.size, 0)
self.assertEqual(f.mode, "w")
content = utils.make_random_data()
f.write(content)
self.assertEqual(f.size, len(content))
with self.fs.open_file(path) as f:
self.assertTrue(f.name.endswith(path))
self.assertEqual(f.size, len(content))
self.assertEqual(f.mode, "r")
def get_path_info(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
info = self.fs.get_path_info(path)
self.__check_path_info(info, kind="file", size=len(content))
self.assertTrue(info['name'].endswith(path))
path = self._make_random_dir()
info = self.fs.get_path_info(path)
self.__check_path_info(info, kind="directory")
self.assertTrue(info['name'].endswith(path))
self.assertRaises(IOError, self.fs.get_path_info,
self._make_random_path())
self.assertRaises(ValueError, self.fs.get_path_info, "")
def _make_random_file(self, where=None, content=None, **kwargs):
kwargs["flags"] = "w"
content = content or utils.make_random_data(printable=True)
path = self._make_random_path(where=where)
with self.fs.open_file(path, **kwargs) as fo:
i = 0
bytes_written = 0
bufsize = 24 * 1024 * 1024
while i < len(content):
bytes_written += fo.write(content[i:i + bufsize])
i += bufsize
self.assertEqual(bytes_written, len(content))
return path
def read(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(-1), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(3), content[:3])
self.assertEqual(f.read(3), content[3:6])
if not _is_py3 and not self.fs.host:
self.assertRaises(ValueError, f.write, content)
else:
self.assertRaises(IOError, f.write, content)
def read(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(-1), content)
with self.fs.open_file(path) as f:
self.assertEqual(f.read(3), content[:3])
self.assertEqual(f.read(3), content[3:6])
if not _is_py3 and not self.fs.host:
self.assertRaises(ValueError, f.write, content)
else:
self.assertRaises(IOError, f.write, content)
def _make_random_file(self, where=None, content=None, **kwargs):
kwargs["flags"] = "w"
content = content or utils.make_random_data(printable=True)
path = self._make_random_path(where=where)
with self.fs.open_file(path, **kwargs) as fo:
i = 0
bytes_written = 0
bufsize = 24 * 1024 * 1024
while i < len(content):
bytes_written += fo.write(content[i : i + bufsize])
i += bufsize
self.assertEqual(bytes_written, len(content))
return path
def pread(self):
content = utils.make_random_data()
offset, length = 2, 3
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.pread(offset, length), content[offset : offset + length])
self.assertEqual(f.tell(), 0)
self.assertEqual(content[1:], f.pread(1, -1))
self.assertRaises(ValueError, f.pread, -1, 10)
# read starting past end of file
self.assertRaises(IOError, f.pread, len(content) + 1, 10)
# read past end of file
buf = f.pread(len(content) - 2, 10)
self.assertEqual(2, len(buf))
def pread(self):
content = utils.make_random_data()
offset, length = 2, 3
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(f.pread(offset, length),
content[offset:offset + length])
self.assertEqual(f.tell(), 0)
self.assertEqual(content[1:], f.pread(1, -1))
self.assertRaises(ValueError, f.pread, -1, 10)
# read starting past end of file
self.assertRaises(IOError, f.pread, len(content) + 1, 10)
# read past end of file
buf = f.pread(len(content) - 2, 10)
self.assertEqual(2, len(buf))
def setUp(self):
wd = tempfile.mkdtemp(suffix='_%s' % UNI_CHR)
wd_bn = os.path.basename(wd)
self.local_wd = "file:%s" % wd
fs = hdfs.hdfs("default", 0)
fs.create_directory(wd_bn)
self.hdfs_wd = fs.get_path_info(wd_bn)["name"]
fs.close()
basenames = ["test_path_%d" % i for i in xrange(2)]
self.local_paths = ["%s/%s" % (self.local_wd, bn) for bn in basenames]
self.hdfs_paths = ["%s/%s" % (self.hdfs_wd, bn) for bn in basenames]
self.data = make_random_data(4*BUFSIZE + BUFSIZE/2, printable=False)
for path in self.local_paths:
self.assertTrue(path.startswith("file:"))
for path in self.hdfs_paths:
if not hdfs.default_is_local():
self.assertTrue(path.startswith("hdfs:"))
def write(self):
content = utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(content)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(bytearray(content))
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
chunk = create_string_buffer(content, len(content))
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(chunk)
self.assertEqual(bytes_written, len(content))
def write(self):
content = utils.make_random_data()
path = self._make_random_path()
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(content)
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(bytearray(content))
self.assertEqual(bytes_written, len(content))
with self.fs.open_file(path) as fo:
self.assertEqual(content, fo.read())
chunk = create_string_buffer(content, len(content))
with self.fs.open_file(path, "w") as fo:
bytes_written = fo.write(chunk)
self.assertEqual(bytes_written, len(content))
def setUp(self):
wd = tempfile.mkdtemp(suffix='_%s' % UNI_CHR)
wd_bn = os.path.basename(wd)
self.local_wd = "file:%s" % wd
fs = hdfs.hdfs("default", 0)
fs.create_directory(wd_bn)
self.hdfs_wd = fs.get_path_info(wd_bn)["name"]
fs.close()
basenames = ["test_path_%d" % i for i in range(2)]
self.local_paths = ["%s/%s" % (self.local_wd, bn) for bn in basenames]
self.hdfs_paths = ["%s/%s" % (self.hdfs_wd, bn) for bn in basenames]
self.data = make_random_data(4 * BUFSIZE + BUFSIZE // 2,
printable=False)
for path in self.local_paths:
self.assertTrue(path.startswith("file:"))
for path in self.hdfs_paths:
if not hdfs.default_is_local():
self.assertTrue(path.startswith("hdfs:"))
def file_attrs(self):
path = self._make_random_path()
content = utils.make_random_data()
for mode in "wb", "wt":
with self.fs.open_file(path, mode) as f:
self.assertTrue(f.name.endswith(path))
self.assertTrue(f.fs is self.fs)
self.assertEqual(f.size, 0)
self.assertEqual(f.mode, mode)
self.assertTrue(f.writable())
f.write(content if mode == "wb" else content.decode("utf-8"))
self.assertEqual(f.size, len(content))
for mode in "rb", "rt":
with self.fs.open_file(path, mode) as f:
self.assertTrue(f.name.endswith(path))
self.assertTrue(f.fs is self.fs)
self.assertEqual(f.size, len(content))
self.assertEqual(f.mode, mode)
self.assertFalse(f.writable())
def file_attrs(self):
path = self._make_random_path()
content = utils.make_random_data()
for mode in "wb", "wt":
with self.fs.open_file(path, mode) as f:
self.assertTrue(f.name.endswith(path))
self.assertTrue(f.fs is self.fs)
self.assertEqual(f.size, 0)
self.assertEqual(f.mode, mode)
self.assertTrue(f.writable())
f.write(content if mode == "wb" else content.decode("utf-8"))
self.assertEqual(f.size, len(content))
for mode in "rb", "rt":
with self.fs.open_file(path, mode) as f:
self.assertTrue(f.name.endswith(path))
self.assertTrue(f.fs is self.fs)
self.assertEqual(f.size, len(content))
self.assertEqual(f.mode, mode)
self.assertFalse(f.writable())
def flush(self):
path = self._make_random_path()
with self.fs.open_file(path, "w") as f:
f.write(utils.make_random_data())
f.flush()
def available(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(len(content), f.available())
def available(self):
content = utils.make_random_data()
path = self._make_random_file(content=content)
with self.fs.open_file(path) as f:
self.assertEqual(len(content), f.available())
def flush(self):
path = self._make_random_path()
with self.fs.open_file(path, "w") as f:
f.write(utils.make_random_data())
f.flush()
