def append(self): replication = 1 # see https://issues.apache.org/jira/browse/HDFS-3091 content, update = utils.make_random_data(), utils.make_random_data() path = self._make_random_path() with self.fs.open_file(path, "w", replication=replication) as fo: fo.write(content) try: with utils.silent_call(self.fs.open_file, path, "a") as fo: fo.write(update) except IOError: sys.stderr.write("NOT SUPPORTED ... ") return else: with self.fs.open_file(path) as fi: self.assertEqual(fi.read(), content + update)
def write(self): content = utils.make_random_data() path = self._make_random_path() with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(content) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(bytearray(content)) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) chunk = create_string_buffer(len(content)) chunk[:] = content with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(chunk) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(u"some unicode data") # try to write a unicode object with self.fs.open_file(path, "w") as fo: u = u"a string" + utils.UNI_CHR data = u.encode("utf-8") bytes_written = fo.write(u) self.assertEqual(bytes_written, len(data))
def copy_on_self(self): content = utils.make_random_data() path = self._make_random_file(content=content) path1 = self._make_random_path() self.fs.copy(path, self.fs, path1) with self.fs.open_file(path1) as f: self.assertEqual(f.read(), content)
def write(self): content = utils.make_random_data() path = self._make_random_path() with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(content) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(bytearray(content)) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) chunk = create_string_buffer(len(content)) chunk[:] = content with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(chunk) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(u'some unicode data') # try to write a unicode object with self.fs.open_file(path, "w") as fo: u = u'a string' + utils.UNI_CHR data = u.encode('utf-8') bytes_written = fo.write(u) self.assertEqual(bytes_written, len(data))
def write_chunk(self): content = utils.make_random_data() chunk = create_string_buffer(len(content)) chunk[:] = content path = self._make_random_path() with self.fs.open_file(path, "w") as fo: bytes_written = fo.write_chunk(chunk) self.assertEqual(bytes_written, len(content))
def move(self): content = utils.make_random_data(printable=True) from_path = self._make_random_file(content=content) to_path = self._make_random_path() self.fs.move(from_path, self.fs, to_path) self.assertFalse(self.fs.exists(from_path)) with self.fs.open_file(to_path) as f: self.assertEqual(f.read(), content) self.assertRaises(ValueError, self.fs.move, "", self.fs, "")
def pread_chunk(self): content = utils.make_random_data() offset, length = 2, 3 chunk = create_string_buffer(length) path = self._make_random_file(content=content) with self.fs.open_file(path) as f: bytes_read = f.pread_chunk(offset, chunk) self.assertEqual(bytes_read, length) self.assertEqual(chunk.value, content[offset : offset + length]) self.assertEqual(f.tell(), 0)
def __read_chunk(self, chunk_factory): content = utils.make_random_data() path = self._make_random_file(content=content) size = len(content) for chunk_size in size - 1, size, size + 1: with self.fs.open_file(path) as f: chunk = chunk_factory(chunk_size) bytes_read = f.read_chunk(chunk) self.assertEqual(bytes_read, min(size, chunk_size)) self.assertEqual(_get_value(chunk), content[:bytes_read])
def pread_chunk(self): content = utils.make_random_data() offset, length = 2, 3 chunk = create_string_buffer(length) path = self._make_random_file(content=content) with self.fs.open_file(path) as f: bytes_read = f.pread_chunk(offset, chunk) self.assertEqual(bytes_read, length) self.assertEqual(chunk.value, content[offset:offset + length]) self.assertEqual(f.tell(), 0)
def read(self): content = utils.make_random_data() path = self._make_random_file(content=content) with self.fs.open_file(path) as f: self.assertEqual(f.read(), content) with self.fs.open_file(path) as f: self.assertEqual(f.read(-1), content) with self.fs.open_file(path) as f: self.assertEqual(f.read(3), content[:3]) self.assertEqual(f.read(3), content[3:6]) self.assertRaisesExternal(IOError, f.write, content)
def get_path_info(self): content = utils.make_random_data() path = self._make_random_file(content=content) info = self.fs.get_path_info(path) self.__check_path_info(info, kind="file", size=len(content)) self.assertTrue(info["name"].endswith(path)) path = self._make_random_dir() info = self.fs.get_path_info(path) self.__check_path_info(info, kind="directory") self.assertTrue(info["name"].endswith(path)) self.assertRaises(IOError, self.fs.get_path_info, self._make_random_path()) self.assertRaises(ValueError, self.fs.get_path_info, "")
def file_attrs(self): path = self._make_random_path() with self.fs.open_file(path, os.O_WRONLY) as f: self.assertTrue(f.name.endswith(path)) self.assertEqual(f.size, 0) self.assertEqual(f.mode, "w") content = utils.make_random_data() f.write(content) self.assertEqual(f.size, len(content)) with self.fs.open_file(path) as f: self.assertTrue(f.name.endswith(path)) self.assertEqual(f.size, len(content)) self.assertEqual(f.mode, "r")
def get_path_info(self): content = utils.make_random_data() path = self._make_random_file(content=content) info = self.fs.get_path_info(path) self.__check_path_info(info, kind="file", size=len(content)) self.assertTrue(info['name'].endswith(path)) path = self._make_random_dir() info = self.fs.get_path_info(path) self.__check_path_info(info, kind="directory") self.assertTrue(info['name'].endswith(path)) self.assertRaises(IOError, self.fs.get_path_info, self._make_random_path()) self.assertRaises(ValueError, self.fs.get_path_info, "")
def _make_random_file(self, where=None, content=None, **kwargs): kwargs["flags"] = "w" content = content or utils.make_random_data(printable=True) path = self._make_random_path(where=where) with self.fs.open_file(path, **kwargs) as fo: i = 0 bytes_written = 0 bufsize = 24 * 1024 * 1024 while i < len(content): bytes_written += fo.write(content[i:i + bufsize]) i += bufsize self.assertEqual(bytes_written, len(content)) return path
def read(self): content = utils.make_random_data() path = self._make_random_file(content=content) with self.fs.open_file(path) as f: self.assertEqual(f.read(), content) with self.fs.open_file(path) as f: self.assertEqual(f.read(-1), content) with self.fs.open_file(path) as f: self.assertEqual(f.read(3), content[:3]) self.assertEqual(f.read(3), content[3:6]) if not _is_py3 and not self.fs.host: self.assertRaises(ValueError, f.write, content) else: self.assertRaises(IOError, f.write, content)
def _make_random_file(self, where=None, content=None, **kwargs): kwargs["flags"] = "w" content = content or utils.make_random_data(printable=True) path = self._make_random_path(where=where) with self.fs.open_file(path, **kwargs) as fo: i = 0 bytes_written = 0 bufsize = 24 * 1024 * 1024 while i < len(content): bytes_written += fo.write(content[i : i + bufsize]) i += bufsize self.assertEqual(bytes_written, len(content)) return path
def pread(self): content = utils.make_random_data() offset, length = 2, 3 path = self._make_random_file(content=content) with self.fs.open_file(path) as f: self.assertEqual(f.pread(offset, length), content[offset : offset + length]) self.assertEqual(f.tell(), 0) self.assertEqual(content[1:], f.pread(1, -1)) self.assertRaises(ValueError, f.pread, -1, 10) # read starting past end of file self.assertRaises(IOError, f.pread, len(content) + 1, 10) # read past end of file buf = f.pread(len(content) - 2, 10) self.assertEqual(2, len(buf))
def pread(self): content = utils.make_random_data() offset, length = 2, 3 path = self._make_random_file(content=content) with self.fs.open_file(path) as f: self.assertEqual(f.pread(offset, length), content[offset:offset + length]) self.assertEqual(f.tell(), 0) self.assertEqual(content[1:], f.pread(1, -1)) self.assertRaises(ValueError, f.pread, -1, 10) # read starting past end of file self.assertRaises(IOError, f.pread, len(content) + 1, 10) # read past end of file buf = f.pread(len(content) - 2, 10) self.assertEqual(2, len(buf))
def setUp(self): wd = tempfile.mkdtemp(suffix='_%s' % UNI_CHR) wd_bn = os.path.basename(wd) self.local_wd = "file:%s" % wd fs = hdfs.hdfs("default", 0) fs.create_directory(wd_bn) self.hdfs_wd = fs.get_path_info(wd_bn)["name"] fs.close() basenames = ["test_path_%d" % i for i in xrange(2)] self.local_paths = ["%s/%s" % (self.local_wd, bn) for bn in basenames] self.hdfs_paths = ["%s/%s" % (self.hdfs_wd, bn) for bn in basenames] self.data = make_random_data(4*BUFSIZE + BUFSIZE/2, printable=False) for path in self.local_paths: self.assertTrue(path.startswith("file:")) for path in self.hdfs_paths: if not hdfs.default_is_local(): self.assertTrue(path.startswith("hdfs:"))
def write(self): content = utils.make_random_data() path = self._make_random_path() with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(content) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(bytearray(content)) self.assertEqual(bytes_written, len(content)) with self.fs.open_file(path) as fo: self.assertEqual(content, fo.read()) chunk = create_string_buffer(content, len(content)) with self.fs.open_file(path, "w") as fo: bytes_written = fo.write(chunk) self.assertEqual(bytes_written, len(content))
def setUp(self): wd = tempfile.mkdtemp(suffix='_%s' % UNI_CHR) wd_bn = os.path.basename(wd) self.local_wd = "file:%s" % wd fs = hdfs.hdfs("default", 0) fs.create_directory(wd_bn) self.hdfs_wd = fs.get_path_info(wd_bn)["name"] fs.close() basenames = ["test_path_%d" % i for i in range(2)] self.local_paths = ["%s/%s" % (self.local_wd, bn) for bn in basenames] self.hdfs_paths = ["%s/%s" % (self.hdfs_wd, bn) for bn in basenames] self.data = make_random_data(4 * BUFSIZE + BUFSIZE // 2, printable=False) for path in self.local_paths: self.assertTrue(path.startswith("file:")) for path in self.hdfs_paths: if not hdfs.default_is_local(): self.assertTrue(path.startswith("hdfs:"))
def file_attrs(self): path = self._make_random_path() content = utils.make_random_data() for mode in "wb", "wt": with self.fs.open_file(path, mode) as f: self.assertTrue(f.name.endswith(path)) self.assertTrue(f.fs is self.fs) self.assertEqual(f.size, 0) self.assertEqual(f.mode, mode) self.assertTrue(f.writable()) f.write(content if mode == "wb" else content.decode("utf-8")) self.assertEqual(f.size, len(content)) for mode in "rb", "rt": with self.fs.open_file(path, mode) as f: self.assertTrue(f.name.endswith(path)) self.assertTrue(f.fs is self.fs) self.assertEqual(f.size, len(content)) self.assertEqual(f.mode, mode) self.assertFalse(f.writable())
def flush(self): path = self._make_random_path() with self.fs.open_file(path, "w") as f: f.write(utils.make_random_data()) f.flush()
def available(self): content = utils.make_random_data() path = self._make_random_file(content=content) with self.fs.open_file(path) as f: self.assertEqual(len(content), f.available())