def testWriteEmpty(self):
temp_dir = self._CreateDeepS3Structure()
ckpt_path = os.path.join(temp_dir, "model2.ckpt")
ckpt_content = ""
with gfile.GFile(ckpt_path, "w") as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testWrite(self):
temp_dir = self._CreateDeepS3Structure()
ckpt_path = os.path.join(temp_dir, 'model2.ckpt')
ckpt_content = u'asdfasdfasdffoobarbuzz'
with gfile.GFile(ckpt_path, 'w') as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, 'r') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testWriteBinary(self):
temp_dir = self._CreateDeepS3Structure()
ckpt_path = os.path.join(temp_dir, "model.ckpt")
ckpt_content = b"asdfasdfasdffoobarbuzz"
with gfile.GFile(ckpt_path, "wb") as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, "rb") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testReadingIterator(self):
file_path = os.path.join(self._base_dir, "temp_file")
data = ["testing1\n", "testing2\n", "testing3\n", "\n", "testing5"]
with gfile.GFile(file_path, mode="w") as f:
f.write("".join(data))
with gfile.GFile(file_path, mode="r") as f:
actual_data = []
for line in f:
actual_data.append(line)
self.assertSequenceEqual(actual_data, data)
def testMultipleFiles(self):
file_prefix = os.path.join(self._base_dir, "temp_file")
for i in range(5000):
with gfile.GFile(file_prefix + str(i), mode="w") as f:
f.write("testing")
f.flush()
with gfile.GFile(file_prefix + str(i), mode="r") as f:
self.assertEqual("testing", f.read())
def testRead(self):
file_path = os.path.join(self._base_dir, "temp_file")
with gfile.GFile(file_path, mode="w") as f:
f.write("testing1\ntesting2\ntesting3\n\ntesting5")
with gfile.GFile(file_path, mode="r") as f:
self.assertEqual(36, gfile.stat(file_path).length)
self.assertEqual("testing1\n", f.read(9))
self.assertEqual("testing2\n", f.read(9))
self.assertEqual("t", f.read(1))
self.assertEqual("esting3\n\ntesting5", f.read())
def testEof(self):
"""Test that reading past EOF does not raise an exception."""
file_path = os.path.join(self._base_dir, "temp_file")
with gfile.GFile(file_path, mode="w") as f:
content = "testing"
f.write(content)
f.flush()
with gfile.GFile(file_path, mode="r") as f:
self.assertEqual(content, f.read(len(content) + 1))
def testOverwrite(self):
temp_dir = self._CreateDeepS3Structure()
ckpt_path = os.path.join(temp_dir, "model2.ckpt")
ckpt_content = "asdfasdfasdffoobarbuzz"
with gfile.GFile(ckpt_path, "w") as f:
f.write("original")
with gfile.GFile(ckpt_path, "w") as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testOverwrite(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = posixpath.join(temp_dir, "model2.ckpt")
ckpt_content = "asdfasdfasdffoobarbuzz"
with gfile.GFile(ckpt_path, "w") as f:
f.write("original")
with gfile.GFile(ckpt_path, "w") as f:
f.write(ckpt_content)
with fsspec.open(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testOverwrite(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model2.ckpt')
ckpt_content = u'asdfasdfasdffoobarbuzz'
with gfile.GFile(ckpt_path, 'w') as f:
f.write(u'original')
with gfile.GFile(ckpt_path, 'w') as f:
f.write(ckpt_content)
with open(ckpt_path, 'r') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testWriteMultipleBinary(self):
temp_dir = self._CreateDeepS3Structure()
ckpt_path = os.path.join(temp_dir, "model2.ckpt")
ckpt_content = b"asdfasdfasdffoobarbuzz" * 5
with gfile.GFile(ckpt_path, "wb") as f:
for i in range(0, len(ckpt_content), 3):
f.write(ckpt_content[i : i + 3])
# Test periodic flushing of the file
if i % 9 == 0:
f.flush()
with gfile.GFile(ckpt_path, "rb") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testListDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
gfile.makedirs(dir_path)
files = ["file1.txt", "file2.txt", "file3.txt"]
for name in files:
file_path = os.path.join(dir_path, name)
gfile.GFile(file_path, mode="w").write("testing")
subdir_path = os.path.join(dir_path, "sub_dir")
gfile.makedirs(subdir_path)
subdir_file_path = os.path.join(subdir_path, "file4.txt")
gfile.GFile(subdir_file_path, mode="w").write("testing")
dir_list = gfile.listdir(dir_path)
self.assertItemsEqual(files + ["sub_dir"], dir_list)
def _setupWalkDirectories(self, dir_path):
# Creating a file structure as follows
# test_dir -> file: file1.txt; dirs: subdir1_1, subdir1_2, subdir1_3
# subdir1_1 -> file: file3.txt
# subdir1_2 -> dir: subdir2
gfile.makedirs(dir_path)
gfile.GFile(os.path.join(dir_path, "file1.txt"),
mode="w").write("testing")
sub_dirs1 = ["subdir1_1", "subdir1_2", "subdir1_3"]
for name in sub_dirs1:
gfile.makedirs(os.path.join(dir_path, name))
gfile.GFile(os.path.join(dir_path, "subdir1_1/file2.txt"),
mode="w").write("testing")
gfile.makedirs(os.path.join(dir_path, "subdir1_2/subdir2"))
def testMultipleWrites(self):
file_path = os.path.join(self._base_dir, "temp_file")
with gfile.GFile(file_path, mode="w") as f:
f.write("line1\n")
f.write("line2")
file_contents = gfile._read_file_to_string(file_path)
self.assertEqual("line1\nline2", file_contents)
def testRead(self):
ckpt_content = 'asdfasdfasdffoobarbuzz'
temp_dir = self._CreateDeepS3Structure(ckpt_content=ckpt_content)
ckpt_path = self._PathJoin(temp_dir, 'model.ckpt')
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testReadLines(self):
ckpt_lines = ["\n"] + ["line {}\n".format(i) for i in range(10)] + [" "]
ckpt_content = "".join(ckpt_lines)
temp_dir = self._CreateDeepS3Structure(ckpt_content=ckpt_content)
ckpt_path = self._PathJoin(temp_dir, "model.ckpt")
with gfile.GFile(ckpt_path, "r") as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read_lines = list(f)
self.assertEqual(ckpt_lines, ckpt_read_lines)
def testReadLines(self):
ckpt_lines = ([u'\n'] + [u'line {}\n'.format(i)
for i in range(10)] + [u' '])
ckpt_content = u''.join(ckpt_lines)
temp_dir = self._CreateDeepS3Structure(ckpt_content=ckpt_content)
ckpt_path = self._PathJoin(temp_dir, 'model.ckpt')
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read_lines = list(f)
self.assertEqual(ckpt_lines, ckpt_read_lines)
def testWriteEmpty(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model2.ckpt')
ckpt_content = u''
with gfile.GFile(ckpt_path, 'w') as f:
f.write(ckpt_content)
with open(ckpt_path, 'r') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testWriteEmpty(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = posixpath.join(temp_dir, "model2.ckpt")
ckpt_content = ""
with gfile.GFile(ckpt_path, "w") as f:
f.write(ckpt_content)
with fsspec.open(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testWrite(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, "model2.ckpt")
ckpt_content = u"asdfasdfasdffoobarbuzz"
with gfile.GFile(ckpt_path, "w") as f:
f.write(ckpt_content)
with open(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testIsDirectory(self):
dir_path = os.path.join(self._base_dir, "test_dir")
# Failure for a non-existing dir.
self.assertFalse(gfile.isdir(dir_path))
gfile.makedirs(dir_path)
self.assertTrue(gfile.isdir(dir_path))
file_path = os.path.join(dir_path, "test_file")
gfile.GFile(file_path, mode="w").write("test")
# False for a file.
self.assertFalse(gfile.isdir(file_path))
def testReadWithOffset(self):
ckpt_content = "asdfasdfasdffoobarbuzz"
ckpt_b_content = b"asdfasdfasdffoobarbuzz"
temp_dir = self._CreateDeepS3Structure(ckpt_content=ckpt_content)
ckpt_path = self._PathJoin(temp_dir, "model.ckpt")
with gfile.GFile(ckpt_path, "r") as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read(12)
self.assertEqual("asdfasdfasdf", ckpt_read)
ckpt_read = f.read(6)
self.assertEqual("foobar", ckpt_read)
ckpt_read = f.read(1)
self.assertEqual("b", ckpt_read)
ckpt_read = f.read()
self.assertEqual("uzz", ckpt_read)
ckpt_read = f.read(1000)
self.assertEqual("", ckpt_read)
with gfile.GFile(ckpt_path, "rb") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_b_content, ckpt_read)
def testRead(self):
temp_dir = tempfile.mkdtemp(prefix=self.base_temp_dir)
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model.ckpt')
ckpt_content = 'asdfasdfasdffoobarbuzz'
with open(ckpt_path, 'w') as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testRead(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, "model.ckpt")
ckpt_content = "asdfasdfasdffoobarbuzz"
with open(ckpt_path, "w") as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, "r") as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testReadWithOffset(self):
ckpt_content = 'asdfasdfasdffoobarbuzz'
ckpt_b_content = b'asdfasdfasdffoobarbuzz'
temp_dir = self._CreateDeepS3Structure(ckpt_content=ckpt_content)
ckpt_path = self._PathJoin(temp_dir, 'model.ckpt')
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read(12)
self.assertEqual('asdfasdfasdf', ckpt_read)
ckpt_read = f.read(6)
self.assertEqual('foobar', ckpt_read)
ckpt_read = f.read(1)
self.assertEqual('b', ckpt_read)
ckpt_read = f.read()
self.assertEqual('uzz', ckpt_read)
ckpt_read = f.read(1000)
self.assertEqual('', ckpt_read)
with gfile.GFile(ckpt_path, 'rb') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_b_content, ckpt_read)
def testTextMode(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = posixpath.join(temp_dir, "model.ckpt")
# Write out newlines as given (i.e., \r\n) regardless of OS, so as to
# test translation on read.
with fsspec.open(ckpt_path, "w", newline="") as f:
data = "asdf\nasdf\nasdf\n"
f.write(data)
with gfile.GFile(ckpt_path, "r") as f:
f.buff_chunk_size = 6 # Test buffering by reducing chunk size
f.read()
def testReadLines(self):
temp_dir = tempfile.mkdtemp(prefix=self.base_temp_dir)
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model.ckpt')
ckpt_lines = ([u'\n'] + [u'line {}\n'.format(i)
for i in range(10)] + [u' '])
# Write out \n as newline even on Windows
with io.open(ckpt_path, 'w', newline='') as f:
f.write(u''.join(ckpt_lines))
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read_lines = list(f)
self.assertEqual(ckpt_lines, ckpt_read_lines)
def testWriteMultiple(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = posixpath.join(temp_dir, "model2.ckpt")
ckpt_content = "asdfasdfasdffoobarbuzz" * 5
with gfile.GFile(ckpt_path, "w") as f:
for i in range(0, len(ckpt_content), 3):
f.write(ckpt_content[i:i + 3])
# Test periodic flushing of the file
if i % 9 == 0:
f.flush()
with fsspec.open(ckpt_path, "r") as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)
def testReadWithOffset(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model.ckpt')
ckpt_content = 'asdfasdfasdffoobarbuzz'
ckpt_b_content = b'asdfasdfasdffoobarbuzz'
with open(ckpt_path, 'w') as f:
f.write(ckpt_content)
with gfile.GFile(ckpt_path, 'r') as f:
f.buff_chunk_size = 4 # Test buffering by reducing chunk size
ckpt_read = f.read(12)
self.assertEqual('asdfasdfasdf', ckpt_read)
ckpt_read = f.read(6)
self.assertEqual('foobar', ckpt_read)
ckpt_read = f.read(1)
self.assertEqual('b', ckpt_read)
ckpt_read = f.read()
self.assertEqual('uzz', ckpt_read)
ckpt_read = f.read(1000)
self.assertEqual('', ckpt_read)
with gfile.GFile(ckpt_path, 'rb') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_b_content, ckpt_read)
def testWriteMultipleBinary(self):
temp_dir = self.get_temp_dir()
self._CreateDeepDirectoryStructure(temp_dir)
ckpt_path = os.path.join(temp_dir, 'model2.ckpt')
ckpt_content = b'asdfasdfasdffoobarbuzz' * 5
with gfile.GFile(ckpt_path, 'wb') as f:
for i in range(0, len(ckpt_content), 3):
f.write(ckpt_content[i:i + 3])
# Test periodic flushing of the file
if i % 9 == 0:
f.flush()
with open(ckpt_path, 'rb') as f:
ckpt_read = f.read()
self.assertEqual(ckpt_content, ckpt_read)