def test_gettable_file_from_disk_file(self):
data = self.data
# create temporary file
fd, path = tempfile.mkstemp()
os.write(fd, data)
os.close(fd)
# test GettableFile from file on disk (read access)
with io.open(path, 'rb') as fp:
gf = GettableFile(fp)
self.assertEqual(len(gf), len(data))
self.verify_getitem(gf, data)
self.verify_getinto(gf, data)
# works also for read+write access
with io.open(path, 'r+b') as fp:
gf = GettableFile(fp)
self.assertEqual(len(gf), len(data))
self.verify_getitem(gf, data)
self.verify_getinto(gf, data)
# fail without read access
with io.open(path, 'wb') as fp:
with self.assertRaises(TypeError):
GettableFile(fp)
# remove temp file
os.unlink(path)
def test_file_interface(self):
data = self.data
size = len(data)
fp = io.BytesIO(data)
gf = GettableFile(fp)
fv = FileView(gf)
# partial read
self.assertEqual(fv.read(1), data[0:1])
# read all, also check continuity
self.assertEqual(fv.read(), data[1:])
# seek and tell
self.assertEqual(len(fv), size)
self.assertEqual(fv.seek(2), 2)
self.assertEqual(fv.tell(), 2)
self.assertEqual(fv.seek(2, io.SEEK_CUR), 4)
self.assertEqual(fv.tell(), 4)
self.assertEqual(fv.seek(0, io.SEEK_END), size)
self.assertEqual(fv.tell(), size)
# IOBase derived methods
fv.seek(0)
self.assertEqual(fv.readlines(), [data])
def test_gettable_file_from_memory_bytes(self):
data = self.data
fp = io.BytesIO(data)
gf = GettableFile(fp)
self.assertEqual(len(gf), len(data))
self.verify_getitem(gf, data)
self.verify_getinto(gf, data)
def test_gettable_file_from_file_like(self):
data = self.data
# create file-like temporary object (on POSIX this is a tmp file)
with tempfile.TemporaryFile() as fp:
fp.write(data)
fp.seek(0)
gf = GettableFile(fp, strict=False)
self.assertEqual(len(gf), len(data))
self.verify_getitem(gf, data)
self.verify_getinto(gf, data)
def test_view_interface(self):
data = self.data
size = len(data)
fp = io.BytesIO(data)
gf = GettableFile(fp)
fv = FileView(gf)
# view, slice index
subfv = fv[1:-1]
self.assertEqual(subfv.read(), data[1:-1])
self.assertEqual(len(subfv), size - 2)
# view, integer index
self.assertEqual(fv[2], data[2])
# view, out of bounds index
with self.assertRaises(IndexError):
fv[size]
# view are independent
self.assertEqual(fv[:2].read(), data[:2])
self.assertEqual(fv[-2:].read(), data[-2:])
def test_gettable_file_critical_section_respected(self):
# setup a shared file view
data = self.data
fp = io.BytesIO(data)
gf = GettableFile(fp)
# file slices
slice_a = slice(0, 7)
slice_b = slice(3, 5)
# add a noticeable sleep inside the critical section (on `file.seek`),
# resulting in minimal runtime equal to (N runs * sleep in crit sect)
sleep = 0.25
def blocking_seek(start):
time.sleep(sleep)
return io.BytesIO.seek(gf._fp, start)
gf._fp.seek = blocking_seek
# define the worker
def worker(slice_):
return gf[slice_]
# run the worker a few times in parallel
executor = ThreadPoolExecutor(max_workers=3)
slices = [slice_a, slice_b, slice_a]
futures = [executor.submit(worker, s) for s in slices]
with tictoc() as timer:
wait(futures)
# verify results
results = [f.result() for f in futures]
expected = [data[s] for s in slices]
self.assertEqual(results, expected)
# verify runtime is consistent with a blocking critical section
self.assertGreaterEqual(timer.dt, 0.9 * len(results) * sleep)
def test_gettable_file_from_memory_string(self):
data = self.data.decode()
fp = io.StringIO(data)
with self.assertRaises(TypeError):
GettableFile(fp)