def check_decompression(self, compress_ext, compress_func):
test_text = 'decompression\ntest\n'
test_data = compress_func(test_text)
stream = tutil.MockStreamReader('.', test_data)
reader = StreamFileReader(stream, [Range(0, 0, len(test_data))],
'test.' + compress_ext)
self.assertEqual(test_text, ''.join(reader.readall_decompressed()))
def make_count_reader(self, nocache=False):
stream = []
n = 0
blocks = {}
for d in ['01234', '34567', '67890']:
loc = '{}+{}'.format(hashlib.md5(d).hexdigest(), len(d))
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
af = ArvadosFile(
ArvadosFileReaderTestCase.MockParent(blocks, nocache),
stream=stream,
segments=[Range(1, 0, 3),
Range(6, 3, 3),
Range(11, 6, 3)])
return ArvadosFileReader(af, "count.txt")
def make_count_reader(self, nocache=False):
stream = []
n = 0
blocks = {}
for d in [b'01234', b'34567', b'67890']:
loc = tutil.str_keep_locator(d)
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
af = ArvadosFile(
ArvadosFileReaderTestCase.MockParent(blocks, nocache),
"count.txt",
stream=stream,
segments=[Range(1, 0, 3),
Range(6, 3, 3),
Range(11, 6, 3)])
return ArvadosFileReader(af, mode="rb")
def reader_for(self, coll_name, **kwargs):
stream = []
segments = []
n = 0
for d in self.manifest_for(coll_name).split():
try:
k = KeepLocator(d)
segments.append(Range(n, n, k.size))
stream.append(Range(d, n, k.size))
n += k.size
except ValueError:
pass
blockmanager = arvados.arvfile._BlockManager(self.keep_client())
blockmanager.prefetch_enabled = False
col = Collection(keep_client=self.keep_client(),
block_manager=blockmanager)
af = ArvadosFile(col, "test", stream=stream, segments=segments)
return ArvadosFileReader(af, **kwargs)
def make_count_reader(self):
stream = tutil.MockStreamReader('.', '01234', '34567', '67890')
return StreamFileReader(
stream,
[Range(1, 0, 3), Range(6, 3, 3),
Range(11, 6, 3)], 'count.txt')
def check_decompressed_name(self, filename, expect):
stream = tutil.MockStreamReader('.', '')
reader = StreamFileReader(stream, [Range(0, 0, 0)], filename)
self.assertEqual(expect, reader.decompressed_name())
def test_name_attribute(self):
# Test both .name and .name() (for backward compatibility)
stream = tutil.MockStreamReader()
sfile = StreamFileReader(stream, [Range(0, 0, 0)], 'nametest')
self.assertEqual('nametest', sfile.name)
self.assertEqual('nametest', sfile.name())
def make_newlines_reader(self):
stream = tutil.MockStreamReader('.', 'one\ntwo\n\nth', 'ree\nfour\n\n')
return StreamFileReader(
stream, [Range(0, 0, 11), Range(11, 11, 10)], 'count.txt')