def read(name): """Read given file and split it in normalized chunks. :param name: Path to the file to split in chunks. :type name: python:str :returns: A three items tuple with the chunk start index, the chunk length and the chunk data. :rtype: ~typing.Tuple[python:int, python:int, python:bytes] """ fp_idx = 0 # Current position in the file. remain = bytearray() # Data remaining from the chunking process. buffer = bytearray(c.GHASH_CHUNK_HI) # Buffer to keep file data. with open(name, 'rb') as fp: while True: # Load up to :data:`~kado.constants.GHASH_CHUNK_HI` file data into # the buffer. read_bytes = fp.readinto(buffer) if read_bytes == 0: # No more data? We're getting out of here. break # We prepend remaining data from previous iteration before chopping. buffer = remain + buffer # Keep in mind read data can be lower than actual buffer size. bf_end = read_bytes + len(remain) # :func:`~kado.utils.ghash.chop` will cut all given data, however # the last chunk may not be the end of the file and we may still # have some more data to read. # # Therefore, we keep the last chunk aside looking for a potential # bigger chunk. ck_idx = 0 for ck_start, ck_end, ck_data in iterator.onexlast( chop(buffer[:bf_end])): yield fp_idx + ck_start, fp_idx + ck_end, bytes(ck_data) ck_idx = ck_end # Saving the last chunk of data for the next iteration. remain = buffer[ck_idx:bf_end] buffer = bytearray(c.GHASH_CHUNK_HI - len(remain)) # Raising our file index. fp_idx += ck_idx # We're done reading the file, chopping remaining data. if remain: for ck_start, ck_end, ck_data in chop(remain): yield fp_idx + ck_start, fp_idx + ck_end, bytes(ck_data)
def test_onexlast_string_empty(self): """Given an empty string no items should be returned.""" TEST_DATA = '' EXPECTED = '' self.assertEqual(''.join(iterator.onexlast(TEST_DATA)), EXPECTED)
def test_onexlast_list_two(self): """A two item list should return one item.""" TEST_DATA = [1, 2] EXPECTED = [1] self.assertEqual(list(iterator.onexlast(TEST_DATA)), EXPECTED)
def test_onexlast_list_empty(self): """Given an empty list no items should be returned.""" TEST_DATA = [] EXPECTED = [] self.assertEqual(list(iterator.onexlast(TEST_DATA)), EXPECTED)
def test_onexlast_string_two(self): """A two item string should return one item.""" TEST_DATA = '12' EXPECTED = '1' self.assertEqual(''.join(iterator.onexlast(TEST_DATA)), EXPECTED)
def test_onexlast_string_one(self): """A one character strin should return no item.""" TEST_DATA = '1' EXPECTED = '1' self.assertEqual(''.join(iterator.onexlast(TEST_DATA)), EXPECTED)