def load_streams(chunks):
"""
Given a gzipped stream of data, yield streams of decompressed data.
"""
chunks = peekable(chunks)
while chunks:
dc = zlib.decompressobj(wbits=zlib.MAX_WBITS | 16)
yield load_stream(dc, chunks)
if dc.unused_data:
chunks = peekable(itertools.chain((dc.unused_data, ), chunks))
def parse_packet(binary: Iterable[Binary]) -> Packet:
version = binary_to_int(take(3, binary))
type_id = binary_to_int(take(3, binary))
if type_id == 4: # literal value
value = 0
for c in chunked(binary, 5):
for b in c[1:]:
value <<= 1
value |= b
if c[0] == 0:
break
return LiteralValuePacket(version, type_id, value)
# operator packet
length_type_id = next(binary)
children = []
if length_type_id == 0:
# next 15 bits are a number that represents the
# total length in bits of the sub-packets contained by this packet
total_length_in_bits = binary_to_int(take(15, binary))
bits = peekable(take(total_length_in_bits, binary))
while bits:
children.append(parse_packet(bits))
else:
# next 11 bits are a number that represents the
# number of sub-packets immediately contained by this packet
number_of_sub_packets = binary_to_int(take(11, binary))
for _ in range(number_of_sub_packets):
children.append(parse_packet(binary))
return OperatorPacket(version, type_id, children)
def glob_data(extension='.csv', folder=getcwd()):
"""Globs data files in a folder into an iterator
Arguments:
----------
extension: str, default='.csv'
File type of data files. Currently does not support mixed types
Should contain leading .
folder: str, default=current working directory
Path name to data. User specified paths should be delimited with
/ or \\ and should not end with a separator
Returns:
--------
files: generator containing all data files found in the folder
Example:
--------
files = globdata(extension='.txt', folder='/my/favorite/dir)
"""
if type(extension) != str:
raise TypeError('Extension must be a string')
if not extension.startswith('.'):
raise ValueError('Extension must start with "."')
if type(folder) != str:
raise TypeError('Folder must be a string')
if folder != getcwd():
if ('\\' not in folder):
raise ValueError('Path should be specified with / or \\ only')
if folder.endswith(('\\', '/')):
raise ValueError('Path should not end with a separator')
files = peekable(iglob(fr'{folder}\*{extension}'))
if files.peek('empty') == 'empty': # Returns empty if files contains no items
raise ValueError(f'No {extension} files found at {folder}')
return files