def _read_page_header(self, fo):
"""Reads the page_header from the given fo"""
tin = TFileObjectTransport(fo)
pin = TCompactProtocol(tin)
ph = PageHeader()
ph.read(pin)
return ph
def write_thrift(fobj, thrift):
"""Write binary compact representation of thiftpy structured object
Parameters
----------
fobj: open file-like object (binary mode)
thrift: thriftpy object to write
Returns
-------
Number of bytes written
"""
t0 = fobj.tell()
pout = TCompactProtocol(fobj)
try:
thrift.write(pout)
fail = False
except TProtocolException as e:
typ, val, tb = sys.exc_info()
frames = []
while tb is not None:
frames.append(tb)
tb = tb.tb_next
frame = [
tb for tb in frames if 'write_struct' in str(tb.tb_frame.f_code)
]
variables = frame[0].tb_frame.f_locals
obj = variables['obj']
name = variables['fname']
fail = True
if fail:
raise ParquetException('Thrift parameter validation failure %s'
' when writing: %s-> Field: %s' %
(val.args[0], obj, name))
return fobj.tell() - t0
def _read_footer(self, fo):
"""Reads the footer from the given file object, returning a FileMetaData
object. This method assumes that the fo references a valid parquet file"""
footer_size = self._get_footer_size(fo)
fo.seek(-(8 + footer_size), 2) # seek to beginning of footer
tin = TFileObjectTransport(fo)
pin = TCompactProtocol(tin)
fmd = FileMetaData()
fmd.read(pin)
return fmd
def read_thrift(file_obj, ttype):
"""Read a thrift structure from the given fo."""
pin = TCompactProtocol(file_obj, True)
page_header = ttype()
page_header.read(pin)
return page_header