def __init__(self,
source: IO,
modname: str,
srcname: str,
decoded: bool = False) -> None:
self.modname = modname # name of the module
self.srcname = srcname # name of the source file
# cache the source code as well
pos = source.tell()
if not decoded:
warnings.warn('decode option for ModuleAnalyzer is deprecated.',
RemovedInSphinx40Warning,
stacklevel=2)
self._encoding, _ = tokenize.detect_encoding(source.readline)
source.seek(pos)
self.code = source.read().decode(self._encoding)
else:
self._encoding = None
self.code = source.read()
# will be filled by analyze()
self.annotations = None # type: Dict[Tuple[str, str], str]
self.attr_docs = None # type: Dict[Tuple[str, str], List[str]]
self.finals = None # type: List[str]
self.overloads = None # type: Dict[str, List[Signature]]
self.tagorder = None # type: Dict[str, int]
self.tags = None # type: Dict[str, Tuple[str, int, int]]
self._analyzed = False
def _from_io(self, source: IO):
"""
Loads an existing JVM ClassFile from any file-like object.
"""
read = source.read
if unpack('>I', source.read(4))[0] != ClassFile.MAGIC:
raise ValueError('invalid magic number')
# The version is swapped on disk to (minor, major), so swap it back.
self.version = unpack('>HH', source.read(4))[::-1]
self._constants.unpack(source)
# ClassFile access_flags, see section #4.1 of the JVM specs.
self.access_flags.unpack(read(2))
# The CONSTANT_Class indexes for "this" class and its superclass.
# Interfaces are a simple list of CONSTANT_Class indexes.
self._this, self._super, interfaces_count = unpack('>HHH', read(6))
self._interfaces = unpack(
f'>{interfaces_count}H',
read(2 * interfaces_count)
)
self.fields.unpack(source)
self.methods.unpack(source)
self.attributes.unpack(source)
def _from_io(self, source: IO):
"""
Loads an existing JVM ClassFile from any file-like object.
"""
read = source.read
if unpack('>I', source.read(4))[0] != ClassFile.MAGIC:
raise ValueError('invalid magic number')
# The version is swapped on disk to (minor, major), so swap it back.
self.version = unpack('>HH', source.read(4))[::-1]
self._constants.unpack(source)
# ClassFile access_flags, see section #4.1 of the JVM specs.
self.access_flags.unpack(read(2))
# The CONSTANT_Class indexes for "this" class and its superclass.
# Interfaces are a simple list of CONSTANT_Class indexes.
self._this, self._super, interfaces_count = unpack('>HHH', read(6))
self._interfaces = unpack(f'>{interfaces_count}H',
read(2 * interfaces_count))
self.fields.unpack(source)
self.methods.unpack(source)
self.attributes.unpack(source)
def to_model(data_file: typing.IO, config: typing.Optional[STLReaderConfiguration] = None, progress_callback=lambda _: None):
"""Converts an STL document to the data model"""
m = DataFile(
data_file.read(1024),
disable_fill_line_gap=False if config is None else config.disable_fill_line_gap,
disable_line_padding=False if config is None else config.disable_line_padding,
start_tc=None if config is None else config.program_start_tc,
font_stack=None if config is None else config.font_stack,
max_row_count=None if config is None else config.max_row_count
)
for i in itertools.count():
buf = data_file.read(128)
if not buf:
break
try:
m.process_tti_block(buf)
except:
LOGGER.error("Bad TTI block")
raise
progress_callback(i/m.get_tti_count())
return m.get_document()
def __init__(self, f: IO):
"""
Init
"""
super().__init__()
self.length: int = read_u32(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Length: {self.length}")
self.reserved_1: bytes = f.read(1)
self.logger.debug(f"Reserved 1: {hex_log_str(self.reserved_1)}")
self.sub_path_type: SubPathType = SubPathType(
read_u8(f, endianess=Endianess.BIG_ENDIAN))
self.logger.debug(f"Sub Path Type: {self.sub_path_type}")
self.flags_1: bytes = f.read(2)
self.logger.debug(f"Flags 1: {hex_log_str(self.flags_1)}")
self.is_repeat_sub_path: bool = (self.flags_1[1] & 0b00000001) == 1
self.logger.debug(f"Is Repeat Sub Path: {self.is_repeat_sub_path}")
self.reserved_2: bytes = f.read(1)
self.logger.debug(f"Reserved 2: {hex_log_str(self.reserved_2)}")
self.num_sub_play_items: int = read_u8(f,
endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Num Sub Play Items: {self.length}")
self.sub_play_items: List[SubPlayItem] = []
for index in range(self.num_sub_play_items):
self.logger.debug(f"Reading Sub Play Item {index}")
self.sub_play_items.append(SubPlayItem(f))
def __init__(self, f: IO):
super().__init__(f)
#: IRD file format version
self.version: int = read_u8(f)
self.logger.info(f"IRD Version: {self.version}")
#: Game ID (ex. BLUS12354)
self.game_id: str = f.read(GAME_ID_SIZE).decode('UTF-8')
self.logger.info(f"Game ID: {self.game_id}")
#: Game name length
self.game_name_size: int = read_u8(f)
self.logger.debug(f"Game Name Size: {self.game_name_size}")
#: Game name
self.game_name: str = f.read(self.game_name_size).decode('UTF-8')
self.logger.info(f"Game Name: {self.game_name}")
#: Update version
self.update_version: str = f.read(4)
self.logger.info(f"Update Version: {self.update_version}")
#: Game version
self.game_version: str = f.read(5)
self.logger.info(f"Game Version: {self.game_version}")
#: App version
self.app_version: str = f.read(5)
self.logger.info(f"App Version: {self.app_version}")
def parse_block_info(self, fp: IO) -> None:
num_blocks, = struct.unpack(">i", b"\x00" + fp.read(3))
for i in range(num_blocks):
encoded_size, decoded_size, md5 = struct.unpack(
">ii16s", fp.read(4 + 4 + 16))
self.block_table.append(
(encoded_size, decoded_size, hexlify(md5).decode()))
def __init__(self, f: IO):
super().__init__()
self.f: IO = f
self.entirety_data_offset: int = read_u32(
f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Entirety Data Offset: {self.entirety_data_offset}")
self.entirety_data_size: int = read_u32(f,
endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Entirety Data Size: {self.entirety_data_size}")
self.flags: int = read_u16(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Flags: {hex(self.flags)}")
self.unk_1: int = read_u32(f, endianess=Endianess.BIG_ENDIAN)
constant_check(self.logger, "Unknown 1", self.unk_1, 0x00)
self.unk_2: int = read_u32(f, endianess=Endianess.BIG_ENDIAN)
self.logger.info(f'Unknown 2: {self.unk_2}')
self.unk_3: bytes = f.read(0x8)
self.logger.info(f'Unknown 3: {hexlify(self.unk_3)}')
self.sha_256_hash: bytes = f.read(0x20)
self.logger.info(f'SHA-256 Hash: {hexlify(self.sha_256_hash)}')
def read_buffer(
self, f: IO, attrs: Optional[Dict[str, Any]] = None
) -> 'core.MeshNeuron':
"""Read buffer into a MeshNeuron.
Parameters
----------
f : IO
Readable buffer - must be bytes.
attrs : dict | None
Arbitrary attributes to include in the MeshNeuron.
Returns
-------
core.MeshNeuron
"""
if not isinstance(f.read(0), bytes):
raise ValueError(f'Expected bytes, got {type(f.read(0))}')
num_vertices = np.frombuffer(f.read(4), np.uint32)[0]
vertices = np.frombuffer(f.read(int(3 * 4 * num_vertices)),
np.float32).reshape(-1, 3)
faces = np.frombuffer(f.read(),
np.uint32).reshape(-1, 3)
return core.MeshNeuron({'vertices': vertices, 'faces': faces},
**(self._make_attributes({'name': self.name_fallback,
'origin': 'DataFrame'}, attrs)))
def __init__(self, f: IO):
super().__init__(f)
self.version: BluRayVersion = BluRayVersion(f.read(4).decode("ASCII"))
self.logger.debug(f'Version: {self.version.value}')
self.sequence_info_start_address: int = read_u32(
f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(
f'Sequence Info Start Address: {self.sequence_info_start_address}')
self.program_info_start_address: int = read_u32(
f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(
f'Program Info Start Address: {self.program_info_start_address}')
self.cpi_start_address: int = read_u32(f,
endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f'CPI Start Address: {self.cpi_start_address}')
self.clip_mark_start_address: int = read_u32(
f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(
f'Clip Mark Start Address: {self.clip_mark_start_address}')
self.extension_data_start_address: int = read_u32(
f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(
f'Extension Data Start: {self.extension_data_start_address}')
self.reserved: bytes = f.read(96 // 8)
self.logger.debug(f'Reserved: {hexlify(self.reserved)}')
def handle_WAVE_fmt(self, size: int, fp: IO) -> None:
if 'WAVE/fmt' in self.__seen:
raise FormatError("Duplicate fmt chunk")
self.__seen.add('WAVE/fmt')
if size not in (16, 18, 40):
raise FormatError("Invalid fmt chunk size %d" % size)
# pylint: disable=unused-variable
wFormatTag, nChannels, nSamplesPerSec, nAvgBytesPerSec, nBlockAlign, wBitsPerSample = (
struct.unpack('<HHIIHH', fp.read(16)))
if wFormatTag == 0x0001:
self.data_format = 'pcm'
elif wFormatTag == 0x0003:
self.data_format = 'float'
elif wFormatTag == 0x0006:
self.data_format = 'a-law'
elif wFormatTag == 0x0007:
self.data_format = 'u-law'
else:
self.data_format = '0x%04X' % wFormatTag
self.channels = nChannels
self.sample_rate = nSamplesPerSec
self.bits_per_sample = wBitsPerSample
self.__bytes_per_sample = nBlockAlign
if size >= 18:
cbSize, = struct.unpack('<H', fp.read(2))
if cbSize == 22:
# pylint: disable=unused-variable
wValidBitsPerSample, dwChannelMast, SubFormat = struct.unpack(
'<HI16s', fp.read(cbSize))
def _read_file(deb: IO, file: ARRecord) -> bytes:
data: bytes = deb.read(file.size)
if file.size % 2 != 0:
deb.read(1)
return data
def __init__(self,
f: IO,
width: int,
height: int):
"""
:param f: filehandle to file storing BMP
:param width: width of image
:param height: height of image
"""
self._data = []
row = []
count_height: int = 0
count_width: int = 0
RGB_BYTES = 3
# Padding of rows
diff = (RGB_BYTES * width) % BYTE4
padding = 0
if diff:
padding = BYTE4 - diff
for _ in range(height):
row = []
for _ in range(width):
B = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
G = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
R = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
row.append([R, G, B])
count_width += 1
# row is finished, discard padding from file handle
for _ in range(padding):
f.read(1)
self._data.insert(0, row)
count_height += 1
def decompress_raw_data(raw_data_fd: IO, add_bracket: bool = True) -> bytes:
"""Decompresses raw data from file like object with zlib.
Args:
raw_data_fd: File descriptor object.
add_bracket: Whether, or not to add brackets around the output. (Default value = True)
Returns:
A byte array of the decompressed file.
"""
obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
writer = io.BytesIO()
if add_bracket:
writer.write(b'[')
d = raw_data_fd.read(CHUNK_SIZE)
while d:
writer.write(obj.decompress(d))
while obj.unused_data != b'':
unused_data = obj.unused_data
obj = zlib.decompressobj(MAGIC_NUMBER + zlib.MAX_WBITS)
writer.write(obj.decompress(unused_data))
d = raw_data_fd.read(CHUNK_SIZE)
writer.write(obj.flush())
if add_bracket:
writer.write(b']')
return writer.getvalue()
def read_cover(stream: IO, offset: int = None) -> Tuple[bytes, int]:
# offset == None : no offset
# offset >= 0 : offset
# offset < 0 : auto detect
if offset is not None:
if offset >= 0:
stream.seek(offset)
else:
_, offset = UMDFile.read_content(stream, offset=-1)
stream.seek(offset)
splitter = stream.read(1)
if not splitter:
cover = None
umd_io_logger.info(f"No cover.")
else:
next_type = struct.unpack("<h", stream.read(2))[0]
if next_type == 0x82:
b1, b2, b3, i1, b4, i2, raw_cover_length = struct.unpack(
"<bbbibii", stream.read(16))
cover = stream.read(raw_cover_length - 9)
umd_io_logger.info(f"Read cover success.")
else:
cover = None
umd_io_logger.info(f"No cover.")
return cover, stream.tell()
def iot_send_image_message(self,
chat_type: str,
chat_uin: str,
file: IO,
content: Union[str, None] = None):
image_base64 = base64.b64encode(file.read()).decode("UTF-8")
md5_sum = hashlib.md5(file.read()).hexdigest()
content = content if content else ""
if chat_type == 'private':
user_info = chat_uin.split('_')
chat_uin = int(user_info[0])
chat_origin = int(user_info[1])
self.action.sendPrivatePic(user=chat_uin,
group=chat_origin,
picBase64Buf=image_base64,
content=content)
elif chat_type == 'friend':
chat_uin = int(chat_uin)
self.action.sendFriendPic(user=chat_uin,
picBase64Buf=image_base64,
content=content)
elif chat_type == 'group':
chat_uin = int(chat_uin)
self.action.sendGroupPic(group=chat_uin,
picBase64Buf=image_base64,
content=content)
def read_midi_var_length(fp: _typing.IO):
"""Reads a length using MIDI's variable length format."""
length = 0
b = u8(fp.read(1))
while b & 0x80:
length = length * 0x80 + (b & 0x7f)
b = u8(fp.read(1))
return length * 0x80 + b
def __init__(self, f: IO):
"""
Init
"""
super().__init__()
self.length: int = read_u16(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Length: {self.length}")
self.clip_information_file_name: str = f.read(5).decode("ASCII")
self.logger.debug(
f"Clip Information File Name: {self.clip_information_file_name}")
self.clip_codec_identifier: str = f.read(4).decode("ASCII")
self.logger.debug(
f"Clip Codec Identifier: {self.clip_codec_identifier}")
self.flags_1: bytes = f.read(2)
self.logger.debug(f"Flags 1: {hex_log_str(self.flags_1)}")
self.is_multi_angle: bool = ((self.flags_1[1] & 0b00010000) >> 4) == 1
self.logger.debug(f"Is Multi Angle: {self.is_multi_angle}")
self.connection_condition: int = self.flags_1[1] & 0b00001111
self.logger.debug(f"Connection Condition: {self.connection_condition}")
self.ref_to_stc_id: int = read_u8(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Reference to STC ID: {self.ref_to_stc_id}")
self.in_time: int = read_u32(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"In Time: {self.in_time}")
self.out_time: int = read_u32(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Out Time: {self.out_time}")
self.u0_mask_table: U0MaskTable = U0MaskTable(f)
self.flags_2: bytes = f.read(1)
self.logger.debug(f"Flags 2: {hex_log_str(self.flags_2)}")
self.random_access_flag: bool = (
(self.flags_2[0] & 0b10000000) >> 7) == 1
self.logger.debug(f"Random Access Flag: {self.random_access_flag}")
self.still_mode: int = read_u8(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Still Mode: {self.still_mode}")
if self.still_mode == 0x01:
self.still_time: int = read_u16(f, endianess=Endianess.BIG_ENDIAN)
self.logger.debug(f"Still Time: {self.still_time}")
else:
self.reserved: bytes = f.read(2)
self.logger.debug(f"Reserved: {hex_log_str(self.reserved)}")
if self.is_multi_angle:
self.multi_angle_entries: MultiAngleEntries = MultiAngleEntries(f)
self.stn_table: StnTable = StnTable(f)
def write(self, data: IO, fp: IO) -> Tuple[int, str]:
encoded_data = BytesIO()
num_blocks = 0
block_table = []
if isinstance(self.spec.frame, espec.BlockTableFrame):
for block_size, repeat, subframe in self.spec.frame.frame_info:
if isinstance(subframe, espec.RawFrame):
callback = lambda d: b"N" + d # noqa
elif isinstance(subframe, espec.ZipFrame):
def callback(d: bytes) -> bytes:
compressor = zlib.compressobj(level=subframe.level,
wbits=subframe.bits)
return b"Z" + compressor.compress(
d) + compressor.flush()
else:
raise NotImplementedError
if repeat == -1:
while True:
data_block = data.read(block_size)
if not data_block:
break
num_blocks += 1
encoded_block = callback(data_block)
digest = md5(encoded_block).digest()
encoded_data.write(encoded_block)
block_table.append(
(len(encoded_block), len(data_block), digest))
else:
for i in range(repeat):
num_blocks += 1
data_block = data.read(block_size)
encoded_block = callback(data_block)
digest = md5(encoded_block).digest()
encoded_data.write(encoded_block)
block_table.append(
(len(encoded_block), len(data_block), digest))
else:
raise NotImplementedError
header = BytesIO()
header.write(b"BLTE")
subheader = BytesIO()
subheader.write(b"\x0f")
subheader.write(struct.pack(">i", num_blocks)[1:])
for block_info in block_table:
subheader.write(struct.pack(">ii16s", *block_info))
header_size = len(subheader.getvalue()) + 8
header.write(struct.pack(">i", header_size))
header.write(subheader.getvalue())
header_data = header.getvalue()
blte_key = md5(header_data).hexdigest()
return fp.write(header_data + encoded_data.getvalue()), blte_key
def scan_jpeg_structure(stream: IO, include_eoi: bool) -> List[JpegSegment]:
offset = stream.tell()
check_jpeg_signature(stream)
segment = JpegSegment.create(marker=SOI,
stream=stream,
offset=offset,
size=JpegMarker.MARKER_SIZE)
segment.log()
structure = [segment]
offset += JpegMarker.MARKER_SIZE
# scan segments
segment_marker = stream.read(JpegMarker.MARKER_SIZE)
while len(segment_marker) == JpegMarker.MARKER_SIZE:
segment_marker = endianess.convert_big_endian(segment_marker)
segment_marker = JpegMarker.detect(segment_marker)
if segment_marker == EOI:
raise RuntimeError('unexpected EOI marker before SOS marker')
segment_size = tools.read_bytes_strict(stream, JpegMarker.LENGTH_SIZE)
segment_size = endianess.convert_big_endian(
segment_size) + JpegMarker.MARKER_SIZE
segment = JpegSegment.create(marker=segment_marker,
stream=stream,
offset=offset,
size=segment_size)
segment.log()
structure.append(segment)
offset += segment_size
stream.seek(
segment_size - JpegMarker.MARKER_SIZE - JpegMarker.LENGTH_SIZE,
SEEK_CUR)
if segment_marker == SOS:
break
segment_marker = stream.read(JpegMarker.MARKER_SIZE)
if include_eoi:
eoi_offset = scan_for_eoi(stream)
if eoi_offset == 0:
raise RuntimeError('EOI is not found')
segment = JpegSegment.create(marker=EOI,
stream=stream,
offset=offset + eoi_offset,
size=JpegMarker.MARKER_SIZE)
segment.log()
structure.append(segment)
return structure
def read(cls, file: IO) -> FullStats:
"""Read a full stats response."""
type_ = Type.read(file)
session_id = BigEndianSignedInt32.read(file)
file.read(11) # Discard padding.
stats = read_stats(file)
file.read(10) # Discard padding.
players = list(read_players(file))
return cls(type_, session_id, *stats_from_dict(stats), players)
def read_file_full(file: IO) -> Generator[ByteString, None, None]:
buffer = file.read()
logger.debug("read {} bytes from file {}".format(len(buffer), file))
if not buffer:
sleep_ms(FILE_READ_DELAY_MS)
buffer = file.read()
logger.debug("read {} bytes from file {} after pause!".format(
len(buffer), file))
yield buffer
def _handle_stream(self, input_stream: IO, output_stream: IO, buffer_size: Optional[int] = None) -> str:
if buffer_size:
bytes_chunk = input_stream.read(buffer_size)
else:
bytes_chunk = input_stream.read()
chunk = '' if bytes_chunk is None else bytes_chunk.decode()
if self._print_streams:
output_stream.write(chunk)
return chunk
def iterchunks(self, file_object: IO) -> Iterator[bytes]:
data = file_object.read(self.chunk_size)
if isinstance(data, str):
while data != "":
yield cast(str, data).encode("utf-8")
data = file_object.read(self.chunk_size)
else:
while data != b"":
yield cast(bytes, data)
data = file_object.read(self.chunk_size)
def in_read(rfile: IO, content_length=None, close: bool = True):
if content_length is None:
content = rfile.read()
else:
content = rfile.read(content_length)
logging.trace("Finished reading %d bytes from input stream. ",
content_length)
if close:
rfile.close()
return content
def extract_appimage_version(basename, appimg_io: t.IO):
"""
Saves 'data' to a temporary file with the given basename, executes it (in a sandbox)
with --appimage-extract to unpack it, and scrapes the version number out of the
first .desktop file it finds.
"""
with tempfile.TemporaryDirectory() as tmpdir:
appimage_path = os.path.join(tmpdir, basename)
header = ELFFile(appimg_io).header
offset = header["e_shoff"] + header["e_shnum"] * header["e_shentsize"]
appimg_io.seek(offset)
log.info("Writing %s to %s with offset %i", basename, appimage_path,
offset)
with open(appimage_path, "wb") as fp:
chunk = appimg_io.read(1024**2)
while chunk:
fp.write(chunk)
chunk = appimg_io.read(1024**2)
bwrap = check_bwrap()
bwrap_args = [
"--bind",
tmpdir,
tmpdir,
"--die-with-parent",
"--new-session",
]
args = ["unsquashfs", "-no-progress", appimage_path]
log.debug("$ %s", " ".join(args))
p = run_command(
args,
cwd=tmpdir,
env=clear_env(os.environ),
bwrap=bwrap,
bwrap_args=bwrap_args,
)
if p.returncode != 0:
log.error("AppImage extraction failed\n%s", p.stderr)
p.check_returncode()
for desktop in glob.glob(
os.path.join(tmpdir, "squashfs-root", "*.desktop")):
kf = GLib.KeyFile()
kf.load_from_file(desktop, GLib.KeyFileFlags.NONE)
return kf.get_string(GLib.KEY_FILE_DESKTOP_GROUP,
"X-AppImage-Version")
def __init__(self, f: IO):
"""
:attr _red: red intensity
:attr _green: green intensity
:attr _blue: blue intensity
:attr _reserved: unused (=0)
"""
self._red: int = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
self._green: int = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
self._blue: int = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
self._reserved: int = self.unpack(f.read(BYTE1), L_ENDIAN, UCHAR)
pass
def fread(io: IO, typ: str, byteorder=sys.byteorder):
"""
Read from (buffered) IO reader in the c type typ, eg
"""
if typ == "bool":
return bool.from_bytes(io.read(1), byteorder)
elif "int" in typ:
signed = typ[0] != "u"
size = sizeof_int(typ)
return int.from_bytes(io.read(size), sys.byteorder, signed=signed)
else:
raise NotImplementedError(f"Don't know how to read '{typ}'")
def read(cls, file: IO) -> Packet:
"""Reads a packet from a file-like object."""
size = LittleEndianSignedInt32.read(file)
id_ = LittleEndianSignedInt32.read(file)
type_ = Type.read(file)
payload = file.read(size - 10)
terminator = file.read(2)
if terminator != TERMINATOR:
LOGGER.warning('Unexpected terminator: %s', terminator)
return cls(id_, type_, payload, terminator)
def read(cls, file: IO) -> Packet:
"""Reads a packet from a file-like object."""
header = Header.read(file)
size = file.read(4)
if header.big_endian:
size = int.from_bytes(size, 'big', signed=False)
else:
size = int.from_bytes(size, 'little', signed=False)
payload = file.read(size)
return cls(header, payload)
def encode(input: IO, output: IO) -> None:
"""Encode a file; input and output are binary files."""
while True:
s = input.read(MAXBINSIZE)
if not s:
break
while len(s) < MAXBINSIZE:
ns = input.read(MAXBINSIZE - len(s))
if not ns:
break
s += ns
line = binascii.b2a_base64(s)
output.write(line)
def encode(input: IO, output: IO) -> None:
"""Encode a file; input and output are binary files."""
while True:
s = input.read(MAXBINSIZE)
if not s:
break
while len(s) < MAXBINSIZE:
ns = input.read(MAXBINSIZE-len(s))
if not ns:
break
s += ns
line = binascii.b2a_base64(s)
output.write(line)
def unpack(self, source: IO):
"""
Read the Field from the file-like object `fio`.
.. note::
Advanced usage only. You will typically never need to call this
method as it will be called for you when loading a ClassFile.
:param source: Any file-like object providing `read()`
"""
self.access_flags.unpack(source.read(2))
self._name_index, self._descriptor_index = unpack('>HH', source.read(4))
self.attributes.unpack(source)
def unpack(self, source: IO):
"""
Read the ConstantPool from the file-like object `source`.
.. note::
Advanced usage only. You will typically never need to call this
method as it will be called for you when loading a ClassFile.
:param source: Any file-like object providing `read()`
"""
count = unpack('>H', source.read(2))[0]
for _ in repeat(None, count):
name_index, length = unpack('>HI', source.read(6))
info_blob = source.read(length)
self._table.append((name_index, info_blob))
def html_table_to_csv(input_f: IO, output_f: IO, table_num: int) -> None:
doc = bs4.BeautifulSoup(input_f.read(), 'html5lib')
tables = doc.find_all('table')
try:
table = tables[table_num]
trows = table.find_all('tr')
csv_writer = csv.writer(output_f)
for trow in trows:
cells = trow.find_all(RX_TH_OR_TD)
csv_writer.writerow([cell.text.strip() for cell in cells])
except IndexError:
sys.stderr.write('ERROR: no table at index {}\n'.format(table_num))
sys.exit(1)
def get_length(stream: IO) -> int:
"""Gets the number of bytes in the stream."""
old_position = stream.tell()
stream.seek(0)
length = 0
try:
while True:
r = stream.read(1024)
if not r:
break
length += len(r)
finally:
stream.seek(old_position)
return length
def unpack(self, source: IO):
"""
Read the FieldTable from the file-like object `source`.
.. note::
Advanced usage only. You will typically never need to call this
method as it will be called for you when loading a ClassFile.
:param source: Any file-like object providing `read()`
"""
field_count = unpack('>H', source.read(2))[0]
for _ in repeat(None, field_count):
field = Field(self._cf)
field.unpack(source)
self.append(field)
def parse_html(self, fh: IO) -> Dict[str, Any]:
'''Return head and content elements of the document.'''
capsule = html_parser.parse(fh.read(), maybe_xhtml=True)
doc = etree.adopt_external_document(capsule).getroot()
result = {}
result['head'] = doc.cssselect('head')[0]
for candidate in ('.main-column .section', '.main__content'):
elements = doc.cssselect(candidate)
if elements:
result['main_content'] = elements[0]
break
if 'main_content' not in result:
raise ValueError('No main content element found')
return result
def iter_ceph_ops(fd: IO):
data = fd.read()
offset = 0
while offset < len(data):
op, offset = CephOp.unpack(data, offset)
yield op
