def convDataToPointer(self, wav_data): tmpBytes = BytesIO() f_size = tmpBytes.write(wav_data) array = (c_byte * f_size)() tmpBytes.seek(0, 0) tmpBytes.readinto(array) return array
def __init__(self, data=None): super(UserConfiguration, self).__init__() if data is not None: buf = BytesIO(data) buf.readinto(self) self.calculated_checksum = self.generate_checksum(data) else: self.sync = 165 self.id = 0 self.size = 256
def get_player(player_num=0): ''' Return the given player struct. player_num -- player number in range [0, 4) ''' assert 0 <= player_num and player_num < players_n addr = get_elem_addr(players_addr, player_size, player_num, players_n) player_buf = get_data(addr, player_size) player = ffi.new('Player *') stream = BytesIO(player_buf) stream.readinto(ffi.buffer(player)) return player
def get_item(item_num=0): ''' Return the given item struct. item_num -- item number in range [0, 4) ''' assert 0 <= item_num and item_num < items_n addr = get_elem_addr(items_addr, item_size, item_num, items_n) item_buf = get_data(addr, item_size) item = ffi.new('Item *') stream = BytesIO(item_buf) stream.readinto(ffi.buffer(item)) return item
def display(self): """Method to display on the screen what the camera 'sees'""" while True: # self.camera.resolution = (1024, 768) """Buffer for screen color data""" rgb = bytearray(WIDTH * HEIGHT * 3) stream = BytesIO() self.camera.capture(stream, use_video_port=True, format='rgb') stream.seek(0) stream.readinto(rgb) stream.close() image = pygame.image.frombuffer(rgb[0:(WIDTH * HEIGHT * 3)], (WIDTH, HEIGHT), 'RGB') self.screen.blit(image, (0, 0)) pygame.display.update()
def test_invoke_success_by_stepfunctions(aws_lambda_endpoint_url: str, my_case: MyCase) -> None: """ AWS StepFunctions Local uses `Transfer-Encoding: chunked` instead of a fixed `Content-Length` to transfer the payload. """ parsed_result: ParseResult = urlparse(aws_lambda_endpoint_url) assert parsed_result.hostname is not None conn = http.client.HTTPConnection(parsed_result.hostname, port=parsed_result.port) conn.connect() conn.putrequest( "POST", f"/2015-03-31/functions/{my_case.function_name}/invocations") conn.putheader("Content-Type", "application/json") conn.putheader("Transfer-Encoding", "chunked") conn.endheaders() payload = BytesIO(json.dumps(my_case.payload).encode()) buffer = bytearray(5) while True: length: int = payload.readinto(buffer) conn.send(b"%x\r\n" % length) if length > 0: conn.send(bytes(buffer[:length]) + b"\r\n") else: break response: http.client.HTTPResponse = conn.getresponse() assert 200 == response.status assert my_case.expected_payload == json.load(response) conn.close()
class FakeCGIBody(io.RawIOBase): def __init__(self, vars, content_type): if content_type.startswith('multipart/form-data'): if not _get_multipart_boundary(content_type): raise ValueError('Content-type: %r does not contain boundary' % content_type) self.vars = vars self.content_type = content_type self.file = None def __repr__(self): inner = repr(self.vars) if len(inner) > 20: inner = inner[:15] + '...' + inner[-5:] return '<%s at 0x%x viewing %s>' % ( self.__class__.__name__, abs(id(self)), inner) def fileno(self): return None @staticmethod def readable(): return True def readinto(self, buff): if self.file is None: if self.content_type.startswith('application/x-www-form-urlencoded'): data = urllib.urlencode(self.vars.items()) elif self.content_type.startswith('multipart/form-data'): data = _encode_multipart(self.vars.iteritems(), self.content_type)[1] else: assert 0, ('Bad content type: %r' % self.content_type) self.file = BytesIO(data) return self.file.readinto(buff)
def main(): host = '0.pool.ntp.org' host = 'ntp1.stratum1.ru' port = 123 # connect to server s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.settimeout(5) # prepare packet ntpv4_header = NTPv4Header() ntpv4_header.leap_indicator = 0 ntpv4_header.version_number = 4 ntpv4_header.mode = 3 # client # leave all other fields empty # get packet bytes ntpv4_header_bytes = BytesIO() ntpv4_header_bytes.write(ntpv4_header) ntpv4_header_bytes.seek(0) payload = ntpv4_header_bytes.read() # send payload s.sendto(payload, (host, port)) response, address = s.recvfrom(256) # read response bytes into NTPv4 header bytes_obj = BytesIO(response) bytes_obj.readinto(ntpv4_header) print('=' * 80) print('NTPv4 header') print('=' * 80) # print NTPv4 header print_ntpv4_header(ntpv4_header) # get time JAN_1970 = 2208988800 # 1970 - 1900 in seconds secs = ((ntpv4_header.transmit_timestamp >> 32) & 0xffffffff) - JAN_1970 print('=' * 80) print('NTP secs: {}'.format(secs)) print('NTP time: {:02d}:{:02d}:{:02d} (UTC)'.format( (secs % 86400) // 3600, (secs % 3600) // 60, (secs % 60))) print('Local date: {}'.format(datetime.datetime.fromtimestamp(secs))) print('=' * 80)
class MockSerial(RawIOBase): def __init__(self, timeout=None): self._read_buf = BytesIO() self._write_buf = BytesIO() self.timeout = timeout self.test_closed = False self.test_input_flushed = False self.test_read_cancelled = False def readable(self) -> bool: return True def writable(self) -> bool: return True def close(self) -> None: super().close() self.test_closed = True def readinto(self, b: bytearray) -> Optional[int]: return self._read_buf.readinto(b) def write(self, b: Union[bytes, bytearray]) -> Optional[int]: return self._write_buf.write(b) def reset_input_buffer(self): self.test_input_flushed = True self._read_buf.seek(0, SEEK_END) def cancel_read(self): self.test_read_cancelled = True def test_put_read_data(self, data: bytes) -> None: cur_pos = self._read_buf.tell() self._read_buf.seek(0, SEEK_END) self._read_buf.write(data) self._read_buf.seek(cur_pos, SEEK_SET) def test_get_remaining_read_data(self) -> bytes: return self._read_buf.read() def test_clear_read_buffer(self) -> None: self._read_buf = BytesIO() def test_get_write_data(self) -> bytes: self._write_buf.seek(0, SEEK_SET) data = self._write_buf.read() self._write_buf = BytesIO() return data
def _split_color_alpha(self, png): bytedepth = png.bitdepth // 8 num_color_bytes = png.color_planes * bytedepth idat = BytesIO() for idat_chunk in png.idatdecomp(): idat.write(idat_chunk) row_num_bytes = 1 + (png.color_planes + 1) * bytedepth * png.width pixel_color_fmt = '{}B{}x'.format(num_color_bytes, bytedepth) pixel_alpha_fmt = '{}x{}B'.format(num_color_bytes, bytedepth) row_color_struct = Struct('B' + pixel_color_fmt * png.width) row_alpha_struct = Struct('B' + pixel_alpha_fmt * png.width) idat.seek(0) row_bytes = bytearray(row_num_bytes) for i in range(png.height): idat.readinto(row_bytes) color_values = row_color_struct.unpack(row_bytes) alpha_values = row_alpha_struct.unpack(row_bytes) yield bytes(color_values), bytes(alpha_values) assert idat.read() == b''
def startup(): with datafile('VSWAP') as handle: _, chunks_in_file = readctype(handle, ctypes.c_ushort) _, pm_sprite_start = readctype(handle, ctypes.c_ushort) _, pm_sound_start = readctype(handle, ctypes.c_ushort) t_page_offsets = ctypes.c_uint32 * (chunks_in_file + 1) _, page_offsets = readctype(handle, t_page_offsets) t_page_lengths = ctypes.c_ushort * chunks_in_file _, page_lengths = readctype(handle, t_page_lengths) # load textures, sprites and sounds into memory # all fit in memory so we skip the paging mechanism for i in range(chunks_in_file): if not page_offsets[i]: # sparse page continue # Use specified page length, when next page is sparse page. # Otherwise, calculate size from the offset difference between this and the next page. size = page_offsets[i + 1] - page_offsets[i] if not page_offsets[i + 1]: size = page_lengths[i] handle.seek(page_offsets[i]) value = handle.read(size) if i < pm_sprite_start: state.textures.append(value) elif i < pm_sound_start: # for sprites we parse the CompShape struct as well bio = BytesIO(value) comp_shape = CompShape() bio.readinto(comp_shape) state.sprites.append((comp_shape, value)) else: value = handle.read(size) state.sounds.append(value)
def ReceiveObservationAsync(self, channel, data): """Receive the observation from sensors. This function is called once per step. The observations are only updated when this function is called. """ del channel # unused stream = BytesIO(data) state = comm.LowState() stream.readinto(state) # pytype: disable=wrong-arg-types self._last_raw_state = self._raw_state self._raw_state = state # Convert quaternion from wxyz to xyzw, which is default for Pybullet. q = state.imu.quaternion self._base_orientation = np.array([q[1], q[2], q[3], q[0]]) self._motor_angles = np.array([motor.q for motor in state.motorState[:12]]) self._motor_velocities = np.array( [motor.dq for motor in state.motorState[:12]]) self._joint_states = np.array( list(zip(self._motor_angles, self._motor_velocities))) if self._init_complete: self._SetMotorAnglesInSim(self._motor_angles, self._motor_velocities) self._velocity_estimator.update(self._raw_state)
def setup_graphics_file(): with datafile('VGADICT') as handle: node = HuffNode() while handle.readinto(node): state.grhuffman.append(node) node = HuffNode() with datafile('VGAHEAD') as handle: data = handle.read() grstarts = [] di = 0 for i in range(gfx.NUMCHUNKS + 1): val = data[di] | data[di + 1] << 8 | data[ di + 2] << 16; grstarts.append(-1 if val == 0x00FFFFFF else val) di += 3 with datafile('VGAGRAPH') as grhandle: # load the pic and sprite headers into the arrays in the data segment grhandle.seek(grstarts[0]) chunkexplen = readctype(grhandle) chunkcomplen = grstarts[1] - grstarts[0] - 4 source = grhandle.read(chunkcomplen) pictable_bytes = huff_expand(source, gfx.NUMPICS * ctypes.sizeof(Picture)) pictable_bytes = BytesIO(pictable_bytes) pic = Picture() while pictable_bytes.readinto(pic): state.pictable.append(pic) pic = Picture() # instead of loading graphics on demand, load them all on startup for chunk in range(gfx.NUMCHUNKS): if chunk == 135: # this one is not defined in gfx continue grseg = load_graphic(grhandle, grstarts, chunk) state.grsegs.append(grseg)
class FakeCGIBody(io.RawIOBase): def __init__(self, vars, content_type): if content_type.startswith('multipart/form-data'): if not _get_multipart_boundary(content_type): raise ValueError('Content-type: %r does not contain boundary' % content_type) self.vars = vars self.content_type = content_type self.file = None def __repr__(self): inner = repr(self.vars) if len(inner) > 20: inner = inner[:15] + '...' + inner[-5:] return '<%s at 0x%x viewing %s>' % (self.__class__.__name__, abs(id(self)), inner) def fileno(self): return None @staticmethod def readable(): return True def readinto(self, buff): if self.file is None: if self.content_type.startswith( 'application/x-www-form-urlencoded'): data = urllib.urlencode(self.vars.items()) elif self.content_type.startswith('multipart/form-data'): data = _encode_multipart(self.vars.iteritems(), self.content_type)[1] else: assert 0, ('Bad content type: %r' % self.content_type) self.file = BytesIO(data) return self.file.readinto(buff)
class BinaryStreamBase: def __init__(self, client, buf=None): self.client = client self.stream = BytesIO(buf) if buf else BytesIO() self._buffer = None @property def compact_footer(self) -> bool: return self.client.compact_footer @compact_footer.setter def compact_footer(self, value: bool): self.client.compact_footer = value def read(self, size): buf = bytearray(size) self.stream.readinto(buf) return buf def read_ctype(self, ctype_class, position=None, direction=READ_FORWARD): ctype_len = ctypes.sizeof(ctype_class) if position is not None and position >= 0: init_position = position else: init_position = self.tell() if direction == READ_FORWARD: start, end = init_position, init_position + ctype_len else: start, end = init_position - ctype_len, init_position with self.getbuffer()[start:end] as buf: return ctype_class.from_buffer_copy(buf) def write(self, buf): self._release_buffer() return self.stream.write(buf) def tell(self): return self.stream.tell() def seek(self, *args, **kwargs): return self.stream.seek(*args, **kwargs) def getbuffer(self): if self._buffer: return self._buffer self._buffer = self.stream.getbuffer() return self._buffer def getvalue(self): return self.stream.getvalue() def slice(self, start=-1, offset=0): start = start if start >= 0 else self.tell() with self.getbuffer()[start:start + offset] as buf: return bytes(buf) def hashcode(self, start, bytes_len): with self.getbuffer()[start:start + bytes_len] as buf: return ignite_utils.hashcode(buf) def _release_buffer(self): if self._buffer: self._buffer.release() self._buffer = None def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self._release_buffer() self.stream.close()
def unserialize(self, data: BytesIO): data.readinto(self)
class KLVParser(object): """Return key, value pairs parsed from an SMPTE ST 336 source.""" def __init__(self, source, key_length): if isinstance(source, IOBase): self.source = source else: self.source = BytesIO(source) self.key_length = key_length def __iter__(self): return self def __next__(self): key = self.__align_to_key() byte_length = bytes_to_int(self.__read(1)) if byte_length < 128: # BER Short Form length = byte_length else: # BER Long Form length = bytes_to_int(self.__read(byte_length - 128)) value = self.__read(length) return key, value def __align_to_key(self): b = bytearray(self.key_length) n = self.source.readinto(b) if n == 0 or n < len(b): raise StopIteration while True: idx = b.find(LV_KEY_HEADER) if idx == 0: return bytes(b) if idx == -1: # not found but could be across boundaries # expand the buffer. keep the last 3 bytes of the buffer len_keep_buffer = 4 - 1 b[-len_keep_buffer:] += self.__read(16 - 3) continue # if idx != 0 && idx != -1 # we need to read more data available = n - idx missing = 16 - available bb = bytearray(missing) nn = self.source.readinto(bb) if nn != missing: # Data stream terminated raise StopIteration key = b[idx:] + b[:nn] return bytes(key) def __read(self, size): if size == 0: return b'' assert size > 0, size assert size < sys.maxsize, size data = self.source.read(size) if data: return data else: raise StopIteration
class VerifiableStream(BinaryIO): """A binary stream whose contents can be verified to not have changed. The stream does not accept a HMAC key, but generates it randomly as a nonce. While unusual, this is intentional -- these streams are meant to be used as part of model serialization, where their nonces and HMAC codes are stored in a cryptographically signed metadata file. In other words, the HMAC simply ensures that stream's data has not changed, and does not guarantee the data's origin -- that's the metadata signature's job. The stream is meant to be used in the following sequence: - instantiate the stream - write all data to the stream (the stream is not readable yet!) - call "finalize()" on the stream, saving the returned nonce and HMAC code - read data from the stream (the stream is not writable any more!) """ def __init__(self): """Create a new VerifiableStream with a random nonce.""" self._finalized = False self._random_nonce = os.urandom( 16) # this is bytes, be careful trying to add strings to it self._underlying_stream = BytesIO() self._hmac_state = hmac.new(self._random_nonce, digestmod=HASHER) def _ensure_finalized(self): """Raise an error if the stream has not already been finalized.""" if not self._finalized: raise AssertionError( "Expected the stream to be finalized, but it was not!") def _ensure_not_finalized(self): """Raise an error if the stream has already been finalized.""" if self._finalized: raise AssertionError( "Expected the stream to not be finalized, but it was!") def finalize(self): """Calculate the HMAC code for the stream, disable writing and enable reading. Returns: tuple (nonce, HMAC code) (both of type string) """ self._ensure_not_finalized() self._finalized = True nonce_string = _convert_base64_bytes_to_string(self._random_nonce) hmac_string = _convert_base64_bytes_to_string( self._hmac_state.digest()) return nonce_string, hmac_string # methods for writing require that the stream not be finalized def writable(self) -> bool: """Return True if the stream is writable, and False otherwise.""" if self._finalized: return False else: return self._underlying_stream.writable() @validate(b=bytes) def write(self, b: bytes) -> int: """Write the given binary data to the stream, and include it in the HMAC calculation.""" self._ensure_not_finalized() num_bytes = self._underlying_stream.write(b) self._hmac_state.update(b) return num_bytes def writelines(self, lines: Iterable[bytes]) -> None: """Write lines to a stream""" self._ensure_not_finalized( ) # technically done by `write` but doesn't hurt to be safe for line in lines: self.write(line) return None # methods for reading require that the stream is finalized def readable(self) -> bool: """Return True if the stream is readable, and False otherwise.""" if self._finalized: return self._underlying_stream.readable() else: return False def read(self, size=None) -> bytes: """Read bytes from stream""" self._ensure_finalized() return self._underlying_stream.read(size) def readall(self) -> bytes: """Read lines from stream""" raise NotImplementedError( "`VerifiablStream` does not implement `readall` since the underlying BtytesIO does not " "implement it.") def readline(self, size=None) -> bytes: """Read a line from stream""" self._ensure_finalized() return self._underlying_stream.readline(size) def readlines(self, size=None) -> List[bytes]: """Read lines from stream""" self._ensure_finalized() return self._underlying_stream.readlines(size) def read1(self, size) -> bytes: """Read bytes from stream""" self._ensure_finalized() return self._underlying_stream.read1(size) def readinto(self, b) -> Optional[int]: """Read bytes into another buffer""" self._ensure_finalized() return self._underlying_stream.readinto(b) def readinto1(self, b) -> Optional[int]: """Read bytes into another buffer""" self._ensure_finalized() return self._underlying_stream.readinto1(b) # seeking requires a finalized stream def seekable(self): """Return True if the read pointer in the stream can be moved, and False otherwise.""" if self._finalized: return self._underlying_stream.seekable() else: return False def seek(self, *args, **kwargs) -> int: """Seek to a new position. Return the new position""" self._ensure_finalized() return self._underlying_stream.seek(*args, **kwargs) def truncate(self, size: Optional[int] = ...) -> None: """Truncate the stream""" raise NotImplementedError( "`VerifiableStream` does not support truncation. It is too " "complicated to keep track of the hmac digests") def close(self): """Close the stream, discarding its data. Will raise an error if not finalized yet.""" if self._finalized: return self._underlying_stream.close() else: raise AssertionError( "Attempting to close an unfinalized VerifiableStream. This is " "almost certainly a bug.") # a bunch of attributes/methods that are always accessible def isatty(self) -> bool: """Determine whether this is a terminal""" return self._underlying_stream.isatty() @property def closed(self) -> bool: """Determine whether the stream is closed""" return self._underlying_stream.closed def fileno(self) -> int: """Return the underlying file descriptor""" # this will technically raise UnsuportedOperation, but better to let BytesIO do that return self._underlying_stream.fileno() def mode(self) -> str: """Return the underlying file descriptor""" # this doesn't exist for the underlying stream raise AssertionError( "`VerifiableStream` does not have a mode. This is probably a bug in " "something assuming that the stream is a backed by a file") def name(self) -> str: """Return the underlying file descriptor""" # this doesn't exist for the underlying stream raise AssertionError( "`VerifiableStream` does not have a name. This is probably a bug in " "something assuming the stream is a file descriptor") def flush(self) -> None: """Flush the underlying stream""" # this technically does nothing in BytesIO return self._underlying_stream.flush() def tell(self) -> int: """Tell the current position""" return self._underlying_stream.tell() # context manager methods def __enter__(self) -> "VerifiableStream": """Enter""" return self def __exit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> bool: """Exit""" return self._underlying_stream.__exit__(exc_type, exc_val, exc_tb)
def decode_from_binary(cls, data : bytes): self = cls() buf = BytesIO(data) buf.seek(8) checksummable_data = buf.read() buf.seek(0) # Read and check the header header = RawSignatureHeader() buf.readinto(header) assert header.magic1 == 0xcafe2580 assert header.size_minus_header == len(data) - 48 assert crc32(checksummable_data) & 0xffffffff == header.crc32 assert header.magic2 == 0x94119c00 self.sample_rate_hz = int(SampleRate(header.shifted_sample_rate_id >> 27).name.strip('_')) self.number_samples = int(header.number_samples_plus_divided_sample_rate - self.sample_rate_hz * 0.24) # Read the type-length-value sequence that follows the header # The first chunk is fixed and has no value, but instead just repeats # the length of the message size minus the header: assert int.from_bytes(buf.read(4), 'little') == 0x40000000 assert int.from_bytes(buf.read(4), 'little') == len(data) - 48 # Then, lists of frequency peaks for respective bands follow self.frequency_band_to_sound_peaks = {} while True: tlv_header = buf.read(8) if not tlv_header: break frequency_band_id = int.from_bytes(tlv_header[:4], 'little') frequency_peaks_size = int.from_bytes(tlv_header[4:], 'little') frequency_peaks_padding = -frequency_peaks_size % 4 frequency_peaks_buf = BytesIO(buf.read(frequency_peaks_size)) buf.read(frequency_peaks_padding) # Decode frequency peaks frequency_band = FrequencyBand(frequency_band_id - 0x60030040) fft_pass_number = 0 self.frequency_band_to_sound_peaks[frequency_band] = [] while True: raw_fft_pass : bytes = frequency_peaks_buf.read(1) if not raw_fft_pass: break fft_pass_offset : int = raw_fft_pass[0] if fft_pass_offset == 0xff: fft_pass_number = int.from_bytes(frequency_peaks_buf.read(4), 'little') continue else: fft_pass_number += fft_pass_offset peak_magnitude = int.from_bytes(frequency_peaks_buf.read(2), 'little') corrected_peak_frequency_bin = int.from_bytes(frequency_peaks_buf.read(2), 'little') self.frequency_band_to_sound_peaks[frequency_band].append( FrequencyPeak(fft_pass_number, peak_magnitude, corrected_peak_frequency_bin, self.sample_rate_hz) ) return self
def ping(addr): """Send echo request (ping) to addr and get reply""" dest = '' try: dest = socket.gethostbyname(addr) except socket.gaierror: print('unknown host: {}'.format(addr)) return print('PING {} ({})'.format(addr, dest)) icmp_header = make_ping_request() # prepare payload icmp_header_bytes = BytesIO() icmp_header_bytes.write(icmp_header) icmp_header_bytes.seek(0) full_payload = icmp_header_bytes.read() print('request: {}'.format( full_payload.hex() if PY3 else full_payload.encode('hex'))) response = '' s = None try: s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.getprotobyname('icmp')) s.settimeout(1) start_time = time.time() s.sendto(full_payload, (dest, 0)) response, _ = s.recvfrom(256) end_time = time.time() except Exception as ex: print(ex) return finally: if s: s.close() if not response: return print('response: {}'.format( response.hex() if PY3 else response.encode('hex'))) print('time = {:.2f} ms'.format((end_time - start_time) * 1000)) # read IPv4 header response_bytes = BytesIO(response) ipv4_header = IPv4Header() response_bytes.readinto(ipv4_header) # read ICMP header icmp_echo_header = ICMPEchoHeader() response_bytes.readinto(icmp_echo_header) print('=====================================================') print('IPv4 header') print('=====================================================') print(ipv4_header) print('=====================================================') print('ICMP Echo header') print('=====================================================') print(icmp_echo_header)
class VCRHTTPResponse(HTTPResponse): """ Stub response class that gets returned instead of a HTTPResponse """ def __init__(self, recorded_response): self.fp = None self.recorded_response = recorded_response self.reason = recorded_response["status"]["message"] self.status = self.code = recorded_response["status"]["code"] self.version = None self._content = BytesIO(self.recorded_response["body"]["string"]) self._closed = False headers = self.recorded_response["headers"] # Since we are loading a response that has already been serialized, our # response is no longer chunked. That means we don't want any # libraries trying to process a chunked response. By removing the # transfer-encoding: chunked header, this should cause the downstream # libraries to process this as a non-chunked response. te_key = [h for h in headers.keys() if h.upper() == "TRANSFER-ENCODING"] if te_key: del headers[te_key[0]] self.headers = self.msg = parse_headers(headers) self.length = compat.get_header(self.msg, "content-length") or None @property def closed(self): # in python3, I can't change the value of self.closed. So I' # twiddling self._closed and using this property to shadow the real # self.closed from the superclas return self._closed def read(self, *args, **kwargs): return self._content.read(*args, **kwargs) def readall(self): return self._content.readall() def readinto(self, *args, **kwargs): return self._content.readinto(*args, **kwargs) def readline(self, *args, **kwargs): return self._content.readline(*args, **kwargs) def readlines(self, *args, **kwargs): return self._content.readlines(*args, **kwargs) def seekable(self): return self._content.seekable() def tell(self): return self._content.tell() def isatty(self): return self._content.isatty() def seek(self, *args, **kwargs): return self._content.seek(*args, **kwargs) def close(self): self._closed = True return True def getcode(self): return self.status def isclosed(self): return self.closed def info(self): return parse_headers(self.recorded_response["headers"]) def getheaders(self): message = parse_headers(self.recorded_response["headers"]) return list(compat.get_header_items(message)) def getheader(self, header, default=None): values = [v for (k, v) in self.getheaders() if k.lower() == header.lower()] if values: return ", ".join(values) else: return default def readable(self): return self._content.readable()
def from_bytearray(self, bin): s = BytesIO(bin) s.seek(0) s.readinto(self)
for i in p: i.join() fs = SharedValue(FileSystem, lock=False) f = open(args.file, 'rb') bytes_all = f.read() if args.n: img_bytes = bytes_all else: data_len = bytes_all[-sizeof(c_size_t):] fs_size = int.from_bytes( data_len, byteorder='little', signed=False) + sizeof(c_size_t) img_bytes = bytes_all[:-fs_size] fs_bytes = zlib.decompress(bytes_all[-fs_size:-sizeof(c_size_t)]) fakefile = BytesIO(fs_bytes) fakefile.readinto(fs) f.close() try: MyShell().cmdloop() except: pass f = open(args.file, 'wb') f.write(img_bytes) fakefile = BytesIO() fakefile.write(fs) fs_bytes = zlib.compress(fakefile.getvalue()) f.write(fs_bytes) fakefile = BytesIO() fakefile.write(c_size_t(len(fs_bytes))) f.write(c_size_t(len(fs_bytes)))
def decode(cls, line): fakefile = BytesIO(my_encoded_c_struct) history_item = HistoryItem() fakefile.readinto(history_item) return history_item
class LSFReader: """ Implements reading of LSF files. The class creates an index by default, but this can be disabled if the file will only be read once. It is also possible to disable the storing of this index to file, e.g. if the LSF-file is located in a read-only filesystem. Must either be used through the one-off static method or using 'with LSFReader(..) as x:' """ @staticmethod def read(lsf: Union[str, bytes], types: List[Type[pyimc.Message]] = None, use_index=True, save_index=True): """ Read all messages of the specified type(s) :param lsf: The path to an LSF-file on the filesystem, or an in-memory lsf-file (bytes) :param types: List of types to return :param use_index: If true, generates an index of the message types (speeds up subsequent reads) :param save_index: If true, the generated index is saved to a pyimc_idx file :return: Message generator object """ with LSFReader(lsf, use_index=use_index, save_index=save_index) as lsf_reader: for message in lsf_reader.read_message(types=types): yield message def __init__(self, lsf: Union[str, bytes], use_index=True, save_index=True): """ Reads an LSF file. :param lsf: The path to an LSF-file on the filesystem, or an in-memory lsf-file (bytes) :param types: The message types to return. List of pyimc message classes. :param use_index: If true, generates an index of the message types (speeds up subsequent reads) :param save_index: If true, the generated index is saved to a pyimc_idx file """ self.lsf = lsf self.f = None # type: io.BufferedIOBase self.header = IMCHeader() # Preallocate header buffer self.idx = {} # type: Dict[Union[int, str], List[int]] self.use_index = use_index self.save_index = save_index def __enter__(self): # Open file/stream if type(self.lsf) is str: self.f = open(self.lsf, mode='rb') elif type(self.lsf) is bytes: self.f = BytesIO(self.lsf) elif type(self.lsf) in (io.BytesIO, io.FileIO, gzip.GzipFile): self.f = self.lsf else: raise ValueError('LSF file must be passed as raw data (bytes), path (string) or file object.') # Attempt to read an pre-existing index file if type(self.lsf) is str: fbase, ext = os.path.splitext(self.lsf) if os.path.isfile(fbase + '.pyimc_idx') and os.path.getsize(fbase + '.pyimc_idx') > 0: self.read_index(fbase + '.pyimc_idx') # Generate/save index if not self.idx and self.use_index: self.generate_index() if self.save_index and type(self.lsf) is str: self.write_index(os.path.splitext(self.lsf)[0] + '.pyimc_idx') return self def __exit__(self, exc_type, exc_val, exc_tb): # Close file/stream self.f.close() def peek_header(self): bytes_read = self.f.readinto(self.header) if bytes_read < ctypes.sizeof(IMCHeader): raise RuntimeError('LSF file ended abruptly.') # Return file position to before header self.f.seek(-ctypes.sizeof(IMCHeader), io.SEEK_CUR) def generate_index(self): """ Run through the lsf-file and generate a message index (in-memory). Speeds up when the file is parsed multiple times, e.g. to output different messages :return:a """ self.f.seek(0) # Timestamp of first message is used to avoid index/lsf mismatch on load self.peek_header() self.idx['timestamp'] = self.header.timestamp # Check for file end while self.f.read(ctypes.sizeof(IMCHeader)): self.f.seek(-ctypes.sizeof(IMCHeader), io.SEEK_CUR) self.peek_header() # Store position for this message try: self.idx[self.header.mgid].append(self.f.tell()) except (KeyError, AttributeError): self.idx[self.header.mgid] = [self.f.tell()] # Go to next message self.f.seek(ctypes.sizeof(IMCHeader) + self.header.size + ctypes.sizeof(IMCFooter), io.SEEK_CUR) self.f.seek(0) def write_index(self, fpath): """ Write message index to pyimc_idx file. Generates index if not already present :param fpath: The file path to write (typically lsf_name.pyimc_idx) :return: """ if not self.idx: self.generate_index() # Store index with open(fpath, mode='wb') as f: pickle.dump(self.idx, f) def read_index(self, fpath): """ :param fpath: The path to write (typically lsf_name.pyimc_idx) :return: """ with open(fpath, 'rb') as f_idx: self.idx = pickle.load(f_idx) # Verify that timestamp matches first message self.peek_header() if self.header.timestamp != self.idx['timestamp']: self.idx = {} # Remove timestamp entry del self.idx['timestamp'] def count_index(self, msg_type: Type[pyimc.Message]) -> int: """ Get the number of messages for a given type, useful for preallocation. Note: generates an index, but only saves if make_index is true :param msg_type: The message type to return the index count for. :return: The number of messages of a given type """ return len(self.idx[pyimc.Factory.id_from_abbrev(msg_type.__name__)]) def count_messages(self): """ Returns a dictionary of all message types with their counts """ return {type(pyimc.Factory.produce(k)): len(v) for k, v in self.idx.items() if type(k) is int} def sorted_idx_iter(self, types: List[int]) -> Iterable[int]: """ Returns an iterator of file positions sorted by file position (across different message types) :param types: The message types to return, None returns all types :return: Generator object for sorted file positions """ if types: idx_iters = [self.idx[key] for key in types if key in self.idx] else: idx_iters = [val for key, val in self.idx.items()] # Use the heapq.merge function to return sorted iterator of file indices return heapq.merge(*idx_iters) def read_message(self, types: List[Type[pyimc.Message]] = None): """ Returns a generator that yields the messages in the currently open LSF file. This requires the LSFReader object to be opened using the "with" statement. See read(), where this is done automatically. :return: """ msg_types = None if types is None else [pyimc.Factory.id_from_abbrev(x.__name__) for x in types] if self.idx and msg_types is not None: # Read using index for pos in self.sorted_idx_iter(msg_types): self.f.seek(pos) self.peek_header() if self.header.sync != pyimc.constants.SYNC: warnings.warn('Invalid synchronization number. Stopping parsing') break b = self.f.read(self.header.size + ctypes.sizeof(IMCHeader) + ctypes.sizeof(IMCFooter)) msg = pyimc.Packet.deserialize(b) yield msg else: # Reset file pointer to start of file self.f.seek(0) # Read file without index # Check for file end while self.f.read(ctypes.sizeof(IMCHeader)): self.f.seek(-ctypes.sizeof(IMCHeader), io.SEEK_CUR) self.peek_header() if self.header.sync != pyimc.constants.SYNC: warnings.warn('Invalid synchronization number. Stopping parsing') break if msg_types is None or self.header.mgid in msg_types: b = self.f.read(self.header.size + ctypes.sizeof(IMCHeader) + ctypes.sizeof(IMCFooter)) msg = pyimc.Packet.deserialize(b) yield msg else: self.f.seek(ctypes.sizeof(IMCHeader) + self.header.size + ctypes.sizeof(IMCFooter), io.SEEK_CUR)
def main(): mac_bytes = [((getnode() >> 8 * i) & 0xff) for i in range(5, -1, -1)] dhcp_header = DHCPHeader() dhcp_header.op = DHCP_BOOTREQUEST dhcp_header.htype = 1 # Ethernet dhcp_header.hlen = 6 # 6 for Ethernet dhcp_header.hops = 0 dhcp_header.xid = random.randint(0, 0xffffffff) dhcp_header.secs = 0 dhcp_header.flags = 0x8000 # first bit = 1 (broadcast), all other bits = 0 dhcp_header.ciaddr = ip_to_int('0.0.0.0') # client IP dhcp_header.yiaddr = ip_to_int('0.0.0.0') # your (client) IP dhcp_header.siaddr = ip_to_int('0.0.0.0') # next server IP dhcp_header.giaddr = ip_to_int('0.0.0.0') # relay agent IP dhcp_header.chaddr = set_bytes(dhcp_header.chaddr, ''.join( map(chr, mac_bytes))) # client HW address (MAC) dhcp_header.sname = set_bytes(dhcp_header.sname, '') # server host name dhcp_header.file = set_bytes(dhcp_header.file, '') # boot file name print(dhcp_header) # prepare DHCP header payload dhcp_header_bytes = BytesIO() dhcp_header_bytes.write(dhcp_header) dhcp_header_bytes.seek(0) dhcp_header_payload = dhcp_header_bytes.read() # prepare DHCP options payload dhcp_options_payload = DHCP_MAGIC_COOKIE # DHCP Message # code = 53 (DHCP_MESSAGE_TYPE) # len = 1 # type = 1 (DHCP_DISCOVER) dhcp_options_payload += struct.pack('BBB', DHCP_MESSAGE_TYPE, 1, DHCP_DISCOVER) # Parameter Request List # code = 55 (DHCP_PARAMETER_REQUEST_LIST) # len = 1 # option codes = 1 (DHCP_SUBNET_MASK) dhcp_options_payload += struct.pack('BBB', DHCP_PARAMETER_REQUEST_LIST, 1, DHCP_SUBNET_MASK) # End Option dhcp_options_payload += struct.pack('B', DHCP_END_OPTION) # write to bytes dhcp_options_bytes = BytesIO() dhcp_options_bytes.write(dhcp_options_payload) dhcp_options_bytes.seek(0) # parse and print DHCP options parse_dhcp_options(dhcp_options_bytes) full_payload = dhcp_header_payload + dhcp_options_payload print('request: {}'.format( full_payload.hex() if PY3 else full_payload.encode('hex'))) # create socket s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.settimeout(3) try: s.bind(('', 68)) except (OSError, socket.error): print('Unable to bind to port 68') s.close() return # send DHCP payload s.sendto(full_payload, ('<broadcast>', 67)) response = '' try: response = s.recv(4096) except (OSError, socket.error) as ex: print(ex) return finally: s.close() print('response: {}'.format( response.hex() if PY3 else response.encode('hex'))) # read DHCP header response_bytes = BytesIO(response) dhcp_response_header = DHCPHeader() response_bytes.readinto(dhcp_response_header) print('=====================================================') print('DHCP header') print('=====================================================') print(dhcp_response_header) # parse and print DHCP options parse_dhcp_options(response_bytes)