def unpack(file_path: Union[Path, str]): if isinstance(file_path, str): file_path = Path(file_path) result = {'spout': list(), 'lever': list(), 'event': dict()} for attr_type in ("design", "hardware"): result[attr_type] = toml.load(file_path.joinpath(attr_type + ".toml")) packet = Struct("<BIi") with file_path.joinpath("temp.raw").open('rb') as fp: for pack in packet.iter_unpack(fp.read()): if 1 == pack[0]: result['spout'].append(pack[1:]) elif 2 == pack[0]: result['lever'].append(pack[1:]) elif 3 == pack[0]: event_type = event_types[pack[2]] if event_type in result['event']: result['event'][event_type].append(pack[1]) else: result['event'][event_type] = [pack[1]] else: raise ValueError(f"wrong packet! {pack}") result['lever'] = np.array(result['lever'], dtype=[('timestamp', '<I'), ('value', '<i')]) result['spout'] = np.array(result['spout'], dtype=[('timestamp', '<I'), ('value', '<i')]) result['event'] = { key: np.array(value, dtype='<I') for key, value in result['event'].items() } return result
class StatsVector: '''A class representing a VPP vector''' def __init__(self, stats, ptr, fmt): self.vec_start = ptr - stats.base self.vec_len = get_vec_len(stats, ptr - stats.base) self.struct = Struct(fmt) self.fmtlen = len(fmt) self.elementsize = self.struct.size self.statseg = stats.statseg self.stats = stats if self.vec_start + self.vec_len * self.elementsize >= stats.size: raise ValueError('Vector overruns stats segment') def __iter__(self): with self.stats.lock: return self.struct.iter_unpack(self.statseg[self.vec_start:self.vec_start + self.elementsize*self.vec_len]) def __getitem__(self, index): if index > self.vec_len: raise ValueError('Index beyond end of vector') with self.stats.lock: if self.fmtlen == 1: return self.struct.unpack_from(self.statseg, self.vec_start + (index * self.elementsize))[0] return self.struct.unpack_from(self.statseg, self.vec_start + (index * self.elementsize))
def deinterleave(data, nbytes, nsplit): """Deinterleaves one bytearray into nsplit many bytearrays on a nbytes basis. Returns a list of bytearrays. """ deinterleaved = [[] for n in range(nsplit)] deinterleave_s = Struct('c' * nbytes) try: deinterleave_iter = deinterleave_s.iter_unpack(data) except error as err: #this error can be many things, handling generically until otherwise print('ERROR:', err, 'CLOSING', file=sys.stderr) raise err #this could cause rounding errors? iterlen = int(len(data) / (nbytes * nsplit)) for _ in range(iterlen): for i, _ in enumerate(deinterleaved): try: next_ = next(deinterleave_iter) except StopIteration: pass deinterleaved[i].extend([*next_]) return [b''.join(delist) for delist in deinterleaved]
def _tile_to_bitplanes(self, data): bitplanes = [] row_fmt = Struct(8 * 'c') row_iter = row_fmt.iter_unpack(data) bitplanes = [self._pixel_row_to_4bpp(row) for row in row_iter] return b''.join(bitplanes)
def _interleave(self, subtile1, subtile2): """Interleaves two 8x8 tiles like [subtile1-row1] [subtile2-row1] ... [subtile1-row16] [subtile2-row16] Returns bytes() """ interleaved = [] interleave_fmt = Struct(4 * 'c') left_iter = interleave_fmt.iter_unpack(subtile1) right_iter = interleave_fmt.iter_unpack(subtile2) for i in left_iter: right_next = next(right_iter) interleaved.extend([*i, *right_next]) return interleaved
def unpack(self): """Unpacks a 4bpp planar tile. Returns a byte() of pixel values. """ tile_fmt = Struct(32 * 'c') tile_iter = tile_fmt.iter_unpack(self._tile_data) tiles = [self._bitplanes_to_tile(b''.join(tile)) for tile in tile_iter] return b''.join(tiles)
def deinterleave_subtiles(self): tile_fmt = Struct(64 * 'c') tile_iter = tile_fmt.iter_unpack(self._tile_data) subtiles = [] for subtile in tile_iter: subtiles.extend(self._deinterleave(b''.join(subtile))) deinterleaved = [subtiles[0], subtiles[2], subtiles[1], subtiles[3]] return Tile(self._tile_addr, b''.join(deinterleaved), self._tile_dimensions)
def _deinterleave(self, data): deinterleaved = [[], []] deinterleave_fmt = Struct(4 * 'c') deinterleave_iter = deinterleave_fmt.iter_unpack(data) for i in deinterleave_iter: deinterleaved[0].extend([*i]) deinterleaved[1].extend([*next(deinterleave_iter)]) return [b''.join(data) for data in deinterleaved]
def pack(self, data): """Converts pixel values into 4bpp and packs it for Tile use. Returns bytes() """ tile_fmt = Struct(32 * 'c') tile_iter = tile_fmt.iter_unpack(data) bitplane_values = [ self._tile_to_bitplanes(b''.join(tile)) for tile in tile_iter ] return b''.join(bitplane_values)
def deinterleave(data, num_bytes): """Deinterleaves a bytearray. Returns two bytearrays. """ evens = [] odds = [] deinterleave_s = Struct('c' * num_bytes) deinterleave_iter = deinterleave_s.iter_unpack(data) for i in deinterleave_iter: evens.extend([*i]) odds.extend([*next(deinterleave_iter)]) return b''.join(evens), b''.join(odds)
def interleave(file1, file2, num_bytes): """Interleaves two bytearray buffers together. Returns a bytearray. """ interleaved = [] interleave_s = Struct('c' * num_bytes) file1_iter = interleave_s.iter_unpack(file1) file2_iter = interleave_s.iter_unpack(file2) for i in file1_iter: file2_next = next(file2_iter) interleave_temp = [*i, *file2_next] interleaved.extend(interleave_temp) return b''.join(interleaved)
def interleave_subtiles(self): """Row interleaves the 4 8x8 subtiles in a 16x16 tile. Returns a new packed Tile. """ tile_fmt = Struct(32 * 'c') tile_iter = tile_fmt.iter_unpack(self._tile_data) subtiles = [b''.join(subtile) for subtile in tile_iter] top = self._interleave(subtiles[0], subtiles[2]) bottom = self._interleave(subtiles[1], subtiles[3]) interleaved = [*top, *bottom] return Tile(self._tile_addr, b''.join(interleaved), self._tile_dimensions)
def deinterleave(data, nbytes, nsplit): """Deinterleaves one bytearray into nsplit many bytearrays on a nbytes basis. Returns a list of bytearrays. """ deinterleaved = [[] for n in range(nsplit)] deinterleave_s = Struct('c' * nbytes) deinterleave_iter = deinterleave_s.iter_unpack(data) for i in deinterleave_iter: deinterleaved[0].extend([*i]) for j in range(1, len(deinterleaved[1:]) + 1): next_ = next(deinterleave_iter) deinterleaved[j].extend([*next_]) return [b''.join(delist) for delist in deinterleaved]
def interleave(data, nbytes): """Interleaves a list of bytearrays together on a nbytes basis. Returns a bytearray. """ interleave_s = Struct('c' * nbytes) iters = [interleave_s.iter_unpack(inter) for inter in data] interleaved = [] #this could cause rounding errors? iterlen = int(len(data[0]) / nbytes) for i in range(iterlen): nexts = [next(iter_) for iter_ in iters] # print('nexts is:', nexts) interleaved.extend([b''.join(val) for val in nexts]) # print('interleaved is:', interleaved) return b''.join(interleaved)
def bool_ptr_map_of_coredump(cdump): struct_uint64 = Struct('=Q') ptr_map = [] with MachO(args.coredump) as cdump: va_ranges = SortedDict((s.vmaddr, s.vmaddr + s.vmsize) for s in cdump.get_segments()) for seg in (s for s in cdump.get_segments() if not s.initprot & VM_PROT_EXECUTE): # Ensure 8-byte alignment of the segment's vmaddr and vmsize vmaddr_8byte_aligned_up = (seg.vmaddr + 0x7) & ~0x7 vmaddr_delta = vmaddr_8byte_aligned_up - seg.vmaddr vmsize_8byte_aligned_down = (seg.vmsize - vmaddr_delta) & ~0x7 vmsize_delta = vmsize_8byte_aligned_down - seg.vmsize # Get 8-byte aligned segment data seg_data = cdump._get_data(seg.fileoff + vmaddr_delta, seg.filesize + vmsize_delta) seg_data_bytes = len(seg_data) for i, (uint64,) in enumerate(struct_uint64.iter_unpack(seg_data)): is_ptr = va_range_set_contains(va_ranges, uint64) ptr_map.append(is_ptr) return ptr_map, 8
def read_type_tree_blob(self): reader = self.reader number_of_nodes = self.reader.read_int() string_buffer_size = self.reader.read_int() type = f"{reader.endian}hb?IIiii" keys = [ "m_Version", "m_Level", "m_IsArray", "m_TypeStrOffset", "m_NameStrOffset", "m_ByteSize", "m_Index", "m_MetaFlag", ] if self.header.version >= 19: type += "Q" keys.append("m_RefTypeHash") node_struct = Struct(type) struct_data = reader.read(node_struct.size * number_of_nodes) string_buffer_reader = EndianBinaryReader( reader.read(string_buffer_size), reader.endian) if not config.SERIALIZED_FILE_PARSE_TYPETREE: return [], string_buffer_reader.bytes type_tree = [ TypeTreeNode( **dict(zip(keys, raw_node)), m_Type=read_string(string_buffer_reader, raw_node[3]), m_Name=read_string(string_buffer_reader, raw_node[4]), ) for i, raw_node in enumerate(node_struct.iter_unpack(struct_data)) ] return type_tree, string_buffer_reader.bytes
def interleave(data, nbytes): """Interleaves a list of bytearrays together on a nbytes basis. Returns a bytearray. """ interleave_s = Struct('c' * nbytes) iters = [] for inter in data: try: iters.append(interleave_s.iter_unpack(inter)) except error as err: print('ERROR:', err, 'CLOSING', file=sys.stderr) raise err interleaved = [] #this could cause rounding errors? iterlen = int(len(data[0]) / nbytes) for _ in range(iterlen): nexts = [next(iter_) for iter_ in iters] interleaved.extend([b''.join(val) for val in nexts]) return b''.join(interleaved)
class NamedStruct(object): """ A type that fuse namedtuple and Struct together. """ __fields__ = ('names', 'format') def __init__(self, name, fmt, *fields): self.format = Struct(fmt) self.names = namedtuple(name, fields) @classmethod def from_namedtuple(cls, ntuple, fmt): """ Build a NamedStruct from a namedtuple Author: Gabriel Dube """ named_struct = super(NamedStruct, cls).__new__(cls) named_struct.format = Struct(fmt) named_struct.names = ntuple return named_struct def unpack(self, data): x, y = len(data), self.format.size if (x > y) and (x % y == 0): return tuple(self.iter_unpack(data)) return self.names(*self.format.unpack(data)) def unpack_from(self, data, offset): return self.names(*self.format.unpack_from(data, offset)) def iter_unpack(self, data): return (self.names(*d) for d in self.format.iter_unpack(data))
class IndexSection(BaseSection): ''' A section containing an index / lookup table with offsets into another section. (Think git packfile indexes.) ''' typeid = SectionType.Index datatype = dict def __init__(self, *args, othersec=None, othersec_idx=None, keysize=32, fanout=True, off64=False, unc_size=True, varint=False, endian='<', **kwargs): # TODO: flag for whether or not there's a full fanout table # (so we can skip it for small indexes) # TODO: flag for varint encoding of offsets/sizes BaseSection.__init__(self, *args, **kwargs) if not (othersec_idx or isinstance(othersec, BaseSection)): raise ValueError("expected BaseSection, got {type(othersec)}") # these control the output encoding and can be set/changed whenever self.endian = endian self.fanout = fanout self.varint = varint # off64 is an output encoding setting that gets set automatically # if a 64-bit offset/size is added self._off64 = off64 # keysize and unc_size can't be changed once an index is created self._keysize = keysize self._unc_size = unc_size # references to the section we're an index over self._othersec = othersec self._othersec_idx = othersec_idx # set up self._key_s = Struct(f'{self._keysize}s') valfmt = ('L' if off64 else 'I') * (3 if unc_size else 2) self._val_s = Struct(f'{self.endian}{valfmt}') self._fanout_s = Struct(f'{self.endian}256I') if unc_size: self.add = self.add3 else: self.add = self.add2 @property def keysize(self): return self._keysize @staticmethod def parse_info(info): return IndexInfo.from_int(info) @classmethod def from_hdr(cls, shdr): info = cls.parse_info(shdr.info) return cls(name_idx=shdr.name, flags=shdr.flags, othersec_idx=info.othersec, keysize=info.keysize, fanout=info.fanout, off64=info.off64, unc_size=info.unc_size, varint=bool(shdr.flags & SectionFlags.VARINT)) @property def count(self): return len(self._data) @property def size(self): if self.varint: return ( len(self.make_fanout()) + (self.count * self.keysize) + sum(len(varint_encode(i)) for v in self.values() for i in v)) else: return (self._fanout_s.size + self.count * (self.keysize + self._val_s.size)) @property def info(self): return IndexInfo(keysize=self.keysize, othersec=self.othersec.idx if self.othersec else 0xff, fanout=self.fanout, unc_size=self._unc_size, off64=self._off64).to_int() @property def othersec(self): if self._othersec is None: if self._dino and self._othersec_idx: self._othersec = self._dino.sectab[self._othersec_idx] return self._othersec def keys(self): return self._data.keys() def values(self): return self._data.values() def items(self): return self._data.items() def get(self, key, default=None): return self._data.get(key, default) def __contains__(self, key): return key in self._data def add2(self, key, offset, size): self._data[self._key_s.pack(key)] = (offset, size) def add3(self, key, offset, size, uncsize): self._data[self._key_s.pack(key)] = (offset, size, uncsize) def remove(self, key): del self._data[key] def make_fanout(self): counts = Counter(k[0] for k in self.keys()) if self.varint: # varint-encoded fanout just gives the counts for each byte return self._varint_pack(*[counts[b] for b in range(256)]) else: fanout = [0] * 257 for i in range(256): fanout[i + 1] = fanout[i] + counts[i] return self._fanout_s.pack(*fanout[1:]) def _varint_pack(self, *values): return b''.join(varint_encode(i) for i in values) @property def keysize(self): return self._key_s.size def write_to(self, fobj): if self.count == 0: return 0 dprint(f"writing index: fanout={self.fanout} varint={self.varint} " f"unc_size={self._unc_size} keysize={self.keysize} " f"count={self.count}") wrote = 0 if self.fanout: wrote += fobj.write(self.make_fanout()) keys, vals = zip(*(sorted(self.items()))) dprint(f" fanout: {wrote:7} bytes") prevpos = wrote for k in keys: wrote += fobj.write(self._key_s.pack(k)) dprint(f" keys: {wrote-prevpos:7} bytes") if self.varint: valpack = self._varint_pack else: valpack = self._val_s.pack prevpos = wrote for v in vals: wrote += fobj.write(valpack(*v)) dprint(f" vals: {wrote-prevpos:7} bytes") dprint(f" total: {wrote:7} bytes") return wrote def from_file(self, fobj, size, count=0): # It's a little silly that we unpack this data structure into native # python data structures rather than using it directly, but the # native structures *seem* to perform better, and this is really # just a rapid-devel prototype anyway. # A real implementation of this would be native code in a library # that we use via the FFI or something. if size == 0: self._data = self.datatype() return dprint(f"reading index: fanout={self.fanout} varint={self.varint} " f"unc_size={self._unc_size} keysize={self.keysize} " f"count={self.count}") data = fobj.read(size) keypos = 0 if self.fanout: if self.varint: # NOTE: varint-encoded fanout is a sequence of counts, not a # running count.. fv = 0 for v, n in varint_iter_decode(data, 256): fv += v keypos += n fanout.append(fv) else: keypos = self._fanout_s.size fanout = self._fanout_s.unpack(data[0:keypos]) if count: assert count == fanout[-1] dprint(f" fanout: {keypos:7} bytes, count={fanout[-1]}") keylen = self.keysize * count valpos = keypos + keylen keydata = data[keypos:valpos] valdata = data[valpos:] dprint(f" keys: {valpos-keypos:7} bytes") dprint(f" vals: {len(valdata):7} bytes") keys = [i[0] for i in self._key_s.iter_unpack(keydata)] if self.varint: vals = [i[0] for i in varint_iter_decode(valdata)] n, m = divmod(len(vals), count) assert (m == 0), "Incorrect/corrupt index" vals = [tuple(vals[i:i + n]) for i in range(0, len(vals), n)] else: if (len(valdata) % self._val_s.size): print( f"wtf: size {self._val_s.size} * count {count} != {len(valdata)}" ) vals = self._val_s.iter_unpack(valdata) self._data = self.datatype(zip(keys, vals))
def parser(buffer): struct = Struct(struct_pattern) for chunk in struct.iter_unpack(buffer): yield event_maker(*chunk)
def extract_version1(file): # consts KEYS_COUNT = 8 PADDING_LENGTH = 432 PAYLOAD_SIZE = 512 # sizeof (u32) * 128 # prepare structs key_struct = Struct(">LL32sLL") header_struct = Struct(">6sH32s32s32sLL20s32sL40s" + str(key_struct.size * KEYS_COUNT) + "s" + str(PADDING_LENGTH) + "x") # read header header = file.read(header_struct.size) assert len( header) == header_struct.size, "File contains less data than needed" # convert bytes into temporary header header = header_struct.unpack(header) header = TmpHeaderVersion1(*header) # convert bytes into temporary keys tmp_keys = [ TmpKeyVersion1(*key) for key in key_struct.iter_unpack(header.keys) ] # read keys' af keys = [] for key in tmp_keys: file.seek(key.material_offset * SECTOR_SIZE, SEEK_SET) af = file.read(header.key_bytes * key.stripes) assert len(af) == (header.key_bytes * key.stripes), "File contains less data than needed" key = KeyVersion1(key.active, key.iterations, key.salt, af) keys.append(key) # read payload file.seek(header.payload_offset * SECTOR_SIZE, SEEK_SET) payload = file.read(PAYLOAD_SIZE) assert len(payload) == PAYLOAD_SIZE, "File contains less data than needed" # convert into header header = HeaderVersion1( header.magic, header.version, header.cipher, header.mode, header.hash, payload, header.key_bytes * 8, header.digest, header.salt, header.iterations, header.uuid, keys, ) # check for any active key for key in header.keys: if key.active not in [ KeyVersion1.Active.ENABLED, KeyVersion1.Active.ENABLED_OLD ]: continue hash = SIGNATURE + "$".join( map( str, [ header.version, header.hash, header.cipher, header.mode, int(header.key_size), key.iterations, key.salt.hex(), key.af.hex(), header.payload.hex(), ], )) print(hash) break else: # all keys are disabled raise ValueError("All keys are disabled")
def read_iter(structure: Struct, stream: IO[bytes]) -> Iterator[Tuple[Any, ...]]: return structure.iter_unpack(stream.read())
from struct import Struct struct1 = Struct('@i13sf') sendBytes = b'\x7f\x00\x00\x00Hello Struct!\x00\x00\x00\xc3\xf5H@\x80\x00\x00\x00Hello Python!\x00\x00\x00\x85\xebQ@' originalIter = struct1.iter_unpack(sendBytes) print('Object type :', originalIter) for item in originalIter: print(item)
line = f.read(record_struct.size) if line == b'': break yield decode_record(record_struct, line) def _decode_record(record_struct, line): return tuple(s.decode() for s in record_struct.unpack_from(line)) def decode_record(rec): return tuple(s.decode() for s in rec) if __name__ == '__main__': # Will throw an AssertionError if the Length variable within the control file is wrong check_ctl('/some/dir/to/keep.csv') field_widths, keep_fields = import_ctl('/some/dir/to/keep.csv') fmt_string = create_fmt(field_widths, keep_fields) record_struct = Struct(fmt_string) with open('/some/dir/to/fixedfield/split1_sample', 'rb') as infile: with open('/some/dir/to/fixedfield/split1_sample.csv', 'w', newline='') as outfile: csv_writer = csv.writer(outfile, delimiter=',') for rec in record_struct.iter_unpack( infile.read(record_struct.size * 10)): # for rec in read_records(record_struct, infile): csv_writer.writerow(decode_record(rec))