def extractFile(filename, start, end, palette, names): dims = [] with open(filename, 'rb') as artFile: artFile.seek(16) count = end - start + 1 widths = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))] if len(widths) < count: raise BadArtFileException("Couldn't read enough widths.") heights = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))] if len(heights) < count: raise BadArtFileException("Couldn't read enough heights.") dims = zip(widths, heights) #don't need the rest of this artFile.seek(count * 4, 1) for dim in enumerate(dims): width, height = dim[1] if width == 0 or height == 0: continue data = artFile.read(width * height) if len(data) < width * height: raise BadArtFileException("Not enough data.") data = rearrangeData(data, width, height) name = "" try: name = names[start + dim[0]] except KeyError: name = "TILE{:04d}".format(start + dim[0]) print(" -> Writing out {:s}...".format(name)) writeBMP(name, width, height, palette, data)
def readPlanes(hdr, data): planedata = (x[0] for x in iter_unpack("24s", data[hdr.planeDataOffset:][:hdr.planeCount * 24])) normals = (Point(*x) for x in iter_unpack("<3i", data[hdr.normalListOffset:][:hdr.planeCount * 24])) data = BytesIO(data[hdr.planeListOffset:]) for plane, normal in zip(planedata, normals): planePointCount, unknown1, texture, unknown2 = unpack("<2BH6s", data.read(10)) planePoints = [PlanePoint(int(point[0]/12), *point[1:]) for point in iter_unpack("<IHH", data.read(planePointCount * 8))] yield Plane(unknown1, texture, unknown2, planePoints, normal, plane)
def parse_cmap(tree, node, data): i = -1 for g, b, r in iter_unpack("3B", data): i += 1 if r == g == b == 0: continue node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))
def test_module_func(self): # Sanity check for the global struct.iter_unpack() it = struct.iter_unpack('>IB', bytes(range(1, 11))) self.assertEqual(next(it), (0x01020304, 5)) self.assertEqual(next(it), (0x06070809, 10)) self.assertRaises(StopIteration, next, it) self.assertRaises(StopIteration, next, it)
def parse_6int32(name, node, data): node.data = [Point(*x) for x in iter_unpack("<3i", data)] for point in node.data: corner = element("corner") for k, v in zip(point._fields, point): corner.append(element(k, text=str(v))) node.append(corner)
def packets_from_file(filename, recording_time, chunksize=102): with open(filename, "rb") as f: while True: chunk = f.read(chunksize) if chunk: sensor_time = datetime.timedelta(milliseconds=struct.unpack("<I", chunk[:4])[0]) # ms since start of capturing system fused_time = recording_time + sensor_time packet = { 'timestamp': fused_time, 'data': {} } counter = 0 for qw, qx, qy, qz, ax, ay, az in struct.iter_unpack("<hhhhhhh", chunk[4:]): scale_q = (1.0 / (1 << 14)) scale_a = 1.0 / 100 packet['data'][sensor_id[counter]] = { # Decode Quaternion: Orientation data (unit-less) 'orientation': [round(scale_q * qw, 6), round(scale_q * qx, 6), round(scale_q * qy, 6), round(scale_q * qz, 6)], # Decode Vector: Acceleration data (m/s^2) 'acceleration': [round(scale_a * ax, 2), round(scale_a * ay, 2), round(scale_a * az, 2)] } counter += 1 yield packet else: break
def _decode_header(status_dict, names_list, data): names_list = [name[0].split(b"\x00", 1)[0] for name in struct.iter_unpack(f"{FGC_MAX_DEV_LEN + 1}s", data[0 : FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1)])] offset = FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1) (status_dict["hostname"], status_dict["recv_time_sec"], status_dict["recv_time_usec"], status_dict["id"], status_dict["sequence"], status_dict["send_time_sec"], status_dict["send_time_usec"], status_dict["time_sec"], status_dict["time_usec"]) = struct.unpack(f"!{HOST_NAME_MAX}sllllllll", data[offset:]) status_dict["hostname"] = status_dict["hostname"].split(b"\x00", 1)[0].decode() if not status_dict["hostname"]: status_dict.clear() return None return status_dict, names_list
def give_motion_frame(self): """Read the next sample from the POS file and assign to coil objects. :returns: error code, packed bytestring with dataframe response """ if self.motion_lines_read < self.max_num_frames: measurements = self.file.read(self.bytes_in_frame) # there appears to be no timestamp in the data timestamp = MocapParent.timestamp_to_microsecs(self.motion_lines_read * self.frame_time) # 7I is integers for: x, y, z, phi, theta, rms, extra measurements_by_coil = struct.iter_unpack('< 7f', measurements[:]) #print('&&&',measurements_by_coil) for coilvals, coilobj in zip(measurements_by_coil, self.component.coils): floatvals = MocapParent.make_float_or_zero(coilvals) coilobj.set_args_to_vars(floatvals, mapping=self.mappings) self.component.timestamp = timestamp self.latest_timestamp = timestamp print('test timestamp!!', self.latest_timestamp) self.motion_lines_read += 1 return 3, DataFrame(components=[self.component]).pack_components_to_df(), None # no more data available (static, give up) else: print("All the frames have been read") return 4, "No more frames left in mocap file"
def give_motion_frame(self): """Read the next sample from the POS file and assign to coil objects. :returns: error code, packed bytestring with dataframe response """ if self.motion_lines_read < self.max_num_frames: # update the timestamp timestamp = self.timestamp_to_microsecs(self.motion_lines_read * self.frame_time) self.component.timestamp, self.latest_timestamp = timestamp, timestamp print('BVH latest timestamp:', self.latest_timestamp) # 7f is floats for: x, y, z, phi, theta, rms, extra measurements_by_coil = struct.iter_unpack('< 7f', self.file.read(self.bytes_in_frame)) # for m in measurements_by_coil: # print('POS measurement by coil:', m) # # print([m for m in measurements_by_coil]) self.component.coils = [CoilBuilder6D.build_from_mapping(self.mappings, coilvals) for coilvals in measurements_by_coil] # print([(self.mappings, coilvals)for coilvals in measurements_by_coil]) assert len(self.component.coils) > 0 # at least one coil must be made # update the line statistics self.motion_lines_read += 1 return 3, DataFrame(components=[self.component]).pack_all(), None # no more data available (static, give up) else: print("All the frames have been read") return 4, "No more frames left in mocap file"
def parse_dict(data): dct = {} mode = 'pad' key = [] value = [] for c in struct.iter_unpack('c', data): c = c[0] if mode == 'pad': if c in (bytes([0]), bytes([3])): continue else: mode = 'key' if mode == 'key': if c == bytes([0]): mode = 'value' else: key.append(c.decode()) elif mode == 'value': if c == bytes([0]): dct[''.join(key)] = ''.join(value) key = [] value = [] mode = 'pad' else: value.append(c.decode()) return dct
def parse_compact_node_info(data): """ Unpack a list of node id, IPv4 address and port returned by a find_node or get_peers request. """ for frame in struct.iter_unpack("!20s4BH", data): # (id, (ip, port)) yield {'id':frame[0], 'addr':("{}.{}.{}.{}".format(frame[1], frame[2], frame[3], frame[4]), frame[5])}
def parse_minor_mesh_form(self, mesh_form, lod_lev=0): if lod_lev not in self.lods: self.lods[lod_lev] = {} self.lods[lod_lev]["mats"] = [] self.lods[lod_lev]["altmats"] = [] self.lods[lod_lev]["lightflags"] = [] vers_form = self.iff.read_data() mesh_vers = int(vers_form["name"].decode("ascii")) self.lods[lod_lev]["version"] = mesh_vers mnrmsh_read = 16 # Bytes for version form header while mnrmsh_read < mesh_form["length"]: mdat = self.iff.read_data() mnrmsh_read += 8 + mdat["length"] mnrmsh_read += 1 if mdat["length"] % 2 == 1 else 0 if mdat["name"] == b"NAME": self.lods[lod_lev]["name"] = ( self.parse_cstr(mdat["data"], 0)) elif mdat["name"] == b"FACE": # This isn't part of Wing Blender, so I'm using features # from newer versions of Python 3. for f in struct.iter_unpack(self.FACE_FMT, mdat["data"]): if f[2] not in self.lods[lod_lev]["mats"]: self.lods[lod_lev]["mats"].append(f[2]) if f[5] not in self.lods[lod_lev]["lightflags"]: self.lods[lod_lev]["lightflags"].append(f[5]) if f[6] not in self.lods[lod_lev]["altmats"]: self.lods[lod_lev]["altmats"].append(f[6])
def get_intlist(elf): intdata = elf.get_section_by_name("intList").data() header_sz = struct.calcsize(intlist_header_fmt) header = struct.unpack_from(intlist_header_fmt, intdata, 0) intdata = intdata[header_sz:] spurious_code = header[0] spurious_nocode = header[1] debug("spurious handler (code) : %s" % hex(header[0])) debug("spurious handler (no code) : %s" % hex(header[1])) intlist = [i for i in struct.iter_unpack(intlist_entry_fmt, intdata)] debug("Configured interrupt routing") debug("handler irq pri vec dpl") debug("--------------------------") for irq in intlist: debug("{0:<10} {1:<3} {2:<3} {3:<3} {4:<2}".format( hex(irq[0]), "-" if irq[1] == -1 else irq[1], "-" if irq[2] == -1 else irq[2], "-" if irq[3] == -1 else irq[3], irq[4])) return (spurious_code, spurious_nocode, intlist)
def decode_tilemap(tilemap, tiles, palettes): """ Creates a list of colored tiles from a tilemap, raw tiles and a color palette """ tilemap_tiles = [] for char in struct.iter_unpack("<H", tilemap): char = char[0] # Layout: Section 9.3 at http://www.coranac.com/tonc/text/regbg.htm tile_id = char & ((1 << 10) - 1) flip_h = bool(char & (1 << 10)) flip_v = bool(char & (1 << 11)) pal_id = char >> 12 pal = palettes[pal_id] tile = color_tile(tiles[tile_id], pal) tile_img = tile_image(tile) if flip_h: tile_img = tile_img.transpose(Image.FLIP_LEFT_RIGHT) if flip_v: tile_img = tile_img.transpose(Image.FLIP_TOP_BOTTOM) tilemap_tiles.append(tile_img) return tilemap_tiles
def test_extract_palette(self): extracted = assets.files.backgrounds['mindscape'].extract_palette() with open(os.path.join(test_dir, 'mindscap_palette_extract_1.bin'), 'rb') as f: test_data = f.read() assert extracted == [i[0] for i in iter_unpack('<H', test_data)]
def parse_compact_node6_info(data): """ Unpack a list of node id, IPv6 address and port returned by a find_node or get_peers request. """ for frame in struct.iter_unpack("!20s8HH", data): # (id, (ipv6, port)) yield {'id':frame[0], 'addr':('{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}'.format(*frame[1:9]), frame[9])}
def unpack_data(self, buf): unpacked_data = {} payload = struct.iter_unpack('iffffffff', buf[5:]) for item in payload: if item != None: unpacked_data[item[0]] = item[1:] return unpacked_data
def parse_htbl(tree, node, data): i = -1 res = [] for x in iter_unpack("256s", data): i += 1 node.append(element("unknown", text=to_hex(x[0]), attrib={"id": str(i)})) res.append(x[0]) node.data = res
def parse_hicl(tree, node, data): i = -1 for x in iter_unpack(">H", data): i += 1 r, g, b = unpack_15bpp(x[0]) if r == g == b == 0: continue node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))
def unpack_data_from_file(file, fmt, offset=0): """Retrieve data from given file and unpack them according to specified format. file: The path name of the file or an opened file object fmt: Format specification offset: Offset from the start of the file from where to start the unpacking operation """ if isinstance(file, str): if not os.path.isfile(file): raise ValueError('"{}" is not a regular file'.format(file)) else: if not hasattr(file, 'read'): raise ValueError('Invalid file object') if hasattr(file, 'mode') and file.mode != 'rb': raise ValueError('Invalid opening mode') if not isinstance(offset, int) or offset < 0: raise ValueError('Invalid offset value') with fileutil.open_file(file, 'rb') as fp: file_len = fileutil.file_size(fp) pack_size = struct.calcsize(fmt) if file_len <= offset: return fp.seek(offset, os.SEEK_SET) if (file_len - offset) % pack_size == 0: if sys.version_info >= (3, 3): #return struct.iter_unpack(fmt, fp.read()) # Fix issue #1 yield from struct.iter_unpack(fmt, fp.read()) else: for unpacked_data in struct.iter_unpack(fmt, fp.read()): yield unpacked_data else: # The length of the file isn't the multiple of struct.calcsize(fmt), so # don't call struct.iter_unpack directly. data = fp.read(pack_size) while data: if len(data) == pack_size: yield struct.unpack(fmt, data) else: break data = fp.read(pack_size)
def read_intlist(intlist_path): """read a binary file containing the contents of the kernel's .intList section. This is an instance of a header created by include/linker/intlist.ld: struct { u32_t num_vectors; <- typically CONFIG_NUM_IRQS struct _isr_list isrs[]; <- Usually of smaller size than num_vectors } Followed by instances of struct _isr_list created by IRQ_CONNECT() calls: struct _isr_list { /** IRQ line number */ s32_t irq; /** Flags for this IRQ, see ISR_FLAG_* definitions */ s32_t flags; /** ISR to call */ void *func; /** Parameter for non-direct IRQs */ void *param; }; """ intlist = {} prefix = endian_prefix() intlist_header_fmt = prefix + "II" intlist_entry_fmt = prefix + "iiII" with open(intlist_path, "rb") as fp: intdata = fp.read() header_sz = struct.calcsize(intlist_header_fmt) header = struct.unpack_from(intlist_header_fmt, intdata, 0) intdata = intdata[header_sz:] debug(str(header)) intlist["num_vectors"] = header[0] intlist["offset"] = header[1] intlist["interrupts"] = [i for i in struct.iter_unpack(intlist_entry_fmt, intdata)] debug("Configured interrupt routing") debug("handler irq flags param") debug("--------------------------") for irq in intlist["interrupts"]: debug("{0:<10} {1:<3} {2:<3} {3}".format( hex(irq[2]), irq[0], irq[1], hex(irq[3]))) return intlist
def query_values(self, string, nValues=0): if nValues != 0: nBytes = 2 + 16*nValues + 1; buf = b'' if self.write(string): result = self.read(string=False, nBytes=nBytes) unpacker = struct.iter_unpack('d',result) return numpy.array(list(unpacker)) else: return None
def splitByteIntoNArrays(lineList, N, block_size, type_ = 'B'): # print("type: " + type_) # assert( len(lineList) == (1 + 2*rangel)*N ), 'wrong blob length: %u' % len(lineList) " split into N int numpy sub-arrays within list intList" it = struct.iter_unpack(type_, lineList) intList = [[] for Null in range(N)] for i, temp_char in enumerate(it): # print('ind %u\t arr %u' % (i, i//(2*rangel+1)) ) intList[i // block_size].append(temp_char[0]) return intList
def extractData(data): if len(data)<5 or data[0:2] != b'aa': return None size = int.from_bytes(data[2:4],byteorder='little') if data[4]==ord(AP_TRUE): return True elif data[4]==ord(AP_FALSE): return False elif data[4]==ord(AP_DVECTOR): tam = int.from_bytes(data[5:7],byteorder='little') if tam*8+7>len(data): print("BAD FORMATED MESSAGE") else: return [x[0] for x in struct.iter_unpack('d',data[7:])] elif data[4]==ord(AP_LM_INFO): tam = int.from_bytes(data[5:7], byteorder='little') if tam * (8+8+2) + 7 > len(data): print("BAD FORMATED MESSAGE") else: return [x for x in struct.iter_unpack('<Hdd', data[7:])]
def getLUTs(filename): LUTs = {} with open(filename, 'rb') as LUTFile: count = struct.unpack('<B', LUTFile.read(1))[0] for i in range(count): index = struct.unpack('<B', LUTFile.read(1))[0] LUT = [x[0] for x in struct.iter_unpack('<B', LUTFile.read(256))] LUTs[index] = LUT return LUTs
def unpack_coils(self, coilbytes): """Make n coil objects, based on the coil data given with this component.""" coils = [] toolcount = int(struct.unpack('>I', coilbytes[:4])[0]) coilstruct = Coil.struct_for_componenttype[self.comp_type] toolsforeseen = (len(coilbytes)-4) / struct.calcsize(coilstruct) for vals in struct.iter_unpack(coilstruct, coilbytes[4:]): # interpret the values according to the component type coils.append(Coil(loc_dict=vals, dfct=self.comp_type)) return coils
def load_map(map_path): map_data = open(map_path, 'rb').read() eb = (len(map_data) - 3) % 4 if eb > 0: #print("{}: {} extra bytes".format(map_path, eb)) # many maps seem to have an extra byte tacked on. # must be a bug in some editor pass return MapStruct( [MapRec(*rec) for rec in struct.iter_unpack('4B', map_data[3:-eb])], *struct.unpack('3B', map_data[:3]))
def getPalette(filename): palette = [] with open(filename, 'rb') as palFile: for r, g, b in struct.iter_unpack('<BBB', palFile.read(768)): if r > 63 or g > 63 or b > 63: raise BadPaletteException("Color value out of range 0 - 63.") palette.append((linearTable[r], linearTable[g], linearTable[b])) if len(palette) != 256: raise BadPaletteException("Couldn't read enough colors.") return palette
def load_world_dat(planet_dat): "http://www.ufopaedia.org/index.php/WORLD.DAT" dat = open(planet_dat, "rb").read() dat_t = struct.Struct('<hhhhhhhhhh') rv = [] # 8 for rec in struct.iter_unpack('<hhhhhhhhhh', dat): if rec[6] == -1: # tri rv.append([rec[8]] + list(map(lambda x: x/8.0, rec[:6]))) else: # quad rv.append([rec[8]] + list(map(lambda x: x/8.0, rec[:8]))) return rv
def unpackGRPImage(data): width, height, bpp = struct.unpack_from('<HHH', data, 0) if bpp == 8: pass elif bpp == 24: pass elif bpp == 32: pixels = [(r, g, b, a) for b, g, r, a in struct.iter_unpack('<BBBB', data[16:])] im = Image.new('RGBA', (width, height)) im.putdata(pixels) out = io.BytesIO() im.save(out, format='PNG') return out.getvalue()
def _parse_stream_cap_struct(self, bytes_data): tplg_stream_caps_fields = [ "size", "name", "formats", "rates", "rate_min", "rate_max", "channels_min", "channels_max", "periods_min", "periods_max", "period_size_min", "period_size_max", "buffer_size_min", "buffer_size_max", "sig_bits" ] stream_cap_value = [] stream_cap_value.append(struct.unpack("I", bytes_data[:4])[0]) stream_cap_value.append(self._parse_char_array(bytes_data[4:48])) stream_cap_value.append(struct.unpack("Q", bytes_data[48:56])[0]) # rates ... sig_bits are all u32 type for i in list(struct.iter_unpack("I", bytes_data[56:])): stream_cap_value.append(i[0]) stream_cap_struct = dict(zip(tplg_stream_caps_fields, stream_cap_value)) return stream_cap_struct
async def object_sid_to_string(self, objectsid): version = struct.unpack('B', objectsid[0:1])[0] if version != 1: raise CallError(f"{version}: Invalid SID version") sid_length = struct.unpack('B', objectsid[1:2])[0] authority = struct.unpack(b'>Q', b'\x00\x00' + objectsid[2:8])[0] objectsid = objectsid[8:] if len(objectsid) != 4 * sid_length: raise CallError("Invalid SID length") output_sid = f'S-{version}-{authority}' for v in struct.iter_unpack('<L', objectsid): output_sid += f'-{v[0]}' return output_sid
def read_binary_file(self, file_name, obj_list): if self.fixed_format: format_str = '=' + ''.join( [v.file_type * v.num for v in self.variables if v.active]) num_bytes = struct.calcsize(format_str) with open(file_name, 'rb') as file: file_iter = struct.iter_unpack(format_str, file.read()) for obj, data_values in zip(obj_list, file_iter): ind = 0 for v in self.variables: if v.active: if isinstance(v.py_type, list): setattr(obj, v.name, list(data_values[ind:(ind + v.num)])) else: setattr(obj, v.name, data_values[ind]) ind += v.num else: with open(file_name, 'rb') as file: for obj in obj_list: for v in self.variables: if v.active: if isinstance(v.py_type, list): if v.num < 0: input_num = struct.unpack( '=I', file.read(struct.calcsize('I')))[0] format_str = '=' + v.file_type * input_num data_values = struct.unpack( format_str, file.read(struct.calcsize(format_str))) setattr(obj, v.name, list(data_values)) else: format_str = '=' + v.file_type * v.num data_values = struct.unpack( format_str, file.read(struct.calcsize(format_str))) setattr(obj, v.name, list(data_values)) else: format_str = '=' + v.file_type * v.num data_values = struct.unpack( format_str, file.read(struct.calcsize(format_str)))[0] setattr(obj, v.name, data_values)
def _parse_pcm_struct(self, bytes_data): pcm_fields = ["size", "pcm_name", "dai_name", "pcm_id", "dai_id", "playback", "capture", "compress", "stream", "num_streams", "caps", "flag_mask", "flags", "priv"] values = [] values.append(struct.unpack("I",bytes_data[:4])[0]) values.append(self._parse_char_array(bytes_data[4:48])) values.append(self._parse_char_array(bytes_data[48:92])) for i in list(struct.iter_unpack("I", bytes_data[92:112])): values.append(i[0]) # parse snd_soc_tplg_stream array stream_list = [] tplg_stream_size = 72 # tplg_stream size is 72 stream_data = bytes_data[112:688] for idx in range(8): stream_start = tplg_stream_size * idx stream_list.append(self._parse_stream_struct(stream_data[stream_start: stream_start + 72])) values.append(stream_list) values.append(struct.unpack("I", bytes_data[688:692])[0]) # parse snc_soc_tplg_stream_caps stream_cap_list = [] tplg_stream_caps_size = 104 stream_cap_data = bytes_data[692:900] for idx in range(2): start = tplg_stream_caps_size * idx stream_cap_list.append(self._parse_stream_cap_struct(stream_cap_data[start:start+tplg_stream_caps_size])) values.append(stream_cap_list) values.append(struct.unpack("I", bytes_data[900:904])[0]) values.append(struct.unpack("I", bytes_data[904:908])[0]) priv_size = struct.unpack("I", bytes_data[908:912])[0] priv = {"size":priv_size} if priv_size == 0: priv["data"] = None else : priv["data"] = bytes_data[908:908+priv_size] values.append(priv) pcm = dict(zip(pcm_fields, values)) if len(bytes_data[912 + priv_size -1:]) < 4: return pcm, None return pcm, bytes_data[912+priv_size:]
def decode_legacy(src, dst): with open(src, "rb") as f: signature = f.read(4) assert (signature == b"\xffTCI") q, w, h = struct.unpack("<HII", f.read(10)) lines = ceil(h, 2 * q) blocks = ceil(w, q) + 1 data = list(struct.iter_unpack("<BBB", f.read())) assert (len(data) == lines * blocks * 3) color = iter(data) im = PIL.Image.new("RGB", (w, h)) px = im.load() for j in range(lines): triangles = [ tuple(next(color) for _ in range(3)) for _ in range(blocks) ] for y in range(2 * j * q, min(2 * (j + 1) * q, h)): for x in range(0, w): t = x // q if (t + j) % 2 == 0: a = x % q b = y % (2 * q) if b < 2 * a: t += 1 else: a = q - x % q b = y % (2 * q) if b >= 2 * a: t += 1 px[x, y] = get_gradient((x, y), get_dots(t, j, q), triangles[t]) im.save(dst)
def LoadData(self, fnData, dev): fid = open(fnData,'rb') head = fid.read(16) data = fid.read() fid.close() res = struct.unpack(">iiii", head) data1 = struct.iter_unpack(">"+"B"*784,data) self.d = torch.zeros(res[1],1,res[2],res[3]) for idx,k in enumerate(data1): tmp = torch.Tensor(k) tmp = tmp.view(1,res[2],res[3]) if self.transform: tmp = self.transform(tmp) self.d[idx,:,:,:] = tmp self.d = self.d.to(dev)
def _load_4byte_le_words(self, data: bytes) -> None: '''Replace the start of memory with data The bytes loaded should represent each 32-bit word with 4 bytes in little-endian format. ''' if len(data) > 32 * len(self.data): raise ValueError('Trying to load {} bytes of data, but DMEM ' 'is only {} bytes long.'.format( len(data), 32 * len(self.data))) # Zero-pad bytes up to the next multiple of 32 bits (because things # are little-endian, is like zero-extending the last word). if len(data) % 4: data = data + b'0' * (32 - (len(data) % 32)) for idx32, u32 in enumerate(struct.iter_unpack('<I', data)): self.data[idx32] = u32[0]
def read(path, last_days=7, fmt='<Lff'): size = calcsize(fmt) with open(path, 'rb') as f: # search back given number of days epoch, _, _ = tail(path, fmt=fmt) since = datetime.timestamp( datetime.fromtimestamp(epoch) - timedelta(days=last_days)) # binary search for starting index n = int(getsize(path) / size) l, r = 0, n - 1 while not l == r: m = ceil((l + r) / 2) f.seek(m * size) if unpack(fmt, f.read(size))[0] > since: r = m - 1 else: l = m return iter_unpack(fmt, f.read())
def getAltPalettes(filename): altPalettes = [] with open(filename, 'rb') as LUTFile: count = struct.unpack('<B', LUTFile.read(1))[0] LUTFile.seek(count * 257, 1) while True: palData = LUTFile.read(768) if len(palData) < 768: break palette = [] for r, g, b in struct.iter_unpack('<BBB', palData): if r > 63 or g > 63 or b > 63: raise BadPaletteException("Color value out of range 0 - 63.") palette.append((linearTable[r], linearTable[g], linearTable[b])) altPalettes.append(palette) return altPalettes
def stream_callback(self, indata, frames, time, status): peak = min([x[0] for x in struct.iter_unpack('h', indata)]) if self.multi_press_timer > 0 and self.multi_press_timer < multi_press_blocks: self.multi_press_timer += 1 elif self.multi_press_timer >= multi_press_blocks: self.pressed(self.press_count) self.multi_press_timer = 0 self.press_count = 0 if peak < press_amplitude_threshold and self.press_timer == 0: self.press_timer += 1 self.press_count += 1 self.multi_press_timer = 1 elif self.press_timer > 0 and self.press_timer < press_duration_blocks: self.press_timer += 1 else: self.press_timer = 0
def _read_data(self): while True: data_head = self.socket.recv(12) if len(data_head) != 0: break size = int.from_bytes(data_head[8:12], byteorder='big') # 4400 data = bytearray() while len(data) < size: data += self.socket.recv(size - len(data)) data = [ i[0] * self.BResolution for i in struct.iter_unpack(self.pattern, data) ] # (ch_num*20,) 1100 self.signal += [ data[i:i + self.ch_num - 1] for i in range(0, len(data), self.ch_num) ] # remove label column self.data_time.append(time())
def _parse(self, block, format): #format = [(32,'s'),(4,'i'),(2,'d')] """Translate a *block* of binary data into list of values in specified *format*. *format* should be a list of pairs *(a,t)* where *t* is one of the following characters: ``'s'`` for string (bytes), ``'i'`` for 32-bit integer, ``'q'`` for 64-bit integer and *a* is the number of occurrences (or length of a string). For example, if *format* is equal to ``[(32,'s'),(4,'i'),(2,'d'),(2,'i')]``, the contents of *block* are divided into 72 bytes (32*1 + 4*4 + 2*8 + 2*4 = 72) chunks (possibly droping the last one, if it's shorter than 72 bytes). Then each chunk is translated to a 9-tuple of bytes, 4 ints, 2 floats and 2 ints. List of such tuples is the returned value. """ step = 0 formatstring = self.endian for a, t in format: step += a * self._sizes[t] formatstring += str(a) + t if step > 0: end = (len(block) // step) * step return list(struct.iter_unpack(formatstring, block[:end])) else: return []
def get_trial_bin_iter(trial_size, num_trials): trials_returned = 0 file_index = 0 while trials_returned < num_trials: with open( os.path.join('trials', 't_' + str(trial_size) + "bin", "trials" + str(file_index) + ".bin"), 'rb') as file: data = file.read() # instruct = "Q" * ((len(data) ) // 8) arr_iter = struct.iter_unpack("!Q", data) for a in arr_iter: if trials_returned >= num_trials: break yield a[0] trials_returned += 1 file_index += 1
def parse(cls, stream): num, protocol, spi_size, n_transforms = struct.unpack( '>BBBB', stream.read(4)) spi = stream.read(spi_size) transforms = [] more = True while more: more, length, type, id = struct.unpack('>BxHBxH', stream.read(8)) keylen = None for attr_type, value in struct.iter_unpack('>HH', stream.read(length - 8)): if attr_type & 0x7FFF == 0xe: keylen = value transforms.append( Transform(enums.Transform(type), enums.TransformTable[type](id), keylen)) return Proposal(num, protocol, spi, transforms)
def get(cls, bytestream, count): import struct if cls.FORMAT[0] == '*': # have variant-sized or un-type-able objects return [cls(bytestream) for _ in range(count)] else: size = struct.calcsize("<" + cls.FORMAT) fmt = "<" + cls.FORMAT if sys.version_info >= (3, ): items = struct.iter_unpack(fmt, bytestream.read(count * size)) return [cls(bytestream, item) for item in items] else: return [ cls(bytestream, struct.unpack(fmt, bytestream.read(size))) for _ in range(count) ]
def split_into_windows_by_framerate(frames, framerate, num_of_frames, sample_width, channels): windows = [] format_character = FORMAT_CHARACTERS[str(sample_width)] frames_iterator = struct.iter_unpack(format_character, frames) for window in range(num_of_frames // framerate): frames = [] for frame in range(framerate): data = 0 for i in range(channels): data += next(frames_iterator)[0] data /= channels frames.append(data) windows.append(frames) return windows
def rawWavFile(self,fileName): # open wav file without librosa raw = wave.open(fileName,'rb'); params = raw.getparams(); print(params); nFrames = raw.getnframes(); nChannels = raw.getnchannels(); frames = raw.readframes(nFrames); nSampleSize = raw.getsampwidth(); frameRate = raw.getframerate(); print(len(frames)); data_per_channel = [frames[offset::nChannels] for offset in range(nChannels)]; chnlIdx = 0 for s in data_per_channel: print ((chnlIdx,audioop.max(s,nSampleSize))); l = int(len(s)/nSampleSize) print('stft start transform...chnlIdx=' + str(chnlIdx)); if nSampleSize==2: print('start unpacking bytes...') start = timeit.timeit() sg = struct.iter_unpack('>h',s) nums = map(lambda x:x[0],sg) end = timeit.timeit() print('time for unpacking =', end - start) sig = numpy.fromiter(nums,numpy.int16,count=-1) print (sig.shape) print ('start stft...') start = timeit.timeit() stftTransform = scipy.signal.stft(sig, fs=frameRate, window='hann', nperseg=2048, noverlap=2048 / 4) end = timeit.timeit() print('time for stft =', end - start) start = timeit.timeit() stftTransform2 = librosa.core.stft(sig,n_fft=1024) end = timeit.timeit() print ('time for librosa stft =',end-start) librosa.display.specshow(stftTransform2) chnlIdx = chnlIdx + 1; raw.close();
async def async_solve(self, constraints: List[str], variables_length: Dict[str, int]): # print out the values for the symbolic variables constraints.append('(check-sat)') for variable, length in variables_length.items(): for index in range(length): constraints.append( f'(get-value ((select {variable} (_ bv{index} 32))))') constraints.append('(exit)') smt2_file = (self._output_dir / 'minmax.smt2') with smt2_file.open('w') as fp: fp.write('\n'.join(constraints)) process = await asyncio.subprocess.create_subprocess_exec( str(self._binary), str(smt2_file.resolve()), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) out, err = await process.communicate() await process.wait() # parse the output variable_bytes: Dict[str, bytearray] = { variable: bytearray(length) for variable, length in variables_length.items() } is_sat = out.decode('utf-8').split('\n', 1)[0] is_sat = True if is_sat == 'sat' else False for variable, byte_index, value in re.findall( r'\(\(\(select (\w+)\s\(_ bv(\d+) 32\)\) #x([0-9a-f]{2})\)\)', out.decode('utf-8')): variable_bytes[variable][int(byte_index)] = int(value, 16) # convert the bytearray into tuple of ints or int objects = {variable: None for variable in variable_bytes.keys()} for variable, byte_value in variable_bytes.items(): value = tuple(element[0] for element in struct.iter_unpack('i', byte_value)) objects[variable] = value return is_sat, objects
def derive_dwords_sub(self): def get_sub_encoding_bytes(target): """ target x y z 0x100 - (0x21+0x21) = 0xbe We need to select x, y, z such that it gives target when summed and all of x, y, z is ASCII and non-badchar """ # Get all possible solutions all_xy = list( itertools.combinations_with_replacement(self.charset, 2)) results = [] for x, y in all_xy: z = target - (x + y) # Get only bytes which are ASCII and non-badchar if (0 < z < 256) and (z in self.charset): results.append({ 'x': x, 'y': y, 'z': z, 'of': True if target >= 0x100 else False }) # Choose random solution return random.choice(results) for dword in struct.iter_unpack('<L', self.shellcode): # 32-bit 2's complement twos_comp = (dword[0] ^ 0xffffffff) + 1 self.twos_comps.append(twos_comp) encoded_block = [] for byte_ in struct.pack('>L', twos_comp): # Will overflow be used when calculating this byte using 3 SUB instructions? if byte_ / 3 < min(self.charset): byte_ += 0x100 encoded_block.append(get_sub_encoding_bytes(byte_)) pass self.encoded_dwords.append(encoded_block)
def insert(data, tamayio): query = 'INSERT INTO registro (Instante, OUI, OUA, Canal, RSSI) VALUES (%s, %s, %s, %s, %s)' try: c, conector = connection() print('Conectado a base de datos') except Exception as e: print('Error al intentar conectar a base', str(e)) finally: i = 1 for dataII in struct.iter_unpack('B B B B B B c c c c c c B B', data): if i == tamayio + 1: break fecha = datetime.datetime.strptime( str(datetime.datetime(*dataII[0:6])), "00%y-%m-%d %H:%M:%S") OUI = [ bytes.hex(dataII[6]), bytes.hex(dataII[7]), bytes.hex(dataII[8]) ] separator = '' OUA = [ bytes.hex(dataII[9]), bytes.hex(dataII[10]), bytes.hex(dataII[11]) ] separator = '' fechaTratada = fecha.strftime('%Y-%m-%d %H:%M:%S') OUITratado = str(separator.join(OUI)) OUATratado = str(separator.join(OUA)) try: param = (fechaTratada, OUITratado, OUATratado, dataII[12], -dataII[13]) c.execute(query, param) conector.commit() except Exception as e: print('Error al intentar guardar dato ', str(e)) i = i + 1 c.close() conector.close()
def parse_weapons(filename): with open(filename, 'rb') as f: count = 0 for record in struct.iter_unpack('<25s6sdh15s6s5h', f.read()): count += 1 is_deleted = record[10] if is_deleted: continue yield {'id': count, 'name': record[0].decode().strip(), 'damage': record[1].decode().strip(), 'space': record[2], 'criticals': record[3], 'range': record[4].decode().strip(), 'tohit': record[5].decode().strip(), 'maxnum': record[6], 'techbase': get_techbase(record[7]), 'locations': get_locations(record[8]), 'options': get_options(record[9])}
def _read_file(self, mode='r'): """Read in the directory information for the pak file.""" self.fp.seek(0) header = Header.read(self.fp) if header.identity != IDENTITY: raise BadPakFile(f'Bad magic number: {header.identity}') self.end_of_data = header.directory_offset size_of_directory = header.directory_size self.fp.seek(self.end_of_data) data = self.fp.read(size_of_directory) entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)] for entry in entries: info = PakInfo(entry.filename, entry.file_offset, entry.file_size) self.file_list.append(info) self.NameToInfo[info.filename] = info
def _breakdown(self, line, ordering): ''' Convert hex metadata to science useable data. Steps: 1. Throw away away first part of the line 2. Start processing the rest of the line 3. Return a sequence (tuple? list?) Input: line - a sequence specified by format in _parse_scans_meta ordering - the order the string comes in Output: output - a string of converted data according to format Format: Lat, Lon, pressure temp, bottle fire status, NMEA time, Scan time ''' output = '' for x, y in zip(ordering, line): if x == 'nmea_pos': tokens = [] for t in struct.iter_unpack("2s2s2s2s2s2s2s", y): for tt in t: tokens.append(tt) output = output + str( self._location_fix(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6])) + ',' elif x == 'nmea_time': output = output + str( self._sbe_time(self._reverse_bytes(y), 'nmea')) + ',' elif x == 'scan_time': output = output + str( self._sbe_time(self._reverse_bytes(y), 'scan')) elif x == 'btl_status': output = output + str(self._bottle_fire(y)) + ',' elif x == 'pressure_temp': output = output + str(int(y, 16)) + ',' return output
def main(): with open('colors.bin', 'rb') as f: buffer = f.read() print("buffer: {} bytes".format(len(buffer))) indexes = ' '.join(str(n).zfill(2) for n in range(len(buffer))) print(indexes) hex_buffer = hexlify(buffer).decode('ascii') hex_pairs = ' '.join(hex_buffer[i:i+2] for i in range(0, len(hex_buffer), 2)) print(hex_pairs) vertices = [] for fields in struct.iter_unpack('fffHHHxx', buffer): vertex = make_colored_vertex(*fields) vertices.append(vertex) pp(vertices)
def load(cls, filename): ins = cls() f = bread(filename) hdr = f[0:44] data = f[44:] hdata = struct.unpack('<4sI4s4sIHHIIHH4sI', hdr) ins.format = hdata[5] ins.channels = hdata[6] ins.fech = hdata[7] ins.bitspersample = hdata[10] s = '<%d%s' % (ins.channels, ('B' if ins.bitspersample == 8 else 'h')) rawsamples = [u for u in struct.iter_unpack(s, data)] if ins.bitspersample == 8: ins.samples = [[(s - 127) / 127 for s in sample] for sample in rawsamples] elif ins.bitspersample == 16: ins.samples = [[s / 32767 for s in sample] for sample in rawsamples] return ins
def _parse_pe_group_icon_directory( data: bytes) -> Mapping[int, Mapping[str, int]]: #TODO: Use dataclass struct_format = '<BBBBHHIH' _, _, count = struct.unpack('<HHH', data[:6]) #don't need type I think return { entry_id: { 'width': width, 'height': height, 'colour_count': colour_count, 'planes': planes, 'bit_count': bit_count, 'bytes_in_res': bytes_in_res } for width, height, colour_count, _, planes, bit_count, bytes_in_res, entry_id in struct.iter_unpack( struct_format, data[6:6 + (struct.calcsize(struct_format) * count)]) }
def get_header_info(): header = {} header_block = ole.ReadBlock(fp, -1) header['magic_number'] = hexdump.Dump(header_block, 0, 8) # d0 cf 11 e0 a1 b1 1a e1 header['number_bbat_depot'] = struct.unpack( '<I', hexdump.Dump(header_block, 44, 4))[0] header['start_entry_of_property'] = struct.unpack( '<I', hexdump.Dump(header_block, 48, 4))[0] header['start_cluster_of_sbat'] = struct.unpack( '<I', hexdump.Dump(header_block, 60, 4))[0] header['number_sbat_depot'] = struct.unpack( '<I', hexdump.Dump(header_block, 64, 4))[0] iter_bbat = struct.iter_unpack( '<I', hexdump.Dump(header_block, 76, 4 * header['number_bbat_depot'])) header['array_bbat'] = [] for i in range(0, header['number_bbat_depot']): header['array_bbat'].append(next(iter_bbat)[0]) return header
def decode_packets(packet1, packet2) -> str: """ Decodes a pair Parameters ---------- packet1, packet2 : packets (buffers) A pair of ordered packets containing one frame captured by HTPA 32x32d. Returns ------- str Decoded space-delimited temperature values in [1e2 deg. Celsius] (consistent with Heimann's data structure) """ packet = packet1 + packet2 packet_txt = "" for byte in struct.iter_unpack(HTPA32x32d_BYTE_FORMAT, packet): packet_txt += str(byte[0]) + " " return packet_txt
def get_window(data, mono, framerate): window = [] current_sample = 0 data_it = struct.iter_unpack('h', data) for data_tuple in data_it: channel_1 = data_tuple[0] channel_2 = 0 if not mono: try: channel_2 = next(data_it)[0] except StopIteration: print('Invalid source file.', StopIteration) break window.append((channel_1 + channel_2) / 2) current_sample += 1 if current_sample == framerate: yield window del window[:] current_sample = 0
def ReceiveConvolutionInternal(freq_list): data_size = 7 * 4 assert (len(freq_list) <= data_size) print("freq_list=", freq_list) data = [] for freq in freq_list: data += struct.pack("II", 0xABDE0001, freq) ser.write(data) print("len_sended=", len(data)) request_received = len(freq_list) * data_size is_empty = False data_sum = bytearray() while True: data = ser.read_all() if len(data) == 0: print(".", end='') sys.stdout.flush() time.sleep(0.1) continue is_empty = False data_sum += data if len(data_sum) >= request_received: break time.sleep(0.01) freq_offset = 0 out = [] print(" ") for (real_freq, re, im, a_re, a_im, b_re, b_im) in struct.iter_unpack("fffffff", data_sum): s = {} s["f"] = real_freq s["re"] = re s["im"] = im s["a_re"] = a_re s["a_im"] = a_im s["b_re"] = b_re s["b_im"] = b_im print("f=", s["f"], "Zx=", s["re"], s["im"]) freq_offset += 1 out.append(s) print("ReceiveConvolution complete. size=", len(data_sum)) return out