def extractFile(filename, start, end, palette, names):
dims = []
with open(filename, 'rb') as artFile:
artFile.seek(16)
count = end - start + 1
widths = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))]
if len(widths) < count:
raise BadArtFileException("Couldn't read enough widths.")
heights = [x[0] for x in struct.iter_unpack('<H', artFile.read(count * 2))]
if len(heights) < count:
raise BadArtFileException("Couldn't read enough heights.")
dims = zip(widths, heights)
#don't need the rest of this
artFile.seek(count * 4, 1)
for dim in enumerate(dims):
width, height = dim[1]
if width == 0 or height == 0:
continue
data = artFile.read(width * height)
if len(data) < width * height:
raise BadArtFileException("Not enough data.")
data = rearrangeData(data, width, height)
name = ""
try:
name = names[start + dim[0]]
except KeyError:
name = "TILE{:04d}".format(start + dim[0])
print(" -> Writing out {:s}...".format(name))
writeBMP(name, width, height, palette, data)
def readPlanes(hdr, data):
planedata = (x[0] for x in iter_unpack("24s", data[hdr.planeDataOffset:][:hdr.planeCount * 24]))
normals = (Point(*x) for x in iter_unpack("<3i", data[hdr.normalListOffset:][:hdr.planeCount * 24]))
data = BytesIO(data[hdr.planeListOffset:])
for plane, normal in zip(planedata, normals):
planePointCount, unknown1, texture, unknown2 = unpack("<2BH6s", data.read(10))
planePoints = [PlanePoint(int(point[0]/12), *point[1:])
for point in iter_unpack("<IHH", data.read(planePointCount * 8))]
yield Plane(unknown1, texture, unknown2, planePoints, normal, plane)
def parse_cmap(tree, node, data):
i = -1
for g, b, r in iter_unpack("3B", data):
i += 1
if r == g == b == 0:
continue
node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))
def test_module_func(self):
# Sanity check for the global struct.iter_unpack()
it = struct.iter_unpack('>IB', bytes(range(1, 11)))
self.assertEqual(next(it), (0x01020304, 5))
self.assertEqual(next(it), (0x06070809, 10))
self.assertRaises(StopIteration, next, it)
self.assertRaises(StopIteration, next, it)
def parse_6int32(name, node, data):
node.data = [Point(*x) for x in iter_unpack("<3i", data)]
for point in node.data:
corner = element("corner")
for k, v in zip(point._fields, point):
corner.append(element(k, text=str(v)))
node.append(corner)
def packets_from_file(filename, recording_time, chunksize=102):
with open(filename, "rb") as f:
while True:
chunk = f.read(chunksize)
if chunk:
sensor_time = datetime.timedelta(milliseconds=struct.unpack("<I", chunk[:4])[0]) # ms since start of capturing system
fused_time = recording_time + sensor_time
packet = {
'timestamp': fused_time,
'data': {}
}
counter = 0
for qw, qx, qy, qz, ax, ay, az in struct.iter_unpack("<hhhhhhh", chunk[4:]):
scale_q = (1.0 / (1 << 14))
scale_a = 1.0 / 100
packet['data'][sensor_id[counter]] = {
# Decode Quaternion: Orientation data (unit-less)
'orientation': [round(scale_q * qw, 6),
round(scale_q * qx, 6),
round(scale_q * qy, 6),
round(scale_q * qz, 6)],
# Decode Vector: Acceleration data (m/s^2)
'acceleration': [round(scale_a * ax, 2),
round(scale_a * ay, 2),
round(scale_a * az, 2)]
}
counter += 1
yield packet
else:
break
def _decode_header(status_dict, names_list, data):
names_list = [name[0].split(b"\x00", 1)[0] for name in
struct.iter_unpack(f"{FGC_MAX_DEV_LEN + 1}s",
data[0 : FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1)])]
offset = FGCD_MAX_DEVS * (FGC_MAX_DEV_LEN + 1)
(status_dict["hostname"],
status_dict["recv_time_sec"],
status_dict["recv_time_usec"],
status_dict["id"],
status_dict["sequence"],
status_dict["send_time_sec"],
status_dict["send_time_usec"],
status_dict["time_sec"],
status_dict["time_usec"]) = struct.unpack(f"!{HOST_NAME_MAX}sllllllll",
data[offset:])
status_dict["hostname"] = status_dict["hostname"].split(b"\x00", 1)[0].decode()
if not status_dict["hostname"]:
status_dict.clear()
return None
return status_dict, names_list
def give_motion_frame(self):
"""Read the next sample from the POS file and assign to coil objects.
:returns: error code, packed bytestring with dataframe response
"""
if self.motion_lines_read < self.max_num_frames:
measurements = self.file.read(self.bytes_in_frame)
# there appears to be no timestamp in the data
timestamp = MocapParent.timestamp_to_microsecs(self.motion_lines_read * self.frame_time)
# 7I is integers for: x, y, z, phi, theta, rms, extra
measurements_by_coil = struct.iter_unpack('< 7f', measurements[:])
#print('&&&',measurements_by_coil)
for coilvals, coilobj in zip(measurements_by_coil, self.component.coils):
floatvals = MocapParent.make_float_or_zero(coilvals)
coilobj.set_args_to_vars(floatvals, mapping=self.mappings)
self.component.timestamp = timestamp
self.latest_timestamp = timestamp
print('test timestamp!!', self.latest_timestamp)
self.motion_lines_read += 1
return 3, DataFrame(components=[self.component]).pack_components_to_df(), None
# no more data available (static, give up)
else:
print("All the frames have been read")
return 4, "No more frames left in mocap file"
def give_motion_frame(self):
"""Read the next sample from the POS file and assign to coil objects.
:returns: error code, packed bytestring with dataframe response
"""
if self.motion_lines_read < self.max_num_frames:
# update the timestamp
timestamp = self.timestamp_to_microsecs(self.motion_lines_read * self.frame_time)
self.component.timestamp, self.latest_timestamp = timestamp, timestamp
print('BVH latest timestamp:', self.latest_timestamp)
# 7f is floats for: x, y, z, phi, theta, rms, extra
measurements_by_coil = struct.iter_unpack('< 7f', self.file.read(self.bytes_in_frame))
# for m in measurements_by_coil:
# print('POS measurement by coil:', m)
#
# print([m for m in measurements_by_coil])
self.component.coils = [CoilBuilder6D.build_from_mapping(self.mappings, coilvals)
for coilvals in measurements_by_coil]
# print([(self.mappings, coilvals)for coilvals in measurements_by_coil])
assert len(self.component.coils) > 0 # at least one coil must be made
# update the line statistics
self.motion_lines_read += 1
return 3, DataFrame(components=[self.component]).pack_all(), None
# no more data available (static, give up)
else:
print("All the frames have been read")
return 4, "No more frames left in mocap file"
def parse_dict(data):
dct = {}
mode = 'pad'
key = []
value = []
for c in struct.iter_unpack('c', data):
c = c[0]
if mode == 'pad':
if c in (bytes([0]), bytes([3])):
continue
else:
mode = 'key'
if mode == 'key':
if c == bytes([0]):
mode = 'value'
else:
key.append(c.decode())
elif mode == 'value':
if c == bytes([0]):
dct[''.join(key)] = ''.join(value)
key = []
value = []
mode = 'pad'
else:
value.append(c.decode())
return dct
def parse_compact_node_info(data):
"""
Unpack a list of node id, IPv4 address and port returned by a find_node or get_peers request.
"""
for frame in struct.iter_unpack("!20s4BH", data):
# (id, (ip, port))
yield {'id':frame[0], 'addr':("{}.{}.{}.{}".format(frame[1], frame[2], frame[3], frame[4]), frame[5])}
def parse_minor_mesh_form(self, mesh_form, lod_lev=0):
if lod_lev not in self.lods:
self.lods[lod_lev] = {}
self.lods[lod_lev]["mats"] = []
self.lods[lod_lev]["altmats"] = []
self.lods[lod_lev]["lightflags"] = []
vers_form = self.iff.read_data()
mesh_vers = int(vers_form["name"].decode("ascii"))
self.lods[lod_lev]["version"] = mesh_vers
mnrmsh_read = 16 # Bytes for version form header
while mnrmsh_read < mesh_form["length"]:
mdat = self.iff.read_data()
mnrmsh_read += 8 + mdat["length"]
mnrmsh_read += 1 if mdat["length"] % 2 == 1 else 0
if mdat["name"] == b"NAME":
self.lods[lod_lev]["name"] = (
self.parse_cstr(mdat["data"], 0))
elif mdat["name"] == b"FACE":
# This isn't part of Wing Blender, so I'm using features
# from newer versions of Python 3.
for f in struct.iter_unpack(self.FACE_FMT, mdat["data"]):
if f[2] not in self.lods[lod_lev]["mats"]:
self.lods[lod_lev]["mats"].append(f[2])
if f[5] not in self.lods[lod_lev]["lightflags"]:
self.lods[lod_lev]["lightflags"].append(f[5])
if f[6] not in self.lods[lod_lev]["altmats"]:
self.lods[lod_lev]["altmats"].append(f[6])
def get_intlist(elf):
intdata = elf.get_section_by_name("intList").data()
header_sz = struct.calcsize(intlist_header_fmt)
header = struct.unpack_from(intlist_header_fmt, intdata, 0)
intdata = intdata[header_sz:]
spurious_code = header[0]
spurious_nocode = header[1]
debug("spurious handler (code) : %s" % hex(header[0]))
debug("spurious handler (no code) : %s" % hex(header[1]))
intlist = [i for i in
struct.iter_unpack(intlist_entry_fmt, intdata)]
debug("Configured interrupt routing")
debug("handler irq pri vec dpl")
debug("--------------------------")
for irq in intlist:
debug("{0:<10} {1:<3} {2:<3} {3:<3} {4:<2}".format(
hex(irq[0]),
"-" if irq[1] == -1 else irq[1],
"-" if irq[2] == -1 else irq[2],
"-" if irq[3] == -1 else irq[3],
irq[4]))
return (spurious_code, spurious_nocode, intlist)
def decode_tilemap(tilemap, tiles, palettes):
"""
Creates a list of colored tiles from a tilemap, raw tiles and a
color palette
"""
tilemap_tiles = []
for char in struct.iter_unpack("<H", tilemap):
char = char[0]
# Layout: Section 9.3 at http://www.coranac.com/tonc/text/regbg.htm
tile_id = char & ((1 << 10) - 1)
flip_h = bool(char & (1 << 10))
flip_v = bool(char & (1 << 11))
pal_id = char >> 12
pal = palettes[pal_id]
tile = color_tile(tiles[tile_id], pal)
tile_img = tile_image(tile)
if flip_h:
tile_img = tile_img.transpose(Image.FLIP_LEFT_RIGHT)
if flip_v:
tile_img = tile_img.transpose(Image.FLIP_TOP_BOTTOM)
tilemap_tiles.append(tile_img)
return tilemap_tiles
def test_extract_palette(self):
extracted = assets.files.backgrounds['mindscape'].extract_palette()
with open(os.path.join(test_dir, 'mindscap_palette_extract_1.bin'), 'rb') as f:
test_data = f.read()
assert extracted == [i[0] for i in iter_unpack('<H', test_data)]
def parse_compact_node6_info(data):
"""
Unpack a list of node id, IPv6 address and port returned by a find_node or get_peers request.
"""
for frame in struct.iter_unpack("!20s8HH", data):
# (id, (ipv6, port))
yield {'id':frame[0], 'addr':('{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}:{:04X}'.format(*frame[1:9]), frame[9])}
def unpack_data(self, buf):
unpacked_data = {}
payload = struct.iter_unpack('iffffffff', buf[5:])
for item in payload:
if item != None:
unpacked_data[item[0]] = item[1:]
return unpacked_data
def parse_htbl(tree, node, data):
i = -1
res = []
for x in iter_unpack("256s", data):
i += 1
node.append(element("unknown", text=to_hex(x[0]), attrib={"id": str(i)}))
res.append(x[0])
node.data = res
def parse_hicl(tree, node, data):
i = -1
for x in iter_unpack(">H", data):
i += 1
r, g, b = unpack_15bpp(x[0])
if r == g == b == 0:
continue
node.append(element("colour", text="%d %d %d" % (r, g, b), attrib={"id": str(i)}))
def unpack_data_from_file(file, fmt, offset=0):
"""Retrieve data from given file and unpack them according to specified format.
file: The path name of the file or an opened file object
fmt: Format specification
offset: Offset from the start of the file from where to start the unpacking operation
"""
if isinstance(file, str):
if not os.path.isfile(file):
raise ValueError('"{}" is not a regular file'.format(file))
else:
if not hasattr(file, 'read'):
raise ValueError('Invalid file object')
if hasattr(file, 'mode') and file.mode != 'rb':
raise ValueError('Invalid opening mode')
if not isinstance(offset, int) or offset < 0:
raise ValueError('Invalid offset value')
with fileutil.open_file(file, 'rb') as fp:
file_len = fileutil.file_size(fp)
pack_size = struct.calcsize(fmt)
if file_len <= offset:
return
fp.seek(offset, os.SEEK_SET)
if (file_len - offset) % pack_size == 0:
if sys.version_info >= (3, 3):
#return struct.iter_unpack(fmt, fp.read()) # Fix issue #1
yield from struct.iter_unpack(fmt, fp.read())
else:
for unpacked_data in struct.iter_unpack(fmt, fp.read()):
yield unpacked_data
else:
# The length of the file isn't the multiple of struct.calcsize(fmt), so
# don't call struct.iter_unpack directly.
data = fp.read(pack_size)
while data:
if len(data) == pack_size:
yield struct.unpack(fmt, data)
else:
break
data = fp.read(pack_size)
def read_intlist(intlist_path):
"""read a binary file containing the contents of the kernel's .intList
section. This is an instance of a header created by
include/linker/intlist.ld:
struct {
u32_t num_vectors; <- typically CONFIG_NUM_IRQS
struct _isr_list isrs[]; <- Usually of smaller size than num_vectors
}
Followed by instances of struct _isr_list created by IRQ_CONNECT()
calls:
struct _isr_list {
/** IRQ line number */
s32_t irq;
/** Flags for this IRQ, see ISR_FLAG_* definitions */
s32_t flags;
/** ISR to call */
void *func;
/** Parameter for non-direct IRQs */
void *param;
};
"""
intlist = {}
prefix = endian_prefix()
intlist_header_fmt = prefix + "II"
intlist_entry_fmt = prefix + "iiII"
with open(intlist_path, "rb") as fp:
intdata = fp.read()
header_sz = struct.calcsize(intlist_header_fmt)
header = struct.unpack_from(intlist_header_fmt, intdata, 0)
intdata = intdata[header_sz:]
debug(str(header))
intlist["num_vectors"] = header[0]
intlist["offset"] = header[1]
intlist["interrupts"] = [i for i in
struct.iter_unpack(intlist_entry_fmt, intdata)]
debug("Configured interrupt routing")
debug("handler irq flags param")
debug("--------------------------")
for irq in intlist["interrupts"]:
debug("{0:<10} {1:<3} {2:<3} {3}".format(
hex(irq[2]), irq[0], irq[1], hex(irq[3])))
return intlist
def query_values(self, string, nValues=0):
if nValues != 0:
nBytes = 2 + 16*nValues + 1;
buf = b''
if self.write(string):
result = self.read(string=False, nBytes=nBytes)
unpacker = struct.iter_unpack('d',result)
return numpy.array(list(unpacker))
else:
return None
def splitByteIntoNArrays(lineList, N, block_size, type_ = 'B'):
# print("type: " + type_)
# assert( len(lineList) == (1 + 2*rangel)*N ), 'wrong blob length: %u' % len(lineList)
" split into N int numpy sub-arrays within list intList"
it = struct.iter_unpack(type_, lineList)
intList = [[] for Null in range(N)]
for i, temp_char in enumerate(it):
# print('ind %u\t arr %u' % (i, i//(2*rangel+1)) )
intList[i // block_size].append(temp_char[0])
return intList
def extractData(data):
if len(data)<5 or data[0:2] != b'aa':
return None
size = int.from_bytes(data[2:4],byteorder='little')
if data[4]==ord(AP_TRUE):
return True
elif data[4]==ord(AP_FALSE):
return False
elif data[4]==ord(AP_DVECTOR):
tam = int.from_bytes(data[5:7],byteorder='little')
if tam*8+7>len(data):
print("BAD FORMATED MESSAGE")
else:
return [x[0] for x in struct.iter_unpack('d',data[7:])]
elif data[4]==ord(AP_LM_INFO):
tam = int.from_bytes(data[5:7], byteorder='little')
if tam * (8+8+2) + 7 > len(data):
print("BAD FORMATED MESSAGE")
else:
return [x for x in struct.iter_unpack('<Hdd', data[7:])]
def getLUTs(filename):
LUTs = {}
with open(filename, 'rb') as LUTFile:
count = struct.unpack('<B', LUTFile.read(1))[0]
for i in range(count):
index = struct.unpack('<B', LUTFile.read(1))[0]
LUT = [x[0] for x in struct.iter_unpack('<B', LUTFile.read(256))]
LUTs[index] = LUT
return LUTs
def unpack_coils(self, coilbytes):
"""Make n coil objects, based on the coil data given with this component."""
coils = []
toolcount = int(struct.unpack('>I', coilbytes[:4])[0])
coilstruct = Coil.struct_for_componenttype[self.comp_type]
toolsforeseen = (len(coilbytes)-4) / struct.calcsize(coilstruct)
for vals in struct.iter_unpack(coilstruct, coilbytes[4:]):
# interpret the values according to the component type
coils.append(Coil(loc_dict=vals, dfct=self.comp_type))
return coils
def load_map(map_path):
map_data = open(map_path, 'rb').read()
eb = (len(map_data) - 3) % 4
if eb > 0:
#print("{}: {} extra bytes".format(map_path, eb))
# many maps seem to have an extra byte tacked on.
# must be a bug in some editor
pass
return MapStruct(
[MapRec(*rec) for rec in struct.iter_unpack('4B', map_data[3:-eb])],
*struct.unpack('3B', map_data[:3]))
def getPalette(filename):
palette = []
with open(filename, 'rb') as palFile:
for r, g, b in struct.iter_unpack('<BBB', palFile.read(768)):
if r > 63 or g > 63 or b > 63:
raise BadPaletteException("Color value out of range 0 - 63.")
palette.append((linearTable[r], linearTable[g], linearTable[b]))
if len(palette) != 256:
raise BadPaletteException("Couldn't read enough colors.")
return palette
def load_world_dat(planet_dat):
"http://www.ufopaedia.org/index.php/WORLD.DAT"
dat = open(planet_dat, "rb").read()
dat_t = struct.Struct('<hhhhhhhhhh')
rv = []
# 8
for rec in struct.iter_unpack('<hhhhhhhhhh', dat):
if rec[6] == -1: # tri
rv.append([rec[8]] + list(map(lambda x: x/8.0, rec[:6])))
else: # quad
rv.append([rec[8]] + list(map(lambda x: x/8.0, rec[:8])))
return rv
def unpackGRPImage(data):
width, height, bpp = struct.unpack_from('<HHH', data, 0)
if bpp == 8:
pass
elif bpp == 24:
pass
elif bpp == 32:
pixels = [(r, g, b, a) for b, g, r, a in struct.iter_unpack('<BBBB', data[16:])]
im = Image.new('RGBA', (width, height))
im.putdata(pixels)
out = io.BytesIO()
im.save(out, format='PNG')
return out.getvalue()
def _parse_stream_cap_struct(self, bytes_data):
tplg_stream_caps_fields = [
"size", "name", "formats", "rates", "rate_min", "rate_max",
"channels_min", "channels_max", "periods_min", "periods_max",
"period_size_min", "period_size_max", "buffer_size_min",
"buffer_size_max", "sig_bits"
]
stream_cap_value = []
stream_cap_value.append(struct.unpack("I", bytes_data[:4])[0])
stream_cap_value.append(self._parse_char_array(bytes_data[4:48]))
stream_cap_value.append(struct.unpack("Q", bytes_data[48:56])[0])
# rates ... sig_bits are all u32 type
for i in list(struct.iter_unpack("I", bytes_data[56:])):
stream_cap_value.append(i[0])
stream_cap_struct = dict(zip(tplg_stream_caps_fields,
stream_cap_value))
return stream_cap_struct
async def object_sid_to_string(self, objectsid):
version = struct.unpack('B', objectsid[0:1])[0]
if version != 1:
raise CallError(f"{version}: Invalid SID version")
sid_length = struct.unpack('B', objectsid[1:2])[0]
authority = struct.unpack(b'>Q', b'\x00\x00' + objectsid[2:8])[0]
objectsid = objectsid[8:]
if len(objectsid) != 4 * sid_length:
raise CallError("Invalid SID length")
output_sid = f'S-{version}-{authority}'
for v in struct.iter_unpack('<L', objectsid):
output_sid += f'-{v[0]}'
return output_sid
def read_binary_file(self, file_name, obj_list):
if self.fixed_format:
format_str = '=' + ''.join(
[v.file_type * v.num for v in self.variables if v.active])
num_bytes = struct.calcsize(format_str)
with open(file_name, 'rb') as file:
file_iter = struct.iter_unpack(format_str, file.read())
for obj, data_values in zip(obj_list, file_iter):
ind = 0
for v in self.variables:
if v.active:
if isinstance(v.py_type, list):
setattr(obj, v.name,
list(data_values[ind:(ind + v.num)]))
else:
setattr(obj, v.name, data_values[ind])
ind += v.num
else:
with open(file_name, 'rb') as file:
for obj in obj_list:
for v in self.variables:
if v.active:
if isinstance(v.py_type, list):
if v.num < 0:
input_num = struct.unpack(
'=I',
file.read(struct.calcsize('I')))[0]
format_str = '=' + v.file_type * input_num
data_values = struct.unpack(
format_str,
file.read(struct.calcsize(format_str)))
setattr(obj, v.name, list(data_values))
else:
format_str = '=' + v.file_type * v.num
data_values = struct.unpack(
format_str,
file.read(struct.calcsize(format_str)))
setattr(obj, v.name, list(data_values))
else:
format_str = '=' + v.file_type * v.num
data_values = struct.unpack(
format_str,
file.read(struct.calcsize(format_str)))[0]
setattr(obj, v.name, data_values)
def _parse_pcm_struct(self, bytes_data):
pcm_fields = ["size", "pcm_name", "dai_name", "pcm_id", "dai_id", "playback", "capture",
"compress", "stream", "num_streams", "caps", "flag_mask", "flags", "priv"]
values = []
values.append(struct.unpack("I",bytes_data[:4])[0])
values.append(self._parse_char_array(bytes_data[4:48]))
values.append(self._parse_char_array(bytes_data[48:92]))
for i in list(struct.iter_unpack("I", bytes_data[92:112])):
values.append(i[0])
# parse snd_soc_tplg_stream array
stream_list = []
tplg_stream_size = 72 # tplg_stream size is 72
stream_data = bytes_data[112:688]
for idx in range(8):
stream_start = tplg_stream_size * idx
stream_list.append(self._parse_stream_struct(stream_data[stream_start: stream_start + 72]))
values.append(stream_list)
values.append(struct.unpack("I", bytes_data[688:692])[0])
# parse snc_soc_tplg_stream_caps
stream_cap_list = []
tplg_stream_caps_size = 104
stream_cap_data = bytes_data[692:900]
for idx in range(2):
start = tplg_stream_caps_size * idx
stream_cap_list.append(self._parse_stream_cap_struct(stream_cap_data[start:start+tplg_stream_caps_size]))
values.append(stream_cap_list)
values.append(struct.unpack("I", bytes_data[900:904])[0])
values.append(struct.unpack("I", bytes_data[904:908])[0])
priv_size = struct.unpack("I", bytes_data[908:912])[0]
priv = {"size":priv_size}
if priv_size == 0:
priv["data"] = None
else :
priv["data"] = bytes_data[908:908+priv_size]
values.append(priv)
pcm = dict(zip(pcm_fields, values))
if len(bytes_data[912 + priv_size -1:]) < 4:
return pcm, None
return pcm, bytes_data[912+priv_size:]
def decode_legacy(src, dst):
with open(src, "rb") as f:
signature = f.read(4)
assert (signature == b"\xffTCI")
q, w, h = struct.unpack("<HII", f.read(10))
lines = ceil(h, 2 * q)
blocks = ceil(w, q) + 1
data = list(struct.iter_unpack("<BBB", f.read()))
assert (len(data) == lines * blocks * 3)
color = iter(data)
im = PIL.Image.new("RGB", (w, h))
px = im.load()
for j in range(lines):
triangles = [
tuple(next(color) for _ in range(3)) for _ in range(blocks)
]
for y in range(2 * j * q, min(2 * (j + 1) * q, h)):
for x in range(0, w):
t = x // q
if (t + j) % 2 == 0:
a = x % q
b = y % (2 * q)
if b < 2 * a:
t += 1
else:
a = q - x % q
b = y % (2 * q)
if b >= 2 * a:
t += 1
px[x, y] = get_gradient((x, y), get_dots(t, j, q),
triangles[t])
im.save(dst)
def LoadData(self, fnData, dev):
fid = open(fnData,'rb')
head = fid.read(16)
data = fid.read()
fid.close()
res = struct.unpack(">iiii", head)
data1 = struct.iter_unpack(">"+"B"*784,data)
self.d = torch.zeros(res[1],1,res[2],res[3])
for idx,k in enumerate(data1):
tmp = torch.Tensor(k)
tmp = tmp.view(1,res[2],res[3])
if self.transform:
tmp = self.transform(tmp)
self.d[idx,:,:,:] = tmp
self.d = self.d.to(dev)
def _load_4byte_le_words(self, data: bytes) -> None:
'''Replace the start of memory with data
The bytes loaded should represent each 32-bit word with 4 bytes in
little-endian format.
'''
if len(data) > 32 * len(self.data):
raise ValueError('Trying to load {} bytes of data, but DMEM '
'is only {} bytes long.'.format(
len(data), 32 * len(self.data)))
# Zero-pad bytes up to the next multiple of 32 bits (because things
# are little-endian, is like zero-extending the last word).
if len(data) % 4:
data = data + b'0' * (32 - (len(data) % 32))
for idx32, u32 in enumerate(struct.iter_unpack('<I', data)):
self.data[idx32] = u32[0]
def read(path, last_days=7, fmt='<Lff'):
size = calcsize(fmt)
with open(path, 'rb') as f:
# search back given number of days
epoch, _, _ = tail(path, fmt=fmt)
since = datetime.timestamp(
datetime.fromtimestamp(epoch) - timedelta(days=last_days))
# binary search for starting index
n = int(getsize(path) / size)
l, r = 0, n - 1
while not l == r:
m = ceil((l + r) / 2)
f.seek(m * size)
if unpack(fmt, f.read(size))[0] > since:
r = m - 1
else:
l = m
return iter_unpack(fmt, f.read())
def getAltPalettes(filename):
altPalettes = []
with open(filename, 'rb') as LUTFile:
count = struct.unpack('<B', LUTFile.read(1))[0]
LUTFile.seek(count * 257, 1)
while True:
palData = LUTFile.read(768)
if len(palData) < 768:
break
palette = []
for r, g, b in struct.iter_unpack('<BBB', palData):
if r > 63 or g > 63 or b > 63:
raise BadPaletteException("Color value out of range 0 - 63.")
palette.append((linearTable[r], linearTable[g], linearTable[b]))
altPalettes.append(palette)
return altPalettes
def stream_callback(self, indata, frames, time, status):
peak = min([x[0] for x in struct.iter_unpack('h', indata)])
if self.multi_press_timer > 0 and self.multi_press_timer < multi_press_blocks:
self.multi_press_timer += 1
elif self.multi_press_timer >= multi_press_blocks:
self.pressed(self.press_count)
self.multi_press_timer = 0
self.press_count = 0
if peak < press_amplitude_threshold and self.press_timer == 0:
self.press_timer += 1
self.press_count += 1
self.multi_press_timer = 1
elif self.press_timer > 0 and self.press_timer < press_duration_blocks:
self.press_timer += 1
else:
self.press_timer = 0
def _read_data(self):
while True:
data_head = self.socket.recv(12)
if len(data_head) != 0:
break
size = int.from_bytes(data_head[8:12], byteorder='big') # 4400
data = bytearray()
while len(data) < size:
data += self.socket.recv(size - len(data))
data = [
i[0] * self.BResolution
for i in struct.iter_unpack(self.pattern, data)
] # (ch_num*20,) 1100
self.signal += [
data[i:i + self.ch_num - 1]
for i in range(0, len(data), self.ch_num)
] # remove label column
self.data_time.append(time())
def _parse(self, block, format): #format = [(32,'s'),(4,'i'),(2,'d')]
"""Translate a *block* of binary data into list of values in specified *format*.
*format* should be a list of pairs *(a,t)* where *t* is one of the following characters: ``'s'`` for string (bytes), ``'i'`` for 32-bit integer, ``'q'`` for 64-bit integer and *a* is the number of occurrences (or length of a string).
For example, if *format* is equal to ``[(32,'s'),(4,'i'),(2,'d'),(2,'i')]``, the contents of *block* are divided into 72 bytes (32*1 + 4*4 + 2*8 + 2*4 = 72) chunks (possibly droping the last one, if it's shorter than 72 bytes). Then each chunk is translated to a 9-tuple of bytes, 4 ints, 2 floats and 2 ints. List of such tuples is the returned value.
"""
step = 0
formatstring = self.endian
for a, t in format:
step += a * self._sizes[t]
formatstring += str(a) + t
if step > 0:
end = (len(block) // step) * step
return list(struct.iter_unpack(formatstring, block[:end]))
else:
return []
def get_trial_bin_iter(trial_size, num_trials):
trials_returned = 0
file_index = 0
while trials_returned < num_trials:
with open(
os.path.join('trials', 't_' + str(trial_size) + "bin",
"trials" + str(file_index) + ".bin"),
'rb') as file:
data = file.read()
# instruct = "Q" * ((len(data) ) // 8)
arr_iter = struct.iter_unpack("!Q", data)
for a in arr_iter:
if trials_returned >= num_trials:
break
yield a[0]
trials_returned += 1
file_index += 1
def parse(cls, stream):
num, protocol, spi_size, n_transforms = struct.unpack(
'>BBBB', stream.read(4))
spi = stream.read(spi_size)
transforms = []
more = True
while more:
more, length, type, id = struct.unpack('>BxHBxH', stream.read(8))
keylen = None
for attr_type, value in struct.iter_unpack('>HH',
stream.read(length -
8)):
if attr_type & 0x7FFF == 0xe:
keylen = value
transforms.append(
Transform(enums.Transform(type),
enums.TransformTable[type](id), keylen))
return Proposal(num, protocol, spi, transforms)
def get(cls, bytestream, count):
import struct
if cls.FORMAT[0] == '*':
# have variant-sized or un-type-able objects
return [cls(bytestream) for _ in range(count)]
else:
size = struct.calcsize("<" + cls.FORMAT)
fmt = "<" + cls.FORMAT
if sys.version_info >= (3, ):
items = struct.iter_unpack(fmt,
bytestream.read(count * size))
return [cls(bytestream, item) for item in items]
else:
return [
cls(bytestream,
struct.unpack(fmt, bytestream.read(size)))
for _ in range(count)
]
def split_into_windows_by_framerate(frames, framerate, num_of_frames,
sample_width, channels):
windows = []
format_character = FORMAT_CHARACTERS[str(sample_width)]
frames_iterator = struct.iter_unpack(format_character, frames)
for window in range(num_of_frames // framerate):
frames = []
for frame in range(framerate):
data = 0
for i in range(channels):
data += next(frames_iterator)[0]
data /= channels
frames.append(data)
windows.append(frames)
return windows
def rawWavFile(self,fileName):
# open wav file without librosa
raw = wave.open(fileName,'rb');
params = raw.getparams();
print(params);
nFrames = raw.getnframes();
nChannels = raw.getnchannels();
frames = raw.readframes(nFrames);
nSampleSize = raw.getsampwidth();
frameRate = raw.getframerate();
print(len(frames));
data_per_channel = [frames[offset::nChannels] for offset in range(nChannels)];
chnlIdx = 0
for s in data_per_channel:
print ((chnlIdx,audioop.max(s,nSampleSize)));
l = int(len(s)/nSampleSize)
print('stft start transform...chnlIdx=' + str(chnlIdx));
if nSampleSize==2:
print('start unpacking bytes...')
start = timeit.timeit()
sg = struct.iter_unpack('>h',s)
nums = map(lambda x:x[0],sg)
end = timeit.timeit()
print('time for unpacking =', end - start)
sig = numpy.fromiter(nums,numpy.int16,count=-1)
print (sig.shape)
print ('start stft...')
start = timeit.timeit()
stftTransform = scipy.signal.stft(sig, fs=frameRate, window='hann', nperseg=2048, noverlap=2048 / 4)
end = timeit.timeit()
print('time for stft =', end - start)
start = timeit.timeit()
stftTransform2 = librosa.core.stft(sig,n_fft=1024)
end = timeit.timeit()
print ('time for librosa stft =',end-start)
librosa.display.specshow(stftTransform2)
chnlIdx = chnlIdx + 1;
raw.close();
async def async_solve(self, constraints: List[str],
variables_length: Dict[str, int]):
# print out the values for the symbolic variables
constraints.append('(check-sat)')
for variable, length in variables_length.items():
for index in range(length):
constraints.append(
f'(get-value ((select {variable} (_ bv{index} 32))))')
constraints.append('(exit)')
smt2_file = (self._output_dir / 'minmax.smt2')
with smt2_file.open('w') as fp:
fp.write('\n'.join(constraints))
process = await asyncio.subprocess.create_subprocess_exec(
str(self._binary),
str(smt2_file.resolve()),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
out, err = await process.communicate()
await process.wait()
# parse the output
variable_bytes: Dict[str, bytearray] = {
variable: bytearray(length)
for variable, length in variables_length.items()
}
is_sat = out.decode('utf-8').split('\n', 1)[0]
is_sat = True if is_sat == 'sat' else False
for variable, byte_index, value in re.findall(
r'\(\(\(select (\w+)\s\(_ bv(\d+) 32\)\) #x([0-9a-f]{2})\)\)',
out.decode('utf-8')):
variable_bytes[variable][int(byte_index)] = int(value, 16)
# convert the bytearray into tuple of ints or int
objects = {variable: None for variable in variable_bytes.keys()}
for variable, byte_value in variable_bytes.items():
value = tuple(element[0]
for element in struct.iter_unpack('i', byte_value))
objects[variable] = value
return is_sat, objects
def derive_dwords_sub(self):
def get_sub_encoding_bytes(target):
"""
target x y z
0x100 - (0x21+0x21) = 0xbe
We need to select x, y, z such that it gives target when summed and all of
x, y, z is ASCII and non-badchar
"""
# Get all possible solutions
all_xy = list(
itertools.combinations_with_replacement(self.charset, 2))
results = []
for x, y in all_xy:
z = target - (x + y)
# Get only bytes which are ASCII and non-badchar
if (0 < z < 256) and (z in self.charset):
results.append({
'x': x,
'y': y,
'z': z,
'of': True if target >= 0x100 else False
})
# Choose random solution
return random.choice(results)
for dword in struct.iter_unpack('<L', self.shellcode):
# 32-bit 2's complement
twos_comp = (dword[0] ^ 0xffffffff) + 1
self.twos_comps.append(twos_comp)
encoded_block = []
for byte_ in struct.pack('>L', twos_comp):
# Will overflow be used when calculating this byte using 3 SUB instructions?
if byte_ / 3 < min(self.charset):
byte_ += 0x100
encoded_block.append(get_sub_encoding_bytes(byte_))
pass
self.encoded_dwords.append(encoded_block)
def insert(data, tamayio):
query = 'INSERT INTO registro (Instante, OUI, OUA, Canal, RSSI) VALUES (%s, %s, %s, %s, %s)'
try:
c, conector = connection()
print('Conectado a base de datos')
except Exception as e:
print('Error al intentar conectar a base', str(e))
finally:
i = 1
for dataII in struct.iter_unpack('B B B B B B c c c c c c B B', data):
if i == tamayio + 1:
break
fecha = datetime.datetime.strptime(
str(datetime.datetime(*dataII[0:6])), "00%y-%m-%d %H:%M:%S")
OUI = [
bytes.hex(dataII[6]),
bytes.hex(dataII[7]),
bytes.hex(dataII[8])
]
separator = ''
OUA = [
bytes.hex(dataII[9]),
bytes.hex(dataII[10]),
bytes.hex(dataII[11])
]
separator = ''
fechaTratada = fecha.strftime('%Y-%m-%d %H:%M:%S')
OUITratado = str(separator.join(OUI))
OUATratado = str(separator.join(OUA))
try:
param = (fechaTratada, OUITratado, OUATratado, dataII[12],
-dataII[13])
c.execute(query, param)
conector.commit()
except Exception as e:
print('Error al intentar guardar dato ', str(e))
i = i + 1
c.close()
conector.close()
def parse_weapons(filename):
with open(filename, 'rb') as f:
count = 0
for record in struct.iter_unpack('<25s6sdh15s6s5h', f.read()):
count += 1
is_deleted = record[10]
if is_deleted:
continue
yield {'id': count,
'name': record[0].decode().strip(),
'damage': record[1].decode().strip(),
'space': record[2],
'criticals': record[3],
'range': record[4].decode().strip(),
'tohit': record[5].decode().strip(),
'maxnum': record[6],
'techbase': get_techbase(record[7]),
'locations': get_locations(record[8]),
'options': get_options(record[9])}
def _read_file(self, mode='r'):
"""Read in the directory information for the pak file."""
self.fp.seek(0)
header = Header.read(self.fp)
if header.identity != IDENTITY:
raise BadPakFile(f'Bad magic number: {header.identity}')
self.end_of_data = header.directory_offset
size_of_directory = header.directory_size
self.fp.seek(self.end_of_data)
data = self.fp.read(size_of_directory)
entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]
for entry in entries:
info = PakInfo(entry.filename, entry.file_offset, entry.file_size)
self.file_list.append(info)
self.NameToInfo[info.filename] = info
def _breakdown(self, line, ordering):
'''
Convert hex metadata to science useable data.
Steps:
1. Throw away away first part of the line
2. Start processing the rest of the line
3. Return a sequence (tuple? list?)
Input:
line - a sequence specified by format in _parse_scans_meta
ordering - the order the string comes in
Output:
output - a string of converted data according to format
Format:
Lat, Lon, pressure temp, bottle fire status, NMEA time, Scan time
'''
output = ''
for x, y in zip(ordering, line):
if x == 'nmea_pos':
tokens = []
for t in struct.iter_unpack("2s2s2s2s2s2s2s", y):
for tt in t:
tokens.append(tt)
output = output + str(
self._location_fix(tokens[0], tokens[1], tokens[2],
tokens[3], tokens[4], tokens[5],
tokens[6])) + ','
elif x == 'nmea_time':
output = output + str(
self._sbe_time(self._reverse_bytes(y), 'nmea')) + ','
elif x == 'scan_time':
output = output + str(
self._sbe_time(self._reverse_bytes(y), 'scan'))
elif x == 'btl_status':
output = output + str(self._bottle_fire(y)) + ','
elif x == 'pressure_temp':
output = output + str(int(y, 16)) + ','
return output
def main():
with open('colors.bin', 'rb') as f:
buffer = f.read()
print("buffer: {} bytes".format(len(buffer)))
indexes = ' '.join(str(n).zfill(2) for n in range(len(buffer)))
print(indexes)
hex_buffer = hexlify(buffer).decode('ascii')
hex_pairs = ' '.join(hex_buffer[i:i+2] for i in range(0, len(hex_buffer), 2))
print(hex_pairs)
vertices = []
for fields in struct.iter_unpack('fffHHHxx', buffer):
vertex = make_colored_vertex(*fields)
vertices.append(vertex)
pp(vertices)
def load(cls, filename):
ins = cls()
f = bread(filename)
hdr = f[0:44]
data = f[44:]
hdata = struct.unpack('<4sI4s4sIHHIIHH4sI', hdr)
ins.format = hdata[5]
ins.channels = hdata[6]
ins.fech = hdata[7]
ins.bitspersample = hdata[10]
s = '<%d%s' % (ins.channels, ('B' if ins.bitspersample == 8 else 'h'))
rawsamples = [u for u in struct.iter_unpack(s, data)]
if ins.bitspersample == 8:
ins.samples = [[(s - 127) / 127 for s in sample]
for sample in rawsamples]
elif ins.bitspersample == 16:
ins.samples = [[s / 32767 for s in sample]
for sample in rawsamples]
return ins
def _parse_pe_group_icon_directory(
data: bytes) -> Mapping[int, Mapping[str, int]]:
#TODO: Use dataclass
struct_format = '<BBBBHHIH'
_, _, count = struct.unpack('<HHH', data[:6]) #don't need type I think
return {
entry_id: {
'width': width,
'height': height,
'colour_count': colour_count,
'planes': planes,
'bit_count': bit_count,
'bytes_in_res': bytes_in_res
}
for width, height, colour_count, _, planes, bit_count, bytes_in_res,
entry_id in struct.iter_unpack(
struct_format, data[6:6 +
(struct.calcsize(struct_format) * count)])
}
def get_header_info():
header = {}
header_block = ole.ReadBlock(fp, -1)
header['magic_number'] = hexdump.Dump(header_block, 0,
8) # d0 cf 11 e0 a1 b1 1a e1
header['number_bbat_depot'] = struct.unpack(
'<I', hexdump.Dump(header_block, 44, 4))[0]
header['start_entry_of_property'] = struct.unpack(
'<I', hexdump.Dump(header_block, 48, 4))[0]
header['start_cluster_of_sbat'] = struct.unpack(
'<I', hexdump.Dump(header_block, 60, 4))[0]
header['number_sbat_depot'] = struct.unpack(
'<I', hexdump.Dump(header_block, 64, 4))[0]
iter_bbat = struct.iter_unpack(
'<I', hexdump.Dump(header_block, 76, 4 * header['number_bbat_depot']))
header['array_bbat'] = []
for i in range(0, header['number_bbat_depot']):
header['array_bbat'].append(next(iter_bbat)[0])
return header
def decode_packets(packet1, packet2) -> str:
"""
Decodes a pair
Parameters
----------
packet1, packet2 : packets (buffers)
A pair of ordered packets containing one frame captured by HTPA 32x32d.
Returns
-------
str
Decoded space-delimited temperature values in [1e2 deg. Celsius] (consistent with Heimann's data structure)
"""
packet = packet1 + packet2
packet_txt = ""
for byte in struct.iter_unpack(HTPA32x32d_BYTE_FORMAT, packet):
packet_txt += str(byte[0]) + " "
return packet_txt
def get_window(data, mono, framerate):
window = []
current_sample = 0
data_it = struct.iter_unpack('h', data)
for data_tuple in data_it:
channel_1 = data_tuple[0]
channel_2 = 0
if not mono:
try:
channel_2 = next(data_it)[0]
except StopIteration:
print('Invalid source file.', StopIteration)
break
window.append((channel_1 + channel_2) / 2)
current_sample += 1
if current_sample == framerate:
yield window
del window[:]
current_sample = 0
def ReceiveConvolutionInternal(freq_list):
data_size = 7 * 4
assert (len(freq_list) <= data_size)
print("freq_list=", freq_list)
data = []
for freq in freq_list:
data += struct.pack("II", 0xABDE0001, freq)
ser.write(data)
print("len_sended=", len(data))
request_received = len(freq_list) * data_size
is_empty = False
data_sum = bytearray()
while True:
data = ser.read_all()
if len(data) == 0:
print(".", end='')
sys.stdout.flush()
time.sleep(0.1)
continue
is_empty = False
data_sum += data
if len(data_sum) >= request_received:
break
time.sleep(0.01)
freq_offset = 0
out = []
print(" ")
for (real_freq, re, im, a_re, a_im, b_re,
b_im) in struct.iter_unpack("fffffff", data_sum):
s = {}
s["f"] = real_freq
s["re"] = re
s["im"] = im
s["a_re"] = a_re
s["a_im"] = a_im
s["b_re"] = b_re
s["b_im"] = b_im
print("f=", s["f"], "Zx=", s["re"], s["im"])
freq_offset += 1
out.append(s)
print("ReceiveConvolution complete. size=", len(data_sum))
return out
