def _get_data_subblocks(name):
"""Return Adapter to parse GIF data sub-blocks."""
return construct.ExprAdapter(
construct.Struct(
name,
construct.RepeatUntil(
lambda obj, ctx: obj.block_size == 0x00,
construct.Struct(
'blocks',
construct.ULInt8('block_size'),
construct.Bytes('data_values', lambda ctx: ctx.block_size),
),
),
),
# from comment string, build Containers
encoder=lambda obj, ctx: construct.Container(blocks=[
construct.Container(
block_size=len(chunk),
data_values=chunk,
) for chunk in [obj[i:i + 255] for i in xrange(0, len(obj), 255)]
] + [construct.Container(block_size=0, data_values='')], ),
# from Containers, build comment string
decoder=lambda obj, ctx: ''.join(dsb.data_values
for dsb in obj.blocks),
)
def ibm2ieee32(ibm_float):
# IBM Float to IEEE Float (32 bit)
# IBM bit pattern:
# SEEEEEEE MMMMMMMM MMMMMMMM MMMMMMMM
#
ibm_s = ibm()
b = ibm_s.parse(ibm_float)
ieee_s = ieee()
#s = sign (0x00000001, n.s)
# This returns a python float which might not be what we want,
# but it shows how we use the bits.
#valuep = s * (16 ** (n.e - 64)) * (float (n.m) / float (2 ** 24))
#
mapi = 0x00800000
i = b.e
i = i - 64
i = i * 4
m = b.m
if m == 0:
return ieee_s.build(construct.Container(s=0, e=0, m=0))
while (m & mapi) == 0:
m = m << 1
i -= 1
i -= 1
c = construct.Container(s=b.s, e=i + 127, m=m)
value = ieee_s.build(c)
#
return value
def getNewGrfs (self, host, port, grf_list) :
# self.log("sending newgrfs query to %s:%s for: %s", host, port, ' '.join([grf.grfid for grf in grf_list]))
header = c.Container(type='UDP_CLIENT_GET_NEWGRFS', size=4+(20)*len(grf_list))
req = c.Container(header=header, newgrf_count=len(grf_list), newgrfs=grf_list)
self.transport.write(NewGrfQuery.build(req), (host, port))
return self.deferred('newgrf', host, port, 2)
def test_flatten():
test_sub = struct.XStruct('c' / construct.Int16ub)
test_struct = struct.XStruct('a' / construct.Int32ub, 'b' / test_sub)
obj = test_struct(a=1, b=test_sub(c=2))
flat = struct.flatten(obj.container)
assert flat == construct.Container(a=1, b=construct.Container(c=2))
def build_command(name, **kw):
if name == 'keep_alive':
return _message.build(c.Container(length=0, payload=''))
s = _commands[name]
obj = c.Container(**kw)
payload = s.build(obj)
return _message.build(c.Container(length=len(payload), payload=payload))
def __init__(self, stream):
self.stream = stream
pos = stream.tell()
stream.seek(64, 1)
maybe_swgb_header = stream.read(4)
stream.seek(pos)
if maybe_swgb_header == "swbg":
self.header = SWGB_HEADER._parse(stream, cons.Container(drs_file=self))
else:
self.header = AOE_HEADER._parse(stream, cons.Container(drs_file=self))
def _getServers_haveAddr (self, host) :
# self.log("getting server list from masterserver (%s)", host)
req = c.Container(type='UDP_CLIENT_GET_LIST', size=3)
self.transport.write(Header.build(req), (host, MASTER_SERVER_PORT))
return self.deferred('servers', host, MASTER_SERVER_PORT, 10, False)
def getInfo (self, host, port) :
# self.log("sending info query to %s:%s", host, port)
req = c.Container(type='UDP_CLIENT_FIND_SERVER', size=3)
self.transport.write(Header.build(req), (host, port))
return self.deferred('info', host, port, 2)
def set_signature(self, signer):
# cds = self.get_codedirectories()
# print "ABOUT TO SIGN", len(cds), "cds"
# for i, cd in enumerate(cds):
# print("cd", i)
# for k, v in cd.iteritems():
# if k == "data":
# print "data:"
# for k, v in v.iteritems():
# if k == "hashes":
# print " ", k, v[:5]
# print " ", "...", len(v)
# elif k != "bytes":
# print " ", k, v
# elif k != "bytes":
# print k, v
# TODO how do we even know this blobwrapper contains the signature?
# seems like this is a coincidence of the structure, where
# it's the only blobwrapper at that level...
# log.debug("sig:")
sigwrapper = self.get_blob('CSMAGIC_BLOBWRAPPER')
oldsig = sigwrapper.bytes.value
# signer._log_parsed_asn1(sigwrapper.data.data.value)
# open("sigrip.der", "wb").write(sigwrapper.data.data.value)
sig = signer.sign(oldsig, self.get_codedirectory_hashes())
# log.debug("sig len: {0}".format(len(sig)))
# log.debug("old sig len: {0}".format(len(oldsig)))
# open("my_sigrip.der", "wb").write(sig)
sigwrapper.data = construct.Container(data=sig)
# signer._log_parsed_asn1(sig)
# sigwrapper.data = construct.Container(data="hahaha")
sigwrapper.length = len(sigwrapper.data.data) + 8
sigwrapper.bytes = sigwrapper.data.data
def getDetails (self, host, port) :
# self.log("sending details query to %s:%s", host, port)
req = c.Container(type='UDP_CLIENT_DETAIL_INFO', size=3)
self.transport.write(Header.build(req), (host, port))
return self.deferred('detail', host, port, 2)
def ieee2ibm32(ieee_float):
# IEEE Float to IBM Float (32 bit)
# IEEE bit pattern:
# SEEEEEEE EMMMMMMM MMMMMMM MMMMMMM
ieee_s = ieee()
e = ieee_s.parse(ieee_float)
ibm_s = ibm()
#s = sign (0x00000001, n.s)
# ...
#valuei = s * (2 ** (n.e - 127)) * (1.0 + (float (n.m) / float (2 ** 23)))
#
mapi = 0x00800000
m = e.m
m = m | mapi
i = e.e
i -= 127
i += 1
while (i % 4) != 0:
m = m >> 1
i += 1
i /= 4
i += 64
c = construct.Container(s=e.s, e=i, m=m)
value = ibm_s.build(c)
#
return value
def _parse(self, stream, context, path):
min = self.min(context) if callable(self.min) else self.min
max = self.max(context) if callable(self.max) else self.max
if not 0 <= min <= max <= sys.maxsize:
raise construct.RangeError("unsane min %s and max %s" % (min, max))
obj = construct.ListContainer()
context = construct.Container(_=context)
try:
# PATCH: add counter to path, so we know the children's index when viewing results.
counter = 0
while len(obj) < max:
fallback = stream.tell()
obj.append(
self.subcon._parse(stream, context._,
path + '[{}]'.format(counter)))
context[len(obj) - 1] = obj[-1]
counter += 1
except StopIteration:
pass
except construct.ExplicitError:
raise
except Exception:
if len(obj) < min:
raise construct.RangeError("expected %d to %d, found %d" %
(min, max, len(obj)))
stream.seek(fallback)
return obj
def _IPv6Format(self, high, low):
"""Provide a readable IPv6 IP having the high and low part in 2 integers.
Args:
high: 64 bits integers number with the high part of the IPv6.
low: 64 bits integers number with the low part of the IPv6.
Returns:
String with a well represented IPv6.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, u'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == u'0000':
if not blank:
address.append(u'')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip(u'0'))
return u':'.join(address)
def ieee2ibm32(ieee_float):
# IEEE Float to IBM Float (32 bit)
# IEEE bit pattern:
# SEEEEEEE EMMMMMMM MMMMMMM MMMMMMM
ieee_s = ieee()
e = ieee_s.parse(ieee_float)
ibm_s = ibm()
mapi = 0x00800000
m = e.m
m = m | mapi
i = e.e
i -= 127
i += 1
while (i % 4) != 0:
m = m >> 1
i += 1
i /= 4
i += 64
c = construct.Container(s=e.s, e=i, m=m)
value = ibm_s.build(c)
return value
def set_signature(self, signer):
# TODO how do we even know this blobwrapper contains the signature?
# seems like this is a coincidence of the structure, where
# it's the only blobwrapper at that level...
# log.debug("sig:")
blob_wrappers = self.get_blobs('CSMAGIC_BLOBWRAPPER',
min_expected=1,
max_expected=1)
sigwrapper = blob_wrappers[0]
# oldsig = sigwrapper.bytes.value
# signer._log_parsed_asn1(sigwrapper.data.data.value)
# open("sigrip.der", "wb").write(sigwrapper.data.data.value)
code_directories = self.get_blobs('CSMAGIC_CODEDIRECTORY',
min_expected=1,
max_expected=2)
cd_data = self.get_blob_data(code_directories[0])
sig = signer.sign(cd_data, 'sha1')
# log.debug("sig len: {0}".format(len(sig)))
# log.debug("old sig len: {0}".format(len(oldsig)))
# open("my_sigrip.der", "wb").write(sig)
sigwrapper.data = construct.Container(data=sig)
# signer._log_parsed_asn1(sig)
# sigwrapper.data = construct.Container(data="hahaha")
sigwrapper.length = len(sigwrapper.data.data) + 8
sigwrapper.bytes = sigwrapper.data.data
def _IPv6Format(self, high, low):
"""Formats an IPv6 address as a human readable string.
Args:
high (int): upper 64-bit part of the IPv6 address.
low (int): lower 64-bit part of the IPv6 address.
Returns:
str: human readable string of IPv6 address.
"""
ipv6_string = self.IPV6_STRUCT.build(
construct.Container(high=high, low=low))
# socket.inet_ntop not supported in Windows.
if hasattr(socket, 'inet_ntop'):
return socket.inet_ntop(socket.AF_INET6, ipv6_string)
# TODO: this approach returns double "::", illegal IPv6 addr.
str_address = binascii.hexlify(ipv6_string)
address = []
blank = False
for pos in range(0, len(str_address), 4):
if str_address[pos:pos + 4] == '0000':
if not blank:
address.append('')
blank = True
else:
blank = False
address.append(str_address[pos:pos + 4].lstrip('0'))
return ':'.join(address)
def __init__(self, stream, palette, image_adapter_cls):
self.header = None
self.palette = palette
self.image_adapter_cls = image_adapter_cls
self.stream = stream
self.header = HEADER._parse(stream, cons.Container(slp_file=self))
self.frames = self.header.frames
def importToContainer(node):
if isinstance(node, dict):
return C.Container(
{k: importToContainer(v)
for (k, v) in node.items()})
if isinstance(node, list):
return C.ListContainer(importToContainer(i) for i in node)
return node
def ack(s, h):
ack = s.header.build(
construct.Container(packet_type=s.PT_REQ_ACK if h.packet_type
== s.PT_REQ else s.PT_RESP_ACK,
cmd_id=h.cmd_id,
payload_size=0,
seq_id=h.seq_id))
CMD_S.sendto(ack, ('192.168.1.10', PORT_CMD))
def make_signature(arch_macho, arch_end, cmds, f, entitlements_file):
raise Exception("Making a signature is not fully implemented. This code was"
"abandoned since we think our customers will only give us signed"
"apps. But, it almost works, so it's preserved here.")
# sign from scratch
log.debug("signing from scratch")
drs = None
drs_lc = cmds.get('LC_DYLIB_CODE_SIGN_DRS')
if drs_lc:
drs = drs_lc.data.blob
codesig_offset = arch_end
# generate code hashes
hashes = []
#log.debug("codesig offset:", codesig_offset)
start_offset = arch_macho.macho_start
end_offset = macho_end
#log.debug("new start-end", start_offset, end_offset)
codeLimit = end_offset - start_offset
#log.debug("new cL:", codeLimit)
nCodeSlots = int(math.ceil(float(end_offset - start_offset) / 0x1000))
#log.debug("new nCS:", nCodeSlots)
for i in range(nCodeSlots):
f.seek(start_offset + 0x1000 * i)
actual_data = f.read(min(0x1000, end_offset - f.tell()))
actual = hashlib.sha1(actual_data).digest()
#log.debug(actual.encode('hex'))
hashes.append(actual)
codesig_cons = make_basic_codesig(entitlements_file,
drs,
codeLimit,
hashes)
codesig_data = macho_cs.Blob.build(codesig_cons)
cmd_data = construct.Container(dataoff=codesig_offset,
datasize=len(codesig_data))
cmd = construct.Container(cmd='LC_CODE_SIGNATURE',
cmdsize=16,
data=cmd_data,
bytes=macho.CodeSigRef.build(cmd_data))
arch_macho.commands.append(cmd)
arch_macho.ncmds += 1
arch_macho.sizeofcmds += len(macho.LoadCommand.build(cmd))
cmds['LC_CODE_SIGNATURE'] = cmd
def ack(self, h):
ack = self.header.build(
construct.Container(packet_type=self.PT_REQ_ACK if h.packet_type
== self.PT_REQ else self.PT_RESP_ACK,
cmd_id=h.cmd_id,
payload_size=0,
seq_id=h.seq_id))
sockets.Sockets.WII_CMD_S.sendto(
ack, ('192.168.1.10', constants.PORT_WII_CMD))
def build_handshake(info_hash, host_id, extensions):
bits = bitarray.bitarray([0]*64, endian='little')
for i in extensions:
bits[i] = True
obj = c.Container(info_hash=info_hash, peer_id=host_id,
reserved=bits.tobytes())
return _handshake.build(obj)
def __init__(self,
message_type: MessageType,
fields: typing.Optional[typing.Mapping] = None,
timestamp: typing.Optional[float] = None):
if not isinstance(message_type, MessageType):
raise TypeError('Expected MessageType not %r' % message_type)
self._type = message_type
self._fields = con.Container(fields or {})
self._timestamp = float(timestamp or 0)
def gen_packet():
while True:
try:
raw = os.urandom(random.randint(6, 127))
mpdu = fix_mpdu(raw)
frame = ct.Container(
phr=ct.Container(
reserved=random.randint(0, 1),
size=len(mpdu),
),
mpdu=mpdu_t.parse(mpdu),
)
built_frame = frame_t.build(frame)
return built_frame
except ct.StreamError:
continue
except ct.ExplicitError:
continue
def _pmtu_probe_do(self):
"""
Periodically probes PMTU.
"""
if not self.manager.config.getboolean("broker", "pmtu_discovery"):
return
probe_interval = 15
while True:
gevent.sleep(probe_interval)
# Reset measured PMTU
self.probed_pmtu = 0
# Transmit PMTU probes of different sizes multiple times
for _ in xrange(4):
for size in [
500, 750, 1000, 1100, 1250, 1300, 1400, 1492, 1500
]:
try:
msg = ControlMessage.build(
cs.Container(magic1=0x80,
magic2=0x73A7,
version=1,
type=CONTROL_TYPE_PMTUD,
data=""))
# We need to subtract 6 because ControlMessage gets auto-padded to 12 bytes
msg += '\x00' * (size - IPV4_HDR_OVERHEAD -
L2TP_CONTROL_SIZE - 6)
self.socket.send(msg)
except gsocket.error:
pass
gevent.sleep(1)
# Collect all acknowledgements
gevent.sleep(1)
detected_pmtu = self.probed_pmtu - L2TP_TUN_OVERHEAD
if detected_pmtu > 0 and detected_pmtu != self.pmtu:
# Alter MTU for all sessions
for session in self.sessions.values():
self.manager.session_set_mtu(self, session, detected_pmtu)
# Invoke MTU change hook for each session
self.manager.hook('session.mtu-changed', self.id,
session.id, session.name, self.pmtu,
detected_pmtu)
logger.debug("Detected PMTU of %d for tunnel %d." %
(detected_pmtu, self.id))
self.pmtu = detected_pmtu
# Increase probe interval until it reaches 10 minutes
probe_interval = min(600, probe_interval * 2)
def _pmtu_probe_do(self):
"""
Periodically probes PMTU.
"""
if not self.manager.config.getboolean("broker", "pmtu_discovery"):
return
probe_interval = 15
while True:
gevent.sleep(probe_interval)
# Reset measured PMTU
self.probed_pmtu = 0
self.num_pmtu_probes = 0
self.num_pmtu_replies = 0
# Transmit PMTU probes of different sizes multiple times
for _ in xrange(4):
for size in [1334, 1400, 1450, 1476, 1492, 1500]:
try:
msg = ControlMessage.build(cs.Container(
magic1 = 0x80,
magic2 = 0x73A7,
version = 1,
type = CONTROL_TYPE_PMTUD,
data = ""
))
# We need to subtract 6 because ControlMessage gets auto-padded to 12 bytes
msg += '\x00' * (size - IPV4_HDR_OVERHEAD - L2TP_CONTROL_SIZE - 6)
self.socket.send(msg)
self.num_pmtu_probes += 1
except gsocket.error:
pass
gevent.sleep(1)
# Collect all acknowledgements
if self.num_pmtu_probes != self.num_pmtu_replies:
gevent.sleep(3)
detected_pmtu = max(self.probed_pmtu - L2TP_TUN_OVERHEAD, 1280)
if not self.probed_pmtu or not self.num_pmtu_replies:
logger.warning("Got no replies to any PMTU probes for tunnel %d." % self.id)
continue
elif detected_pmtu > 0 and detected_pmtu != self.pmtu:
self.pmtu = detected_pmtu
self._update_mtu()
# Notify the client of the detected PMTU
self.handler.send_message(self.socket, CONTROL_TYPE_PMTU_NTFY,
cs.UBInt16("mtu").build(self.pmtu))
# Increase probe interval until it reaches 10 minutes
probe_interval = min(600, probe_interval * 2)
def test_xstruct():
test_struct = struct.XStruct('a' / construct.Int32ub,
'b' / construct.Int16ub)
obj = test_struct(a=1, b=2)
assert test_struct.a.subcon == construct.Int32ub
assert test_struct.b.subcon == construct.Int16ub
assert 'a' in test_struct
assert 'b' in test_struct
assert obj.container == construct.Container(a=1, b=2)
assert obj.a == 1
assert obj.b == 2
assert obj.build() == b'\x00\x00\x00\x01\x00\x02'
obj.parse(b'\x00\x00\x00\x02\x00\x03')
assert obj.container == construct.Container(a=2, b=3)
obj = test_struct.parse(b'\x00\x00\x00\x03\x00\x04')
assert obj.container == construct.Container(a=3, b=4)
def _decode(self, obj: construct.ListContainer, context, path):
result = construct.Container()
last = {}
for i, item in enumerate(obj):
key = item.key
if key in result:
raise construct.ConstructError(
f"Key {key} found twice in object", path)
last[key] = i
result[key] = item.value
return result
def make_requirements(drs, ident, signer):
if signer.is_adhoc():
log.debug("Ad hoc -- using empty requirement set")
reqs = construct.Container(
sb_start=0,
count=0,
BlobIndex=[],
)
return reqs
common_name = signer.get_common_name()
expr = make_expr(
'And',
('Ident', ident),
('AppleGenericAnchor',),
('CertField', 'leafCert', 'subject.CN', ['matchEqual', common_name]),
('CertGeneric', 1, '*\x86H\x86\xf7cd\x06\x02\x01', ['matchExists']))
des_req = construct.Container(kind=1, expr=expr)
des_req_data = macho_cs.Requirement.build(des_req)
reqs = construct.Container(
sb_start=0,
count=1,
BlobIndex=[construct.Container(type='kSecDesignatedRequirementType',
offset=28,
blob=construct.Container(magic='CSMAGIC_REQUIREMENT',
length=len(des_req_data) + 8,
data=des_req,
bytes=des_req_data))])
if drs:
dr_exprs = []
for dr in drs.data.BlobIndex:
if dr.blob is not None:
dr_exprs.append(dr.blob.data.expr)
# make_expr expects at least 2 arguments, need to verify that we pass those in, otherwise just return
if len(dr_exprs) > 1:
expr = make_expr('Or', *dr_exprs)
lib_req = construct.Container(kind=1, expr=expr)
lib_req_data = macho_cs.Requirement.build(lib_req)
reqs.BlobIndex.append(construct.Container(type='kSecLibraryRequirementType',
offset=28 + len(des_req_data) + 8,
blob=construct.Container(magic='CSMAGIC_REQUIREMENT',
length=len(lib_req_data) + 8,
data=lib_req,
bytes=lib_req_data)))
reqs.count += 1
return reqs
def make_arg(data_type, arg):
if data_type.name == 'Data':
return construct.Container(data=arg, length=len(arg))
elif data_type.name.lower() == 'expr':
if isinstance(arg, construct.Container):
# preserve expressions that are already containerized
return arg
return make_expr(*arg)
elif data_type.name == 'slot':
if arg == 'leafCert':
return 0
return arg
elif data_type.name == 'Match':
matchOp = arg[0]
data = None
if len(arg) > 1:
data = construct.Container(data=arg[1], length=len(arg[1]))
return construct.Container(matchOp=matchOp, Data=data)
log.debug(data_type)
log.debug(data_type.name)
log.debug(arg)
assert 0
