def hex_components(self, boxed=False):
result = ['{}:{}'.format(self.name, to_hex(self.number, 4))
] if boxed else ['{}: '.format(self.name)]
for arg in self:
result += [
'{}:{}'.format(arg.name, to_hex(b''.join(arg.to_buffers()), 4))
]
return " ".join(result)
def from_stream(cls, stream):
"""Boxed type combinator loading"""
con_num = stream.read(4)
con = cls.constructors.get(con_num)
if con is None:
if cls is TLType:
raise ValueError(
'constructor with number {} does not exists'.format(
to_hex(con_num)))
else:
raise ValueError(
'{} does not have combinator with number {}'.format(
cls, to_hex(con_num)))
return con.from_stream(stream)
def x15_x03(o, sock, data):
'''
SNAC (x15, x3): From Server
reference: U{http://iserverd.khstu.ru/oscar/snac_15_03.html}
'''
tlv, data = oscar.util.s_tlv(data)
assert not data
data = tlv.v
fmt = '''
size H
uid I
reqtype H
reqid H
'''.split()
fmt = zip(fmt[::2],fmt[1::2])
sz, uid, reqtype, reqid, data = oscar.unpack(fmt, data, '<')
assert len(data) == (sz - 8)
try:
f = getattr(oscar.snac, 'icq_srv_%04x'%reqtype)
except AttributeError:
print 'received ICQ response: %d %04x reqid=%d / %s' % (uid, reqtype, reqid,util.to_hex(data))
return
uid = o.icq_req_cbs.get(reqid, (lambda: None, (), {}))
f(o, uid, data)
def annotation_data(dataset_name):
""" Gets general annotation data for a dataset. Basically everything that the
annotation Web UI needs, like classes and their colors, which keys to press
for each class, and progress.
"""
try:
classnames = get_classnames(dataset_name)
except FileNotFoundError:
return None
else:
# Removing R from this would not be sufficient, and would look like a bug
all_keys = 'abcdefghijklmnopqrstuvwxyz'
colors = class_colors(len(classnames))
out_colors = {}
for cn, cc in zip(classnames, colors):
out_colors[cn] = to_hex(cc)
keys_list = []
keys = set()
for cn in classnames:
cn = cn.lower()
success = False
for letter in cn:
if not (letter in keys):
if not letter == 'r': # 'R' is not okay, since that button is used for cancelling
keys_list.append(letter)
keys.add(letter)
success = True
break
while not success:
letter = choice(all_keys)
if not (letter in keys):
if not letter == 'r': # 'R' is not okay, since that button is used for cancelling
keys.add(letter)
keys_list.append(letter)
success = True
key_codes_list = [ord(x) - 32 for x in keys_list
] # convert to nonsense Javascript keyCodes
keys_list = [x.upper() for x in keys_list
] # are nicer visualized in upper case in the Web UI
train_stats = get_annotation_stats(dataset_name, 'train')
test_stats = get_annotation_stats(dataset_name, 'test')
out = [
classnames, out_colors, keys_list, key_codes_list, train_stats,
test_stats
]
return out
def from_stream(cls, stream):
"""Boxed type combinator loading"""
con_num = stream.read(4)
con = cls.constructors.get(con_num)
if con is None:
if cls is TLType:
raise ValueError('constructor with number {} does not exists'.format(to_hex(con_num)))
else:
raise ValueError('{} does not have combinator with number {}'.format(cls, to_hex(con_num)))
return con.from_stream(stream)
def onkey(e):
wx.CallAfter(lambda: st.SetLabel(to_hex(t.Value)))
if e.ControlDown() and e.AltDown() and e.KeyCode == wx.WXK_SPACE:
from random import choice, randrange
import string
for i in xrange(200):
rstr = ''.join(choice(string.letters) for x in xrange(randrange(5,300)))
t.AppendText(rstr)
onenter()
return
e.Skip()
def x10_x05(o, sock, data):
'''
SNAC (x10, x5): Requested buddy icon
reference: U{http://iserverd.khstu.ru/oscar/snac_10_05.html}
'''
fmt = (('sn', 'pstring'),
('unknown', 'H'),
('flags', 'B'),
('hash', 'pstring'),
('icon_len', 'H'),
('icon', 's', 'icon_len'))
sn, __,__, hash, __, icon, data = oscar.unpack(fmt, data)
assert not data, util.to_hex(data)
return sn, hash, icon
def printTLV(tlvtype, tlvdata):
import util
tlvstring = "0x%04X" % tlvtype + ": "
if tlvtype == 0x006D:
(dateint, ) = struct.unpack("!I", tlvdata[:4])
tlvstring = tlvstring + '"' + time.asctime(
time.localtime(dateint)) + '" + '
tlvdata = tlvdata[4:]
if tlvtype in (0x0067, 0x154, 0x0160, 0x01C3, 0x01C4, 0x01C5):
(dateint, ) = struct.unpack("!I", tlvdata)
tlvstring = tlvstring + '"' + time.asctime(
time.localtime(dateint)) + '"'
tlvdata = None
if tlvtype == 0x006E:
pass
if tlvtype == 0x00C8:
for short in tlvdata:
tlvstring += "%04X" % short + " "
tlvstring = tlvstring[:-1]
tlvdata = None
if tlvtype == 0x0131:
tlvstring += "Alias: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x0137:
tlvstring += "mail: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x0137:
tlvstring += "SMS: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x013C:
tlvstring += "Comment: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x014B:
tlvstring += "Metadata: " + repr(tlvdata)
tlvdata = None
if tlvtype == 0x0D5:
pass
if tlvdata:
try:
tlvstring += '"' + util.to_hex(tlvdata) + '"'
except Exception:
# import traceback;traceback.print_exc()
print tlvtype, tlvdata
raise
return tlvstring
def printTLV(tlvtype, tlvdata):
import util
tlvstring = "0x%04X" % tlvtype + ": "
if tlvtype == 0x006D:
(dateint,) = struct.unpack("!I", tlvdata[:4])
tlvstring = tlvstring + '"' + time.asctime(time.localtime(dateint)) + '" + '
tlvdata = tlvdata[4:]
if tlvtype in (0x0067, 0x154, 0x0160, 0x01C3, 0x01C4, 0x01C5):
(dateint,) = struct.unpack("!I", tlvdata)
tlvstring = tlvstring + '"' + time.asctime(time.localtime(dateint)) + '"'
tlvdata = None
if tlvtype == 0x006E:
pass
if tlvtype == 0x00C8:
for short in tlvdata:
tlvstring += "%04X" % short + " "
tlvstring = tlvstring[:-1]
tlvdata = None
if tlvtype == 0x0131:
tlvstring += "Alias: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x0137:
tlvstring += "mail: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x0137:
tlvstring += "SMS: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x013C:
tlvstring += "Comment: \"" + return_printable(tlvdata)
tlvdata = None
if tlvtype == 0x014B:
tlvstring += "Metadata: " + repr(tlvdata)
tlvdata = None
if tlvtype == 0x0D5:
pass
if tlvdata:
try:
tlvstring += '"' + util.to_hex(tlvdata) + '"'
except Exception:
# import traceback;traceback.print_exc()
print tlvtype, tlvdata
raise
return tlvstring
def make_proxy_init(sn, cookie, port=None):
'Creates a proxy packet.'
# Check for screenname as a string.
if not isinstance(sn, str):
raise TypeError('screenname must be a str object')
# Cookie must be an eight byte string or a long.
if isinstance(cookie, long):
cookie = struct.pack('!Q', cookie)
else:
assert isinstance(cookie, str) and len(cookie) == 8
# If a port was specified, use initreceive, since that is the only way
# that and 'initsend' differ.
command = 'initreceive' if port else 'initsend'
# Length is everything that follows the two bytes of length.
length = len(sn) + 41
if port is None:
# Init sends have two less byes.
length -= 2
info(command + ' length: %d', length)
data = ProxyHeader(length,
ProxyHeader.version,
ProxyHeader.commands[command],
null = 0,
flags = 0).pack()
# Screen name as a pascal string
data += struct.pack('B', len(sn)) + sn
# Add the two byte port if this is an init_receive request
if port:
assert isinstance(port, int)
data += struct.pack('!H', port)
fullpacket = data + cookie + _send_file_tlv #Send file capability on the end
log.info_s('proxy packet assembled (%d bytes): %s', len(fullpacket),to_hex(fullpacket))
return fullpacket
def icq_srv_07da_00c8(o, uid, result, data):
if result != SUCCESS:
log.warning('icq got response %d', result)
return
vals = 'nick first last email city state phone fax street cellular zip country timezone publish webaware data'.split()
fmt = [(x, 'lnts') for x in vals[:11]]
fmt.extend([
(vals[11], 'H'),
(vals[12], 'B'),
(vals[13], 'H'),
(vals[14], 'B')
])
#locals.update(zip(vals, oscar.unpack(fmt, data, '<')))
(nick, first, last, email, city, state, phone, fax, street,
cellular, zip, country_code, timezone, publish, webaware), data = unpack_decode(fmt, data, '<')
country = icq_country.codes.get(country_code, sentinel)
if country is sentinel:
log.warning('Unknown country code %r. Maybe you can guess from these locals? %r', country_code, locals())
country = u'Unknown'
del country_code
if data:
log.warning('icq user info got extra data: %r', util.to_hex(data))
webaware = bool(webaware%2)
b = o.buddies[str(uid)]
try: personal = b.personal
except Exception: personal = b.personal = util.Storage()
set = b.setnotifyif
for x in vals:
set(x, locals()[x])
setattr(personal, x, locals()[x])
def create(input_image, output_path):
"""Create an SVG from the input image."""
conf = config.get()
im = util.get_image(input_image)
(width, height) = im.size
pix = im.load()
dwg = svgwrite.Drawing(
output_path,
profile='full',
width=width,
height=height,
viewBox='0 0 {0} {1}'.format(width * conf['svg']['ratio'], height),
style='font-family:\'{0}\';font-weight:900;font-size:{1}'.format(
conf['svg']['font_family'], conf['svg']['font_size']))
dwg.attribs['xml:space'] = 'preserve'
for h in range(0, height):
colors = []
for w in range(0, width):
try:
colors.append(util.to_hex(pix[w, h]))
except:
colors.append('#FFFFFF')
pass
colors = [(len(list(g)), k) for k, g in groupby(colors)]
x = 0
for c in colors:
t = ''
for l in range(0, c[0]):
t = t + code.get_char()
text = dwg.text(t, fill='{0}'.format(c[1]))
text.attribs['y'] = str(h)
text.attribs['x'] = x * conf['svg']['ratio']
dwg.add(text)
x = x + len(t)
dwg.save()
def _set_dc_info(self, newval):
dc_info, data = oscar.unpack((('_', 'dc_info'),), newval)
if data: print 'extra data on dc_info for %s: %s' % (self.name, util.to_hex(data))
self._dc_info = dc_info
# apdu('00 C0 00 00 15')
apdu('00 20 00 81 10 31 32 33 34 35 36 00 00 00 00 00 00 00 00 00 00',
'Verify PIN')
c = GemaltoCrypto()
c.lib_nonce = os.urandom(32)
c.lib_random = os.urandom(16)
c.lib_constant = from_hex('22 34 00 00 AF 04 E3 A9')
apdu('00 22 41 A4 06 83 01 01 95 01 80', 'MSE')
card_challenge = apdu('80 84 00 00 08', 'get challenge')
print("!! parse card challenge")
c.parse_card_challenge(card_challenge)
print("!! make lib challenge")
lib_challenge = c.make_lib_challenge()
print("!! parse lib challenge")
c.parse_lib_challenge(lib_challenge)
ret = apdu(to_hex(lib_challenge), 'lib challenge')
assert ret == '\x61\x48'
card_ch_response = apdu('80 C0 00 00 48', 'get response')
print("!! parse card ch response")
c.parse_card_ch_response(card_ch_response)
print("!! calc mac params")
c.calc_mac_params()
def addhash(self, level, index, hash, peer=None):
"""
Adds a hash to either the validated tree (when possible) or to the
unvalidated cache, self.purgatory. This will also add any computed parent
hashes recursively. If the hash makes it into the validated tree, this
will also check the nephews of this hash to see if they can now be
validated. However, direct descendents will not be checked, and must be
checked by the caller.
"""
if type(hash) == long:
hash = util.ser_uint256(hash)
key = (level, index)
if key in self.purgatory:
if self.purgatory[key] == hash:
return
else:
oldpeer = self.peerorigins[self.purgatory[key]] if self.purgatory[key] in self.peerorigins else None
debuglog('btnet', 'Warning: received two different hashes for the same part of a tree. Replacing old hash.')
debuglog('btnet', 'Cause is likely either network corruption or a malicious peer. Peers:')
debuglog('btnet', oldpeer, peer)
debuglog('btnet', 'Hash added is (%i, %i): %s. Oldhash: %s.' % (level, index, to_hex(hash), to_hex(self.purgatory[key])))
# fixme: peer banning
# continue to add the new hash and validate
elif self.getnode(level, index):
debuglog('bttree', 'Debug warning: level=%i index=%i already validated in tree' % (level, index))
return
self.purgatory[key] = hash
#self.peerorigins[hash] = peer # fixme: make sure memory growth is bounded
parent = self.getnode(level-1, index//2) # is our parent already valid?
#if parent: print "valid parent of %i,%i is %i,%i:" %(level, index, level-1, index//2), to_hex(parent])
siblingkey = (level, index ^ 1)
if not siblingkey in self.purgatory: # Is this is the right edge of the tree?
if not index & 1: # if even (left sibling)
for hint in self.txcounthints:
height = int(math.ceil(math.log(hint, 2)))
if level > height: continue
edge = (hint-1) >> (height - level)
if index == edge:
self.purgatory[siblingkey] = hash # this can be overwritten later
break
if siblingkey in self.purgatory: # then we can check one level up
sib = self.purgatory[siblingkey]
parenthash = self.calcparent(sib, hash) if (index%2) else self.calcparent(hash, sib) # left sibling goes first
if parent and parent == parenthash:
result = 'connected'
elif parent and parent != parenthash and not sib == hash:
debuglog('btnet', 'Invalid hash(es) encountered when checking (%i, %i): %s.' % (level, index, to_hex(hash)))
debuglog('btnet', 'Parent (%i, %i) = %s not %s' % (level-1, index//2, to_hex(parent), to_hex(parenthash)))
result = 'invalid'
elif parent and parent != parenthash and sib == hash:
debuglog('btnet', 'Found a bad edge: (%i, %i) = %s not %s' % (level-1, index//2, to_hex(parent), to_hex(parenthash)))
result = 'orphan' # incorrect tx count hint
else: # recurse one level up
result = self.addhash(level-1, index//2, parenthash, None)
else:
result = 'orphan'
if result == 'connected':
self.setnode(level, index, hash, edge=(hash==sib))
self.setnode(level, index^1, sib, edge=(hash==sib))
del self.purgatory[key]
del self.purgatory[siblingkey]
if hash == sib and level == self.levels: # right edge, bottom row
self.txcount = index|1-1 # left sib's index
# the recursive caller of addhash will take care of the children of key, but not siblingkey
self.checkchildren(siblingkey[0], siblingkey[1])
elif result == 'invalid':
if sib == hash: # invalid hint about the number of transactions
debuglog('btnet', 'Invalid txcount hint: %i among ' % hint, self.txcounthints)
del self.purgatory[max(siblingkey, key)]
result = 'orphan'
else:
for k in key, siblingkey:
# fixme: for multi-level recursion, there's a good chance we're deleting the wrong txes.
# should we delete all of the decendants of the lowest valid hash to which this resolves?
# or should we leave these hashes all in purgatory? or what? who do we ban?
debuglog('btnet', 'Invalid hash(es) encountered. Deleting: (%i, %i): %s.' % (k[0], k[1], to_hex(self.purgatory[k])))
#del self.purgatory[k]
elif result == 'orphan':
pass # fixme: deal with peer info (and banning) in each of these branches above
return result
def hex_list(self):
return [to_hex(buf, 0) for buf in self.buffers()]
def hexdigest(self, final=None):
return util.to_hex(self.digest(final), '').lstrip('0')
def hex_list(self):
return [to_hex(buf, 0) for buf in self.buffers()]
def hex_components(self, boxed=False):
result = ['{}:{}'.format(self.name, to_hex(self.number, 4))] if boxed else ['{}: '.format(self.name)]
for arg in self:
result += ['{}:{}'.format(arg.name, to_hex(b''.join(arg.to_buffers()), 4))]
return " ".join(result)
def to_hex(self, width=4, boxed=False):
if boxed:
return to_hex(self.to_boxed_bytes(), width)
return to_hex(self.to_bytes(), width)
def hex_components(self):
return ' '.join([to_hex(data) for data in self.to_buffers()])
def ch2cancel(self, data):
log.info('%r cancelled by buddy', self)
self.cancelled = True
self.cancel_by_buddy()
if data:
log.info('data in a ch2 cancel: %s', to_hex(data))
def hexdigest(self, final=None):
return util.to_hex(self.digest(final), '').lstrip('0')
def hex_components(self):
return ' '.join([to_hex(data) for data in self.to_buffers()])
def addhash(self, level, index, hash, peer=None, locked=False):
"""
Adds a hash to either the validated tree (when possible) or to the
unvalidated cache, self.purgatory. This will also add any computed parent
hashes recursively. If the hash makes it into the validated tree, this
will also check the nephews of this hash to see if they can now be
validated. However, direct descendants will not be checked, and must be
checked by the caller.
"""
if type(hash) == long:
hash = util.ser_uint256(hash)
key = (level, index)
if not locked:
self.rwlock.acquire_write(
) # this might benefit from more fine-grained locks when the switch is made to C++
if key in self.purgatory:
if self.purgatory[key] == hash:
if not locked: self.rwlock.release_write()
return
else:
oldpeer = self.peerorigins[self.purgatory[
key]] if self.purgatory[key] in self.peerorigins else None
debuglog(
'btnet',
'Warning: received two different hashes for the same part of a tree. Replacing old hash.'
)
debuglog(
'btnet',
'Cause is likely either network corruption or a malicious peer. Peers:'
)
debuglog('btnet', oldpeer, peer)
debuglog(
'btnet', 'Hash added is (%i, %i): %s. Oldhash: %s.' %
(level, index, to_hex(hash), to_hex(self.purgatory[key])))
# fixme: peer banning
# continue to add the new hash and validate
elif self.getnode(level, index, locked=True):
debuglog(
'bttree',
'Debug warning: level=%i index=%i already validated in tree' %
(level, index))
if not locked: self.rwlock.release_write()
return
self.purgatory[key] = hash
#self.peerorigins[hash] = peer # fixme: make sure memory growth is bounded
parent = self.getnode(level - 1, index // 2,
locked=True) # is our parent already valid?
siblingkey = (level, index ^ 1)
if not siblingkey in self.purgatory: # Is this is the right edge of the tree?
if not index & 1: # if even (left sibling)
if self.txcount:
height = int(math.ceil(math.log(self.txcount, 2)))
assert level <= height
edge = (self.txcount - 1) >> (height - level)
if index == edge:
self.purgatory[
siblingkey] = hash # this can be overwritten later
if siblingkey in self.purgatory: # then we can check one level up
sib = self.purgatory[siblingkey]
parenthash = self.calcparent(sib, hash) if (
index %
2) else self.calcparent(hash, sib) # left sibling goes first
if parent and parent == parenthash:
result = 'connected'
elif parent and parent != parenthash and not sib == hash:
debuglog(
'btnet',
'Invalid hash(es) encountered when checking (%i, %i): %s.'
% (level, index, to_hex(hash)))
debuglog(
'btnet', 'Parent (%i, %i) = %s not %s' %
(level - 1, index // 2, to_hex(parent),
to_hex(parenthash)))
result = 'invalid'
elif parent and parent != parenthash and sib == hash:
debuglog(
'btnet', 'Found a bad edge: (%i, %i) = %s not %s' %
(level - 1, index // 2, to_hex(parent),
to_hex(parenthash)))
result = 'orphan' # incorrect tx count hint
else: # recurse one level up
result = self.addhash(level - 1,
index // 2,
parenthash,
None,
locked=True)
else:
result = 'orphan'
if result == 'connected':
self.setnode(level, index, hash, edge=(hash == sib), locked=True)
self.setnode(level,
index ^ 1,
sib,
edge=(hash == sib),
locked=True)
del self.purgatory[key]
del self.purgatory[siblingkey]
if hash == sib and level == self.levels: # right edge, bottom row
self.txcount = index | 1 # left sib is the last tx, but we start counting from 0, so we want the right sib's index
# the recursive caller of addhash will take care of the children of key, but not siblingkey
if hash != sib:
self.checkchildren(siblingkey[0], siblingkey[1], locked=True)
elif result == 'invalid':
if sib == hash: # invalid hint about the number of transactions
debuglog('btnet', 'Invalid txcount? -- %i ' % self.txcount)
del self.purgatory[max(siblingkey, key)]
result = 'orphan'
else:
for k in key, siblingkey:
# fixme: for multi-level recursion, there's a good chance we're deleting the wrong txes.
# should we delete all of the descendants of the lowest valid hash to which this resolves?
# or should we leave these hashes all in purgatory? or what? who do we ban?
debuglog(
'btnet',
'Invalid hash(es) encountered. Deleting: (%i, %i): %s.'
% (k[0], k[1], to_hex(self.purgatory[k])))
#del self.purgatory[k]
elif result == 'orphan':
pass # fixme: deal with peer info (and banning) in each of these branches above
if not locked: self.rwlock.release_write()
return result
def to_hex(self, width=4, boxed=False):
if boxed:
return to_hex(self.to_boxed_bytes(), width)
return to_hex(self.to_bytes(), width)
