def pack_awb(dir):
file_list = sorted(os.listdir(dir))
file_count = len(file_list)
file_end_offset = 0x10 + (2 * file_count)
data = AWB_MAGIC + \
AWB_UNK + \
bitstring.pack("uintle:32, uintle:32", file_count, AWB_ALIGN) + \
bitstring.pack(", ".join(["uintle:16=%d" % id for id in range(file_count)])) + \
BitStream(uintle = 0, length = (file_count + 1) * 32)
# Plus one for the header.
for i, file in enumerate(file_list):
file_end = data.len / 8
data.overwrite(bitstring.pack("uintle:32", file_end), (file_end_offset + (i * 4)) * 8)
padding = 0
if file_end % AWB_ALIGN > 0:
padding = AWB_ALIGN - (file_end % AWB_ALIGN)
data.append(BitStream(uintle = 0, length = padding * 8))
file_data = ConstBitStream(filename = os.path.join(dir, file))
data.append(file_data)
# One last file end.
file_end = data.len / 8
data.overwrite(bitstring.pack("uintle:32", file_end), (file_end_offset + (file_count * 4)) * 8)
return data
def bytes(self):
binary = bitstring.pack(
"pad:8, uint:8, bool, bool, pad:2",
self.TABLE_ID, self.section_syntax_indicator,
self.private_indicator)
program_info_length = 0
for descriptor in self.descriptors:
program_info_length += descriptor.size
length = 13 + program_info_length
for stream in self.streams.values():
length += stream.size
binary.append(bitstring.pack(
"uint:12, uint:16, pad:2, uint:5, bool, uint:8, uint:8, pad:3," +
"uint:13, pad:4, uint:12",
length, self.program_number, self.version_number,
self.current_next_indicator, self.section_number,
self.last_section_number, self.pcr_pid, program_info_length))
for descriptor in self.descriptors:
binary.append(descriptor.bytes)
for stream in self.streams.values():
binary.append(stream.bytes)
binary.append(bitstring.pack("uint:32", crc32(binary.bytes[1:])))
return binary.bytes
def testPack(self):
s = bitstring.pack('0b11, pad:3=5, 0b1')
self.assertEqual(s.bin, '110001')
d = bitstring.pack('pad:c', c=12)
self.assertEqual(d, Bits(12))
e = bitstring.pack('0xf, uint:12, pad:1, bin, pad:4, 0b10', 0, '111')
self.assertEqual(e.bin, '11110000000000000111000010')
def convert_data(self, weather_data):
"""Converts the weather data into a string of bits
The display is based on a relatively simple programmable interrupt controller (or PIC) when compared to a
Raspberry PI. It only speaks binary, which means that we need to convert the dictionary with weather data into a
sequence of bits, before it can be transmitted.
This conversion is based on four principles:
# The amount of bits available for each data element is fixed
# Each element in the data has a minimum value
# Each element has a maximum value
# Each element can vary only in discrete steps
@precondition: the member 'requested data' is properly set
@param weather_data: a dictionary containing the weatherdata
@return: a bitstring with the data in bits according to the configuration
"""
s = None
t_data = self.transmittable_data(weather_data, self.requested_data)
for i, data in enumerate(self.requested_data):
formatting = self.requested_data[str(i)]
bit_length = int(formatting['length'])
bit_key = formatting['key']
bit_value = t_data[bit_key]
padding_string = '#0{0}b'.format(bit_length + 2) # don't forget to account for '0b' in the length
padded_bit_value = format(bit_value, padding_string)
if s is not None:
s += bitstring.pack("bin:{}={}".format(bit_length, padded_bit_value))
else:
s = bitstring.pack("bin:{}={}".format(bit_length, padded_bit_value))
return s
def decode(self, in_stream, out_stream):
bs = BitStream()
dq = deque()
at_least_three = False
for word in self.words_from_file(in_stream):
if not word or word not in self.word_dict:
continue
#print >> sys.stderr, 'word:"', word, '"'
dq.append(self.word_dict[word])
if at_least_three or len(dq) == 3:
bs.append(pack(self.int_type, dq.popleft()))
at_least_three = True
if bs.len > self.bit_buffer:
cut = 0
for byte in bs.cut(self.bit_buffer):
cut += 1
byte.tofile(out_stream)
del bs[:cut * self.bit_buffer]
# dq has to have exactly 2 elements here, the last is the bit length of the first, unless it's 0
#print >> sys.stderr, 'dq:', dq
extra_bits = dq.pop()
bs.append(pack('uint:' + str(extra_bits), dq.popleft()))
bs.tofile(out_stream)
def get_payload(self):
self.payload_fields.append(("Service", self.service))
self.payload_fields.append(("Port", self.port))
service = little_endian(bitstring.pack("8", self.service))
port = little_endian(bitstring.pack("32", self.port))
payload = service + port
return payload
def pack( self, s, p ):
p._validate()
beginPos = len(s)
# print 'Packing:', p.Name, beginPos >> 3
if p.Encoding == self.TLV:
s.append( bitstring.pack('uintbe:16, uintbe:16', p.Type, 0) )
else:
assert not p.Parameters, 'LLRP TV _parameters cannot contain nested parameters'
s.append( bitstring.pack('uintbe:8', p.Type | 128 ) )
for f in p.FieldDefs: # Was self.FieldDefs. We need to use the "p" to handle the extra fields in the custom messages.
f.write( s, p )
# Fix the length field.
if p.Encoding == self.TLV:
for pp in p.Parameters:
pp.pack( s )
endPos = len(s)
p._Length = (endPos - beginPos) >> 3
s.overwrite( bitstring.pack('uintbe:16', p._Length), beginPos + 16 )
s.pos = endPos
else:
p._Length = self.Length
return s
def clonePage(self, pgno):
origine = self.getPage(pgno)
fp = bitstring.BitString(bytes="\x00" * DEFAULT_PAGESIZE, length=DEFAULT_PAGESIZE * 8)
celloffset = origine.hdroffset + 8 + origine.childPtrSize
end = celloffset + origine.nCell * 2
data = origine.fp.bytes
s = bitstring.pack("bytes:%d" % (end - origine.hdroffset), data[origine.hdroffset : end])
fp.pos = 0
fp.overwrite(s)
origine.fp.pos = (origine.hdroffset + 5) * 8
top = get2byte(origine.fp)
size = self.usableSize - top
self.nPage += 1
s = bitstring.pack("bytes:%d" % size, data[top : self.usableSize])
fp.pos = top * 8
fp.overwrite(s)
page = Page(fp.bytes, self.pagesize, self.nPage, self)
page.overflow = origine.overflow
fp = page.fp
self.init_freelist(fp, page.hdroffset, page.pageno)
fp.pos = page.hdroffset * 8
fp.overwrite(bitstring.pack("uint:8", origine.flag))
return page
def get_payload(self):
self.payload_fields.append(("Label", self.label))
field_len_bytes = 32
label = "".join(little_endian(bitstring.pack("8", ord(c))) for c in self.label)
padding = "".join(little_endian(bitstring.pack("8", 0)) for i in range(field_len_bytes - len(self.label)))
payload = label + padding
return payload
def testUnfoldConcatenatedValues(self):
class Foo(object):
pass
cls = Foo
def unfold_func(name, instance, **kw):
instance = instance or Foo()
instance.a = kw['a']
instance.b = kw['b']
return instance
folds = 'a=uint:8, b=uint:8'
creases = {}
self.crafter.learn_pattern(cls, unfold_func, folds, creases)
a, b = 120, 254
other_a, other_b = 121, 255
data = bitstring.pack('uint:8, uint:8', a, b)
data += bitstring.pack('uint:8, uint:8', other_a, other_b)
foo = self.crafter.unfold(data, Foo)
other_foo = self.crafter.unfold(data, Foo)
assert foo.a == a
assert foo.b == b
assert other_foo.a == other_a
assert other_foo.b == other_b
def check_flag_a(flags, flag_ops, fail_label):
# XX XX 00 YY
# * If there are multiple flags (as many as needed)
# -> WW XX XX 00 YY
#
# * When all the flags have been listed.
# -> 70 3C 70 34 ZZ ZZ
#
# * XX XX = Flag group/ID
# * YY = Flag State
# * 00 = Off
# * 01 = On
#
# * WW = Operator
# * 06 = AND
# * 07 = OR (?)
#
# * ZZ ZZ = Label to jump to if check failed.
command = bitstring.pack("uint:8, uint:8", CMD_MARKER, WRD_CHECKFLAG_A)
for i, (flag_group, flag_id, flag_state) in enumerate(flags):
command += bitstring.pack("uint:8, uint:8, uint:16", flag_group, flag_id, flag_state)
if i < len(flag_ops):
command += bitstring.pack("uint:8", flag_ops[i])
command += bitstring.pack("uint:8, uint:8", CMD_MARKER, WRD_FLAG_CHECK_END)
if not fail_label == None:
command += bitstring.pack("uint:8, uint:8, uint:16", CMD_MARKER, WRD_GOTO_LABEL, fail_label)
return command
def get_payload(self):
self.payload_fields.append(("Count", self.service))
self.payload_fields.append(("Index", self.port))
count = little_endian(bitstring.pack("8", self.service))
index = little_endian(bitstring.pack("8", self.port))
payload = count + index
return payload
def InterpretRcvFrame( self, RawData ):
# Basic sanity check
if RawData[0] != MAGIC_ID:
print "Error: Bad Magic"
return( False )
if RawData[1] != VERSION:
print "Error: Bad Expected Protocol Version/Revision"
return( False )
header = struct.unpack_from("!ccBBH",RawData)
Headers = {}
Headers["frameType"] = header[2]
Headers["sequence"] = header[4]
#print Headers
if Headers["frameType"] == CTRL_FRAMETYPE_IORESP:
body = struct.unpack_from("!IIIIIIIIIIHHHHHHHH", RawData[6:])
quad_val = body[0:10]
adc_val = body[10:18]
for ii in range(10):
self.QEs[ii].updateValue( bitstring.pack('uint:32',quad_val[ii] ))
for ii in range(8):
self.ADCs[ii].updateVoltage( bitstring.pack('uint:16', adc_val[ii]))
else:
print "uninterpreted body"
def get_payload(self):
reserved_8 = little_endian(bitstring.pack("8", self.reserved))
color = "".join(little_endian(bitstring.pack("16", field)) for field in self.color)
duration = little_endian(bitstring.pack("32", self.duration))
payload = reserved_8 + color + duration
payloadUi = " ".join("{:02x}".format(ord(c)) for c in payload)
return payload
def get_payload(self):
self.payload_fields.append(("Start Index", self.start_index))
self.payload_fields.append(("End Index", self.end_index))
start_index = little_endian(bitstring.pack("8", self.start_index))
end_index = little_endian(bitstring.pack("8", self.end_index))
payload = start_index + end_index
return payload
def get_payload(self):
self.payload_fields.append(("Power Level", self.power_level))
self.payload_fields.append(("Duration", self.duration))
power_level = little_endian(bitstring.pack("16", self.power_level))
duration = little_endian(bitstring.pack("32", self.duration))
payload = power_level + duration
return payload
def get_payload(self):
start_index = little_endian(bitstring.pack("8", self.start_index))
end_index = little_endian(bitstring.pack("8", self.end_index))
color = "".join(little_endian(bitstring.pack("16", field)) for field in self.color)
duration = little_endian(bitstring.pack("32", self.duration))
apply = little_endian(bitstring.pack("8", self.apply))
payload = start_index + end_index + color + duration + apply
return payload
def hamdist(str1, str2):
diffs = 0
assert len(str1) == len(str2), "length of arguments are different"
for i in range(len(str1)):
a = bitstring.pack('uint:8', str1[i])
b = bitstring.pack('uint:8', str2[i])
diffs += (a ^ b).count(True)
return diffs
def get_payload(self):
self.payload_fields.append(("Color", self.color))
self.payload_fields.append(("Duration", self.duration))
reserved_8 = little_endian(bitstring.pack("8", self.reserved))
color = b"".join(little_endian(bitstring.pack("16", field)) for field in self.color)
duration = little_endian(bitstring.pack("32", self.duration))
payload = reserved_8 + color + duration
return payload
def get_payload(self):
self.payload_fields.append(("Vendor", self.vendor))
self.payload_fields.append(("Reserved", self.product))
self.payload_fields.append(("Version", self.version))
vendor = little_endian(bitstring.pack("32", self.vendor))
product = little_endian(bitstring.pack("32", self.product))
version = little_endian(bitstring.pack("32", self.version))
payload = vendor + product + version
return payload
def get_protocol_header(self):
reserved_64_format = self.protocol_header_format[0]
message_type_format = self.protocol_header_format[1]
reserved_16_format = self.protocol_header_format[2]
reserved_64 = little_endian(bitstring.pack(reserved_64_format, self.reserved))
message_type = little_endian(bitstring.pack(message_type_format, self.message_type))
reserved_16 = little_endian(bitstring.pack(reserved_16_format, self.reserved))
protocol_header = reserved_64 + message_type + reserved_16
return protocol_header
def get_frame(self):
size_format = self.frame_format[0]
flags_format = self.frame_format[1]
source_id_format = self.frame_format[2]
size = little_endian(bitstring.pack(size_format, self.size))
flags = little_endian(bitstring.pack(flags_format, self.origin, self.tagged, self.addressable, self.protocol))
source_id = little_endian(bitstring.pack(source_id_format, self.source_id))
frame = size + flags + source_id
return frame
def get_payload(self):
self.payload_fields.append(("Count", self.count))
self.payload_fields.append(("Index", self.index))
self.payload_fields.append(("Color (HSBK)", self.color))
count = little_endian(bitstring.pack("8", self.count))
index = little_endian(bitstring.pack("8", self.index))
color = "".join(little_endian(bitstring.pack("16", field)) for field in self.color)
payload = count + index + color
return payload
def get_payload(self):
self.payload_fields.append(("Current Time", self.time))
self.payload_fields.append(("Uptime (ns)", self.uptime))
self.payload_fields.append(("Last Downtime Duration (ns) (5 second error)", self.downtime))
time = little_endian(bitstring.pack("64", self.time))
uptime = little_endian(bitstring.pack("64", self.uptime))
downtime = little_endian(bitstring.pack("64", self.downtime))
payload = time + uptime + downtime
return payload
def get_payload(self):
self.payload_fields.append(("Timestamp of Build", self.build))
self.payload_fields.append(("Reserved", self.reserved1))
self.payload_fields.append(("Version", self.version))
build = little_endian(bitstring.pack("64", self.build))
reserved1 = little_endian(bitstring.pack("64", self.reserved1))
version = little_endian(bitstring.pack("32", self.version))
payload = build + reserved1 + version
return payload
def append_minifloat(self, fmt, unit, value):
if value is None:
e = s = 0
else:
(e, s, rounded) = fmt.encode(value)
if value != rounded:
self.roundings.append("{0} {2} to {1} {2}".format(value, rounded, unit))
self.data.append(pack('uint:n', e, n = fmt.ebits))
self.data.append(pack('uint:n', s, n = fmt.sbits))
def get_payload(self):
self.payload_fields.append(("Label", self.label))
field_len_bytes = 32
try:
label = b"".join(little_endian(bitstring.pack("8", c)) for c in self.label.encode('utf-8'))
except ValueError: # because of differences in Python 2 and 3
label = b"".join(little_endian(bitstring.pack("8", ord(c))) for c in self.label.encode('utf-8'))
padding = b"".join(little_endian(bitstring.pack("8", 0)) for i in range(field_len_bytes-len(self.label)))
payload = label + padding
return payload
def bytes(self):
binary = bitstring.pack("uint:8, uint:8", self.tag, self.length)
if self.tag == self.TAG_CA_DESCRIPTOR:
binary.append(bitstring.pack(
"bytes:2, pad:3, uint:13, bytes",
self.ca_system_id, self.ca_pid, self.private_data_bytes))
else:
binary.append(self.contents)
assert(len(binary) / 8 == self.size)
return binary.bytes
def get_payload(self):
self.payload_fields.append(("Signal (mW)", self.signal))
self.payload_fields.append(("TX (bytes since on)", self.tx))
self.payload_fields.append(("RX (bytes since on)", self.rx))
self.payload_fields.append(("Reserved", self.reserved1))
signal = little_endian(bitstring.pack("32", self.signal))
tx = little_endian(bitstring.pack("32", self.tx))
rx = little_endian(bitstring.pack("32", self.rx))
reserved1 = little_endian(bitstring.pack("16", self.reserved1))
payload = signal + tx + rx + reserved1
return payload
def get_payload(self):
field_len = 64
self.payload_fields.append(("Byte Array", self.byte_array))
byte_array = "".join(little_endian(bitstring.pack("8", b)) for b in self.byte_array)
byte_array_len = len(byte_array)
if byte_array_len < field_len:
byte_array += "".join(little_endian(bitstring.pack("8", 0)) for i in range(field_len - byte_array_len))
elif byte_array_len > field_len:
byte_array = byte_array[:field_len]
payload = byte_array
return payload
def get_payload(self):
power_level = little_endian(bitstring.pack("uint:16",
self.power_level))
duration = little_endian(bitstring.pack("uint:32", self.duration))
payload = power_level + duration
return payload
def calc_similarity_matrix(glyphs, sim_calc, n, p, in_sim_file):
"""Compute similarity matrix.
glyphs --
sim_calc -- method index
n --
p --
in_sim_file -- name of file containing matrix to be read in and returned
"""
#FIXME ... above docstring
if sim_calc == CALC_PACKBITS:
# optimize by packing the bits with numpy.packbits
cast_to_uint8 = np.array([0], dtype=np.uint8)
dummy_ba = np.packbits(np.array([True] * p) + cast_to_uint8)
b_glyphs = [None] * n
print "converting to bitstrings using numpy.packbits",
time_start = time.time()
for i in range(n):
try:
ba = np.packbits(glyphs[i] + cast_to_uint8)
except TypeError:
ba = dummy_ba
b_glyphs[i] = ba
print '.',
print
print "converted to bitstrings in %f seconds" % (time.time() -
time_start)
print "computing similarity matrix with bitcount",
time_start = time.time()
sim = np.zeros((n, n), dtype=float)
for i in range(n):
sim[i, i] = 1.0
for j in range(i + 1, n):
try:
s = jaccard_sim_numpy_packed(b_glyphs[i],
b_glyphs[j],
verbose=(0, 1315) == (i, j))
except TypeError:
# a glyph exceeded the bounding box and therefore
# was probably a non-textual item such as a line
s = 0.0
sim[i, j] = s
sim[j, i] = s
print '.',
print
print "computed (bitstring) similarity matrix in %f seconds" % (
time.time() - time_start)
elif sim_calc == CALC_BITSTRING:
# optimize by packing the bits with bitstring
dummy_ba = [True] * p
b_glyphs = [None] * n
fmt = "%d*bool" % p
print "converting to bitstrings",
time_start = time.time()
for i in range(n):
try:
ba = glyphs[i].ravel().tolist()
except AttributeError:
ba = dummy_ba
b_glyphs[i] = bitstring.pack(fmt, *ba)
print '.',
print
print "converted to bitstrings in %f seconds" % (time.time() -
time_start)
print "computing similarity matrix with bitstring",
time_start = time.time()
sim = np.zeros((n, n), dtype=float)
for i in range(n):
sim[i, i] = 1.0
for j in range(i + 1, n):
try:
s = jaccard_sim_bitstring(b_glyphs[i],
b_glyphs[j],
verbose=(0, 1315) == (i, j))
except TypeError:
# a glyph exceeded the bounding box and therefore
# was probably a non-textual item such as a line
s = 0.0
sim[i, j] = s
sim[j, i] = s
print '.',
print
print "computed (bitstring) similarity matrix in %f seconds" % (
time.time() - time_start)
elif sim_calc == CALC_NUMPY:
print "computing similarity matrix with numpy",
time_start = time.time()
sim = np.zeros((n, n), dtype=float)
for i in range(n):
sim[i, i] = 1.0
for j in range(i + 1, n):
try:
s = jaccard_sim(glyphs[i],
glyphs[j],
verbose=(0, 1315) == (i, j))
except TypeError:
# a glyph exceeded the bounding box and therefore
# was probably a non-textual item such as a line
s = 0.0
sim[i, j] = s
sim[j, i] = s
print ".",
print
print "computed (numpy) similarity matrix in %f seconds" % (
time.time() - time_start)
elif sim_calc == CALC_SCIPY:
print "computing similarity matrix with scipy",
time_start = time.time()
ba_dummy = np.ones(glyphs[0].shape, dtype=np.bool)
f_glyphs = np.zeros((n, p), dtype=np.bool)
for i in range(n):
try:
f_glyphs[i, :] = glyphs[i].ravel()
except AttributeError:
f_glyphs[i, :] = ba_dummy.ravel()
print '.',
dissimilarity = distance.pdist(f_glyphs, 'jaccard')
print '.',
sim = 1 - distance.squareform(dissimilarity)
print "computed (scipy) similarity matrix in %f seconds" % (
time.time() - time_start)
elif sim_calc == CALC_FROMFILE:
with open(in_sim_file) as f:
o = cPickle.load(f)
sim = o.sim
else:
print >> sys.stderr, "sim_calc method %d undefined." % sim_calc
sys.exit(2)
return sim
def format_ip_address(self):
address = socket.gethostbyname(socket.gethostname())
return bitstring.pack('uint:8, uint:8, uint:8, uint:8', *[int(x) for x in address.split('.')]).bytes
def to_bytes(self):
bytes_name = name_to_bytes(self.qname)[0]
bits_packet = BitArray(length=32)
bits_packet[0:16] = pack('uint: 16', self.qtype)
bits_packet[16:32] = pack('uint: 16', self.qclass)
return bytes_name + bits_packet.tobytes()
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('airconSwitch', type=str)
parser.add_argument('barnNo', type=str)
args = parser.parse_args()
print(args)
barnNo = args['barnNo']
airconSwitch = args['airconSwitch']
nodes = db.session.query(LoraNode.node_addr).join(
GrainBarn, GrainBarn.id == LoraNode.grain_barn_id).filter(
GrainBarn.barn_no == barnNo).order_by(
LoraNode.node_addr.asc()).all()
print("nodes are:", nodes)
if airconSwitch == '1':
for node in nodes:
node_addr = node[0]
print('node_addr is:', node_addr)
mqtt_node_addr = bitstring.pack('uint:13', node_addr).bin
node_mqtt_trans_func = db.session.query(
NodeMqttTransFunc.gateway_addr,
NodeMqttTransFunc.node_addr,
NodeMqttTransFunc.trans_direct,
NodeMqttTransFunc.func_code, NodeMqttTransFunc.wind_direct,
NodeMqttTransFunc.wind_speed, NodeMqttTransFunc.model,
NodeMqttTransFunc.on_off, NodeMqttTransFunc.work_mode,
NodeMqttTransFunc.temp).filter(
NodeMqttTransFunc.node_addr == mqtt_node_addr).all()
print('******node_mqtt_trans_func******', node_mqtt_trans_func)
if node_mqtt_trans_func:
print('airconditoner switch to on!')
on_off = '01'
mqtt_auto_control_air(node_mqtt_trans_func, on_off)
elif airconSwitch == '0':
for node in nodes:
node_addr = node[0]
print('node_addr is:', node_addr)
mqtt_node_addr = bitstring.pack('uint:13', node_addr).bin
node_mqtt_trans_func = db.session.query(
NodeMqttTransFunc.gateway_addr,
NodeMqttTransFunc.node_addr,
NodeMqttTransFunc.trans_direct,
NodeMqttTransFunc.func_code, NodeMqttTransFunc.wind_direct,
NodeMqttTransFunc.wind_speed, NodeMqttTransFunc.model,
NodeMqttTransFunc.on_off, NodeMqttTransFunc.work_mode,
NodeMqttTransFunc.temp).filter(
NodeMqttTransFunc.node_addr == mqtt_node_addr).all()
print('******node_mqtt_trans_func******', node_mqtt_trans_func)
if node_mqtt_trans_func:
print('airconditoner switch to off!')
on_off = '00'
mqtt_auto_control_air(node_mqtt_trans_func, on_off)
else:
print('airconSwitch is :', airconSwitch)
return nodes, args
def __init__(self, num_entries: int) -> None:
super().__init__(0x18)
with self._at_offset(20):
self.buffer.overwrite(pack('uintle:32', num_entries))
def __init__(self, offset: int, bitfield: int) -> None:
super().__init__(8)
with self._at_offset(0):
self.buffer.overwrite(pack('uintle:32', offset))
self.buffer.overwrite(pack('uintle:32', bitfield))
def tagged(char, l):
# Tagged fields need to be zero-padded to 5 bits.
while l.len % 5 != 0:
l.append('0b0')
return bitstring.pack("uint:5, uint:5, uint:5", CHARSET.find(char),
(l.len / 5) / 32, (l.len / 5) % 32) + l
def lnencode(addr, privkey):
if addr.amount:
amount = Decimal(str(addr.amount))
# We can only send down to millisatoshi.
if amount * 10**12 % 10:
raise ValueError(
"Cannot encode {}: too many decimal places".format(
addr.amount))
amount = addr.currency + shorten_amount(amount)
else:
amount = addr.currency if addr.currency else ''
hrp = 'ln' + amount
# Start with the timestamp
data = bitstring.pack('uint:35', addr.date)
# Payment hash
data += tagged_bytes('p', addr.paymenthash)
tags_set = set()
for k, v in addr.tags:
# BOLT #11:
#
# A writer MUST NOT include more than one `d`, `h`, `n` or `x` fields,
if k in ('d', 'h', 'n', 'x'):
if k in tags_set:
raise ValueError("Duplicate '{}' tag".format(k))
if k == 'r':
route = bitstring.BitArray()
for step in v:
pubkey, channel, feebase, feerate, cltv = step
route.append(
bitstring.BitArray(pubkey) + bitstring.BitArray(channel) +
bitstring.pack('intbe:32', feebase) +
bitstring.pack('intbe:32', feerate) +
bitstring.pack('intbe:16', cltv))
data += tagged('r', route)
elif k == 'f':
data += encode_fallback(v, addr.currency)
elif k == 'd':
data += tagged_bytes('d', v.encode())
elif k == 'x':
# Get minimal length by trimming leading 5 bits at a time.
expirybits = bitstring.pack('intbe:64', v)[4:64]
while expirybits.startswith('0b00000'):
expirybits = expirybits[5:]
data += tagged('x', expirybits)
elif k == 'h':
data += tagged_bytes('h',
hashlib.sha256(v.encode('utf-8')).digest())
elif k == 'n':
data += tagged_bytes('n', v)
else:
# FIXME: Support unknown tags?
raise ValueError("Unknown tag {}".format(k))
tags_set.add(k)
# BOLT #11:
#
# A writer MUST include either a `d` or `h` field, and MUST NOT include
# both.
if 'd' in tags_set and 'h' in tags_set:
raise ValueError("Cannot include both 'd' and 'h'")
if not 'd' in tags_set and not 'h' in tags_set:
raise ValueError("Must include either 'd' or 'h'")
# We actually sign the hrp, then data (padded to 8 bits with zeroes).
privkey = secp256k1.PrivateKey(bytes(unhexlify(privkey)))
sig = privkey.ecdsa_sign_recoverable(
bytearray([ord(c) for c in hrp]) + data.tobytes())
# This doesn't actually serialize, but returns a pair of values :(
sig, recid = privkey.ecdsa_recoverable_serialize(sig)
data += bytes(sig) + bytes([recid])
return bech32_encode(hrp, bitarray_to_u5(data))
def connectionMade(self):
# Yank out our peer/host addresses as we'll need them to construct our
# header.
peer = self.peer.transport.getPeer()
peer_parsed = ipaddress.ip_address(six.text_type(peer.host))
host = self.peer.transport.getHost()
host_parsed = ipaddress.ip_address(six.text_type(host.host))
# All of our PROXY data needs to start with this header to indicate
# that it is the PROXY v2 protocol.
data = [_PROXY_HEADER]
# We need to communicate what version of the protocol we speak (v2) and
# what kind of command this is. Currently we only support the Proxy
# commands.
data.append(
bitstring.pack(
"int:4, int:4",
_PROXY_VERSION,
_PROXY_COMMANDS["PROXY"],
).bytes)
# We also need to communicate what type of connection this is, like an
# IPv4 over TCP or so.
data.append(
bitstring.pack(
"int:4, int:4",
_PROXY_AF[peer.__class__],
_PROXY_PROTOCOL[peer.type],
).bytes)
# Here we need to add what the length of the rest of our data is,
# however we don't actually *have* the rest of out data yet, so we'll
# just add a junk value here for now and come back to this later.
data.append(None)
# We need to send information about our client and what they connected
# to, and the exact format of that will change based on the address
# family of the connection.
if isinstance(peer, address.IPv4Address):
data.append(
_IPv4Stuct.pack(
peer_parsed.packed,
host_parsed.packed,
peer.port,
host.port,
))
elif isinstance(peer, address.IPv6Address):
data.append(
_IPv6Struct.pack(
peer_parsed.packed,
host_parsed.packed,
peer.port,
host.port,
))
elif isinstance(peer, address.UNIXAddress):
data.append(_UnixStruct.pack(
peer.name,
host.name,
))
# TODO: Insert TLS Stuff
# We can only actually do this for one of our known address types,
# otherwise we have to have a length of zero for the rest of the data.
if isinstance(peer, tuple(_PROXY_AF.keys())):
pass
# Now that we've finished building up our message, we'll figure out
# what the length of our header block is.
data[3] = _LengthStruct.pack(sum(map(len, data[4:])))
# Send this all to the server we're proxying for so that it can set its
# own internal state correctly.
self.transport.writeSequence(data)
# Finally, we'll call the standard connectionMade, which will finish
# constructing our tunnel.
return super(ProxyClient, self).connectionMade()
def u5_to_bitarray(arr):
"""Bech32 spits out array of 5-bit values. Shim here."""
ret = bitstring.BitArray()
for a in arr:
ret += bitstring.pack("uint:5", a)
return ret
def get_payload(self):
self.payload_fields.append(("Start Index", self.start_index))
self.payload_fields.append(("Total Count", self.total_count))
self.payload_fields.append(("Tile Devices", self.tile_devices))
start_idex = little_endian(bitstring.pack("uint:8", self.start_index))
payload = start_idex
for tile in self.tile_devices:
reserved1 = little_endian(bitstring.pack("int:16", tile['reserved1']))
reserved2 = little_endian(bitstring.pack("int:16", tile['reserved2']))
reserved3 = little_endian(bitstring.pack("int:16", tile['reserved3']))
reserved4 = little_endian(bitstring.pack("int:16", tile['reserved4']))
user_x = little_endian(bitstring.pack("float:32", tile['user_x']))
user_y = little_endian(bitstring.pack("float:32", tile['user_y']))
width = little_endian(bitstring.pack("uint:8", tile['width']))
height = little_endian(bitstring.pack("uint:8", tile['height']))
reserved5 = little_endian(bitstring.pack("uint:8", tile['reserved5']))
device_version_vendor = little_endian(bitstring.pack("uint:32", tile['device_version_vendor']))
device_version_product = little_endian(bitstring.pack("uint:32", tile['device_version_product']))
device_version_version = little_endian(bitstring.pack("uint:32", tile['device_version_version']))
firmware_build = little_endian(bitstring.pack("uint:64", tile['firmware_build']))
reserved6 = little_endian(bitstring.pack("uint:64", tile['reserved6']))
firmware_version = little_endian(bitstring.pack("uint:32", tile['firmware_version']))
reserved7 = little_endian(bitstring.pack("uint:32", tile['reserved7']))
payload += reserved1 + reserved2 + reserved3 + reserved4 + user_x + user_y + width + height + reserved5 + device_version_vendor + device_version_product + device_version_version + firmware_build + reserved6 + firmware_version + reserved7
total_count = little_endian(bitstring.pack("uint:8", self.total_count))
payload += total_count
return payload
def get_payload(self):
reserved_8 = little_endian(bitstring.pack("uint:8", self.reserved))
transient = little_endian(bitstring.pack("uint:8", self.transient))
color = b"".join(
little_endian(bitstring.pack("uint:16", field))
for field in self.color)
period = little_endian(bitstring.pack("uint:32", self.period))
cycles = little_endian(bitstring.pack("float:32", self.cycles))
skew_ratio = little_endian(bitstring.pack("int:16", self.skew_ratio))
waveform = little_endian(bitstring.pack("uint:8", self.waveform))
set_hue = little_endian(bitstring.pack("uint:8", self.set_hue))
set_saturation = little_endian(
bitstring.pack("uint:8", self.set_saturation))
set_brightness = little_endian(
bitstring.pack("uint:8", self.set_brightness))
set_kelvin = little_endian(bitstring.pack("uint:8", self.set_kelvin))
payload = reserved_8 + transient + color + period + cycles + skew_ratio + waveform + set_hue + set_saturation + set_brightness + set_kelvin
payloadUi = " ".join("{:02x}".format(c) for c in payload)
return payload
def get_payload(self):
self.payload_fields.append(("Byte Array", self.byte_array))
byte_array = b"".join(little_endian(bitstring.pack("8", b)) for b in self.byte_array)
payload = byte_array
return payload
def get_pehash(pe_file):
""" Return pehash for PE file, sha1 of PE structural properties.
:param pe_file: file name or instance of pefile.PE() class
:rtype : string SHA1 in hexdigest format
"""
if isinstance(pe_file, PE): # minimize mem. usage and time of execution
exe = pe_file
else:
exe = PE(pe_file, fast_load=True)
# Image Characteristics
img_chars = pack('uint:16', exe.FILE_HEADER.Characteristics)
pehash_bin = img_chars[0:8] ^ img_chars[8:16]
# Subsystem
subsystem = pack('uint:16', exe.OPTIONAL_HEADER.Subsystem)
pehash_bin.append(subsystem[0:8] ^ subsystem[8:16])
# Stack Commit Size, rounded up to a value divisible by 4096,
# Windows page boundary, 8 lower bits must be discarded
# in PE32+ is 8 bytes
stack_commit = exe.OPTIONAL_HEADER.SizeOfStackCommit
if stack_commit % 4096:
stack_commit += 4096 - stack_commit % 4096
stack_commit = pack('uint:56', stack_commit >> 8)
pehash_bin.append(
stack_commit[:8] ^ stack_commit[8:16] ^
stack_commit[16:24] ^ stack_commit[24:32] ^
stack_commit[32:40] ^ stack_commit[40:48] ^ stack_commit[48:56])
# Heap Commit Size, rounded up to page boundary size,
# 8 lower bits must be discarded
# in PE32+ is 8 bytes
heap_commit = exe.OPTIONAL_HEADER.SizeOfHeapCommit
if heap_commit % 4096:
heap_commit += 4096 - heap_commit % 4096
heap_commit = pack('uint:56', heap_commit >> 8)
pehash_bin.append(
heap_commit[:8] ^ heap_commit[8:16] ^
heap_commit[16:24] ^ heap_commit[24:32] ^
heap_commit[32:40] ^ heap_commit[40:48] ^ heap_commit[48:56])
# Section structural information
for section in exe.sections:
# Virtual Address, 9 lower bits must be discarded
pehash_bin.append(pack('uint:24', section.VirtualAddress >> 9))
# Size Of Raw Data, 8 lower bits must be discarded
pehash_bin.append(pack('uint:24', section.SizeOfRawData >> 8))
# Section Characteristics, 16 lower bits must be discarded
sect_chars = pack('uint:16', section.Characteristics >> 16)
pehash_bin.append(sect_chars[:8] ^ sect_chars[8:16])
# Kolmogorov Complexity, len(Bzip2(data))/len(data)
# (0..1} ∈ R -> [0..7] ⊂ N
kolmogorov = 0
if section.SizeOfRawData:
kolmogorov = int(round(
len(compress(section.get_data()))
* 7.0 /
section.SizeOfRawData))
if kolmogorov > 7:
kolmogorov = 7
pehash_bin.append(pack('uint:8', kolmogorov))
assert 0 == pehash_bin.len % 8
if not isinstance(pe_file, PE):
exe.close()
return sha1(pehash_bin.tobytes()).hexdigest()
def lnencode(addr: 'LnAddr', privkey) -> str:
if addr.amount:
amount = Decimal(str(addr.amount))
# We can only send down to millisatoshi.
if amount * 10**12 % 10:
raise ValueError("Cannot encode {}: too many decimal places".format(
addr.amount))
amount = addr.currency + shorten_amount(amount)
else:
amount = addr.currency if addr.currency else ''
hrp = 'ln' + amount
# Start with the timestamp
data = bitstring.pack('uint:35', addr.date)
tags_set = set()
# Payment hash
data += tagged_bytes('p', addr.paymenthash)
tags_set.add('p')
if addr.payment_secret is not None:
data += tagged_bytes('s', addr.payment_secret)
tags_set.add('s')
for k, v in addr.tags:
# BOLT #11:
#
# A writer MUST NOT include more than one `d`, `h`, `n` or `x` fields,
if k in ('d', 'h', 'n', 'x', 'p', 's'):
if k in tags_set:
raise ValueError("Duplicate '{}' tag".format(k))
if k == 'r':
route = bitstring.BitArray()
for step in v:
pubkey, channel, feebase, feerate, cltv = step
route.append(bitstring.BitArray(pubkey) + bitstring.BitArray(channel) + bitstring.pack('intbe:32', feebase) + bitstring.pack('intbe:32', feerate) + bitstring.pack('intbe:16', cltv))
data += tagged('r', route)
elif k == 'f':
data += encode_fallback(v, addr.currency)
elif k == 'd':
data += tagged_bytes('d', v.encode())
elif k == 'x':
expirybits = bitstring.pack('intbe:64', v)
expirybits = trim_to_min_length(expirybits)
data += tagged('x', expirybits)
elif k == 'h':
data += tagged_bytes('h', sha256(v.encode('utf-8')).digest())
elif k == 'n':
data += tagged_bytes('n', v)
elif k == 'c':
finalcltvbits = bitstring.pack('intbe:64', v)
finalcltvbits = trim_to_min_length(finalcltvbits)
data += tagged('c', finalcltvbits)
elif k == '9':
if v == 0:
continue
feature_bits = bitstring.BitArray(uint=v, length=v.bit_length())
feature_bits = trim_to_min_length(feature_bits)
data += tagged('9', feature_bits)
else:
# FIXME: Support unknown tags?
raise ValueError("Unknown tag {}".format(k))
tags_set.add(k)
# BOLT #11:
#
# A writer MUST include either a `d` or `h` field, and MUST NOT include
# both.
if 'd' in tags_set and 'h' in tags_set:
raise ValueError("Cannot include both 'd' and 'h'")
if not 'd' in tags_set and not 'h' in tags_set:
raise ValueError("Must include either 'd' or 'h'")
# We actually sign the hrp, then data (padded to 8 bits with zeroes).
msg = hrp.encode("ascii") + data.tobytes()
privkey = ecc.ECPrivkey(privkey)
sig = privkey.sign_message(msg, is_compressed=False, algo=lambda x:sha256(x).digest())
recovery_flag = bytes([sig[0] - 27])
sig = bytes(sig[1:]) + recovery_flag
data += sig
return bech32_encode(hrp, bitarray_to_u5(data))
def prepare_bitstream(self) -> BitStream:
with self._at_offset(12):
self.buffer.overwrite(pack('uintle:32', self.offset - 16))
return super().prepare_bitstream()
def u5_to_bitarray(arr):
ret = bitstring.BitArray()
for a in arr:
ret += bitstring.pack("uint:5", a)
return ret
def parse_address_to(address: str):
result = bitstring.BitArray()
for part in address.split("."):
result += bitstring.pack("uint:8", int(part))
return result.bytes
def _inject_faults_int8_random_bit_position_ecc(tensor,
random,
n_bits,
block_size=64,
t=1,
debug_mode=False):
'''
For the input tensor, apply ecc protection. Input tensor should be 1-d torch tensor.
Total number of bits is num_values * 8. The procedure is:
Step 1: randomly choose n_bits number of bits to flip for tensor
Step 2: get the bit flipps that can be corrected via ECC.
Step 3: apply ECC correction.
Args:
- tensor: torch tensor
- random: numpy random
- n_bits: # bits to be flipped
- block_size: the number of bits to be protected together to form a codeword
- t: protection ability, can be 1 or 2
Returns:
- stats (dict): key-value pairs. The key is the index of a value whose bits are flipped.
The value is a tuple: (value before flipped, flipped bit position, bit after flipped, value after flipped)
- corr (dict): key-value pairs. The key is the index of a value who is detected to be faulty based on parity encoding.
The value is a tuple: (value before correction, value after correction)
'''
assert len(tensor.size(
)) == 1, "The input tensor is not a 1-D vector. Current shape:{}".format(
tensor.size())
# 1. fault injection with correction
start = time.time()
num_values = tensor.nelement()
indexes = random.choice(num_values * 8, size=n_bits, replace=False)
sample_time = time.time() - start
# correct some error: put the indexes into data blocks, check whether a data block has more than two faults.
corrected_indexes = _get_correctable_indexes(indexes,
block_size=block_size,
t=t)
start = time.time()
stats = defaultdict(list)
corr = {}
for index in indexes:
vid, bid = index >> 3, index & 0b111
value = int(tensor[vid])
assert value == tensor[vid], "value is not an integer," + str(
value) + ', ' + str(tensor[vid])
bits = bitstring.pack('>b', value)
bits[bid] ^= 1
value_after_flip = bits.int
# if the flip can be corrected:
if index in corrected_indexes:
corr[vid] = (value_after_flip, value)
else:
tensor[vid] = value_after_flip
if debug_mode:
print('vid: %5d, before: %5d, bid: %d => %s, after: %5d (%s)' %
(vid, value, bid, bits[bid], value_after_flip, bits.bin))
stats[vid].append((value, bid, bits[bid], value_after_flip))
injection_time = time.time() - start
print('sample time (s):%.4f' % (sample_time),
', injection_time (s):%.4f' % (injection_time),
', #faults:%5d' % (n_bits), ', #corr:%5d' % (len(corr)))
return stats, corr
def _u5_to_bitarray(arr: List[int]) -> bitstring.BitArray:
ret = bitstring.BitArray()
for a in arr:
ret += bitstring.pack("uint:5", a)
return ret
def _parity_bit(v):
bits = bitstring.pack('>b', v) # 8 bits
# code = (sum(bits)%2 == 1)
code = reduce(lambda x, y: x ^ y, bits)
return code
msg = open("./enwik8", "rb").read()
# do a pass through to find the cummulative distribution
msg_length = 0
cum = [0] * 256
for c in msg:
cum[c + 1] += 1
msg_length += 1
for i in range(1, len(cum)):
cum[i] = cum[i - 1] + (cum[i]) / msg_length
if cum[i] > 1:
cum[i] = 1
#print(cum)
out.append(bitstring.pack('uint:64', msg_length))
for ch in range(256):
bf = bitstring.pack('float:64', cum[ch])
out.append(bf)
cum[ch] = bf.read('float:64')
#print(msg_length)
#print(cum)
b = 0
l = 1
# we make sure l is always bigger than .5
def renormalize():
# use b as lower
def _parity_bit_sum(v):
bits = bitstring.pack('>b', v) # 8 bits
code = (sum(bits) % 2 == 1)
return code
def encode_pad(value):
return pack('pad:n', n=value)
def get_payload(self):
self.payload_fields.append(("Infrared Brightness", self.infrared_brightness))
infrared_brightness = little_endian(bitstring.pack("16", self.infrared_brightness))
payload = infrared_brightness
return payload
def get_payload(self):
start_index = little_endian(bitstring.pack("uint:8", self.start_index))
end_index = little_endian(bitstring.pack("uint:8", self.end_index))
payload = start_index + end_index
return payload
def get_payload(self):
self.payload_fields.append(("Power Level", self.power_level))
power_level = little_endian(bitstring.pack("uint:16",
self.power_level))
payload = power_level
return payload
def get_payload(self):
infrared_brightness = little_endian(
bitstring.pack("uint:16", self.infrared_brightness))
payload = infrared_brightness
return payload
#!/usr/bin/env python
import pymongo
import bitstring
from hashes.simhash import simhash
connection = pymongo.Connection("localhost", 27017)
db = connection.pace
coll = db.people
for i in coll.find():
h = simhash(i['firstName'] + i['lastName'], hashbits=64)
ha = bitstring.pack('uint:64', long(h))
for j in xrange(0, 8):
i['h%s' % j] = ha.hex
#print ha.hex
ha.ror(64 / 8)
coll.update({'n': i['n']}, i)
