def tobits(n, nbits=8, signed=False):
# convert the integer n to a bit string of n bits
# if signed then the returned value is 2's comp
if not signed:
return BitString(uint=n, length=nbits).bin
else:
return BitString(int=n, length=nbits).bin
def __init__(self, parent, param):
self.category = param.category
self.parent = parent
self.block = param.block
self.word = param.word
self.pos = param.pos
self.write_disable_bit = param.write_disable_bit
self.read_disable_bit = param.read_disable_bit
self.name = param.name
self.efuse_class = param.class_type
self.efuse_type = param.type
self.description = param.description
self.dict_value = param.dictionary
self.fail = False
self.num_errors = 0
if self.efuse_type.startswith("bool"):
field_len = 1
else:
field_len = int(re.search(r"\d+", self.efuse_type).group())
if self.efuse_type.startswith("bytes"):
field_len *= 8
self.bitarray = BitString(field_len)
self.bit_len = field_len
self.bitarray.set(0)
self.update(self.parent.blocks[self.block].bitarray)
def encode(self, message):
#initializing Table[0 to 255] to corresponding ASCII charaters
Table = initialize(encode=True)
#initializing table index and P
index = 256
counter = 0
output = []
P = ''
#LZW compression algorithm
while counter != len(message) + 1:
try:
C = message[counter]
except IndexError: # for the last iteration, C is empty so just return P and exit
output.append(Table[P])
break
if P + C in Table:
P = P + C
else:
output.append(Table[P])
Table[P + C] = index
index += 1
P = C
counter += 1
#if the dictionary outgrew the word_size, re_initialize the dictionary
if index > 2**self.word_size:
Table = initialize(encode=True)
index = 256
packed_output = BitString()
packed_output.pack_numbers(output, self.word_size)
return packed_output
def check_data_block_in_log(self, log, file_path, repeat=1, reverse_order=False, offset=0):
with open(file_path, 'rb') as f:
data = BitString('0x00') * offset + BitString(f)
blk = data.readlist("%d*uint:8" % (data.len // 8))
blk = blk[::-1] if reverse_order else blk
hex_blk = " ".join("{:02x}".format(num) for num in blk)
self.assertEqual(repeat, log.count(hex_blk))
def encode(self, message):
output = []
main_dictionary = {}
for i in range(256):
main_dictionary[chr(i)] = i
string = message[0]
curr_index = 0
l = len(message)
while curr_index < l - 1:
symbol = message[curr_index + 1]
if string + symbol in main_dictionary:
string += symbol
else:
output.append(main_dictionary[string])
max_key = max(main_dictionary, key=main_dictionary.get)
max_val = main_dictionary[max_key]
if max_val == 2**16 - 1:
main_dictionary = {}
for i in range(256):
main_dictionary[chr(i)] = i
#main_dictionary[string+symbol]=0
else:
main_dictionary[string + symbol] = max_val + 1
string = symbol
curr_index += 1
output.append(main_dictionary[string])
bits = BitString()
bits.pack_numbers(output, 16)
return bits
def decodeFlags(flagsByte):
from bitstring import BitString
bits=BitString(bytes=flagsByte)
bools=[]
for x in range(bits.length):
bools.append(bits.readbit().uint==1)
return bools
def __init__(self, parent, name, category, block, word, pos, efuse_type,
write_disable_bit, read_disable_bit, efuse_class, description,
dict_value):
self.category = category
self.parent = parent
self.block = block
self.word = word
self.pos = pos
self.write_disable_bit = write_disable_bit
self.read_disable_bit = read_disable_bit
self.name = name
self.efuse_class = efuse_class
self.efuse_type = efuse_type
self.description = description
self.dict_value = dict_value
if self.efuse_type.startswith("bool"):
field_len = 1
else:
field_len = int(re.search(r'\d+', self.efuse_type).group())
if self.efuse_type.startswith("bytes"):
field_len *= 8
self.bitarray = BitString(field_len)
self.bit_len = field_len
self.bitarray.set(0)
self.update(self.parent.blocks[self.block].bitarray)
def buildcustomrandomdata(orderedrandomlist, lengthin, digits=8):
if digits == 8:
random4 = ''
for x in range(0, lengthin):
random4 += str(orderedrandomlist[random.randint(0, 7)])
ssxwrite = open('orighex_eight.bin', 'wb')
BitString(hex=random4).tofile(ssxwrite)
ssxwrite.close()
with zipfile.ZipFile('orighex_eight.zip', 'w',
zipfile.ZIP_DEFLATED) as myzip:
myzip.write('orighex_eight.bin')
return random4
elif digits == 4:
random4 = ''
for x in range(0, lengthin):
random4 += str(orderedrandomlist[random.randint(0, 3)])
ssxwrite = open('orighex.bin', 'wb')
BitString(hex=random4).tofile(ssxwrite)
ssxwrite.close()
with zipfile.ZipFile('orighex.zip', 'w',
zipfile.ZIP_DEFLATED) as myzip:
myzip.write('orighex.bin')
return random4
def __init__(self, system_manager):
# Fairly certain we don't need this to be a dict, as
# we are unlikely to want to reference specific
# systems by id.
self._systems = []
# A dict of all components inside this entity.
# The keys are the IDs of the components.
self._components = {}
# The components this entity has.
self._component_bitstring = BitString()
# Keep track of the old bitstring so we can see
# what has changed when we call refresh twice.
self._old_component_bitstring = BitString()
# Systems that this entity belongs to.
self._systems_bitstring = BitString()
# Ensure unique IDs.
self.ID = Entity._id_counter
Entity._id_counter = Entity._id_counter + 1
# The system manager the entity belongs to.
# Gives access to the existing systems.
self.system_manager = system_manager
def encodeFlags(bools):
from bitstring import BitString
bits=BitString()
for bool in bools:
if bool:
bits.append(BitString('0b1'))
else:
bits.append(BitString('0b0'))
return bits.bytes
def get_status(self):
self.can_h.send_msg((0x40, 0x00, 0x21, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x4B and msg[1]==0x00 and msg[2]==0x21):
msg, msgId, time = self.can_h.read_msg()
status = BitString("0x%02x " % msg[5])
status.append("0x%02x " % msg[4])
return status.uint
def get_voltage(self):
self.can_h.send_msg((0x40, 0x00, 0x23, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x4B and msg[1]==0x00 and msg[2]==0x23):
msg, msgId, time = self.can_h.read_msg()
voltage = BitString("0x%02x " % msg[5])
voltage.append("0x%02x " % msg[4])
return voltage.uint
def get_temperature(self):
self.can_h.send_msg((0x40, 0x02, 0x23, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x4B and msg[1]==0x02 and msg[2]==0x23):
msg, msgId, time = self.can_h.read_msg()
temp = BitString("0x%02x " % msg[5])
temp.append("0x%02x " % msg[4])
return temp.uint
def wait_till_error(self, timeout):
# self.can_h.send_msg((0x40, 0x41, 0x60, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = None, None, None
while not (msgId==0x081 and msg[0]==0x30 and msg[1]==0x81):
msg, msgId, time = self.can_h.read_msg(timeout)
error_code = BitString("0x%02x " % msg[1])
error_code.append("0x%02x " % msg[0])
return error_code
def get_actual_position(self):
'''returned position in shaft'''
self.can_h.send_msg((0x40, 0x64, 0x60, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x43 and msg[1]==0x64 and msg[2]==0x60):
msg, msgId, time = self.can_h.read_msg()
pos = BitString("0x%02x " % msg[5])
pos.append("0x%02x " % msg[4])
return pos.uint/81.0
def get_actual_speed(self):
'''returned speed in shaft'''
self.can_h.send_msg((0x40, 0x02, 0x22, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x43 and msg[1]==0x02 and msg[2]==0x22):
msg, msgId, time = self.can_h.read_msg()
speed = BitString("0x%02x " % msg[5])
speed.append("0x%02x " % msg[4])
return speed.uint/810.0
def get_motor_current(self):
'''returned in milliamperes '''
self.can_h.send_msg((0x40, 0x01, 0x23, 0x00, 00, 00, 0x00, 0x00), self.node_id)
msg, msgId, time = self.can_h.read_msg()
while not (msg[0]==0x4b and msg[1]==0x01 and msg[2]==0x23):
msg, msgId, time = self.can_h.read_msg()
current = BitString("0x%02x " % msg[5])
current.append("0x%02x " % msg[4])
return current.uint
def encode_datalist(x):
return (NEXT_ELEMENT.join(
MORE_BYTES.join(
BitString(bytes=byte)
for byte in data)
for data in x)
.tobytes()) # zero-pads if needed
def fFunction(k, R):
R_expanded = permute(R, E)
R_xor_k = R_expanded ^ k
pieces6b = list(splitToN(R_xor_k, 8))
pieces4b = map(substitute, pieces6b, S)
R_after_S = BitString('').join(pieces4b)
return permute(R_after_S, P)
def get_correct_answer(bitstring: list, control_bits: int) -> int:
bits = BitString(bitstring)
_ctrl_bits = bits[:control_bits]
_data_bits = bits[control_bits:]
return int(_data_bits[_ctrl_bits.uint])
def subrank3(adj):
"""
subrank2 takes an adjacency dictionary in (nid, pids) format and returns a
dictionary mapping nids to their SubRank. subrank3 uses BitStrings and the
Warshall algorithm, so it is the most scalable python SubRank yet!
"""
# Map adj to a reversed bitstring adjacency matrix, for speed.
matrixmap = {}
A = []
for nid in adj:
matrixmap[nid] = len(A)
A.append(BitString(uint=0, length=len(adj)))
for cid in adj:
for pid in adj[cid]:
A[matrixmap[pid]][matrixmap[cid]] = 1
# Compute SubRank.
progress = ProgressBar('Computing SubRank')
T = A
for j in xrange(len(adj)):
progress.next()
for i in xrange(len(adj)):
if T[i][j]:
T[i] = T[i] | T[j]
sys.stdout.write(' done.\n')
sys.stdout.flush()
return dict([(nid, float(sum(T[matrixmap[nid]]) + 1) / len(adj))
for nid in adj])
def __init__(self, bitstring_or_filename, **kwargs):
"Takes either a filename or a BitString object. Optional: flowcell_layout {}, read_config [{},]"
self.flowcell_layout = kwargs.get('flowcell_layout',
FLOWCELL_LAYOUT_DEFAULTS)
self.read_config = kwargs.get('read_config', READ_CONFIG_DEFAULTS)
# see if it's a filename or a bitstring (aka bitstream)
try:
bitstring_or_filename.all(
1) # attempts to perform the "are these bits all 1s" method
self.bs = bitstring_or_filename
except AttributeError: # assume it's a filename, then.
self.bs = BitString(bytes=open(bitstring_or_filename, 'rb').read())
self.num_tiles = reduce(lambda x, y: x * y,
self.flowcell_layout.values())
self.num_reads = len(self.read_config)
self._init_variables()
if self.bs is None:
raise Exception("bitstring empty; cannot parse metrics for %s" %
self.__class__.__name__)
else:
self.parse_binary()
def check_rd_protection_area(self):
# checks fields which have the read protection bits.
# if the read protection bit is set then we need to reset this field to 0.
def get_read_disable_mask(blk):
mask = 0
if isinstance(blk.read_disable_bit, list):
for i in blk.read_disable_bit:
mask |= (1 << i)
else:
mask = (1 << blk.read_disable_bit)
return mask
read_disable_bit = self.read_field("RD_DIS", bitstring=False)
for b in self.Blocks.BLOCKS:
blk = self.Blocks.get(b)
block = self.read_block(blk.id)
if blk.read_disable_bit is not None and read_disable_bit & get_read_disable_mask(blk):
if isinstance(blk.read_disable_bit, list):
if read_disable_bit & (1 << blk.read_disable_bit[0]):
block.set(0, [i for i in range(blk.len * 32 // 2, blk.len * 32)])
if read_disable_bit & (1 << blk.read_disable_bit[1]):
block.set(0, [i for i in range(0, blk.len * 32 // 2)])
else:
block.set(0)
else:
for e in self.Fields.EFUSES:
field = self.Fields.get(e)
if blk.id == field.block and field.read_disable_bit is not None and read_disable_bit & get_read_disable_mask(field):
raw_data = self.read_field(field.name)
raw_data.set(0)
block.pos = block.length - (field.word * 32 + field.pos + raw_data.length)
block.overwrite(BitString(raw_data.length))
self.overwrite_mem_from_block(blk, block)
def check_rd_protection_area(self):
# checks fields which have the read protection bits.
# if the read protection bit is set then we need to reset this field to 0.
read_disable_bit = self.read_field("RD_DIS", bitstring=False)
for b in self.Blocks.BLOCKS:
blk = self.Blocks.get(b)
block = self.read_block(blk.id)
if blk.read_disable_bit is not None and read_disable_bit & (
1 << blk.read_disable_bit
):
block.set(0)
else:
for e in self.Fields.EFUSES:
field = self.Fields.get(e)
if (
blk.id == field.block
and field.read_disable_bit is not None
and read_disable_bit & (1 << field.read_disable_bit)
):
raw_data = self.read_field(field.name)
raw_data.set(0)
block.pos = block.length - (
field.word * 32 + field.pos + raw_data.length
)
block.overwrite(BitString(raw_data.length))
self.overwrite_mem_from_block(blk, block)
def __init__(self, size=5000, bits=16):
self.all = []
self.dom = []
self.no_dom = []
for _ in range(size):
b = BitString.Random()
# Ordered insert at the left of any possible repeats:
bisect.insort_left(self.all, b)
def __init__(self, parent, param, skip_read=False):
self.parent = parent
self.name = param.name
self.alias = param.alias
self.id = param.id
self.rd_addr = param.rd_addr
self.wr_addr = param.wr_addr
self.write_disable_bit = param.write_disable_bit
self.read_disable_bit = param.read_disable_bit
self.len = param.len
self.key_purpose_name = param.key_purpose
bit_block_len = self.get_block_len() * 8
self.bitarray = BitString(bit_block_len)
self.bitarray.set(0)
self.wr_bitarray = BitString(bit_block_len)
self.wr_bitarray.set(0)
if not skip_read:
self.read()
def __init__(self, parent, param, skip_read=False):
self.parent = parent
self.name = param[0]
self.alias = param[1]
self.id = param[2]
self.rd_addr = param[3]
self.wr_addr = param[4]
self.write_disable_bit = param[5]
self.read_disable_bit = param[6]
self.len = param[7]
self.key_purpose_name = param[8]
bit_block_len = self.get_block_len() * 8
self.bitarray = BitString(bit_block_len)
self.bitarray.set(0)
self.wr_bitarray = BitString(bit_block_len)
self.wr_bitarray.set(0)
if not skip_read:
self.read()
def byte_isMore_tuples(x):
numBits = len(x) * 8
bs = BitString(bytes=x)
pos = 0
while numBits - pos >= 8:
yield (
bs[pos:pos + 8].bytes,
bs[pos + 8] if (numBits - pos >= 9) else False)
pos += 9
def __init__(self, filestr):
try:
self.data = filestr
b = BitString(bytes=filestr[:8])
self.signature = b.read("bytes:3") # FWS/SWS
self.version = b.read("uint:8")
self.filelength = b.read("uintle:32")
self.sizelen = self.xmin = self.xmax = self.ymin = self.ymax = 0
self.framerate = self.framecount = 0
self.tags = []
# print self.signature,self.version,self.filelength
if self.signature == "CWS":
data = zlib.decompress(filestr[8:])
self.parse(data)
else:
self.parse(filestr[8:])
self.parsed = True
except Exception as what:
print what
self.parsed = False
def __init__(self,filestr):
try:
self.data = filestr
b = BitString(bytes=filestr[:8])
self.signature = b.read('bytes:3') #FWS/SWS
self.version = b.read('uint:8')
self.filelength = b.read('uintle:32')
self.sizelen = self.xmin = self.xmax = self.ymin = self.ymax = 0
self.framerate = self.framecount = 0
self.tags = []
#print self.signature,self.version,self.filelength
if self.signature=='CWS':
data = zlib.decompress(filestr[8:])
self.parse(data)
else:
self.parse(filestr[8:])
self.parsed = True
except Exception as what:
print what
self.parsed = False
def make_test_data(codename, infile, outfile, n=1):
'''Takes a "complete" binary source and returns a bitstring suitable
for creating a much smaller, parsable binary suitable for testing.
Currently supported metrics: 'quality'
:param codename: (required) metrics codename (e.g. 'tile', 'quality')
:param infile: (required) path to file to be used as source data.
:param outfile: (optional) path to file to be written (default: ./QualityMetricsTest.bin)
:param n: (optional) integer describing number of complete records to write.
'''
record_length = { 'quality': 153730 }
bs = BitString(bytes=open(infile, 'rb').read())
buf = bs.read(record_length[codename] * n)
outbin = open(outfile, 'wb')
buf.tofile(outbin)
outbin.close()
return outfile
def decodeBitsChannel(self, name: str, key: str = ""):
v = self.rootgrp.variables[name]
vstr = chartostring(v[:]).tostring()
vstr = vstr.decode("US-ASCII")
# 1 is good, everything else is bad
vstr = re.sub('[023456789]', '0', vstr)
bits = BitString(bin="0b" + vstr)
vmissing = v._FillValue
self.channels[self.getKey(key, name)] = Channel(
name, bits, vmissing,
v) # add as a sequence of bits, which is more efficient
pass
def make_test_data(codename, infile, outfile, n=1):
'''Takes a "complete" binary source and returns a bitstring suitable
for creating a much smaller, parsable binary suitable for testing.
Currently supported metrics: 'quality'
:param codename: (required) metrics codename (e.g. 'tile', 'quality')
:param infile: (required) path to file to be used as source data.
:param outfile: (optional) path to file to be written (default: ./QualityMetricsTest.bin)
:param n: (optional) integer describing number of complete records to write.
'''
record_length = {'quality': 153730}
bs = BitString(bytes=open(infile, 'rb').read())
buf = bs.read(record_length[codename] * n)
outbin = open(outfile, 'wb')
buf.tofile(outbin)
outbin.close()
return outfile
def save(self, new_data):
# new_data will be checked by check_wr_data() during burn_all()
# new_data (bytes) = [0][1][2] ... [N] (original data)
# in string format = [0] [1] [2] ... [N] (util.hexify(data, " "))
# in hex format = 0x[N]....[2][1][0] (from bitstring print(data))
# in reg format = [3][2][1][0] ... [N][][][] (as it will be in the device)
# in bitstring = [N] ... [2][1][0] (to get a correct bitstring need to reverse new_data)
# *[x] - means a byte.
data = BitString(bytes=new_data[::-1], length=len(new_data) * 8)
if self.parent.debug:
print("\twritten : {} ->\n\tto write: {}".format(self.get_bitstring(), data))
self.wr_bitarray.overwrite(data, pos=0)
def clear(self, offset, length, bit_offset=None, bit_length=None):
"""
If a whole register, writes ones to the whole register
If a bit field, writes a one to the specified bit with everything else being zero
"""
bs = BitString(length=length)
if bit_offset is None:
bs = ~bs
else:
# Slice addresses are backward, ex: msb is 0, lsb is 31 so bit_offset 32 is slice[0] (for 32 bit)
for i in range(length - (bit_offset + bit_length), length - bit_offset):
bs[i] = 1
self.write(bs.uint, offset, length)
def burn_bit(esp, efuses, args):
num_block = efuses.get_index_block_by_name(args.block)
block = efuses.blocks[num_block]
data_block = BitString(block.get_block_len() * 8)
data_block.set(0)
try:
data_block.set(True, args.bit_number)
except IndexError:
raise esptool.FatalError("%s has bit_number in [0..%d]" % (args.block, data_block.len - 1))
data_block.reverse()
print("bit_number: [%-03d]........................................................[0]" % (data_block.len - 1))
print("BLOCK%-2d :" % block.id, data_block)
block.print_block(data_block, "regs_to_write", debug=True)
block.save(data_block.bytes[::-1])
efuses.burn_all()
print("Successful")
def bitreverse(N=256,nbit=9):
"""
Parameters
----------
N : ideally a power of 2
Returns
-------
t : list of the N integers in time reverse order
Notes
-----
This function is used for example in buildGv.
One error has been fixed by forbidding the value 0
The value 0 is not return
"""
t = []
for k in np.arange(N-1)+1:
b = BitString(uint=k,length=nbit)
b.reverse()
#b.ror(1)
t.append(b.uint)
return(np.array(t))
def hexToBase64(hexString):
byte_string = bytes.fromhex(hexString)
num_bytes = len(byte_string)
ba = BitArray(byte_string)
chunked_byte_string = chunk_list(ba, 6)
s = []
for i in range(len(chunked_byte_string)):
s.append(b64_map[BitString([int(i)
for i in chunked_byte_string[i]]).uint])
encoded_wo_padding = ''.join(s)
ret_val = encoded_wo_padding
if (num_bytes % 3 != 0):
for i in range(3 - (num_bytes % 3)):
ret_val += ('=')
return ret_val
def clear(self, config):
off = self.base_offset + self.offset
data = BitString(length=self.length)
if self.bit_offset is None:
data = ~data
else:
for i in range(self.bit_offset, self.bit_offset+self.bit_length):
data[i] = 1
#print data.bin
# file objects
config.seek(off)
config.write(data.bytes)
def bitreverse(N=256,nbit=9):
"""
Parameters
----------
N : ideally a power of 2
Returns
-------
t : list of the N integers in time reverse order
Notes
-----
This function is used for example in buildGv.
One error has been fixed by forbidding the value 0
The value 0 is not return
"""
t = []
for k in np.arange(N-1)+1:
b = BitString(uint=k,length=nbit)
b.reverse()
b.ror(1)
t.append(b.uint)
return(np.array(t))
def encode(s, storage=BIT_STORAGE, alpha=ALPHABET, char_func=unichr):
"""
Accepts any string/bytes. You can override the storage and alpha
keywords to change to another language set and storage mechanism
Returns a list of encoded bits.
"""
n = s
buf = ''
while len(n) > 0:
b = n[:storage]
n = n[storage:]
d = 11 - len(b)
for i in range(d):
b += '\0'
bs = BitString(data=b)
for i in range(8):
v = bs.readbits(storage).uint
buf += char_func(alpha[v])
return buf.rstrip(char_func(alpha[0]))
def extract_bufr_from_lrit_file(file_path, prefix, output_dir):
""" read lrit file """
file_size = os.path.getsize(file_path)
fp = open(file_path, "rb")
bs = BitString(fp)
cpt = 0
print("==== Looking for Bufrs in %s\n" % (file_path))
# look for BUFR in hex 0x42554652
f_val = bs.find('0x42554652', bytealigned=True)
while f_val:
begin = bs.pos
print("** Found Bufr in pos %s\n" % (begin))
# read size
bufr_value = bs.read(32).hex
size = bs.read('uint:24')
print("size in decimal = %s" % (size))
if size > file_size:
print("Size read in bufr %d is bigger than the file size %d\n" %
(size, file_size))
return
#read all bits (end-begin)+1
bs.pos = begin
read_bits = bs.read(size)
dest_fp = open("%s/%s_bufr_%s.bufr" % (output_dir, prefix, cpt), "wb")
dest_fp.write(read_bits.tobytes())
dest_fp.close()
cpt += 1
#look for the next grib
f_val = bs.find('0x42554652', start=bs.pos, bytealigned=True)
def __init__(self, bytes=None, bits=None):
if bytes:
if isinstance(bytes, str):
io = StringIO(bytes)
elif hasattr(bytes, 'read'):
io = bytes
else:
raise TypeError, 'not readable object'
first_byte = io.read(1)
self.field_bits_length = ord(first_byte) >> 3
bytes = first_byte + io.read(len(self) - 1)
self.bits = BitString(bytes=bytes)
elif bits:
self.bits = bits
self.field_bits_length = self.bits[:5].uint
def _init(self, ID, level_z_index):
self.ID = ID
self.level_z_index = level_z_index
# Stores all current entities in the system.
# The entities are effected by what level they exist in.
self._entities = []
# The bitstring that contains the components mapped to this system.
self._component_bitstring = BitString()
# Stores the IDs of components mapped to it in a list. This allows us to quickly see if an
# entity belongs to the system.
self._component_list = []
# Map all components to this system.
self.mappings()
def uncompressnumberpatter(filename, output, zeros='', hexzeros=''):
with open(filename + '.pickle', 'r') as HI:
buf, hexbuf = HI.read().split(',')
if buf == '':
buf = '0'
if hexbuf == '':
hexbuf = '0'
for c in range(0, int(buf, 10)):
zeros += '0'
for c in range(0, int(hexbuf, 10)):
hexzeros += '0'
zzread = open(filename, 'rb').read()
uncompress = gzip.decompress(zzread)
strunc = zeros + bin(int(uncompress.hex(), 16))[2:]
j = seethat(strunc)
j = '0x' + hexzeros + hex(j)[2:]
zz = open(output, 'wb')
BitString(j).tofile(zz)
zz.close()
def extract_bufr_from_lrit_file(file_path, prefix, output_dir):
""" read lrit file """
file_size = os.path.getsize(file_path)
fp = open(file_path, "rb")
bs = BitString(fp)
cpt=0
print("==== Looking for Bufrs in %s\n" %(file_path))
# look for BUFR in hex 0x42554652
f_val = bs.find('0x42554652', bytealigned=True)
while f_val:
begin= bs.pos
print("** Found Bufr in pos %s\n" % (begin) )
# read size
bufr_value = bs.read(32).hex
size = bs.read('uint:24')
print("size in decimal = %s" %(size))
if size > file_size:
print("Size read in bufr %d is bigger than the file size %d\n" %(size, file_size))
return
#read all bits (end-begin)+1
bs.pos = begin
read_bits = bs.read(size)
dest_fp = open("%s/%s_bufr_%s.bufr" % (output_dir, prefix, cpt), "wb")
dest_fp.write(read_bits.tobytes())
dest_fp.close()
cpt +=1
#look for the next grib
f_val = bs.find('0x42554652', start=bs.pos, bytealigned=True)
def decode(s, storage=BIT_STORAGE, alpha=ALPHABET):
"""
Accepts any iterable object, you can override storage and alpha
keywords to change to another language set and storage mechanism
Returns a string/bytes
"""
n = [ord(a) for a in s if a != TWUUENC_START and a != TWUUENC_START_ZLIB]
bs = BitString()
for a in n:
for pos,l in enumerate(alpha):
if a == l:
bs.append(BitString(uint=pos, length=storage))
bs.seekbyte(0)
return bs.readbytes(len(bs)/8).data.rstrip('\0')
class Entity:
# Keep track of the latest ID for entities to ensure they get unique IDs.
_id_counter = 0
def __init__(self, system_manager):
# Fairly certain we don't need this to be a dict, as
# we are unlikely to want to reference specific
# systems by id.
self._systems = []
# A dict of all components inside this entity.
# The keys are the IDs of the components.
self._components = {}
# The components this entity has.
self._component_bitstring = BitString()
# Keep track of the old bitstring so we can see
# what has changed when we call refresh twice.
self._old_component_bitstring = BitString()
# Systems that this entity belongs to.
self._systems_bitstring = BitString()
# Ensure unique IDs.
self.ID = Entity._id_counter
Entity._id_counter = Entity._id_counter + 1
# The system manager the entity belongs to.
# Gives access to the existing systems.
self.system_manager = system_manager
# Remove an entity from all systems it is hooked into.
def kill(self):
for system in self._systems:
system._remove_entity(self)
# Allow all systems to hook into (or out of) the entity.
def refresh(self):
# If the entity is already hooked in remove them, so we don't add them twice.
# This design can be optimized, but low priority since we likely only refresh
# an entity once.
if not self._old_component_bitstring.is_empty():
self.kill()
self._old_component_bitstring = self._component_bitstring
for system in self.system_manager._systems:
# Add the entity if it maps correctly to the system.
does_map = True
for component_id in system._component_list:
if not self._component_bitstring.contains(component_id):
does_map = False
if does_map:
system._add_entity(self)
self._systems.append(system)
# Return a component that the entity contains based on the class.
def get_component(self, component_class):
try:
return self._components[component_class.ID]
except:
return None
# Add a component to the entity. Be sure to refresh afterwards.
def add_component(self, component):
# Fairly certain we don't want two PositionComponents for example.
if component.__class__.ID in self._components:
raise Exception(component.__class__.__name__ + " already inside of entity!")
self._components[component.__class__.ID] = component
#self._components.append(component)
self._component_bitstring.add_to_set(component.ID)
def get_swf(self):
if not self.parsed:
return ""
b = BitString(bytes="")
b.append("uint:5=%d" % self.sizelen)
if self.sizelen > 0:
b.append("uint:%d=%d" % (self.sizelen, self.xmin))
b.append("uint:%d=%d" % (self.sizelen, self.xmax))
b.append("uint:%d=%d" % (self.sizelen, self.ymin))
b.append("uint:%d=%d" % (self.sizelen, self.ymax))
taillen = (self.sizelen * 4 + 5) % 8 != 0 and 8 - (self.sizelen * 4 + 5) % 8 or 0
else:
taillen = 3
b.append("uint:%d=0" % taillen)
b.append("uint:16=%d" % self.framerate)
b.append("uintle:16=%d" % self.framecount)
for tagcode, tagdata in self.tags:
taglen = len(tagdata)
if taglen >= 0x3F:
b.append("uintle:16=%d" % ((tagcode << 6) | 0x3F))
b.append("intle:32=%d" % len(tagdata))
else:
b.append("uintle:16=%d" % ((tagcode << 6) | taglen))
b.append(BitString(bytes=tagdata))
data = b.tobytes()
self.filelength = len(data)
if self.signature == "CWS":
data = zlib.compress(data)
b = BitString(bytes="")
b.append(BitString(bytes=self.signature))
b.append(BitString("uint:8=%d" % self.version))
b.append(BitString("uintle:32=%d" % self.filelength))
header = b.tobytes()
return header + data
def parse(self, data):
totalbits = len(data) * 8
b = BitString(bytes=data)
self.sizelen = b.read("uint:5")
if self.sizelen > 0:
self.xmin = b.read("uint:%d" % self.sizelen)
self.xmax = b.read("uint:%d" % self.sizelen)
self.ymin = b.read("uint:%d" % self.sizelen)
self.ymax = b.read("uint:%d" % self.sizelen)
taillen = (self.sizelen * 4 + 5) % 8 != 0 and 8 - (self.sizelen * 4 + 5) % 8 or 0
else:
self.xmin = self.xmax = self.ymin = self.ymax = 0
taillen = 3
b.read("uint:%d" % taillen)
self.framerate = b.read("uint:16")
self.framecount = b.read("uintle:16")
# print self.sizelen,self.xmin,self.xmax,self.ymin,self.ymax
# print self.framerate,self.framecount
while b.pos < totalbits:
tagcl = b.read("uintle:16")
tagcode = tagcl >> 6
taglen = tagcl & 0x3F
if taglen == 0x3F:
taglen = b.read("intle:32")
tagdata = b.read("bytes:%d" % taglen)
self.tags.append([tagcode, tagdata])
def extract_grib_from_lrit_file(file_path, prefix, output_dir):
""" read lrit file """
fp = open(file_path, "rb")
bs = BitString(fp)
cpt=0
print("==== Looking for Gribs in %s\n" %(file_path))
#look for HRAA70 in hex 0x485241413730
#f_val = bs.find('0x485241413730', bytealigned=True)
#print("f_val = %s\n" %(f_val))
#bs.pos = f_val
# look for GRIB in hex 0x47524942
f_val = bs.find('0x47524942', bytealigned=True)
print("f_val = %s\n" %(f_val))
while f_val:
begin = bs.pos
print("begin = %d\n" % (begin))
#look for bulletin header
bs.pos -= 21*8
bulletin_header = bs.read(18*8).hex
print("Bulletin Header = %s\n" % bulletin_header)
l = "%s" % bulletin_header
header_name = binascii.unhexlify(l[2:])
header_name = header_name.replace(" ","_")
print("Bulletin Header = %s\n" % header_name)
# look for GRIB in hex 0x47524942
f_val = bs.find('0x47524942', bytealigned=True)
bs.pos = begin
print("** Found Grib in pos %s" % (bs.bytepos) )
# read size
grib_value = bs.read(32).hex
size = bs.read('uint:24')*8
#print("size in decimal = %s" %(size))
#read all bits (end-begin)+1
bs.pos = begin
read_bits = bs.read(size)
dest_fp = open("%s/%s_%s_%s.grb" % (output_dir, prefix, bulletin_header, cpt), "wb")
dest_fp.write(read_bits.tobytes())
dest_fp.close()
cpt +=1
#sys.exit(1)
#look for the next grib
f_val = bs.find('0x47524942', start=bs.pos, bytealigned=True)
class System(object):
def _init(self, ID, level_z_index):
self.ID = ID
self.level_z_index = level_z_index
# Stores all current entities in the system.
# The entities are effected by what level they exist in.
self._entities = []
# The bitstring that contains the components mapped to this system.
self._component_bitstring = BitString()
# Stores the IDs of components mapped to it in a list. This allows us to quickly see if an
# entity belongs to the system.
self._component_list = []
# Map all components to this system.
self.mappings()
# Override this method to handle new mappings.
def mappings(self):
raise Exception("system " + self.__class__.__name__ + " has not defined mappings.")
# Fires when an entity gets added to the system. Can be overridden.
def register_entity(self, entity):
pass
# Fires when an entity gets removed from the system.
def unregister_entity(self, entity):
pass
def _remove_entity(self, entity):
self._entities.remove(entity)
self.unregister_entity(entity)
def _add_entity(self, entity):
self._entities.append(entity)
self.register_entity(entity)
# Map what kind of components the system must have to gain control of an entity.
def map_component_class(self, class_name):
# Ensure the component has ID overridden.
if class_name.ID == -1:
raise Exception(class_name.__name__ + " does not define ID.")
# Ensure ID fits within BitString range.
if not (class_name.ID >= 0 and class_name.ID < BitString.SIZE):
raise Exception(class_name.__name__ + " has an invalid ID must be [0.." \
+ BitString.SIZE + "].")
self._component_bitstring.add_to_set(class_name.ID)
self._component_list.append(class_name.ID)
# Override this method to process entities.
def process(self, entity):
pass
# Process all components.
def update(self):
for entity in self._entities:
self.process(entity)