def save(self, filename, font_type = FONT_TYPES.font01, game = GAMES.dr):
   data = BitStream(SPFT_MAGIC)
   
   data += BitStream(uintle = len(self.data), length = 32)
   
   mapping_table_len = self.find_max_char() + 1 # zero-indexed so +1 for the size.
   mapping_table_start = 0x20
   font_table_start = mapping_table_len * 2 + mapping_table_start
   
   data += BitStream(uintle = font_table_start, length = 32)
   data += BitStream(uintle = mapping_table_len, length = 32)
   data += BitStream(uintle = mapping_table_start, length = 32)
   data += UNKNOWN1[game][font_type] + UNKNOWN2
   
   data += self.gen_mapping_table(mapping_table_len)
   
   data += self.gen_font_table()
   
   padding = BitStream(hex = '0x00') * (16 - ((data.len / 8) % 16))
   
   data += padding
   
   f = open(filename, "wb")
   data.tofile(f)
   f.close()
def uncompress_golomb_coding(coded_bytes, hash_length, M):
    """Given a bytstream produced using golomb_coded_bytes, uncompress it."""
    ret_list = []
    instream = BitStream(
        bytes=coded_bytes, length=len(coded_bytes) * 8)
    hash_len_bits = hash_length * 8
    m_bits = int(math.log(M, 2))
    # First item is a full hash value.
    prev = instream.read("bits:%d" % hash_len_bits)
    ret_list.append(prev.tobytes())

    while (instream.bitpos + m_bits) <= instream.length:
        # Read Unary-encoded value.
        read_prefix = 0
        curr_bit = instream.read("uint:1")
        while curr_bit == 1:
            read_prefix += 1
            curr_bit = instream.read("uint:1")
        assert curr_bit == 0

        # Read r, assuming M bits were used to represent it.
        r = instream.read("uint:%d" % m_bits)
        curr_diff = read_prefix * M + r
        curr_value_int = prev.uint + curr_diff
        curr_value = Bits(uint=curr_value_int, length=hash_len_bits)
        ret_list.append(curr_value.tobytes())
        prev = curr_value

    return ret_list
Esempio n. 3
0
class Mem(object):
    def __init__(self):
        self.real = BitStream(600*16)
        self.jumps = 0

    def load(self, file):
        self.real = BitStream(filename=file)

    def save(self, file):
        self.real.tofile(file)

    def jump(self, pos):
        self.jumps += 1
        self.real.bytepos = pos

    def read(self, size=16):
        return self.real.read(16)

    def get(self, pos, size=16):
        realpos = pos * 8
        return self.real[realpos:realpos+size]

    def set(self, pos, bits):
        realpos = pos * 8
        self.real[realpos:realpos+len(bits)] = bits

    @property
    def pos(self):
        return self.real.bytepos
def adjust_tiles(tiles, size, isd_text):
    ChunkMap = re.split(r"<ChunkMap>", isd_text)
    if len(ChunkMap) != 2:
        e = "Previous split should have resulted in 2 strings. {} found".format(len(ChunkMap))
        raise NotImplementedError(e)
    ChunkMap = ChunkMap[1]
    # first 2 characters after <Width> tag in <ChunkMap> => up to 99 chunks (= 1584 x 1584 tiles per island)
    (width_tiles, height_tiles) = size #@UnusedVariable
    width_chunks = int( re.split(r"<Width>", ChunkMap[:100])[1][:2].strip("<") )
    height_chunks = int( re.split(r"<Height>", ChunkMap[:100])[1][:2].strip("<") ) #@UnusedVariable
    chunks = re.split(r"<Element>", ChunkMap)[1:]
    for i in range(len(chunks)):
        VertexResolution = re.split(r"<VertexResolution>", chunks[i])[1][0]
        if VertexResolution in ("-", "5"): # -1 => empty chunk
            continue
        VertexResolution = int(VertexResolution)
        HeightMap = re.split(r"HeightMap[^C]*CDATA\[", chunks[i])[1:]
        start_x = i%width_chunks
        start_z = i//width_chunks
        resolution = {4: [  4, 4],
                      5: [ 25,15],
                      6: [142,58]}[VertexResolution]
        useful_bytes = 17*17*resolution[1]
        load_bytes = resolution[0] + useful_bytes
        bits_per_tile = resolution[1] * 8
        data = BitStream( bytes=HeightMap [0][:load_bytes][-useful_bytes:] )
        read_string = "uint:{}".format(bits_per_tile)
        for z in range(16):
            for x in range(17):
                position = start_z*16*width_tiles + z*240 + start_x*16 + x
                d = int( data.read(read_string))
                if x != 16 and d == 858993471: #trial and error, 0x3333333f=858993471, 0x33=51, 0x3f=63, 0x3333=13107, 0x333f=13119
                    tiles[position] = 255
        
    return None
Esempio n. 5
0
    def parse(self):
        bs = BitStream(filename=self.file_path)
        [index_start, extension_data_start] = parse_header(bs)
        self.app_info = parse_app_info(bs)

        bs.bytepos = index_start
        self.indexes = parse_index(bs)
        self.parsed = True
 def save(self, filename):
   
   data = BitStream(self.magic) + BitStream(uintle = len(self.lines), length = 16)
   
   for line in self.lines:
     data += line.to_data()
   
   with open(filename, "wb") as f:
     data.tofile(f)
Esempio n. 7
0
    def __init__(self, chunk_id, size, data):
        super(HeaderChunk, self).__init__(chunk_id, size)
        data_stream = BitStream(data)

        # First two bytes are the format type
        self.format_type = data_stream.read('bits:16').int

        # Second two bytes are the number of tracks
        self.num_of_tracks = data_stream.read('bits:16').int

        # Third two bytes are the time division
        self.time_division = Bits(data_stream.read('bits:16'))
Esempio n. 8
0
def decompress(data):
    feed = []
    pos = 0
    binary = BitStream(bytes=data)
    binary_length = len(binary.bin)
    while binary_length - binary.pos >= (WINDOW_BITS + LENGTH_BITS):
        distance = binary.read('uint:%d' % WINDOW_BITS)
        c_or_length = binary.read('uint:%d' % LENGTH_BITS)
        if distance == 0:
            c_or_length = chr(c_or_length)
        feed.append([distance, c_or_length])

    return feed2text(feed)
Esempio n. 9
0
    def decode(self, in_stream, out_stream):
        bs = BitStream()
        dq = deque()
        at_least_three = False
        for word in self.words_from_file(in_stream):
            if not word or word not in self.word_dict:
                continue
            #print >> sys.stderr, 'word:"', word, '"'
            dq.append(self.word_dict[word])
            if at_least_three or len(dq) == 3:
                bs.append(pack(self.int_type, dq.popleft()))
                at_least_three = True
                if bs.len > self.bit_buffer:
                    cut = 0
                    for byte in bs.cut(self.bit_buffer):
                        cut += 1
                        byte.tofile(out_stream)
                    del bs[:cut * self.bit_buffer]

        # dq has to have exactly 2 elements here, the last is the bit length of the first, unless it's 0
        #print >> sys.stderr, 'dq:', dq
        extra_bits = dq.pop()
        bs.append(pack('uint:' + str(extra_bits), dq.popleft()))

        bs.tofile(out_stream)
def uncompress_delta_diff(compressed_input, hash_length):
    ret_list = []
    instream = BitStream(bytes=compressed_input, length=len(compressed_input) * 8)
    hash_len_bits = hash_length * 8
    prev = instream.read("bits:%d" % hash_len_bits)
    ret_list.append(prev.tobytes())
    # Must always have at least 6 bits to read.
    while (instream.bitpos + 6) < instream.length:
        curr_diff_len = instream.read("uint:6") + 1
        curr_diff = instream.read("bits:%d" % curr_diff_len)
        curr_item = prev[:hash_len_bits - curr_diff_len] + curr_diff
        assert curr_item.length == hash_len_bits
        ret_list.append(curr_item.tobytes())
        prev = curr_item
    return ret_list
Esempio n. 11
0
 def write_char_array(self, max_length, value):
     self.write_int(bit_count(max_length), len(value))
     if self._bits.len > 0:
         more = 8 - self._bits.len
         tail = (BitStream(int=0, length=more) + self._bits).tobytes()
         self._bits = BitStream()
         self._bytes += tail
     self._bytes += value
Esempio n. 12
0
def main():
    for datum in DATA:
        as_hex = ":".join("{:02x}".format(h) for h in datum)
        as_bin = ":".join("{:08b}".format(h) for h in datum)

        print(as_hex)
        print(as_bin)

        a = BitStream(datum)
        first_mb_in_slice = a.read('ue')
        slice_type = a.read('ue')
        pic_parameter_set_id = a.read('ue')
        frame_num = a.read(9)
        print("first-mb-in-slice: {}".format(first_mb_in_slice))
        print("slice-type: {}".format(slice_type))
        print("pic-parameter-set-id: {}".format(pic_parameter_set_id))
        print("frame-num: {}".format(frame_num.int))
Esempio n. 13
0
def gen():

    consonants = "bcdfghjklmnpqrstvwxyz"
    vowels = "aeiou"
    generated = ""

    randdata = subprocess.check_output(["openssl", "rand", "9"])
    assert len(randdata) == 9
    bs = BitStream(randdata)

    generated += consonants[bs.read('int:5') % len(consonants)]
    for i in range(5):
        generated += vowels[bs.read('int:3') % len(vowels)]
        generated += consonants[bs.read('int:5') % len(consonants)]
        generated += consonants[bs.read('int:5') % len(consonants)]

    return generated
Esempio n. 14
0
    def decompress(self, data):
        bitstream = BitStream(data)
        pad = bitstream.read(8).int
        # remove pad bits
        if pad > 0:
            bitstream = bitstream[:-pad]
            bitstream.read(8)  # false read 1 B to move read pointer

        tree_len = bitstream.read(16).int
        tree_serial = bitstream.read(tree_len)
        tree = HuffmanNode()
        tree.deserialize(tree_serial)
        dictionary = tree.assign_codes()
        dictionary = {v: k for k, v in dictionary.items()}  # reverse dict
        result = bytearray()
        sequence = ""

        while True:
            try:
                bit = bitstream.read(1)
            except ReadError:
                break

            if bit:
                sequence += '1'
            else:
                sequence += '0'

            if sequence in dictionary:
                result.append(dictionary[sequence])
                sequence = ""

        return result
def decrypt(ciphertext, key):
    bstream = BitStream()

    p, g, y = key[0]
    u = key[1]

    #trying to improve execution speed
    #a_pow_u = mod_exp(ciphertext[1][0], u, p)
    #inv_a_pow_u = modinv(a_pow_u, p)
    for block in ciphertext[1:]:
        #sys.stdout.write(".")
        #trying to improve execution speed
        a_pow_u = mod_exp(block[0], u, p)
        inv_a_pow_u = modinv(a_pow_u, p)
        x = (block[1] * inv_a_pow_u) % p
        block_size = math.floor(math.log(p,2))
        bstream.append('0b' + bin(x)[2:].zfill(int(block_size)))

    return bstream.read(ciphertext[0])
Esempio n. 16
0
def RTCM_converter_thread(server, port, username, password, mountpoint, rtcm_callback = None):
    import subprocess

    nt = subprocess.Popen(["./ntripclient",
                            "--server", server,
                            "--password", password,
                            "--user", username,
                            "--mountpoint", mountpoint ],
                            stdout=subprocess.PIPE)

    """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint],
                            stdout=subprocess.PIPE)"""


    if nt is None or nt.stdout is None:
        indev = sys.stdin
    else:
        indev = nt.stdout

    print("RTCM using input {}".format(indev))

    while True:
        sio = indev

        d = ord(sio.read(1))
        if d != RTCMv3_PREAMBLE:
            continue

        pack_stream = BitStream()

        l1 = ord(sio.read(1))
        l2 = ord(sio.read(1))

        pack_stream.append(bs.pack('2*uint:8', l1, l2))
        pack_stream.read(6)
        pkt_len = pack_stream.read(10).uint

        pkt = sio.read(pkt_len)
        parity = sio.read(3)

        if len(pkt) != pkt_len:
            print "Length error {} {}".format(len(pkt), pkt_len)
            continue

        if True: #TODO check parity
            for d in pkt:
                pack_stream.append(bs.pack('uint:8',ord(d)))

            msg = parse_rtcmv3(pack_stream)

            if msg is not None and rtcm_callback is not None:
                rtcm_callback(msg)
Esempio n. 17
0
def create_sample_sync_frame():

    stream = BitStream(bin="0" * 32)
    stream.set(True, range(0, 12))  # frame sync
    stream.set(True, 14)  # Layer III
    stream.set(True, 15)  # protection bit
    stream.set(True, 17)  # bitrate, 128k

    return stream
 def load_data(self, data, offset = 0):
   if not data[offset * 8 : offset * 8 + GMO_MAGIC.len] == GMO_MAGIC:
     _LOGGER.error("GMO header not found at 0x%04X." % offset)
     return
   
   data.bytepos = offset + GMO_SIZE_OFFSET
   gmo_size = data.read("uintle:32") + GMO_SIZE_DIFF
   
   self.data = BitStream(data[offset * 8 : (offset + gmo_size) * 8])
   
   self.__find_gims()
Esempio n. 19
0
    def __init__(self, filp, pos, utype, size, tid, nosetup=False):
        self.filp = filp
        self.pos = pos
        self.num_bytes_in_nalu = size
        self.nal_unit_type = utype
        self.temporal_id = tid

        self.rbsp_byte = BitStream()
        self.setup_rbsp()
        self.print_bin()
        if not nosetup:
            self.setup()
Esempio n. 20
0
    def serialize(self):
        bitstream = BitStream()
        if self.leaf():
            bitstream.append('bin=1')
            bitstream.append("{0:#0{1}x}".format(self.value, 4))
        else:
            bitstream.append('bin=0')
            bitstream += self.left.serialize() + self.right.serialize()

        return bitstream
def pack_txt(filename):
  
  if os.path.basename(os.path.dirname(filename)) in SCRIPT_NONSTOP:
    is_nonstop = True
  else:
    is_nonstop = False

  script = ScriptFile(filename)
  text = script[common.editor_config.lang_trans]
  if not text:
    text = script[common.editor_config.lang_orig]
  
  # Nonstop Debate lines need an extra newline at the end
  # so they show up in the backlog properly.
  if is_nonstop and not text[-1] == u"\n":
    text += u"\n"
  
  text = SCRIPT_BOM + text + SCRIPT_NULL
  text = BitStream(bytes = text.encode("UTF-16LE"))
  
  return text
Esempio n. 22
0
class BitWriter:
    def __init__(self):
        self._bytes = ""
        self._bits = BitStream()

    def write_int(self, length, value):
        news = BitStream(uint=value, length=length)
        start = 0

        if self._bits.len > 0:
            left = 8 - self._bits.len
            if news.len < left:
                left = news.len
            self._bits = news[:left] + self._bits
            start += left

            if self._bits.len == 8:
                self._bytes += self._bits.tobytes()
                self._bits = BitStream()

        byte_len = (news.len - start) / 8
        if byte_len > 0:
            more = byte_len * 8
            self._bytes += news[start:start+more].tobytes()
            start += more

        if news.len > start:
            self._bits = news[start:]

    def write_int64(self, length, value):
        if length <= 32:
            self.write_int(length, value)
            return
        count = length - 32
        self.write_int(32, value >> count)
        self.write_int(count, value & ((1 << count) - 1))

    def write_char_array(self, max_length, value):
        self.write_int(bit_count(max_length), len(value))
        if self._bits.len > 0:
            more = 8 - self._bits.len
            tail = (BitStream(int=0, length=more) + self._bits).tobytes()
            self._bits = BitStream()
            self._bytes += tail
        self._bytes += value

    def get_bytes(self):
        if self._bits.len > 0:
            more = 8 - self._bits.len
            tail = (BitStream(int=0, length=more) + self._bits).tobytes()
            return self._bytes + tail
        else:
            return self._bytes
Esempio n. 23
0
 def encode(self, in_stream, out_stream):
     extra_bits = self.num_bits
     bs = BitStream()
     try:
         while True:
             chunk = in_stream.read(self.byte_buffer)
             #print >> sys.stderr, 'chunk:', chunk
             if(chunk):
                 bs.append(BitStream(bytes=chunk))
             else:
                 while True:
                     self.print_index(bs.read(self.int_type), out_stream)
             try:
                 while True:
                     self.print_index(bs.read(self.int_type), out_stream)
             except ReadError, e:
                 #print >> sys.stderr, 'inner:', e
                 pass
     except ReadError, e:
         #print >> sys.stderr, 'outer:', e
         extra_bits = bs.len - bs.bitpos
         if extra_bits > 0:
             #print >> sys.stderr, 'extra_bits:', extra_bits
             self.print_index(bs.read('uint:' + str(extra_bits)), out_stream)
         else:
             extra_bits = self.num_bits
Esempio n. 24
0
 def __init__(self, data, base64=False):
     """ Constructor.
     
         :param string data: Binaray data
         :param bool base64: True if data is coded in base64
     """
     if base64:
         data = b64decode(data)
     
     #: raw data size
     self._sz = len(data)
     
     #: bit stream used to read data
     self._bs = BitStream(bytes=data)
Esempio n. 25
0
def uncompress_golomb_coding(coded_bytes, hash_length, M):
    ret_list = []
    instream = BitStream(
            bytes=coded_bytes, length=len(coded_bytes) * hash_length)
    hash_len_bits = hash_length * 8
    m_bits = int(math.log(M, 2))
    prev = instream.read("bits:%d" % hash_len_bits)
    ret_list.append(prev.tobytes())
    while instream.bitpos < instream.length:
        read_prefix = 0
        curr_bit = instream.read("uint:1")
        while curr_bit == 1:
            read_prefix += 1
            curr_bit = instream.read("uint:1")
        assert curr_bit == 0
        r = instream.read("uint:%d" % m_bits)
        curr_diff = read_prefix * M + r
        curr_value_int = prev.uint + curr_diff
        curr_value = Bits(uint=curr_value_int, length=hash_len_bits)
        ret_list.append(curr_value.tobytes())
        prev = curr_value

    return ret_list
def pack_pak(dir, file_list = None, align_toc = 16, align_files = 16, eof = False):
  
  if file_list == None:
    file_list = sorted(os.listdir(dir))
    
  num_files  = len(file_list)
  toc_length = (num_files + 1) * 4
  
  if eof:
    toc_length += 1
  
  if toc_length % align_toc > 0:
    toc_length += align_toc - (toc_length % align_toc)
  
  archive_data = BitStream(uintle = 0, length = toc_length * 8)
  archive_data.overwrite(bitstring.pack("uintle:32", num_files), 0)
  
  for file_num, item in enumerate(file_list):
    full_path = os.path.join(dir, item)
    
    if os.path.isfile(full_path):
      data = pack_file(full_path)
    else:
      data = pack_dir(full_path, align_toc, align_files, eof)
    
    file_size = data.len / 8
    padding = 0
    
    if file_size % align_files > 0:
      padding = align_files - (file_size % align_files)
      data.append(BitStream(uintle = 0, length = padding * 8))
    
    file_pos = archive_data.len / 8
    archive_data.overwrite(bitstring.pack("uintle:32", file_pos), (file_num + 1) * 32)
    archive_data.append(data)
    
    del data
  
  if eof:
    archive_data.overwrite(bitstring.pack("uintle:32", archive_data.len / 8), (num_files + 1) * 32)
  
  return archive_data
Esempio n. 27
0
    def dataEntryAppend(self, eventLogEntryBitStream:BitStream):
        """
        since latset entry is in end of whole ECDA dump,
        collect all entries and store in reverse order

        :param eventLogEntryBitStream:
        :return:
        """
        timestampHex = eventLogEntryBitStream.read('hex:{}'.format(32))
        eventIdHex = eventLogEntryBitStream.read('hex:{}'.format(16))
        eventId = int(eventIdHex, 16)
        extraInfo = eventLogEntryBitStream.read('hex:{}'.format(16))


        self._dataDict['timestampHex'].append(timestampHex)
        ##self._dataDict['timestamp'].append(timestamp)
        ##self._dataDict['timeLast'].append(abs(timestamp-self._time) )
        ##self._dataDict['timeLastInUs'].append(abs(timestamp - self._time )*self._timeGranunityUs)
        ##self._time = timestamp
        self._dataDict['eventId'].append(eventId)
        self._dataDict['eventIdHex'].append(eventIdHex)
        self._dataDict['extraInfo'].append(extraInfo)

        pass
Esempio n. 28
0
    def compress(self, data):
        weight = Counter(data)
        priority_queue = [HuffmanNode(value=byte, weight=weight[byte])
                          for byte in weight]
        heapify(priority_queue)

        while len(priority_queue) > 1:
            left = heappop(priority_queue)
            right = heappop(priority_queue)
            node = HuffmanNode(left, right, weight=left.weight + right.weight)
            heappush(priority_queue, node)

        root = heappop(priority_queue)
        dictionary = root.assign_codes()
        """
        we need to add the tree to the compressed data, so that the
        decompressor can rebuild it in order do to it's work
        """
        tree = root.serialize()
        result = BitStream()
        tree_len_bits = len(bin(len(tree))[2:])
        if tree_len_bits > 16:
            raise ValueError("Huffman tree len is max 10*255-1 bit")

        # this converts len(tree) to hex with zero front pad to two bytes
        result.append("{0:#0{1}x}".format(len(tree), 6))
        result += tree

        for byte in data:
            result.append('bin=' + dictionary[byte])

        pad = 0
        if len(result) % 8 != 0:
            pad = 8 - len(result) % 8
            result.append('bin=' + '0' * pad)

        """
        the compressed data layout is as follows:
        * 1B - number of pad bits (for byte align)
        * 2B- Huffman tree length (which btw = 10*num_of_chars_in_the_tree -1)
        * the Huffman tree itself (not byte aligned)
        * the encoded data paded with 0-7 bits at the end
        """
        result = BitStream("{0:#0{1}x}".format(pad, 4)) + result
        return bytearray(result.bytes)
def to_bin(commands):

  data  = BitStream()
  lines = 0
  
  for op, params in commands:
    if op == WRD_HEADER:
      continue
    
    if not op in OP_PARAMS:
      # raise Exception("Unknown op: 0x%02X" % op)
      print "Unknown op: 0x%02X" % op
      continue
    
    param_info = OP_PARAMS[op]
    
    # If it has a custom parsing function, use the equivalent packing function.
    if isinstance(param_info, basestring):
      command = globals()[OP_FUNCTIONS[op]](**params)
      data.append(command)
    
    else:
      if op == WRD_SHOW_LINE:
        lines += 1
      
      data.append(bitstring.pack("uint:8, uint:8", CMD_MARKER, op))
      
      unnamed_param_id = 0
      
      for param_name, param_type in param_info:
        if param_name == None:
          data.append(bitstring.pack(param_type, params[param_name][unnamed_param_id]))
          unnamed_param_id += 1
          
        else:
          data.append(bitstring.pack(param_type, params[param_name]))
  
  return bitstring.pack("uint:8, uint:8, uintle:16", CMD_MARKER, WRD_HEADER, lines) + data
 def load_data(self, data):
   files = [entry_data for (entry_name, entry_data) in get_pak_files(data)]
   
   # There are always at least four files in a model pak.
   # The first three I don't know a lot about, and then
   # the GMO files come after that.
   if len(files) < 4:
     _LOGGER.error("Invalid model PAK. %d files found, but at least 4 needed." % len(files))
     return
   
   # The name pak contains a list of null-terminated names for
   # each of the models, stored in our standard pak format.
   name_pak = files[0]
   names    = [entry_data.bytes.strip('\0') for (entry_name, entry_data) in get_pak_files(name_pak)]
   
   # Most of the model paks in SDR2 have a fourth unknown file before the models
   # start, so we'll just take everything from the back end and call it a day.
   models = files[-len(names):]
   
   # Now, we don't get file positions from the unpacker, so let's find those
   # and start filling out our internal list of GMO files.
   file_starts, file_ends = parse_pak_toc(data)
   model_starts = file_starts[-len(names):]
   
   for i, model in enumerate(models):
     # First of all, not all of the "models" present are actually GMO files.
     # It's rare, but there is the occasional other unknown format.
     # So let's make sure we have a GMO file.
     if not model[:GMO_MAGIC.len] == GMO_MAGIC:
       # print i, "Not a GMO."
       continue
     
     name  = names[i]
     gmo   = GmoFile(data = model)
     size  = model.len / 8
     start = model_starts[i]
     
     self.__gmo_files.append({
       _NAME:   name,
       _START:  start,
       _SIZE:   size,
       _DATA:   gmo,
     })
   
   self.__data = BitStream(data)