def run(self): import subprocess server, port, username, password, mountpoint, rtcm_callback = self._Thread__args nt = subprocess.Popen([ "./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint ], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout sio = indev while not self._stopevent.isSet(): d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print "Length error {} {}".format(len(pkt), pkt_len) continue if True: #TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream) # MARCO forward unchanged pack_stream.append( bs.pack('3*uint:8', ord(parity[0]), ord(parity[1]), ord(parity[2]))) msg = struct.pack('<B', RTCMv3_PREAMBLE) + pack_stream.tobytes() if msg is not None and rtcm_callback is not None: rtcm_callback(msg) if nt is not None: nt.terminate() nt.wait() nt = None
class bit_file: def __init__(self, f, bits_per_op, mode='read', keep_open=False, read_size=MAX_ENCODING, read_count=1): if mode not in ['read', 'write']: raise Exception('bad bit_file mode. Try "read" or "write"') self.mode = mode if isinstance(f, str): fmode = 'wb' if mode == 'write' else 'rb' self.f = open(f, fmode) self.keep_open = False else: self.f = f self.keep_open = keep_open # determines whether __exit__ is a flush()+close(), or just a flush() self.bits_per_op = bits_per_op self.stream = BitStream() self.read_size = read_size self.read_count = read_count def __enter__(self): return self def __exit__(self, type, value, traceback): if self.mode == 'write' and not self.f.closed: self.save() if not self.keep_open and not self.f.closed: with self.f: # close file pass def write(self, bits): if isinstance(bits, bit_write_buffer): bits = bits.stream if not isinstance(bits, Bits): bits = Bits(uint=bits, length=self.bits_per_op) self.stream.append(bits) def read(self): if self.read_count and self.stream.bitpos == self.stream.length: self.stream.clear() self.stream.append(Bits(bytes=self.f.read(self.read_size))) self.read_count -= 1 try: bits = self.stream.read(f'uint:{self.bits_per_op}') except bitstring.ReadError: try: bits = self.stream.read('uint') except bitstring.InterpretError: bits = 0 return bits def save(self): self.f.write(self.stream.tobytes())
class BitWriter: def __init__(self): self._bytes = "" self._bits = BitStream() def write_int(self, length, value): news = BitStream(uint=value, length=length) start = 0 if self._bits.len > 0: left = 8 - self._bits.len if news.len < left: left = news.len self._bits = news[:left] + self._bits start += left if self._bits.len == 8: self._bytes += self._bits.tobytes() self._bits = BitStream() byte_len = (news.len - start) / 8 if byte_len > 0: more = byte_len * 8 self._bytes += news[start:start+more].tobytes() start += more if news.len > start: self._bits = news[start:] def write_int64(self, length, value): if length <= 32: self.write_int(length, value) return count = length - 32 self.write_int(32, value >> count) self.write_int(count, value & ((1 << count) - 1)) def write_char_array(self, max_length, value): self.write_int(bit_count(max_length), len(value)) if self._bits.len > 0: more = 8 - self._bits.len tail = (BitStream(int=0, length=more) + self._bits).tobytes() self._bits = BitStream() self._bytes += tail self._bytes += value def get_bytes(self): if self._bits.len > 0: more = 8 - self._bits.len tail = (BitStream(int=0, length=more) + self._bits).tobytes() return self._bytes + tail else: return self._bytes
def valid_checksum(source_address, dest_address, bytes_packet): bin_packet = BitStream(bytes=bytes_packet) length = len(bin_packet) % 8 checksum = bin_packet[128:144].bytes pseudo_header = TCPyPacket.create_pseudo_header( source_address, dest_address, length) bin_packet.overwrite(b'\x00\x00', 128) cs_calc = cs.Checksum16() cs_calc.process(bin_packet.tobytes()) #cs_calc.process(pseudo_header) if checksum != cs_calc.finalbytes(): return False return True
def find_mapping(infile, outfile): stream = BitStream() with open(infile) as f: byte = f.read(1) while byte: if byte == '\x01': stream.append('0b1') else: stream.append('0b0') byte = f.read(1) iteration = 0 for mapping in bitmappings: stream.pos = 0 tmpstream = BitStream() for pie in stream.cut(2): index = pie.read('uint:2') tmpstream.append( BitArray(uint=mapping[index], length=2) ) # look for the start of the flag in the stream of bits pos = tmpstream.find(ZULU) if len(pos) > 0: # print("Found the bytes we are looking for on iteration {}".format(iteration)) # print("pos = {}".format(pos)) tmpstream <<= pos[0] % 8 data = tmpstream.tobytes() # print(data) with open(outfile, 'wb') as fh: fh.write(data) break else: # print("Did not find the header") iteration += 1
def decrypt_images(**kwargs): path = kwargs.pop('path', 'herders/static/herders/images') for im_path in iglob(f'{path}/**/*.png', recursive=True): encrypted = BitStream(filename=im_path) # Check if it is 'encrypted'. 8th byte is 0x0B instead of the correct signature 0x0A encrypted.pos = 0x07 * 8 signature = encrypted.peek('uint:8') if signature == 0x0B: print(f'Decrypting {im_path}') # Correct the PNG signature encrypted.overwrite('0x0A', encrypted.pos) # Replace bits with magic decrypted values try: while True: pos = encrypted.pos val = encrypted.peek('uint:8') encrypted.overwrite( Bits(uint=com2us_decrypt_values[val], length=8), pos) except ReadError: # EOF pass # Write it back to the file with open(im_path, 'wb') as f: encrypted.tofile(f) continue # Check for weird jpeg format with extra header junk. Convert to png. encrypted.pos = 0 if encrypted.peek('bytes:5') == b'Joker': print(f'Trimming and converting weird JPEG to PNG {im_path}') del encrypted[0:16 * 8] # Open it as a jpg and resave to disk try: new_imfile = Image.open(io.BytesIO(encrypted.tobytes())) new_imfile.save(im_path) except IOError: print(f'Unable to open {im_path}')
def _writeFile(self, data, fileName, toCompress=False): '''This is an internal method used to write data to the object's folder. Since we are dealing with bits and not bytes (which is the smallest size operating systems can work with), there is a special five-byte header that decodes as an unsigned integer which is the amount of bits to read. ''' bitsAppendage = BitStream(uint=data.len, length=40) bitsAppendageToBytes = bitsAppendage.tobytes() dataToBytes = data.tobytes() if toCompress == True: tempName = self.saveFolder + '\\temp.bin' with open(tempName, 'wb') as writeData: writeData.write(bitsAppendageToBytes) writeData.write(dataToBytes) compressFile(tempName, self.saveFolder + f'\\{fileName}.bin') else: with open(self.saveFolder + f'\\{fileName}.bin', 'wb') as writeData: writeData.write(bitsAppendageToBytes) writeData.write(dataToBytes)
def _attemptAssembly(self): '''This method will check to see if all frames for the stream have been read. If so, they will be assembled into a single binary, it's hash will be validated, and then it will be ran through post-processing, which is what ultimately yields the original files encoded in the stream. ''' if self.isAssembled == False: # If false, assembly will be attempted. Otherwise, we skip to postprocessing. if self.totalFrames == self.framesIngested: logging.info( f'All frame(s) loaded for {self.streamSHA}, attempting assembly...' ) dataLeft = self.sizeInBytes * 8 assembledPath = f'{self.saveFolder}\\assembled.bin' interFrameBitBuffer = BitStream() with open(assembledPath, 'ab') as assemblePackage: for frame in range(self.totalFrames - self.payloadBeginsThisFrame + 1): frameNumber = frame + self.payloadBeginsThisFrame logging.debug( f'Assembling {frameNumber}/{self.totalFrames}') activeFrame = self._readFile(f'frame{frameNumber}') if frameNumber != self.totalFrames: # All frames except the last one. bitMerge = BitStream(interFrameBitBuffer + activeFrame) dataHolder = bitMerge.read( f'bytes: {bitMerge.len // 8}') if bitMerge.len - bitMerge.pos > 0: interFrameBitBuffer = bitMerge.read( f'bits : {bitMerge.len - bitMerge.pos}') else: interFrameBitBuffer = BitStream() if isinstance(dataHolder, bytes): logging.debug('was bytes this frame!') assemblePackage.write(dataHolder) else: logging.debug('bits to bytes this frame') toByteType = dataHolder.tobytes() assemblePackage.write(toByteType) dataLeft -= activeFrame.len else: #This is the last frame bitMerge = interFrameBitBuffer + activeFrame.read( f'bits : {dataLeft}') toByteType = bitMerge.tobytes() assemblePackage.write(toByteType) if returnHashFromFile(assembledPath) != self.assembledSHA: logging.critical( f'Assembled frames do not match self.packageSHA. Cannot continue.' ) return False logging.debug(f'Successfully assembled.') self.isAssembled = True else: logging.info( f'{self.framesIngested} / {self.totalFrames} frames have been loaded for ' f'{self.streamSHA}\n so far, cannot assemble yet.') return False postProcessAttempt = PostProcessor(self.outputPath, self.streamSHA, self.saveFolder, self.encryptionEnabled, self.encryptionKey, self.scryptN, self.scryptR, self.scryptP, self.compressionEnabled) # Did all three stages of PostProcess successfully run? if postProcessAttempt.FullyAssembled != True: return False return True
def _attempt_assembly(self): '''This method will check to see if all frames for the stream have been read. If so, they will be assembled into a single binary, it's hash will be validated, and then it will be ran through post-processing, which is what ultimately yields the original files encoded in the stream. ''' if self.is_assembled == False: # If false, assembly will be attempted. Otherwise, we skip to postprocessing. if self.total_frames == self.frames_ingested: logging.info( f'All frame(s) loaded for {self.stream_sha}, attempting assembly...' ) data_left = self.size_in_bytes * 8 assembled_path = f'{self.save_folder}\\assembled.bin' inter_frame_bit_buffer = BitStream() with open(assembled_path, 'ab') as assemble_package: for frame in range(self.total_frames - self.payload_begins_this_frame + 1): frame_number = frame + self.payload_begins_this_frame logging.debug( f'Assembling {frame_number}/{self.total_frames}') active_frame = self._read_file(f'frame{frame_number}') if frame_number != self.total_frames: # All frames except the last one. bit_merge = BitStream(inter_frame_bit_buffer + active_frame) data_holder = bit_merge.read( f'bytes: {bit_merge.len // 8}') if bit_merge.len - bit_merge.pos > 0: inter_frame_bit_buffer = bit_merge.read( f'bits : {bit_merge.len - bit_merge.pos}') else: inter_frame_bit_buffer = BitStream() if isinstance(data_holder, bytes): logging.debug('was bytes this frame!') assemble_package.write(data_holder) else: logging.debug('bits to bytes this frame') to_byte_type = data_holder.tobytes() assemble_package.write(to_byte_type) data_left -= active_frame.len else: #This is the last frame bit_merge = inter_frame_bit_buffer + active_frame.read( f'bits : {data_left}') to_byte_type = bit_merge.tobytes() assemble_package.write(to_byte_type) if return_hash_from_file(assembled_path) != self.assembled_sha: logging.critical( f'Assembled frames do not match self.packageSHA. Cannot continue.' ) return False logging.debug(f'Successfully assembled.') self.is_assembled = True else: logging.info( f'{self.frames_ingested} / {self.total_frames} frames have been loaded for ' f'{self.stream_sha}\n so far, cannot assemble yet.') return False post_process_attempt = PostProcessor( self.output_path, self.stream_sha, self.save_folder, self.encryption_enabled, self.encryption_key, self.scrypt_n, self.scrypt_r, self.scrypt_p, self.compression_enabled) # Did all three stages of PostProcess successfully run? if post_process_attempt.fully_assembled != True: return False return True
class NALUnit: def __init__(self, filp, pos, utype, size, tid, nosetup=False): self.filp = filp self.pos = pos self.num_bytes_in_nalu = size self.nal_unit_type = utype self.temporal_id = tid self.rbsp_byte = BitStream() self.setup_rbsp() self.print_bin() if not nosetup: self.setup() def rbsp_read(self, fmt): return self.rbsp_byte.read(fmt) def next_bits(self, fmt, forward=False): try: if not forward: ret = self.filp.peek(fmt) else: ret = self.filp.read(fmt) except ReadError: return 0 return ret def setup_rbsp(self): self.filp.bytepos += 2 self.num_bytes_in_rbsp = 0 i = 2 while i < self.num_bytes_in_nalu: if i + 2 < self.num_bytes_in_nalu and self.next_bits('hex: 24') == '000003': self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 # discard emulation_prevention_three_byte self.filp.read(8) i += 3 else: self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 i += 1 self.filp.bytepos = self.pos def more_rbsp_data(self): try: while self.rbsp_read('uint: 1') != 1: pass return True except ReadError: return False def rbsp_trailing_bits(self): pos = self.rbsp_byte.pos bits = self.rbsp_byte.bytealign() self.rbsp_byte.pos = pos if bits == 0 or self.rbsp_read('uint: %d' % bits) != 1 << bits: print('Wrong rbsp_trailing_bits at NALU begined at bytes %d' % self.pos) exit(1) def profile_tier_level(self, ProfilePresentFlag, MaxNumSubLayersMinus1): if ProfilePresentFlag: self.general_profile_space = self.rbsp_read('uint: 2') self.general_tier_flag = self.rbsp_read('uint: 1') self.general_profile_idc = self.rbsp_read('uint: 5') self.general_profile_compatibility_flag = [0] * 32 for i in range(0, 32): self.general_profile_compatibility_flag[i] = self.rbsp_read("uint: 1") self.general_reserved_zero_16bits = self.rbsp_read('uint: 16') self.general_level_idc = self.rbsp_read('uint: 8') self.sub_layer_profile_present_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_level_present_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_space = [0] * MaxNumSubLayersMinus1 self.sub_layer_tier_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_idc = [0] * MaxNumSubLayersMinus1 self.sub_layer_reserved_zero_16bits = [0] * MaxNumSubLayersMinus1 self.sub_layer_level_idc = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_compatibility_flag = [[0] * 32] * MaxNumSubLayersMinus1 for i in range(0, MaxNumSubLayersMinus1): self.sub_layer_profile_present_flag[i] = self.rbsp_read('uint: 1') self.sub_layer_level_present_flag[i] = self.rbsp_read('uint: 1') if ProfilePresentFlag and self.sub_layer_profile_present_flag[i]: self.sub_layer_profile_space[i] = self.rbsp_read('uint: 2') self.sub_layer_tier_flag[i] = self.rbsp_read('uint: 1') self.sub_layer_profile_idc[i] = self.rbsp_read('uint: 5') for j in range(0, 32): self.sub_layer_profile_compatibility_flag[i][j] = self.rbsp_read('uint: 1') self.sub_layer_reserved_zero_16bits[i] = self.rbsp_read('uint: 16') if self.sub_layer_level_present_flag[i]: self.sub_layer_level_idc[i] = self.rbsp_read('uint: 8') def op_point(self, opIdx): self.op_num_layer_id_values_minus1 = [opIdx] = self.rbsp_read('ue') self.op_layer_id[opIdx] = [0] * self.op_num_layer_id_values_minus1 for i in range(0, self.op_num_layer_id_values_minus1): self.op_layer_id[opIdx][i] = self.rbsp_read('uint: 6') def short_term_ref_pic_set(self, idxRps): if idxRps != 0: self.inter_ref__pic_set_prediction_flag = self.rbsp_read('uint: 1') else: self.inter_ref__pic_set_prediction_flag = 0 if self.inter_ref__pic_set_prediction_flag: if idxRps == self.num_short_term_ref_pic_sets: self.delta_idx_minus1 = self.rbsp_read('ue') else: self.delta_idx_minus1 = 0 RIdx = idxRps - self.delta_idx_minus1 - 1 self.delta_rps_sign = self.rbsp_read('uint: 1') self.abs_delta_rps_minus1 = self.rbsp_read('ue') self.used_by_curr_pic_flag = [0] * (self.NumDeltaPocs[RIdx] + 1) self.use_delta_flag = [0] * (self.NumDeltaPocs[RIdx] + 1) for i in range(0, self.NumDeltaPocs[RIdx] + 1): self.used_by_curr_pic_flag[i] = self.rbsp_read('uint: 1') if not self.used_by_currpic_flag[i]: self.use_delta_flag[i] = self.rbsp_read('uint: 1') else: num_negative_pics = self.rbsp_read('ue') num_positive_pics = self.rbsp_read('ue') self.delta_poc_s0_minus1 = [0] * num_negative_pics self.used_by_curr_pic_s0_flag = [0] * num_negative_pics for i in range(0, num_negative_pics): self.delta_poc_s0_minus1[i] = self.rbsp_read('ue') self.used_by_curr_pic_s0_flag[i] = self.rbsp_read('uint: 1') self.delta_poc_s1_minus1 = [0] * num_positive_pics self.used_by_curr_pic_s1_flag = [0] * num_positive_pics for i in range(0, num_positive_pics): self.delta_poc_s1_minus1[i] = self.rbsp_read('ue') self.used_by_curr_pic_s1_flag[i] = self.rbsp_read('uint: 1') self.NumDeltaPocs[idxRps] = num_negative_pics + num_positive_pics def setup(self): pass def __str__(self): return 'NALU: pos=%d, length=%d, type=%s, tid=%d' % (self.pos, self.num_bytes_in_nalu, NAL_UNIT_TYPE[self.nal_unit_type], self.temporal_id) def print_bin(self): i = 1 for abyte in self.rbsp_byte.tobytes(): if i == 16: print('%3s' % hex(abyte)[2: ]) i = 1 else: print('%3s' % hex(abyte)[2: ], end=' ') i += 1 print('\n')
def processSerial(self, rawSerial): """ first collect bits from serial in to a BitStream buffer. Then split at 0xC0 you now have a list of buffers. Discard garbage ones, and keep the last one to accumulate more bits in to as they come in. Don't process that last one, it might still have stuff in it - save it until the next time through. Pass good packets to unkissify """ rawSerial = BitStream(self.serialBuffer + rawSerial) # print(rawSerial) raw = [s for s in rawSerial.split('0xc0', bytealigned=True)] print(raw) # if you have at least 1 whole packet, there >= 3 elements in "raw" # '', [data], and ['0xc0'] if len(raw) < 3: self.serialBuffer = rawSerial.tobytes() return # need more data! for packet in raw[1:-1]: # skip the stuff before the first '0xc0' if len(packet) > 16: unkissPacket = unkissifyPacket(packet) addresses, unkissPacket.pos = unax25ifyAddresses(unkissPacket) if all(self.destFilter not in s for s in addresses): continue controlField = unkissPacket.read(8) PIDField = unkissPacket.read(8) if (unkissPacket.peek(24) == b'{{V'): unkissPacket.read(24) if (isBase91( unkissPacket.peek( len(unkissPacket) - unkissPacket.pos).bytes)): unkissPacket = base91tobytes(unkissPacket) imageID = unkissPacket.read('uint:8') ny = unkissPacket.read('uint:8') * 16 nx = unkissPacket.read('uint:8') * 16 packetNum = unkissPacket.read('uint:16') numYCbCr = unkissPacket.read('uint:8') channelBD = unkissPacket.read('uint:8') + 1 # print([controlField, PIDField, imageID, ny, nx, packetNum, numYCbCr, channelBD]) hashID = addresses[0] + "/" + addresses[1] + "/" + str(imageID) # print(hashID) pixelYData = [] pixelCbData = [] pixelCrData = [] for tmp in range(numYCbCr): pixelYData.append( unkissPacket.read('uint:' + str(channelBD))) pixelCbData.append( unkissPacket.read('uint:' + str(channelBD))) pixelCrData.append( unkissPacket.read('uint:' + str(channelBD))) while unkissPacket.len - unkissPacket.pos >= channelBD: pixelYData.append( unkissPacket.read('uint:' + str(channelBD))) pixelList = shufflePixels(ny, nx) startingPixel = packetNum * (len(pixelYData) ) # last packet might have fewer! pixelID = pixelList[startingPixel:startingPixel + len(pixelYData)] # temporarily display and hold image data # pixels are counted down a column first, so we transpose image # this conversion is "wrong," need to do it as floats #print(pixelID) #print(pixelYData) pixelYData = np.array(pixelYData) / (2**(channelBD) - 1) * (2**8 - 1) pixelYData[pixelYData > 255] = 255 pixelCbData = np.array(pixelCbData) / (2**(channelBD) - 1) * (2**8 - 1) pixelCbData[pixelCbData > 255] = 255 pixelCrData = np.array(pixelCrData) / (2**(channelBD) - 1) * (2**8 - 1) pixelCrData[pixelCrData > 255] = 255 if hashID not in self.Z: self.Z[hashID] = np.zeros((ny, nx, 3), dtype='uint8') self.pixelsY[hashID] = set() self.pixelsCbCr[hashID] = set() self.nynx[hashID] = (ny, nx) self.pixelsPerPacket[hashID] = len(pixelYData) self.Z[hashID][:, :, 0].T.flat[pixelID] = np.around(pixelYData) # self.Z[:,:,0].T.flat[pixelID] <<= (8-channelBD) self.Z[hashID][:, :, 1].T.flat[ pixelID[:len(pixelCbData)]] = np.around(pixelCbData) # self.Z[:,:,1].T.flat[pixelID] <<= (8-channelBD) self.Z[hashID][:, :, 2].T.flat[ pixelID[:len(pixelCrData)]] = np.around(pixelCrData) # self.Z[:,:,2].T.flat[pixelID] <<= (8-channelBD) # self.Z = self.Z << (8-channelBD) # (Z >> (8-channelBD) ) << (8-channelBD) # /(2**channelBD-1)*255 # self.Z = ycbcr2rgb(self.Z.astype(float)) self.pixelsY[hashID].update(pixelID) self.pixelsCbCr[hashID].update(pixelID[:len(pixelCrData)]) self.serialBuffer = raw[-1].tobytes()