def run(self):
import subprocess
server, port, username, password, mountpoint, rtcm_callback = self._Thread__args
nt = subprocess.Popen([
"./ntripclient", "--server", server, "--password", password,
"--user", username, "--mountpoint", mountpoint
],
stdout=subprocess.PIPE)
"""nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint],
stdout=subprocess.PIPE)"""
if nt is None or nt.stdout is None:
indev = sys.stdin
else:
indev = nt.stdout
sio = indev
while not self._stopevent.isSet():
d = ord(sio.read(1))
if d != RTCMv3_PREAMBLE:
continue
pack_stream = BitStream()
l1 = ord(sio.read(1))
l2 = ord(sio.read(1))
pack_stream.append(bs.pack('2*uint:8', l1, l2))
pack_stream.read(6)
pkt_len = pack_stream.read(10).uint
pkt = sio.read(pkt_len)
parity = sio.read(3)
if len(pkt) != pkt_len:
print "Length error {} {}".format(len(pkt), pkt_len)
continue
if True: #TODO check parity
for d in pkt:
pack_stream.append(bs.pack('uint:8', ord(d)))
msg = parse_rtcmv3(pack_stream)
# MARCO forward unchanged
pack_stream.append(
bs.pack('3*uint:8', ord(parity[0]), ord(parity[1]),
ord(parity[2])))
msg = struct.pack('<B',
RTCMv3_PREAMBLE) + pack_stream.tobytes()
if msg is not None and rtcm_callback is not None:
rtcm_callback(msg)
if nt is not None:
nt.terminate()
nt.wait()
nt = None
class bit_file:
def __init__(self,
f,
bits_per_op,
mode='read',
keep_open=False,
read_size=MAX_ENCODING,
read_count=1):
if mode not in ['read', 'write']:
raise Exception('bad bit_file mode. Try "read" or "write"')
self.mode = mode
if isinstance(f, str):
fmode = 'wb' if mode == 'write' else 'rb'
self.f = open(f, fmode)
self.keep_open = False
else:
self.f = f
self.keep_open = keep_open # determines whether __exit__ is a flush()+close(), or just a flush()
self.bits_per_op = bits_per_op
self.stream = BitStream()
self.read_size = read_size
self.read_count = read_count
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
if self.mode == 'write' and not self.f.closed:
self.save()
if not self.keep_open and not self.f.closed:
with self.f: # close file
pass
def write(self, bits):
if isinstance(bits, bit_write_buffer):
bits = bits.stream
if not isinstance(bits, Bits):
bits = Bits(uint=bits, length=self.bits_per_op)
self.stream.append(bits)
def read(self):
if self.read_count and self.stream.bitpos == self.stream.length:
self.stream.clear()
self.stream.append(Bits(bytes=self.f.read(self.read_size)))
self.read_count -= 1
try:
bits = self.stream.read(f'uint:{self.bits_per_op}')
except bitstring.ReadError:
try:
bits = self.stream.read('uint')
except bitstring.InterpretError:
bits = 0
return bits
def save(self):
self.f.write(self.stream.tobytes())
class BitWriter:
def __init__(self):
self._bytes = ""
self._bits = BitStream()
def write_int(self, length, value):
news = BitStream(uint=value, length=length)
start = 0
if self._bits.len > 0:
left = 8 - self._bits.len
if news.len < left:
left = news.len
self._bits = news[:left] + self._bits
start += left
if self._bits.len == 8:
self._bytes += self._bits.tobytes()
self._bits = BitStream()
byte_len = (news.len - start) / 8
if byte_len > 0:
more = byte_len * 8
self._bytes += news[start:start+more].tobytes()
start += more
if news.len > start:
self._bits = news[start:]
def write_int64(self, length, value):
if length <= 32:
self.write_int(length, value)
return
count = length - 32
self.write_int(32, value >> count)
self.write_int(count, value & ((1 << count) - 1))
def write_char_array(self, max_length, value):
self.write_int(bit_count(max_length), len(value))
if self._bits.len > 0:
more = 8 - self._bits.len
tail = (BitStream(int=0, length=more) + self._bits).tobytes()
self._bits = BitStream()
self._bytes += tail
self._bytes += value
def get_bytes(self):
if self._bits.len > 0:
more = 8 - self._bits.len
tail = (BitStream(int=0, length=more) + self._bits).tobytes()
return self._bytes + tail
else:
return self._bytes
def valid_checksum(source_address, dest_address, bytes_packet):
bin_packet = BitStream(bytes=bytes_packet)
length = len(bin_packet) % 8
checksum = bin_packet[128:144].bytes
pseudo_header = TCPyPacket.create_pseudo_header(
source_address, dest_address, length)
bin_packet.overwrite(b'\x00\x00', 128)
cs_calc = cs.Checksum16()
cs_calc.process(bin_packet.tobytes())
#cs_calc.process(pseudo_header)
if checksum != cs_calc.finalbytes():
return False
return True
def find_mapping(infile, outfile):
stream = BitStream()
with open(infile) as f:
byte = f.read(1)
while byte:
if byte == '\x01':
stream.append('0b1')
else:
stream.append('0b0')
byte = f.read(1)
iteration = 0
for mapping in bitmappings:
stream.pos = 0
tmpstream = BitStream()
for pie in stream.cut(2):
index = pie.read('uint:2')
tmpstream.append( BitArray(uint=mapping[index], length=2) )
# look for the start of the flag in the stream of bits
pos = tmpstream.find(ZULU)
if len(pos) > 0:
# print("Found the bytes we are looking for on iteration {}".format(iteration))
# print("pos = {}".format(pos))
tmpstream <<= pos[0] % 8
data = tmpstream.tobytes()
# print(data)
with open(outfile, 'wb') as fh:
fh.write(data)
break
else:
# print("Did not find the header")
iteration += 1
def decrypt_images(**kwargs):
path = kwargs.pop('path', 'herders/static/herders/images')
for im_path in iglob(f'{path}/**/*.png', recursive=True):
encrypted = BitStream(filename=im_path)
# Check if it is 'encrypted'. 8th byte is 0x0B instead of the correct signature 0x0A
encrypted.pos = 0x07 * 8
signature = encrypted.peek('uint:8')
if signature == 0x0B:
print(f'Decrypting {im_path}')
# Correct the PNG signature
encrypted.overwrite('0x0A', encrypted.pos)
# Replace bits with magic decrypted values
try:
while True:
pos = encrypted.pos
val = encrypted.peek('uint:8')
encrypted.overwrite(
Bits(uint=com2us_decrypt_values[val], length=8), pos)
except ReadError:
# EOF
pass
# Write it back to the file
with open(im_path, 'wb') as f:
encrypted.tofile(f)
continue
# Check for weird jpeg format with extra header junk. Convert to png.
encrypted.pos = 0
if encrypted.peek('bytes:5') == b'Joker':
print(f'Trimming and converting weird JPEG to PNG {im_path}')
del encrypted[0:16 * 8]
# Open it as a jpg and resave to disk
try:
new_imfile = Image.open(io.BytesIO(encrypted.tobytes()))
new_imfile.save(im_path)
except IOError:
print(f'Unable to open {im_path}')
def _writeFile(self, data, fileName, toCompress=False):
'''This is an internal method used to write data to the object's folder. Since we are dealing with bits and not
bytes (which is the smallest size operating systems can work with), there is a special five-byte header that
decodes as an unsigned integer which is the amount of bits to read.
'''
bitsAppendage = BitStream(uint=data.len, length=40)
bitsAppendageToBytes = bitsAppendage.tobytes()
dataToBytes = data.tobytes()
if toCompress == True:
tempName = self.saveFolder + '\\temp.bin'
with open(tempName, 'wb') as writeData:
writeData.write(bitsAppendageToBytes)
writeData.write(dataToBytes)
compressFile(tempName, self.saveFolder + f'\\{fileName}.bin')
else:
with open(self.saveFolder + f'\\{fileName}.bin',
'wb') as writeData:
writeData.write(bitsAppendageToBytes)
writeData.write(dataToBytes)
def _attemptAssembly(self):
'''This method will check to see if all frames for the stream have been read. If so, they will be assembled
into a single binary, it's hash will be validated, and then it will be ran through post-processing,
which is what ultimately yields the original files encoded in the stream.
'''
if self.isAssembled == False: # If false, assembly will be attempted. Otherwise, we skip to postprocessing.
if self.totalFrames == self.framesIngested:
logging.info(
f'All frame(s) loaded for {self.streamSHA}, attempting assembly...'
)
dataLeft = self.sizeInBytes * 8
assembledPath = f'{self.saveFolder}\\assembled.bin'
interFrameBitBuffer = BitStream()
with open(assembledPath, 'ab') as assemblePackage:
for frame in range(self.totalFrames -
self.payloadBeginsThisFrame + 1):
frameNumber = frame + self.payloadBeginsThisFrame
logging.debug(
f'Assembling {frameNumber}/{self.totalFrames}')
activeFrame = self._readFile(f'frame{frameNumber}')
if frameNumber != self.totalFrames: # All frames except the last one.
bitMerge = BitStream(interFrameBitBuffer +
activeFrame)
dataHolder = bitMerge.read(
f'bytes: {bitMerge.len // 8}')
if bitMerge.len - bitMerge.pos > 0:
interFrameBitBuffer = bitMerge.read(
f'bits : {bitMerge.len - bitMerge.pos}')
else:
interFrameBitBuffer = BitStream()
if isinstance(dataHolder, bytes):
logging.debug('was bytes this frame!')
assemblePackage.write(dataHolder)
else:
logging.debug('bits to bytes this frame')
toByteType = dataHolder.tobytes()
assemblePackage.write(toByteType)
dataLeft -= activeFrame.len
else: #This is the last frame
bitMerge = interFrameBitBuffer + activeFrame.read(
f'bits : {dataLeft}')
toByteType = bitMerge.tobytes()
assemblePackage.write(toByteType)
if returnHashFromFile(assembledPath) != self.assembledSHA:
logging.critical(
f'Assembled frames do not match self.packageSHA. Cannot continue.'
)
return False
logging.debug(f'Successfully assembled.')
self.isAssembled = True
else:
logging.info(
f'{self.framesIngested} / {self.totalFrames} frames have been loaded for '
f'{self.streamSHA}\n so far, cannot assemble yet.')
return False
postProcessAttempt = PostProcessor(self.outputPath, self.streamSHA,
self.saveFolder,
self.encryptionEnabled,
self.encryptionKey, self.scryptN,
self.scryptR, self.scryptP,
self.compressionEnabled)
# Did all three stages of PostProcess successfully run?
if postProcessAttempt.FullyAssembled != True:
return False
return True
def _attempt_assembly(self):
'''This method will check to see if all frames for the stream have been read. If so, they will be assembled
into a single binary, it's hash will be validated, and then it will be ran through post-processing,
which is what ultimately yields the original files encoded in the stream.
'''
if self.is_assembled == False: # If false, assembly will be attempted. Otherwise, we skip to postprocessing.
if self.total_frames == self.frames_ingested:
logging.info(
f'All frame(s) loaded for {self.stream_sha}, attempting assembly...'
)
data_left = self.size_in_bytes * 8
assembled_path = f'{self.save_folder}\\assembled.bin'
inter_frame_bit_buffer = BitStream()
with open(assembled_path, 'ab') as assemble_package:
for frame in range(self.total_frames -
self.payload_begins_this_frame + 1):
frame_number = frame + self.payload_begins_this_frame
logging.debug(
f'Assembling {frame_number}/{self.total_frames}')
active_frame = self._read_file(f'frame{frame_number}')
if frame_number != self.total_frames: # All frames except the last one.
bit_merge = BitStream(inter_frame_bit_buffer +
active_frame)
data_holder = bit_merge.read(
f'bytes: {bit_merge.len // 8}')
if bit_merge.len - bit_merge.pos > 0:
inter_frame_bit_buffer = bit_merge.read(
f'bits : {bit_merge.len - bit_merge.pos}')
else:
inter_frame_bit_buffer = BitStream()
if isinstance(data_holder, bytes):
logging.debug('was bytes this frame!')
assemble_package.write(data_holder)
else:
logging.debug('bits to bytes this frame')
to_byte_type = data_holder.tobytes()
assemble_package.write(to_byte_type)
data_left -= active_frame.len
else: #This is the last frame
bit_merge = inter_frame_bit_buffer + active_frame.read(
f'bits : {data_left}')
to_byte_type = bit_merge.tobytes()
assemble_package.write(to_byte_type)
if return_hash_from_file(assembled_path) != self.assembled_sha:
logging.critical(
f'Assembled frames do not match self.packageSHA. Cannot continue.'
)
return False
logging.debug(f'Successfully assembled.')
self.is_assembled = True
else:
logging.info(
f'{self.frames_ingested} / {self.total_frames} frames have been loaded for '
f'{self.stream_sha}\n so far, cannot assemble yet.')
return False
post_process_attempt = PostProcessor(
self.output_path, self.stream_sha, self.save_folder,
self.encryption_enabled, self.encryption_key, self.scrypt_n,
self.scrypt_r, self.scrypt_p, self.compression_enabled)
# Did all three stages of PostProcess successfully run?
if post_process_attempt.fully_assembled != True:
return False
return True
class NALUnit:
def __init__(self, filp, pos, utype, size, tid, nosetup=False):
self.filp = filp
self.pos = pos
self.num_bytes_in_nalu = size
self.nal_unit_type = utype
self.temporal_id = tid
self.rbsp_byte = BitStream()
self.setup_rbsp()
self.print_bin()
if not nosetup:
self.setup()
def rbsp_read(self, fmt):
return self.rbsp_byte.read(fmt)
def next_bits(self, fmt, forward=False):
try:
if not forward:
ret = self.filp.peek(fmt)
else:
ret = self.filp.read(fmt)
except ReadError:
return 0
return ret
def setup_rbsp(self):
self.filp.bytepos += 2
self.num_bytes_in_rbsp = 0
i = 2
while i < self.num_bytes_in_nalu:
if i + 2 < self.num_bytes_in_nalu and self.next_bits('hex: 24') == '000003':
self.rbsp_byte.append(self.filp.read(8))
self.num_bytes_in_rbsp += 1
self.rbsp_byte.append(self.filp.read(8))
self.num_bytes_in_rbsp += 1
# discard emulation_prevention_three_byte
self.filp.read(8)
i += 3
else:
self.rbsp_byte.append(self.filp.read(8))
self.num_bytes_in_rbsp += 1
i += 1
self.filp.bytepos = self.pos
def more_rbsp_data(self):
try:
while self.rbsp_read('uint: 1') != 1:
pass
return True
except ReadError:
return False
def rbsp_trailing_bits(self):
pos = self.rbsp_byte.pos
bits = self.rbsp_byte.bytealign()
self.rbsp_byte.pos = pos
if bits == 0 or self.rbsp_read('uint: %d' % bits) != 1 << bits:
print('Wrong rbsp_trailing_bits at NALU begined at bytes %d' % self.pos)
exit(1)
def profile_tier_level(self, ProfilePresentFlag, MaxNumSubLayersMinus1):
if ProfilePresentFlag:
self.general_profile_space = self.rbsp_read('uint: 2')
self.general_tier_flag = self.rbsp_read('uint: 1')
self.general_profile_idc = self.rbsp_read('uint: 5')
self.general_profile_compatibility_flag = [0] * 32
for i in range(0, 32):
self.general_profile_compatibility_flag[i] = self.rbsp_read("uint: 1")
self.general_reserved_zero_16bits = self.rbsp_read('uint: 16')
self.general_level_idc = self.rbsp_read('uint: 8')
self.sub_layer_profile_present_flag = [0] * MaxNumSubLayersMinus1
self.sub_layer_level_present_flag = [0] * MaxNumSubLayersMinus1
self.sub_layer_profile_space = [0] * MaxNumSubLayersMinus1
self.sub_layer_tier_flag = [0] * MaxNumSubLayersMinus1
self.sub_layer_profile_idc = [0] * MaxNumSubLayersMinus1
self.sub_layer_reserved_zero_16bits = [0] * MaxNumSubLayersMinus1
self.sub_layer_level_idc = [0] * MaxNumSubLayersMinus1
self.sub_layer_profile_compatibility_flag = [[0] * 32] * MaxNumSubLayersMinus1
for i in range(0, MaxNumSubLayersMinus1):
self.sub_layer_profile_present_flag[i] = self.rbsp_read('uint: 1')
self.sub_layer_level_present_flag[i] = self.rbsp_read('uint: 1')
if ProfilePresentFlag and self.sub_layer_profile_present_flag[i]:
self.sub_layer_profile_space[i] = self.rbsp_read('uint: 2')
self.sub_layer_tier_flag[i] = self.rbsp_read('uint: 1')
self.sub_layer_profile_idc[i] = self.rbsp_read('uint: 5')
for j in range(0, 32):
self.sub_layer_profile_compatibility_flag[i][j] = self.rbsp_read('uint: 1')
self.sub_layer_reserved_zero_16bits[i] = self.rbsp_read('uint: 16')
if self.sub_layer_level_present_flag[i]:
self.sub_layer_level_idc[i] = self.rbsp_read('uint: 8')
def op_point(self, opIdx):
self.op_num_layer_id_values_minus1 = [opIdx] = self.rbsp_read('ue')
self.op_layer_id[opIdx] = [0] * self.op_num_layer_id_values_minus1
for i in range(0, self.op_num_layer_id_values_minus1):
self.op_layer_id[opIdx][i] = self.rbsp_read('uint: 6')
def short_term_ref_pic_set(self, idxRps):
if idxRps != 0:
self.inter_ref__pic_set_prediction_flag = self.rbsp_read('uint: 1')
else:
self.inter_ref__pic_set_prediction_flag = 0
if self.inter_ref__pic_set_prediction_flag:
if idxRps == self.num_short_term_ref_pic_sets:
self.delta_idx_minus1 = self.rbsp_read('ue')
else:
self.delta_idx_minus1 = 0
RIdx = idxRps - self.delta_idx_minus1 - 1
self.delta_rps_sign = self.rbsp_read('uint: 1')
self.abs_delta_rps_minus1 = self.rbsp_read('ue')
self.used_by_curr_pic_flag = [0] * (self.NumDeltaPocs[RIdx] + 1)
self.use_delta_flag = [0] * (self.NumDeltaPocs[RIdx] + 1)
for i in range(0, self.NumDeltaPocs[RIdx] + 1):
self.used_by_curr_pic_flag[i] = self.rbsp_read('uint: 1')
if not self.used_by_currpic_flag[i]:
self.use_delta_flag[i] = self.rbsp_read('uint: 1')
else:
num_negative_pics = self.rbsp_read('ue')
num_positive_pics = self.rbsp_read('ue')
self.delta_poc_s0_minus1 = [0] * num_negative_pics
self.used_by_curr_pic_s0_flag = [0] * num_negative_pics
for i in range(0, num_negative_pics):
self.delta_poc_s0_minus1[i] = self.rbsp_read('ue')
self.used_by_curr_pic_s0_flag[i] = self.rbsp_read('uint: 1')
self.delta_poc_s1_minus1 = [0] * num_positive_pics
self.used_by_curr_pic_s1_flag = [0] * num_positive_pics
for i in range(0, num_positive_pics):
self.delta_poc_s1_minus1[i] = self.rbsp_read('ue')
self.used_by_curr_pic_s1_flag[i] = self.rbsp_read('uint: 1')
self.NumDeltaPocs[idxRps] = num_negative_pics + num_positive_pics
def setup(self):
pass
def __str__(self):
return 'NALU: pos=%d, length=%d, type=%s, tid=%d' % (self.pos, self.num_bytes_in_nalu, NAL_UNIT_TYPE[self.nal_unit_type], self.temporal_id)
def print_bin(self):
i = 1
for abyte in self.rbsp_byte.tobytes():
if i == 16:
print('%3s' % hex(abyte)[2: ])
i = 1
else:
print('%3s' % hex(abyte)[2: ], end=' ')
i += 1
print('\n')
def processSerial(self, rawSerial):
"""
first collect bits from serial in to a BitStream buffer. Then split at 0xC0
you now have a list of buffers. Discard garbage ones, and keep the last one
to accumulate more bits in to as they come in. Don't process that last one,
it might still have stuff in it - save it until the next time through. Pass
good packets to unkissify
"""
rawSerial = BitStream(self.serialBuffer + rawSerial)
# print(rawSerial)
raw = [s for s in rawSerial.split('0xc0', bytealigned=True)]
print(raw)
# if you have at least 1 whole packet, there >= 3 elements in "raw"
# '', [data], and ['0xc0']
if len(raw) < 3:
self.serialBuffer = rawSerial.tobytes()
return # need more data!
for packet in raw[1:-1]: # skip the stuff before the first '0xc0'
if len(packet) > 16:
unkissPacket = unkissifyPacket(packet)
addresses, unkissPacket.pos = unax25ifyAddresses(unkissPacket)
if all(self.destFilter not in s for s in addresses):
continue
controlField = unkissPacket.read(8)
PIDField = unkissPacket.read(8)
if (unkissPacket.peek(24) == b'{{V'):
unkissPacket.read(24)
if (isBase91(
unkissPacket.peek(
len(unkissPacket) - unkissPacket.pos).bytes)):
unkissPacket = base91tobytes(unkissPacket)
imageID = unkissPacket.read('uint:8')
ny = unkissPacket.read('uint:8') * 16
nx = unkissPacket.read('uint:8') * 16
packetNum = unkissPacket.read('uint:16')
numYCbCr = unkissPacket.read('uint:8')
channelBD = unkissPacket.read('uint:8') + 1
# print([controlField, PIDField, imageID, ny, nx, packetNum, numYCbCr, channelBD])
hashID = addresses[0] + "/" + addresses[1] + "/" + str(imageID)
# print(hashID)
pixelYData = []
pixelCbData = []
pixelCrData = []
for tmp in range(numYCbCr):
pixelYData.append(
unkissPacket.read('uint:' + str(channelBD)))
pixelCbData.append(
unkissPacket.read('uint:' + str(channelBD)))
pixelCrData.append(
unkissPacket.read('uint:' + str(channelBD)))
while unkissPacket.len - unkissPacket.pos >= channelBD:
pixelYData.append(
unkissPacket.read('uint:' + str(channelBD)))
pixelList = shufflePixels(ny, nx)
startingPixel = packetNum * (len(pixelYData)
) # last packet might have fewer!
pixelID = pixelList[startingPixel:startingPixel +
len(pixelYData)]
# temporarily display and hold image data
# pixels are counted down a column first, so we transpose image
# this conversion is "wrong," need to do it as floats
#print(pixelID)
#print(pixelYData)
pixelYData = np.array(pixelYData) / (2**(channelBD) -
1) * (2**8 - 1)
pixelYData[pixelYData > 255] = 255
pixelCbData = np.array(pixelCbData) / (2**(channelBD) -
1) * (2**8 - 1)
pixelCbData[pixelCbData > 255] = 255
pixelCrData = np.array(pixelCrData) / (2**(channelBD) -
1) * (2**8 - 1)
pixelCrData[pixelCrData > 255] = 255
if hashID not in self.Z:
self.Z[hashID] = np.zeros((ny, nx, 3), dtype='uint8')
self.pixelsY[hashID] = set()
self.pixelsCbCr[hashID] = set()
self.nynx[hashID] = (ny, nx)
self.pixelsPerPacket[hashID] = len(pixelYData)
self.Z[hashID][:, :, 0].T.flat[pixelID] = np.around(pixelYData)
# self.Z[:,:,0].T.flat[pixelID] <<= (8-channelBD)
self.Z[hashID][:, :, 1].T.flat[
pixelID[:len(pixelCbData)]] = np.around(pixelCbData)
# self.Z[:,:,1].T.flat[pixelID] <<= (8-channelBD)
self.Z[hashID][:, :, 2].T.flat[
pixelID[:len(pixelCrData)]] = np.around(pixelCrData)
# self.Z[:,:,2].T.flat[pixelID] <<= (8-channelBD)
# self.Z = self.Z << (8-channelBD) # (Z >> (8-channelBD) ) << (8-channelBD) # /(2**channelBD-1)*255
# self.Z = ycbcr2rgb(self.Z.astype(float))
self.pixelsY[hashID].update(pixelID)
self.pixelsCbCr[hashID].update(pixelID[:len(pixelCrData)])
self.serialBuffer = raw[-1].tobytes()
