def test_prim_fcr_long(self) -> None: nn = 48 kk = 34 tt = nn - kk rs = RSCodec(tt, fcr=120, prim=0x187) hex_enc_msg = ('08faa123555555c000000354064432c0280e1b4d090cfc04' '887400000003500000000e1985ff9c6b33066ca9f43d12e8') strf = str enc_msg = bytearray.fromhex(strf(hex_enc_msg)) dec_msg = enc_msg[:kk] tem = rs.encode(dec_msg) self.assertEqual(enc_msg, tem, msg="encoded does not match expected") tdm, rtem = rs.decode(tem) self.assertEqual(tdm, dec_msg, msg="decoded does not match original") self.assertEqual(rtem, tem, msg="decoded mesecc does not match original") tem1 = bytearray(tem) num_errs = tt >> 1 for i in sample(range(nn), num_errs): tem1[i] ^= 0xff tdm, rtem = rs.decode(tem1) self.assertEqual(tdm, dec_msg, msg="decoded with errors does not match original") self.assertEqual(rtem, tem, msg="decoded mesecc with errors does not match original") tem1 = bytearray(tem) num_errs += 1 for i in sample(range(nn), num_errs): tem1[i] ^= 0xff self.assertRaises(ReedSolomonError, rs.decode, tem1)
def test_prim_fcr_basic(self) -> None: nn = 30 kk = 18 tt = nn - kk rs = RSCodec(tt, fcr=120, prim=0x187) hex_enc_msg = ('00faa123555555c000000354064432' 'c02800fe97c434e1ff5365cf8fafe4') strf = str enc_msg = bytearray.fromhex(strf(hex_enc_msg)) dec_msg = enc_msg[:kk] tem = rs.encode(dec_msg) self.assertEqual(enc_msg, tem, msg="encoded does not match expected") tdm, rtem = rs.decode(tem) self.assertEqual(tdm, dec_msg, msg="decoded does not match original") self.assertEqual(rtem, tem, msg="decoded mesecc does not match original") tem1 = bytearray(tem) # Clone a copy # Encoding and decoding intact message seem OK, so test errors num_errs = tt >> 1 # Inject tt/2 errors (expected to recover fully) for i in sample(range(nn), num_errs): # inject errors in random places tem1[i] ^= 0xff # flip all 8 bits tdm, _ = rs.decode(tem1) self.assertEqual(tdm, dec_msg, msg="decoded with errors does not match original") tem1 = bytearray(tem) # Clone another copy num_errs += 1 # Inject tt/2 + 1 errors (expected to fail and detect it) for i in sample(range(nn), num_errs): # Inject errors in random places tem1[i] ^= 0xff # Flip all 8 bits # If this fails, it means excessive errors not detected self.assertRaises(ReedSolomonError, rs.decode, tem1)
def test_multiple_rs_codec(self) -> None: """Test multiple RSCodec instances with different parameters.""" mes = 'A' * 30 rs_256 = RSCodec(102) rs_1024 = RSCodec(900, c_exp=10) bytearray(rs_1024.decode(rs_1024.encode(mes))[0]) rs_256.encode(mes) rs_1024.encode(mes) bytearray(rs_256.decode(rs_256.encode(mes))[0])
def test_check(self) -> None: rs = RSCodec() msg = bytearray("hello world " * 10, "latin1") enc = rs.encode(msg) rmsg, renc = rs.decode(enc) self.assertEqual(rs.check(enc), [True]) self.assertEqual(rs.check(renc), [True]) for i in [27, -3, -9, 7, 0]: enc[i] = 99 rmsg, renc = rs.decode(enc) self.assertEqual(rs.check(enc), [False]) self.assertEqual(rs.check(renc), [True])
def test_long(self) -> None: rs = RSCodec() msg = bytearray("a" * 10000, "latin1") enc = rs.encode(msg) dec, dec_enc = rs.decode(enc) self.assertEqual(dec, msg) self.assertEqual(dec_enc, enc) enc2 = list(enc) enc2[177] = 99 enc2[2212] = 88 dec2, dec_enc2 = rs.decode(bytes(enc2)) self.assertEqual(dec2, msg) self.assertEqual(dec_enc2, enc)
def test_correction(self) -> None: rs = RSCodec() msg = bytearray("hello world " * 10, "latin1") enc = rs.encode(msg) rmsg, renc = rs.decode(enc) self.assertEqual(rmsg, msg) self.assertEqual(renc, enc) for i in [27, -3, -9, 7, 0]: enc[i] = 99 rmsg, renc = rs.decode(enc) self.assertEqual(rmsg, msg) enc[82] = 99 self.assertRaises(ReedSolomonError, rs.decode, enc)
def test_simple(self) -> None: rs = RSCodec() msg = bytearray("hello world " * 10, "latin1") enc = rs.encode(msg) dec, dec_enc = rs.decode(enc) self.assertEqual(dec, msg) self.assertEqual(dec_enc, enc)
class TestRxMOutGoing(unittest.TestCase): def setUp(self): self.settings = Settings() self.gateway = Gateway() self.rs = RSCodec(2 * self.settings.serial_error_correction) self.queues = {TXM_INCOMING_QUEUE: Queue(), RXM_OUTGOING_QUEUE: Queue(), TXM_TO_IM_QUEUE: Queue(), TXM_TO_NH_QUEUE: Queue(), TXM_TO_RXM_QUEUE: Queue(), NH_TO_IM_QUEUE: Queue(), EXIT_QUEUE: Queue()} def tearDown(self): for k in self.queues: while not self.queues[k].empty(): self.queues[k].get() time.sleep(0.1) self.queues[k].close() def test_loop(self): # Setup packet = b'testpacket' self.queues[TXM_TO_RXM_QUEUE].put(packet) self.queues[RXM_OUTGOING_QUEUE].put(packet) time.sleep(0.1) # Test self.assertIsNone(rxm_outgoing(self.queues, self.settings, self.gateway, unittest=True)) self.assertEqual(packet, self.rs.decode(self.gateway.packets[0]))
def receiver_loop(settings: 'Settings', queues: Dict[bytes, 'Queue']) -> None: """Decode and queue received packets.""" rs = RSCodec(2 * settings.session_ec_ratio) gw_queue = queues[GATEWAY_QUEUE] while True: try: if gw_queue.empty(): time.sleep(0.001) packet = gw_queue.get() ts = datetime.datetime.now() try: packet = bytes(rs.decode(bytearray(packet))) except ReedSolomonError: box_print(["Warning! Failed to correct errors in received packet."], head=1, tail=1) continue p_header = packet[:1] if p_header in [PUBLIC_KEY_PACKET_HEADER, MESSAGE_PACKET_HEADER, LOCAL_KEY_PACKET_HEADER, COMMAND_PACKET_HEADER, IMPORTED_FILE_CT_HEADER]: queues[p_header].put((ts, packet)) except (KeyboardInterrupt, EOFError): pass
def txm_incoming(settings: 'Settings', q_to_tip: 'Queue', q_to_rxm: 'Queue', q_to_im: 'Queue', q_to_nh: 'Queue') -> None: """Load messages from TxM and forward them to appropriate process via queue.""" rs = RSCodec(2 * settings.session_ec_ratio) while True: try: if q_to_tip.empty(): time.sleep(0.001) packet = q_to_tip.get() try: packet = bytes(rs.decode(packet)) except ReedSolomonError: box_print( ["Warning! Failed to correct errors in received packet."], head=1, tail=1) continue ts = datetime.datetime.now().strftime(settings.t_fmt) header = packet[:1] if header == UNENCRYPTED_PACKET_HEADER: q_to_nh.put(packet[1:]) elif header in [LOCAL_KEY_PACKET_HEADER, COMMAND_PACKET_HEADER]: p_type = 'local key' if header == LOCAL_KEY_PACKET_HEADER else 'command' print("{} - {} TxM > RxM".format(ts, p_type)) q_to_rxm.put(packet) elif header in [MESSAGE_PACKET_HEADER, PUBLIC_KEY_PACKET_HEADER]: payload_len, p_type = ( 32, 'pub key') if header == PUBLIC_KEY_PACKET_HEADER else ( 344, 'message') payload = packet[1:1 + payload_len] trailer = packet[1 + payload_len:] user, contact = trailer.split(US_BYTE) print("{} - {} TxM > {} > {}".format(ts, p_type, user.decode(), contact.decode())) q_to_im.put((header, payload, user, contact)) q_to_rxm.put(header + payload + ORIGIN_USER_HEADER + contact) elif header == EXPORTED_FILE_CT_HEADER: payload = packet[1:] file_name = os.urandom(16).hex() with open(file_name, 'wb+') as f: f.write(payload) print("{} - Exported file from TxM as {}".format( ts, file_name)) except (EOFError, KeyboardInterrupt): pass
def test_c_exp_12(self) -> None: rsc = RSCodec(12, c_exp=12) rsc2 = RSCodec(12, nsize=4095) self.assertEqual(rsc.c_exp, rsc2.c_exp) self.assertEqual(rsc.nsize, rsc2.nsize) mes = 'a'*(4095-12) mesecc = rsc.encode(mes) mesecc[2] = 1 mesecc[-1] = 1 rmes, rmesecc = rsc.decode(mesecc) self.assertEqual(rsc.check(mesecc), [False]) self.assertEqual(rsc.check(rmesecc), [True]) self.assertEqual([x for x in rmes], [ord(x) for x in mes])
def test_c_exp_9(self): rsc = RSCodec(12, c_exp=9) rsc2 = RSCodec(12, nsize=511) self.assertEqual(rsc.c_exp, rsc2.c_exp) self.assertEqual(rsc.nsize, rsc2.nsize) mes = 'a' * ((511 - 12) * 2) mesecc = rsc.encode(mes) mesecc[2] = 1 mesecc[-1] = 1 rmes, rmesecc = rsc.decode(mesecc) self.assertEqual(rsc.check(mesecc), [False, False]) self.assertEqual(rsc.check(rmesecc), [True, True]) self.assertEqual([x for x in rmes], [ord(x) for x in mes])
def receiver_loop(queues: Dict[bytes, 'Queue'], settings: 'Settings', unittest: bool = False) -> None: """Decode received packets and forward them to packet queues. This function also determines the timestamp for received message. """ rs = RSCodec(2 * settings.session_serial_error_correction) gw_queue = queues[GATEWAY_QUEUE] while True: with ignored(EOFError, KeyboardInterrupt): if gw_queue.qsize() == 0: time.sleep(0.01) packet = gw_queue.get() timestamp = datetime.now() try: packet = bytes(rs.decode(packet)) except ReedSolomonError: box_print( "Error: Failed to correct errors in received packet.", head=1, tail=1) continue p_header = packet[:1] if p_header in [ PUBLIC_KEY_PACKET_HEADER, MESSAGE_PACKET_HEADER, LOCAL_KEY_PACKET_HEADER, COMMAND_PACKET_HEADER, IMPORTED_FILE_HEADER ]: queues[p_header].put((timestamp, packet)) if unittest: break
def txm_incoming(queues: Dict[bytes, 'Queue'], settings: 'Settings', unittest: bool = False) -> None: """Loop that places messages received from TxM to appropriate queues.""" rs = RSCodec(2 * settings.session_serial_error_correction) q_to_tip = queues[TXM_INCOMING_QUEUE] m_to_rxm = queues[RXM_OUTGOING_QUEUE] c_to_rxm = queues[TXM_TO_RXM_QUEUE] q_to_im = queues[TXM_TO_IM_QUEUE] q_to_nh = queues[TXM_TO_NH_QUEUE] while True: with ignored(EOFError, KeyboardInterrupt): while q_to_tip.qsize() == 0: time.sleep(0.01) packet = q_to_tip.get() try: packet = bytes(rs.decode(packet)) except ReedSolomonError: box_print( "Warning! Failed to correct errors in received packet.", head=1, tail=1) continue ts = datetime.now().strftime("%m-%d / %H:%M:%S") header = packet[:1] if header == UNENCRYPTED_PACKET_HEADER: q_to_nh.put(packet[1:]) elif header in [LOCAL_KEY_PACKET_HEADER, COMMAND_PACKET_HEADER]: p_type = 'local key' if header == LOCAL_KEY_PACKET_HEADER else 'command' print("{} - {} TxM > RxM".format(ts, p_type)) c_to_rxm.put(packet) elif header in [MESSAGE_PACKET_HEADER, PUBLIC_KEY_PACKET_HEADER]: payload_len, p_type = { PUBLIC_KEY_PACKET_HEADER: (KEY_LENGTH, 'pub key'), MESSAGE_PACKET_HEADER: (MESSAGE_LENGTH, 'message') }[header] payload = packet[1:1 + payload_len] trailer = packet[1 + payload_len:] user, contact = trailer.split(US_BYTE) print("{} - {} TxM > {} > {}".format(ts, p_type, user.decode(), contact.decode())) q_to_im.put((header, payload, user, contact)) m_to_rxm.put(header + payload + ORIGIN_USER_HEADER + contact) elif header == EXPORTED_FILE_HEADER: payload = packet[1:] file_name = os.urandom(8).hex() while os.path.isfile(file_name): file_name = os.urandom(8).hex() with open(file_name, 'wb+') as f: f.write(payload) print("{} - Exported file from TxM as {}".format( ts, file_name)) if unittest: break
class Gateway(object): """\ Gateway object is a wrapper for interfaces that connect Source/Destination Computer with the Networked Computer. """ def __init__(self, operation: str, local_test: bool, dd_sockets: bool, qubes: bool, ) -> None: """Create a new Gateway object.""" self.settings = GatewaySettings(operation, local_test, dd_sockets, qubes) self.tx_serial = None # type: Optional[serial.Serial] self.rx_serial = None # type: Optional[serial.Serial] self.rx_socket = None # type: Optional[multiprocessing.connection.Connection] self.tx_socket = None # type: Optional[multiprocessing.connection.Connection] # Initialize Reed-Solomon erasure code handler self.rs = RSCodec(2 * self.settings.session_serial_error_correction) # Set True when the serial interface is initially found so that # further interface searches know to announce disconnection. self.init_found = False if self.settings.local_testing_mode: if self.settings.software_operation in [TX, NC]: self.client_establish_socket() if self.settings.software_operation in [NC, RX]: self.server_establish_socket() elif not self.settings.qubes: self.establish_serial() def establish_serial(self) -> None: """Create a new Serial object. By setting the Serial object's timeout to 0, the method `Serial().read_all()` will return 0..N bytes where N is the serial interface buffer size (496 bytes for FTDI FT232R for example). This is not enough for large packets. However, in this case, `read_all` will return a) immediately when the buffer is full b) if no bytes are received during the time it would take to transmit the next byte of the datagram. This type of behaviour allows us to read 0..N bytes from the serial interface at a time, and add them to a bytearray buffer. In our implementation below, if the receiver side stops receiving data when it calls `read_all`, it starts a timer that is evaluated with every subsequent call of `read_all` that returns an empty string. If the timer exceeds the `settings.rx_receive_timeout` value (twice the time it takes to send the next byte with given baud rate), the gateway object will return the received packet. The timeout timer is triggered intentionally by the transmitter side Gateway object, that after each transmission sleeps for `settings.tx_inter_packet_delay` seconds. This value is set to twice the length of `settings.rx_receive_timeout`, or four times the time it takes to send one byte with given baud rate. """ try: self.tx_serial = self.rx_serial = serial.Serial(self.search_serial_interface(), self.settings.session_serial_baudrate, timeout=0) except SerialException: raise CriticalError("SerialException. Ensure $USER is in the dialout group by restarting this computer.") def send_over_qrexec(self, packet: bytes) -> None: """Send packet content over the Qubes qrexec RPC. More information at https://www.qubes-os.org/doc/qrexec/ The packet is encoded with ASCII85 to ensure e.g. 0x0a byte is not interpreted as line feed by the RPC service. """ target_vm = QUBES_NET_VM_NAME if self.settings.software_operation == TX else QUBES_DST_VM_NAME dom0_policy = QUBES_SRC_NET_POLICY if self.settings.software_operation == TX else QUBES_NET_DST_POLICY subprocess.Popen(['/usr/bin/qrexec-client-vm', target_vm, dom0_policy], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ).communicate(base64.b85encode(packet)) def write(self, orig_packet: bytes) -> None: """Add error correction data and output data via socket/serial interface. After outputting the packet via serial, sleep long enough to trigger the Rx-side timeout timer, or if local testing is enabled, add slight delay to simulate that introduced by the serial interface. """ packet = self.add_error_correction(orig_packet) if self.settings.local_testing_mode and self.tx_socket is not None: try: self.tx_socket.send(packet) time.sleep(LOCAL_TESTING_PACKET_DELAY) except BrokenPipeError: raise CriticalError("Relay IPC server disconnected.", exit_code=0) elif self.settings.qubes: self.send_over_qrexec(packet) elif self.tx_serial is not None: try: self.tx_serial.write(packet) self.tx_serial.flush() time.sleep(self.settings.tx_inter_packet_delay) except SerialException: self.establish_serial() self.write(orig_packet) def read_socket(self) -> bytes: """Read packet from socket interface.""" if self.rx_socket is None: raise CriticalError("Socket interface has not been initialized.") while True: try: packet = self.rx_socket.recv() # type: bytes return packet except KeyboardInterrupt: pass except EOFError: raise CriticalError("Relay IPC client disconnected.", exit_code=0) @staticmethod def read_qubes_buffer_file(buffer_file_dir: str = '') -> bytes: """Read packet from oldest buffer file.""" buffer_file_dir = buffer_file_dir if buffer_file_dir else BUFFER_FILE_DIR ensure_dir(f"{buffer_file_dir}/") while not any([f for f in os.listdir(buffer_file_dir) if f.startswith(BUFFER_FILE_NAME)]): time.sleep(0.001) tfc_buffer_file_numbers = [f[(len(BUFFER_FILE_NAME)+len('.')):] for f in os.listdir(buffer_file_dir) if f.startswith(BUFFER_FILE_NAME)] tfc_buffer_file_numbers = [n for n in tfc_buffer_file_numbers if n.isdigit()] tfc_buffer_files_in_order = [f"{BUFFER_FILE_NAME}.{n}" for n in sorted(tfc_buffer_file_numbers, key=int)] try: oldest_buffer_file = tfc_buffer_files_in_order[0] except IndexError: raise SoftError("No packet was available.", output=False) with open(f"{buffer_file_dir}/{oldest_buffer_file}", 'rb') as f: packet = f.read() try: packet = base64.b85decode(packet) except ValueError: raise SoftError("Error: Received packet had invalid Base85 encoding.") os.remove(f"{buffer_file_dir}/{oldest_buffer_file}") return packet def read_serial(self) -> bytes: """Read packet from serial interface. Read 0..N bytes from serial interface, where N is the buffer size of the serial interface. Once `read_buffer` has data, and the interface hasn't returned data long enough for the timer to exceed the timeout value, return received data. """ if self.rx_serial is None: raise CriticalError("Serial interface has not been initialized.") while True: try: start_time = 0.0 read_buffer = bytearray() while True: read = self.rx_serial.read_all() if read: start_time = time.monotonic() read_buffer.extend(read) else: if read_buffer: delta = time.monotonic() - start_time if delta > self.settings.rx_receive_timeout: return bytes(read_buffer) else: time.sleep(0.0001) except (EOFError, KeyboardInterrupt): pass except (OSError, SerialException): self.establish_serial() def read(self, buffer_file_dir: str = '') -> bytes: """Read data via socket/serial interface.""" if self.settings.local_testing_mode: return self.read_socket() if self.settings.qubes: return self.read_qubes_buffer_file(buffer_file_dir) return self.read_serial() def add_error_correction(self, packet: bytes) -> bytes: """Add error correction to packet that will be output. If the error correction setting is set to 1 or higher, TFC adds Reed-Solomon erasure codes to detect and correct errors during transmission over the serial interface. For more information on Reed-Solomon, see https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction https://www.cs.cmu.edu/~guyb/realworld/reedsolomon/reed_solomon_codes.html If error correction is set to 0, errors are only detected. This is done by using a BLAKE2b based, 128-bit checksum. If Qubes is used, Reed-Solomon is not used as it only slows down data transfer. """ if self.settings.session_serial_error_correction and not self.settings.qubes: packet = self.rs.encode(packet) else: packet = packet + hashlib.blake2b(packet, digest_size=PACKET_CHECKSUM_LENGTH).digest() return packet def detect_errors(self, packet: bytes) -> bytes: """Handle received packet error detection and/or correction.""" if self.settings.qubes: try: packet = base64.b85decode(packet) except ValueError: raise SoftError("Error: Received packet had invalid Base85 encoding.") if self.settings.session_serial_error_correction and not self.settings.qubes: try: packet, _ = self.rs.decode(packet) return bytes(packet) except ReedSolomonError: raise SoftError("Error: Reed-Solomon failed to correct errors in the received packet.", bold=True) else: packet, checksum = separate_trailer(packet, PACKET_CHECKSUM_LENGTH) if hashlib.blake2b(packet, digest_size=PACKET_CHECKSUM_LENGTH).digest() != checksum: raise SoftError("Warning! Received packet had an invalid checksum.", bold=True) return packet def search_serial_interface(self) -> str: """Search for a serial interface.""" if self.settings.session_usb_serial_adapter: search_announced = False if not self.init_found: phase("Searching for USB-to-serial interface", offset=len('Found')) while True: for f in sorted(os.listdir('/dev/')): if f.startswith('ttyUSB'): if self.init_found: time.sleep(1) phase('Found', done=True) if self.init_found: print_on_previous_line(reps=2) self.init_found = True return f'/dev/{f}' time.sleep(0.1) if self.init_found and not search_announced: phase("Serial adapter disconnected. Waiting for interface", head=1, offset=len('Found')) search_announced = True else: if self.settings.built_in_serial_interface in sorted(os.listdir('/dev/')): return f'/dev/{self.settings.built_in_serial_interface}' raise CriticalError(f"Error: /dev/{self.settings.built_in_serial_interface} was not found.") # Local testing def server_establish_socket(self) -> None: """Initialize the receiver (IPC server). The multiprocessing connection during local test does not utilize authentication keys* because a MITM-attack against the connection requires endpoint compromise, and in such situation, MITM attack is not nearly as effective as key/screen logging or RAM dump. * https://docs.python.org/3/library/multiprocessing.html#authentication-keys Similar to the case of standard mode of operation, all sensitive data that passes through the socket/serial interface and Relay Program is encrypted. A MITM attack between the sockets could of course be used to e.g. inject public keys, but like with all key exchanges, that would only work if the user neglects fingerprint verification. Another reason why the authentication key is useless, is the key needs to be pre-shared. This means there's two ways to share it: 1) Hard-code the key to source file from where malware could read it. 2) Force the user to manually copy the PSK from one program to another. This would change the workflow that the local test configuration tries to simulate. To conclude, the local test configuration should never be used under a threat model where endpoint security is of importance. """ try: socket_number = RP_LISTEN_SOCKET if self.settings.software_operation == NC else DST_LISTEN_SOCKET listener = multiprocessing.connection.Listener((LOCALHOST, socket_number)) self.rx_socket = listener.accept() except KeyboardInterrupt: graceful_exit() def client_establish_socket(self) -> None: """Initialize the transmitter (IPC client).""" try: target = RECEIVER if self.settings.software_operation == NC else RELAY phase(f"Connecting to {target}") while True: try: if self.settings.software_operation == TX: socket_number = SRC_DD_LISTEN_SOCKET if self.settings.data_diode_sockets else RP_LISTEN_SOCKET else: socket_number = DST_DD_LISTEN_SOCKET if self.settings.data_diode_sockets else DST_LISTEN_SOCKET try: self.tx_socket = multiprocessing.connection.Client((LOCALHOST, socket_number)) except ConnectionRefusedError: time.sleep(0.1) continue phase(DONE) break except socket.error: time.sleep(0.1) except KeyboardInterrupt: graceful_exit()