def set_property(given_props, prop, value): if not isinstance(value, type(MessageProperties.get_default(prop))): shim_print( f"Incompatible value type provided - Needed {type(MessageProperties.get_default(prop))}, got {type(value)}", level='error') else: given_props[prop] = value
def client_on_connected(ops): shim_print("ON CONNECTED RAN (CLIENT)") precon: Preconnection = Preconnection.preconnection_list[ops.preconnection_id] precon.number_of_connections += 1 con: Connection = Preconnection.initiated_connection_list[ops.preconnection_id] con.established_routine(ops) return NEAT_OK
def __init__(self, context, flow, ops, preconnection): self.__ops: neat_flow_operations = ops self.__context = context self.__flow = flow self.preconnection = preconnection self.props = copy.deepcopy(preconnection.transport_properties) self.number_of_connections = 0 self.connection_limit = math.inf self.HANDLE_STATE_LISTENING: Callable[[], None] = None self.HANDLE_STATE_LISTEN_ERROR: Callable[[], None] = None self.HANDLE_STATE_STOPPED: Callable[[], None] = None self.HANDLE_CONNECTION_RECEIVED: Callable[[Connection], None] = None # Todo: Find a more sophisticated way to keep track of listeners (or is it necessary?) Listener.listener_list[0] = self self.__ops.on_connected = self.handle_connected self.__ops.on_readable = handle_readable self.__ops.on_error = self.handle_error neat_set_operations(self.__context, self.__flow, self.__ops) if neat_accept(self.__context, self.__flow, preconnection.local_endpoint.port, None, 0): sys.exit("neat_accept failed") shim_print("A SERVER RUNNING NEAT STARTING FROM PYTHON 🎊")
def get(self, key, scope=None): """Function to retrieve message properties and metadata for framers. :param key: The message property or framer. :param scope: To retrieve message properties the scope parameter is omitted. The type of framer is passed as scope when querying for metadata. :return: If present value for message property / framer metadata """ # If scope is not set, try to get properties if not scope: if not isinstance(key, MessageProperties): shim_print("Invalid message property provided - ignoring", level='error') return None if key not in self.props: shim_print("Given message property is not present") return None else: return self.props[key] # Else a framer is given, try to fetch meta-data else: framer = scope value_dict = self.framer_values if key in value_dict and isinstance(framer, type(value_dict[key][0])): return value_dict[key] else: return None
def received_called(ops): try: ops.on_readable = handle_readable neat_set_operations(ops.ctx, ops.flow, ops) except: shim_print("An error occurred in the Python callback: {} - {}".format(sys.exc_info()[0], inspect.currentframe().f_code.co_name), level='error') backend.stop(ops.ctx) return NEAT_OK
def prohibit(self, prop: SelectionProperties) -> None: """ Set the preference level to prohibit for the passed selection property. :param prop: Selection property to set ``prohibit`` as preference level for. """ if not isinstance(prop, SelectionProperties): shim_print("Property given is not a selection property - ignoring", level='error') else: self.add(prop, PreferenceLevel.PROHIBIT)
def require(self, prop: SelectionProperties) -> None: """Set the preference level to require for the passed selection property. :param prop: Selection property to set ``require`` as preference level for. """ if not isinstance(prop, SelectionProperties): shim_print("Property given is not a selection property - ignoring", level='error') else: self.add(prop, PreferenceLevel.REQUIRE)
def write(ops, message): try: return neat_write(ops.ctx, ops.flow, message, len(message), None, 0) except: shim_print("An error occurred in the Python callback: {} - {}".format( sys.exc_info()[0], inspect.currentframe().f_code.co_name), level='error') stop(ops.ctx)
def default(self, prop: SelectionProperties): """ Set the default preference level for the given selection property. :param prop: Selection property to reset to default preference level for. """ if not isinstance(prop, SelectionProperties): shim_print("Property given is not a selection property - ignoring", level='error') else: self.add(prop, SelectionProperties.get_default(prop))
def set_connection_properties(self): try: (max_send, max_recv) = neat_get_max_buffer_sizes(self.flow) self.transport_properties.connection_properties[ConnectionProperties.MAXIMUM_MESSAGE_SIZE_ON_SEND] = max_send self.transport_properties.connection_properties[ConnectionProperties.MAXIMUM_MESSAGE_SIZE_ON_RECEIVE] = max_recv shim_print(f"Send buffer: {self.transport_properties.connection_properties[ConnectionProperties.MAXIMUM_MESSAGE_SIZE_ON_SEND]} - Receive buffer: {self.transport_properties.connection_properties[ConnectionProperties.MAXIMUM_MESSAGE_SIZE_ON_RECEIVE]}") except: shim_print("An error occurred in the Python callback: {} - {}".format(sys.exc_info()[0], inspect.currentframe().f_code.co_name), level='error')
def new_sent_message(self, connection, message_data, message_context, sent_handler, is_end_of_message): """ To provide an example, a simple protocol that adds a length as a header would receive the NewSentMessage event, create a data representation of the length of the Message data, and then send a block of data that is the concatenation of the length header and the original Message data. """ shim_print(f"Framer got message: {message_data}") new_block = "😘".encode() + message_data connection.message_framer.send(connection, new_block, message_context, sent_handler, is_end_of_message)
def handle_error(ops): listener: Listener = Listener.listener_list[0] shim_print( f"Listner error - {ListenErrorReasons.UNRESOLVED_LOCAL_ENDPOINT.value}" ) if listener.HANDLE_STATE_LISTEN_ERROR: listener.HANDLE_STATE_LISTEN_ERROR( ListenErrorReasons.UNRESOLVED_LOCAL_ENDPOINT) listener.stop() return NEAT_OK
def __filter_protocols(self, protocol_level, preference_level, candidates): remove_list = [] for prop, preference in self.selection_properties.items(): if preference == preference_level: for protocol in candidates: if protocols_services[protocol][prop] == protocol_level: if protocol not in remove_list: shim_print(f'{protocol.name} is removed as it does not satisfy {prop.name} requirements') remove_list.append(protocol) return [protocol for protocol in candidates if protocol not in remove_list]
def set_property(connection_props, prop_to_set, value): if not ConnectionProperties.is_valid_property(prop_to_set): shim_print("Given property not valid - Ignoring...", level='error') elif ConnectionProperties.is_read_only(prop_to_set): shim_print("Given property is read only - Ignoring...", level='error') else: if prop_to_set is ConnectionProperties.USER_TIMEOUT_TCP: ConnectionProperties.set_tcp_uto(connection_props, value) else: connection_props[prop_to_set] = value
def __init__(self, preconnection, connection_type, listener=None, parent=None): # NEAT specific self.ops = None self.context = None self.flow = None self.transport_stack = None # Map connection for later callbacks fired Connection.static_counter += 1 self.connection_id = Connection.static_counter Connection.connection_list[self.connection_id] = self # Python specific self.connection_type = connection_type self.clone_counter = 0 self.parent = parent self.test_counter = 0 self.msg_list: queue.PriorityQueue = queue.PriorityQueue() self.messages_passed_to_back_end = [] self.receive_request_queue = [] self.buffered_message_data_object: MessageDataObject = None self.partitioned_message_data_object: MessageDataObject = None self.message_sent_with_initiate = None self.received_called_before_established = None self.clone_error_handler = {} self.connection_group = [] self.local_endpoint = None self.remote_endpoint = None self.framer_placeholder: FramerPlaceholder = FramerPlaceholder(connection=self) self.stack_supports_message_boundary_preservation = False # if this connection is a result from a clone, add parent if self.parent: shim_print(f"Connection is a result of cloning - Parent {self.parent}") self.connection_group.append(self.parent) self.close_called = False self.batch_in_session = False self.final_message_received = False self.final_message_passed = False self.HANDLE_STATE_READY: Callable[[Connection], None] = None #: Handler for when the connection transitions to ready state self.HANDLE_STATE_CLOSED: Callable[[Connection], None] = None #: Handler for when the connection transitions to closed state self.HANDLE_STATE_CONNECTION_ERROR: Callable[[Connection], None] = None #: Handler for when the connection gets experiences a connection error self.preconnection = preconnection self.transport_properties = self.preconnection.transport_properties self.listener = listener self.message_framer: message_framer.MessageFramer = preconnection.message_framer self.state = ConnectionState.ESTABLISHING
def read(ops, size): shim_print("ON READABLE") buffer = charArr(size) bytes_read = new_uint32_tp() try: neat_read(ops.ctx, ops.flow, buffer, size - 1, bytes_read, None, 0) byte_array = bytearray(uint32_tp_value(bytes_read)) for i in range(uint32_tp_value(bytes_read)): byte_array[i] = buffer[i] return byte_array except: shim_print("An error occurred in the Python callback: {}".format( sys.exc_info()[0]))
def add(self, key, value, scope: Framer = None): # If scope is not set, try to add message property if not scope: if not isinstance(key, MessageProperties): shim_print("Invalid message property provided - ignoring", level='error') else: self.props[key] = value # Else a framer is given, try to add meta-data else: framer = scope if not isinstance(framer, Framer): shim_print("Provided argument is not a valid framer - ignoring", level='error') self.framer_values[key] = (framer, value)
def handle_received_data(self, connection): """ To provide an example, a simple protocol that parses a length as a header value would receive the HandleReceivedData event, and call Parse with a minimum and maximum set to the length of the header field. Once the parse succeeded, it would call AdvanceReceiveCursor with the length of the header field, and then call DeliverAndAdvanceReceiveCursor with the length of the body that was parsed from the header, marking the new Message as complete. """ shim_print("Framer handles received data") header, context, is_end = connection.message_framer.parse(connection, 4, 4) length = int.from_bytes(header, byteorder='big') shim_print(f"Header is {length}") connection.message_framer.advance_receive_cursor(connection, 4) connection.message_framer.deliver_and_advance_receive_cursor(connection, context, length, True)
def handle_closed(ops): try: connection: Connection = Connection.connection_list[ops.connection_id] shim_print(f"HANDLE CLOSED - connection {ops.connection_id}") if connection.HANDLE_STATE_CLOSED: connection.HANDLE_STATE_CLOSED(connection) # If a Connection becomes finished before a requested Receive action can be satisfied, # the implementation should deliver any partial Message content outstanding..." if connection.receive_request_queue: if connection.buffered_message_data_object: handler, min_length, max_length = connection.receive_request_queue.pop(0) shim_print("Dispatching received partial as connection is closing and there is buffered data") message_context = MessageContext() handler(connection, connection.buffered_message_data_object, message_context, True, None) # "...or if none is available, an indication that there will be no more received Messages." else: shim_print("Connection closed, there will be no more received messages") # TODO: Should this be thrown as an error (error event?) #if connection.connection_type == 'active': # should check if there is any cloned connections etc... # backend.stop(ops.ctx) except: shim_print("An error occurred in the Python callback: {} - {}".format(sys.exc_info()[0], inspect.currentframe().f_code.co_name), level='error') backend.stop(ops.ctx) return NEAT_OK
def established_routine(self, ops): self.state = ConnectionState.ESTABLISHED self.ops = ops self.context = ops.ctx self.flow = ops.flow self.transport_stack = SupportedProtocolStacks(ops.transport_protocol) self.transport_properties = copy.deepcopy(self.preconnection.transport_properties) if self.transport_properties.buffer_capacity: self.transport_properties.dispatch_capacity_profile(self.context, self.flow) self.transport_properties.buffer_capacity = None if SupportedProtocolStacks.get_service_level(self.transport_stack, SelectionProperties.PRESERVE_MSG_BOUNDARIES) == ServiceLevel.INTRINSIC_SERVICE: self.stack_supports_message_boundary_preservation = True self.ops.connection_id = self.connection_id shim_print(f"Connection [ID: {self.connection_id}] established - transport used: {self.transport_stack.name}", level='msg') self.set_connection_properties() # Fire off appropriate event handler (if present) if self.HANDLE_STATE_READY: self.HANDLE_STATE_READY(self) ops.on_all_written = handle_all_written ops.on_close = handle_closed neat_set_operations(ops.ctx, ops.flow, ops) #res, res_json = neat_get_stats(self.context) #json_rep = json.loads(res_json) #shim_print(json.dumps(json_rep, indent=4, sort_keys=True)) self.local_endpoint = self.crate_and_populate_endpoint() self.remote_endpoint = self.crate_and_populate_endpoint(local=False) # Protocol stack specific logic # Set TCP UTO if enabled if self.transport_stack is SupportedProtocolStacks.TCP: is_linux = sys.platform == 'linux' uto_enabled = self.transport_properties.connection_properties[ConnectionProperties.USER_TIMEOUT_TCP][TCPUserTimeout.USER_TIMEOUT_ENABLED] if is_linux and uto_enabled: new_timeout = self.transport_properties.connection_properties[ConnectionProperties.USER_TIMEOUT_TCP][TCPUserTimeout.ADVERTISED_USER_TIMEOUT] backend.set_timeout(self.context, self.flow, new_timeout) if self.message_sent_with_initiate: message_data, sent_handler, message_context = self.message_sent_with_initiate self.send(message_data, sent_handler=sent_handler, message_context=message_context) if self.received_called_before_established: handler, min_incomplete_length, max_length = self.received_called_before_established self.receive(handler, min_incomplete_length, max_length)
def handle_all_written(ops): try: close = False connection = Connection.connection_list[ops.connection_id] shim_print(f"ALL WRITTEN - connection {ops.connection_id}") if not connection.messages_passed_to_back_end: return NEAT_OK message, message_context, handler = connection.messages_passed_to_back_end.pop(0) if handler: handler(connection, None) if connection.close_called and len(connection.messages_passed_to_back_end) == 0: shim_print("All messages passed down to the network layer - calling close") close = True elif message_context.props[MessageProperties.FINAL] is True: shim_print("Message marked final has been completely sent, closing connection / sending FIN") close = True if close: neat_close(connection.__ops.ctx, connection.__ops.flow) except: shim_print("An error occurred: {}".format(sys.exc_info()[0])) backend.stop(ops.ctx) return NEAT_OK
def handle_connected(ops): listener = Listener.listener_list[0] if listener.connection_limit > listener.number_of_connections: listener.number_of_connections += 1 new_connection = Connection(listener.preconnection, 'passive', listener) new_connection.established_routine(ops) if listener.HANDLE_CONNECTION_RECEIVED: listener.HANDLE_CONNECTION_RECEIVED(new_connection) else: shim_print("Connection limit is reached!") return NEAT_OK
def pass_candidates_to_back_end(candidates, context, flow): if len(candidates) is 1: candiates_to_backend = json.dumps( {"transport": { "value": candidates.pop(0).name, "precedence": 1 }}) else: candiates_to_backend = json.dumps({ "transport": { "value": [candidate.name for candidate in candidates], "precedence": 2 } }) shim_print(candiates_to_backend) neat_set_property(context, flow, candiates_to_backend)
def initiate(self, timeout=None) -> Connection: """ Initiate (Active open) is the Action of establishing a Connection to a Remote Endpoint presumed to be listening for incoming Connection requests. Active open is used by clients in client-server interactions. Note that start() must be called on the Preconnection. :param timeout: The timeout parameter specifies how long to wait before aborting Active open. """ if not self.remote_endpoint: shim_print("Initiate error - Remote Endpoint MUST be specified if when calling initiate on the preconnection", level="error") backend.clean_up(self.__context) sys.exit(1) self.__ops.on_error = self.on_initiate_error self.__ops.on_timeout = self.on_initiate_timeout self.__ops.on_connected = self.client_on_connected neat_set_operations(self.__context, self.__flow, self.__ops) candidates = self.transport_properties.select_protocol_stacks_with_selection_properties() if candidates is None: shim_print("Unfulfilled error - No stacks meeting the given constraints by properties", level="error") if self.unfulfilled_handler: self.unfulfilled_handler() else: backend.clean_up(self.__context) return backend.pass_candidates_to_back_end(candidates, self.__context, self.__flow) if self.security_parameteres: self.register_security() if backend.initiate(self.__context, self.__flow, self.remote_endpoint.address, self.remote_endpoint.port, 100): pass if timeout: set_initiate_timer(self.__context, self.__flow, self.__ops, timeout) if self.remote_endpoint.interface: send = json.dumps({"interface": {"value": self.remote_endpoint.interface, "precedence": 1}}) neat_set_property(self.__context, self.__flow, send) new_con = Connection(self, 'active') Preconnection.initiated_connection_list[self.id] = new_con return new_con
def listen(self) -> Listener: """Listen (Passive open) is the Action of waiting for Connections from remote Endpoints. Before listening the transport system will resolve transport properties for candidate protocol stacks. A local endpoint must be passed to the Preconnection prior to listen. :return: A listener object. """ if not self.local_endpoint: shim_print("Listen error - Local Endpoint MUST be specified if when calling listen on the preconnection", level="error") backend.clean_up(self.__context) sys.exit(1) candidates = self.transport_properties.select_protocol_stacks_with_selection_properties() if candidates is None: shim_print("Unfulfilled error - No stacks meeting the given constraints by properties", level="error") if self.unfulfilled_handler: self.unfulfilled_handler() else: backend.clean_up(self.__context) backend.pass_candidates_to_back_end(candidates, self.__context, self.__flow) if self.security_parameteres: self.register_security(is_server=True) shim_print("LISTEN!") listener = Listener(self.__context, self.__flow, self.__ops, self) return listener
def add_to_message_queue(self, message_context, message_data, sent_handler, end_of_message): # Check if lifetime is set expired_epoch = None if message_context.props[MessageProperties.LIFETIME] < math.inf: expired_epoch = int(time.time()) + message_context.props[MessageProperties.LIFETIME] shim_print(f"Send Actions expires {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(expired_epoch))}") # "A final message must always be sorted to the end of the list" if message_context.props[MessageProperties.FINAL]: priority = math.inf self.final_message_passed = True else: priority = -message_context.props[MessageProperties.PRIORITY] if self.batch_in_session: # Check if batch_struct is already present (i.e if this is the first send in batch or not) if isinstance(self.msg_list[-1], List): self.msg_list[-1].append(message_data, message_context) # TODO: Remember handler else: self.msg_list.append([message_data, message_context]) # TODO: Remember handler else: item = MessageQueueObject(priority, (message_data, message_context, sent_handler, expired_epoch)) self.msg_list.put(item) message_passed(self.ops)
def handle_writable(ops): try: connection: Connection = Connection.connection_list[ops.connection_id] shim_print(f"ON WRITABLE CALLBACK - connection {connection.connection_id}") # Socket is writable, write if any messages passed to the transport system if not connection.msg_list.empty(): # Todo: Should we do coalescing of batch sends here? message_to_be_sent, context, handler, expired_epoch = connection.msg_list.get().item # If lifetime was set, check if send action has expired if expired_epoch and expired_epoch < int(time.time()): shim_print("""Send action expired: Expired: {} - Now: {}""".format(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(expired_epoch)), time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(time.time())))), level='error') handler(connection, SendErrorReason.MESSAGE_TIMEOUT) return NEAT_OK neat_result = backend.write(ops, message_to_be_sent) if neat_result: if neat_result == NEAT_ERROR_MESSAGE_TOO_BIG: reason = SendErrorReason.MESSAGE_TOO_LARGE elif neat_result == NEAT_ERROR_IO: reason = SendErrorReason.FAILURE_UNDERLYING_STACK shim_print(f"SendError - {reason.value}") handler(connection, reason) return # Keep message until NEAT confirms sending with all_written connection.messages_passed_to_back_end.append((message_to_be_sent, context, handler)) else: pass shim_print("WHAT") ops.on_writable = None neat_set_operations(ops.ctx, ops.flow, ops) return NEAT_OK except: shim_print("An error occurred in the Python callback: {} - {}".format(sys.exc_info()[0], inspect.currentframe().f_code.co_name), level='error') backend.stop(ops.ctx) return NEAT_OK
def set_tcp_uto(connection_props, values): prev_timeout_val = connection_props[ ConnectionProperties.USER_TIMEOUT_TCP][ TCPUserTimeout.ADVERTISED_USER_TIMEOUT] prev_enabled_bool = connection_props[ ConnectionProperties.USER_TIMEOUT_TCP][ TCPUserTimeout.USER_TIMEOUT_ENABLED] if sys.platform != 'linux': shim_print("TCP UTO is only available on Linux", level='error') return for tcp_uto_parameter, tcp_uto_value in values.items(): if tcp_uto_parameter is TCPUserTimeout.ADVERTISED_USER_TIMEOUT: connection_props[ConnectionProperties.USER_TIMEOUT_TCP][ TCPUserTimeout.ADVERTISED_USER_TIMEOUT] = tcp_uto_value elif tcp_uto_parameter is TCPUserTimeout.USER_TIMEOUT_ENABLED: connection_props[ConnectionProperties.USER_TIMEOUT_TCP][ TCPUserTimeout.USER_TIMEOUT_ENABLED] = tcp_uto_value elif tcp_uto_parameter is TCPUserTimeout.CHANGEABLE: connection_props[ConnectionProperties.USER_TIMEOUT_TCP][ TCPUserTimeout.CHANGEABLE] = tcp_uto_value else: shim_print("No valid TCP UTO parameter", level='error')
def receive(self, handler: Callable[['Connection', 'MessageDataObject', MessageContext, bool, bool], None], min_incomplete_length: int = None, max_length: int = math.inf) -> None: """ As with sending, data is received in terms of Messages. Receiving is an asynchronous operation, in which each call to Receive enqueues a request to receive new data from the connection. Once data has been received, or an error is encountered, an event will be delivered to complete the Receive request. :param handler: The function to handle the event delivered during completion, which includes both potential errors and successfully received data. :param min_incomplete_length: The default ``None`` value indicates that only complete messages should be delivered. Setting it to anything other than this will trigger a receive event only when at least that many bytes are available. :param max_length: Indicates the maximum size of a message in bytes the application is prepared to receive. Incoming messages larger than this will be delivered in received partial events. To determine whether the received event is a partial event the application is able to check whether the variable ``is_end_of_message`` holds the boolean value `False`, which indicates a partial event, while a None value indicates a complete message being delivered. """ shim_print("RECEIVED CALLED") if self.state == ConnectionState.ESTABLISHING: self.received_called_before_established = (handler, min_incomplete_length, max_length) return if self.partitioned_message_data_object: if self.partitioned_message_data_object.length > max_length: new_partitioned_message_object = self.partitioned_message_data_object.partition(max_length) to_be_sent = self.partitioned_message_data_object self.partitioned_message_data_object = new_partitioned_message_object handler(self, to_be_sent, False, None) else: to_be_sent = self.partitioned_message_data_object self.partitioned_message_data_object = None handler(self, to_be_sent, True, None) else: if self.close_called: shim_print("Closed is called, no further reception is possible") return self.receive_request_queue.append((handler, min_incomplete_length, max_length)) # If there is only one request in the queue, this means it was empty and we need to set callback if len(self.receive_request_queue) is 1: received_called(self.ops)
def clone(self, clone_error_handler: Callable[['Connection'], None]) -> None: """Calling Clone on a Connection yields a group of two Connections: the parent Connection on which Clone was called, and the resulting cloned Connection. These connections are "entangled" with each other, and become part of a Connection Group. Calling Clone on any of these two Connections adds a third Connection to the Connection Group, and so on. Connections in a Connection Group generally share :py:class:`connection_properties`. However, there are exceptions, such as the priority property, which obviously will not trigger a change for all connections in the connection group. As with all other properties, priority is copied to the new Connection when calling Clone(). :param clone_error_handler: A function to handle the event which fires when the cloning operation fails. The connection which clone was called on is sent with the handler. """ try: shim_print("CLONE") # NEAT boilerplate flow = neat_new_flow(self.context) ops = neat_flow_operations() # Create new connection and register eventual error_handler new_connection = Connection(self.preconnection, 'active', parent=self) new_connection.clone_error_handler = clone_error_handler # Set connection id for the new clone in the operations struct ops.connection_id = new_connection.connection_id # Set callbacks to properly handle clone establishment / error ops.on_error = on_clone_error ops.on_connected = handle_clone_ready neat_set_operations(self.context, flow, ops) backend.pass_candidates_to_back_end([self.transport_stack], self.context, flow) # Asynchronously call NEAT to create a new connection neat_open(self.context, flow, self.preconnection.remote_endpoint.address, self.preconnection.remote_endpoint.port, None, 0) return new_connection except: shim_print("An error occurred in the Python callback: {} - {}".format(sys.exc_info()[0], inspect.currentframe().f_code.co_name),level='error') backend.stop(self.context)