from twisted.web._stan import Tag, slot, Comment, CDATA, CharRef from twisted.web.iweb import ITemplateLoader from twisted.logger import Logger TEMPLATE_NAMESPACE = 'http://twistedmatrix.com/ns/twisted.web.template/0.1' # Go read the definition of NOT_DONE_YET. For lulz. This is totally # equivalent. And this turns out to be necessary, because trying to import # NOT_DONE_YET in this module causes a circular import which we cannot escape # from. From which we cannot escape. Etc. glyph is okay with this solution for # now, and so am I, as long as this comment stays to explain to future # maintainers what it means. ~ C. # # See http://twistedmatrix.com/trac/ticket/5557 for progress on fixing this. NOT_DONE_YET = 1 _moduleLog = Logger() class _NSContext: """ A mapping from XML namespaces onto their prefixes in the document. """ def __init__(self, parent=None): """ Pull out the parent's namespaces, if there's no parent then default to XML. """ self.parent = parent if parent is not None: self.nss = OrderedDict(parent.nss) else:
class H2Connection(Protocol, TimeoutMixin): """ A class representing a single HTTP/2 connection. This implementation of L{IProtocol} works hand in hand with L{H2Stream}. This is because we have the requirement to register multiple producers for a single HTTP/2 connection, one for each stream. The standard Twisted interfaces don't really allow for this, so instead there's a custom interface between the two objects that allows them to work hand-in-hand here. @ivar conn: The HTTP/2 connection state machine. @type conn: L{h2.connection.H2Connection} @ivar streams: A mapping of stream IDs to L{H2Stream} objects, used to call specific methods on streams when events occur. @type streams: L{dict}, mapping L{int} stream IDs to L{H2Stream} objects. @ivar priority: A HTTP/2 priority tree used to ensure that responses are prioritised appropriately. @type priority: L{priority.PriorityTree} @ivar _consumerBlocked: A flag tracking whether or not the L{IConsumer} that is consuming this data has asked us to stop producing. @type _consumerBlocked: L{bool} @ivar _sendingDeferred: A L{Deferred} used to restart the data-sending loop when more response data has been produced. Will not be present if there is outstanding data still to send. @type _consumerBlocked: A L{twisted.internet.defer.Deferred}, or L{None} @ivar _outboundStreamQueues: A map of stream IDs to queues, used to store data blocks that are yet to be sent on the connection. These are used both to handle producers that do not respect L{IConsumer} but also to allow priority to multiplex data appropriately. @type _outboundStreamQueues: A L{dict} mapping L{int} stream IDs to L{collections.deque} queues, which contain either L{bytes} objects or C{_END_STREAM_SENTINEL}. @ivar _sender: A handle to the data-sending loop, allowing it to be terminated if needed. @type _sender: L{twisted.internet.task.LoopingCall} @ivar abortTimeout: The number of seconds to wait after we attempt to shut the transport down cleanly to give up and forcibly terminate it. This is only used when we time a connection out, to prevent errors causing the FD to get leaked. If this is L{None}, we will wait forever. @type abortTimeout: L{int} @ivar _abortingCall: The L{twisted.internet.base.DelayedCall} that will be used to forcibly close the transport if it doesn't close cleanly. @type _abortingCall: L{twisted.internet.base.DelayedCall """ factory = None site = None abortTimeout = 15 _log = Logger() _abortingCall = None def __init__(self, reactor=None): self.conn = h2.connection.H2Connection(client_side=False, header_encoding=None) self.streams = {} self.priority = priority.PriorityTree() self._consumerBlocked = None self._sendingDeferred = None self._outboundStreamQueues = {} self._streamCleanupCallbacks = {} self._stillProducing = True if reactor is None: from twisted.internet import reactor self._reactor = reactor # Start the data sending function. self._reactor.callLater(0, self._sendPrioritisedData) # Implementation of IProtocol def connectionMade(self): """ Called by the reactor when a connection is received. May also be called by the L{twisted.web.http._GenericHTTPChannelProtocol} during upgrade to HTTP/2. """ self.setTimeout(self.timeOut) self.conn.initiate_connection() self.transport.write(self.conn.data_to_send()) def dataReceived(self, data): """ Called whenever a chunk of data is received from the transport. @param data: The data received from the transport. @type data: L{bytes} """ self.resetTimeout() try: events = self.conn.receive_data(data) except h2.exceptions.ProtocolError: # A remote protocol error terminates the connection. dataToSend = self.conn.data_to_send() self.transport.write(dataToSend) self.transport.loseConnection() self.connectionLost("Protocol error from peer.") return for event in events: if isinstance(event, h2.events.RequestReceived): self._requestReceived(event) elif isinstance(event, h2.events.DataReceived): self._requestDataReceived(event) elif isinstance(event, h2.events.StreamEnded): self._requestEnded(event) elif isinstance(event, h2.events.StreamReset): self._requestAborted(event) elif isinstance(event, h2.events.WindowUpdated): self._handleWindowUpdate(event) elif isinstance(event, h2.events.PriorityUpdated): self._handlePriorityUpdate(event) elif isinstance(event, h2.events.ConnectionTerminated): self.transport.loseConnection() self.connectionLost("Shutdown by remote peer") dataToSend = self.conn.data_to_send() if dataToSend: self.transport.write(dataToSend) def timeoutConnection(self): """ Called when the connection has been inactive for L{self.timeOut<twisted.protocols.policies.TimeoutMixin.timeOut>} seconds. Cleanly tears the connection down, attempting to notify the peer if needed. We override this method to add two extra bits of functionality: - We want to log the timeout. - We want to send a GOAWAY frame indicating that the connection is being terminated, and whether it was clean or not. We have to do this before the connection is torn down. """ self._log.info("Timing out client {client}", client=self.transport.getPeer()) # Check whether there are open streams. If there are, we're going to # want to use the error code PROTOCOL_ERROR. If there aren't, use # NO_ERROR. if (self.conn.open_outbound_streams > 0 or self.conn.open_inbound_streams > 0): error_code = h2.errors.PROTOCOL_ERROR else: error_code = h2.errors.NO_ERROR self.conn.close_connection(error_code=error_code) self.transport.write(self.conn.data_to_send()) # Don't let the client hold this connection open too long. if self.abortTimeout is not None: # We use self.callLater because that's what TimeoutMixin does, even # though we have a perfectly good reactor sitting around. See # https://twistedmatrix.com/trac/ticket/8488. self._abortingCall = self.callLater(self.abortTimeout, self.forceAbortClient) # We're done, throw the connection away. self.transport.loseConnection() def forceAbortClient(self): """ Called if C{abortTimeout} seconds have passed since the timeout fired, and the connection still hasn't gone away. This can really only happen on extremely bad connections or when clients are maliciously attempting to keep connections open. """ self._log.info("Forcibly timing out client: {client}", client=self.transport.getPeer()) self.transport.abortConnection() def connectionLost(self, reason): """ Called when the transport connection is lost. Informs all outstanding response handlers that the connection has been lost, and cleans up all internal state. """ self._stillProducing = False self.setTimeout(None) for stream in self.streams.values(): stream.connectionLost(reason) for streamID in list(self.streams.keys()): self._requestDone(streamID) # If we were going to force-close the transport, we don't have to now. if self._abortingCall is not None: self._abortingCall.cancel() self._abortingCall = None # Implementation of IPushProducer # # Here's how we handle IPushProducer. We have multiple outstanding # H2Streams. Each of these exposes an IConsumer interface to the response # handler that allows it to push data into the H2Stream. The H2Stream then # writes the data into the H2Connection object. # # The H2Connection needs to manage these writes to account for: # # - flow control # - priority # # We manage each of these in different ways. # # For flow control, we simply use the equivalent of the IPushProducer # interface. We simply tell the H2Stream: "Hey, you can't send any data # right now, sorry!". When that stream becomes unblocked, we free it up # again. This allows the H2Stream to propagate this backpressure up the # chain. # # For priority, we need to keep a backlog of data frames that we can send, # and interleave them appropriately. This backlog is most sensibly kept in # the H2Connection object itself. We keep one queue per stream, which is # where the writes go, and then we have a loop that manages popping these # streams off in priority order. # # Logically then, we go as follows: # # 1. Stream calls writeDataToStream(). This causes a DataFrame to be placed # on the queue for that stream. It also informs the priority # implementation that this stream is unblocked. # 2. The _sendPrioritisedData() function spins in a tight loop. Each # iteration it asks the priority implementation which stream should send # next, and pops a data frame off that stream's queue. If, after sending # that frame, there is no data left on that stream's queue, the function # informs the priority implementation that the stream is blocked. # # If all streams are blocked, or if there are no outstanding streams, the # _sendPrioritisedData function waits to be awoken when more data is ready # to send. # # Note that all of this only applies to *data*. Headers and other control # frames deliberately skip this processing as they are not subject to flow # control or priority constraints. def stopProducing(self): """ Stop producing data. This tells the L{H2Connection} that its consumer has died, so it must stop producing data for good. """ self.connectionLost("stopProducing") def pauseProducing(self): """ Pause producing data. Tells the L{H2Connection} that it has produced too much data to process for the time being, and to stop until resumeProducing() is called. """ self._consumerBlocked = Deferred() def resumeProducing(self): """ Resume producing data. This tells the L{H2Connection} to re-add itself to the main loop and produce more data for the consumer. """ if self._consumerBlocked is not None: d = self._consumerBlocked self._consumerBlocked = None d.callback(None) def _sendPrioritisedData(self, *args): """ The data sending loop. This function repeatedly calls itself, either from L{Deferred}s or from L{reactor.callLater<twisted.internet.interfaces.IReactorTime.callLater>} This function sends data on streams according to the rules of HTTP/2 priority. It ensures that the data from each stream is interleved according to the priority signalled by the client, making sure that the connection is used with maximal efficiency. This function will execute if data is available: if all data is exhausted, the function will place a deferred onto the L{H2Connection} object and wait until it is called to resume executing. """ # If producing has stopped, we're done. Don't reschedule ourselves if not self._stillProducing: return stream = None while stream is None: try: stream = next(self.priority) except priority.DeadlockError: # All streams are currently blocked or not progressing. Wait # until a new one becomes available. assert self._sendingDeferred is None self._sendingDeferred = Deferred() self._sendingDeferred.addCallback(self._sendPrioritisedData) return # Wait behind the transport. if self._consumerBlocked is not None: self._consumerBlocked.addCallback(self._sendPrioritisedData) return remainingWindow = self.conn.local_flow_control_window(stream) frameData = self._outboundStreamQueues[stream].popleft() maxFrameSize = min(self.conn.max_outbound_frame_size, remainingWindow) if frameData is _END_STREAM_SENTINEL: # There's no error handling here even though this can throw # ProtocolError because we really shouldn't encounter this problem. # If we do, that's a nasty bug. self.conn.end_stream(stream) self.transport.write(self.conn.data_to_send()) # Clean up the stream self._requestDone(stream) else: # Respect the max frame size. if len(frameData) > maxFrameSize: excessData = frameData[maxFrameSize:] frameData = frameData[:maxFrameSize] self._outboundStreamQueues[stream].appendleft(excessData) # There's deliberately no error handling here, because this just # absolutely should not happen. # If for whatever reason the max frame length is zero and so we # have no frame data to send, don't send any. if frameData: self.conn.send_data(stream, frameData) self.transport.write(self.conn.data_to_send()) # If there's no data left, this stream is now blocked. if not self._outboundStreamQueues[stream]: self.priority.block(stream) # Also, if the stream's flow control window is exhausted, tell it # to stop. if self.remainingOutboundWindow(stream) <= 0: self.streams[stream].flowControlBlocked() self._reactor.callLater(0, self._sendPrioritisedData) # Internal functions. def _requestReceived(self, event): """ Internal handler for when a request has been received. @param event: The Hyper-h2 event that encodes information about the received request. @type event: L{h2.events.RequestReceived} """ stream = H2Stream(event.stream_id, self, event.headers, self.requestFactory, self.site, self.factory) self.streams[event.stream_id] = stream self._streamCleanupCallbacks[event.stream_id] = Deferred() self._outboundStreamQueues[event.stream_id] = deque() # Add the stream to the priority tree but immediately block it. try: self.priority.insert_stream(event.stream_id) except priority.DuplicateStreamError: # Stream already in the tree. This can happen if we received a # PRIORITY frame before a HEADERS frame. Just move on: we set the # stream up properly in _handlePriorityUpdate. pass else: self.priority.block(event.stream_id) def _requestDataReceived(self, event): """ Internal handler for when a chunk of data is received for a given request. @param event: The Hyper-h2 event that encodes information about the received data. @type event: L{h2.events.DataReceived} """ stream = self.streams[event.stream_id] stream.receiveDataChunk(event.data, event.flow_controlled_length) def _requestEnded(self, event): """ Internal handler for when a request is complete, and we expect no further data for that request. @param event: The Hyper-h2 event that encodes information about the completed stream. @type event: L{h2.events.StreamEnded} """ stream = self.streams[event.stream_id] stream.requestComplete() def _requestAborted(self, event): """ Internal handler for when a request is aborted by a remote peer. @param event: The Hyper-h2 event that encodes information about the reset stream. @type event: L{h2.events.StreamReset} """ stream = self.streams[event.stream_id] stream.connectionLost("Stream reset") self._requestDone(event.stream_id) def _handlePriorityUpdate(self, event): """ Internal handler for when a stream priority is updated. @param event: The Hyper-h2 event that encodes information about the stream reprioritization. @type event: L{h2.events.PriorityUpdated} """ try: self.priority.reprioritize( stream_id=event.stream_id, depends_on=event.depends_on or None, weight=event.weight, exclusive=event.exclusive, ) except priority.MissingStreamError: # A PRIORITY frame arrived before the HEADERS frame that would # trigger us to insert the stream into the tree. That's fine: we # can create the stream here and mark it as blocked. self.priority.insert_stream( stream_id=event.stream_id, depends_on=event.depends_on or None, weight=event.weight, exclusive=event.exclusive, ) self.priority.block(event.stream_id) def writeHeaders(self, version, code, reason, headers, streamID): """ Called by L{twisted.web.http.Request} objects to write a complete set of HTTP headers to a stream. @param version: The HTTP version in use. Unused in HTTP/2. @type version: L{bytes} @param code: The HTTP status code to write. @type code: L{bytes} @param reason: The HTTP reason phrase to write. Unused in HTTP/2. @type reason: L{bytes} @param headers: The headers to write to the stream. @type headers: L{twisted.web.http_headers.Headers} @param streamID: The ID of the stream to write the headers to. @type streamID: L{int} """ headers.insert(0, (b':status', code)) try: self.conn.send_headers(streamID, headers) except h2.exceptions.StreamClosedError: # Stream was closed by the client at some point. We need to not # explode here: just swallow the error. That's what write() does # when a connection is lost, so that's what we do too. return else: self.transport.write(self.conn.data_to_send()) def writeDataToStream(self, streamID, data): """ May be called by L{H2Stream} objects to write response data to a given stream. Writes a single data frame. @param streamID: The ID of the stream to write the data to. @type streamID: L{int} @param data: The data chunk to write to the stream. @type data: L{bytes} """ self._outboundStreamQueues[streamID].append(data) # There's obviously no point unblocking this stream and the sending # loop if the data can't actually be sent, so confirm that there's # some room to send data. if self.conn.local_flow_control_window(streamID) > 0: self.priority.unblock(streamID) if self._sendingDeferred is not None: d = self._sendingDeferred self._sendingDeferred = None d.callback(streamID) if self.remainingOutboundWindow(streamID) <= 0: self.streams[streamID].flowControlBlocked() def endRequest(self, streamID): """ Called by L{H2Stream} objects to signal completion of a response. @param streamID: The ID of the stream to write the data to. @type streamID: L{int} """ self._outboundStreamQueues[streamID].append(_END_STREAM_SENTINEL) self.priority.unblock(streamID) if self._sendingDeferred is not None: d = self._sendingDeferred self._sendingDeferred = None d.callback(streamID) def abortRequest(self, streamID): """ Called by L{H2Stream} objects to request early termination of a stream. This emits a RstStream frame and then removes all stream state. @param streamID: The ID of the stream to write the data to. @type streamID: L{int} """ self.conn.reset_stream(streamID) self.transport.write(self.conn.data_to_send()) self._requestDone(streamID) def _requestDone(self, streamID): """ Called internally by the data sending loop to clean up state that was being used for the stream. Called when the stream is complete. @param streamID: The ID of the stream to clean up state for. @type streamID: L{int} """ del self._outboundStreamQueues[streamID] self.priority.remove_stream(streamID) del self.streams[streamID] cleanupCallback = self._streamCleanupCallbacks.pop(streamID) cleanupCallback.callback(streamID) def remainingOutboundWindow(self, streamID): """ Called to determine how much room is left in the send window for a given stream. Allows us to handle blocking and unblocking producers. @param streamID: The ID of the stream whose flow control window we'll check. @type streamID: L{int} @return: The amount of room remaining in the send window for the given stream, including the data queued to be sent. @rtype: L{int} """ # TODO: This involves a fair bit of looping and computation for # something that is called a lot. Consider caching values somewhere. windowSize = self.conn.local_flow_control_window(streamID) sendQueue = self._outboundStreamQueues[streamID] alreadyConsumed = sum( len(chunk) for chunk in sendQueue if chunk is not _END_STREAM_SENTINEL) return windowSize - alreadyConsumed def _handleWindowUpdate(self, event): """ Manage flow control windows. Streams that are blocked on flow control will register themselves with the connection. This will fire deferreds that wake those streams up and allow them to continue processing. @param event: The Hyper-h2 event that encodes information about the flow control window change. @type event: L{h2.events.WindowUpdated} """ streamID = event.stream_id if streamID: if not self._streamIsActive(streamID): # We may have already cleaned up our stream state, making this # a late WINDOW_UPDATE frame. That's fine: the update is # unnecessary but benign. We'll ignore it. return # If we haven't got any data to send, don't unblock the stream. If # we do, we'll eventually get an exception inside the # _sendPrioritisedData loop some time later. if self._outboundStreamQueues.get(streamID): self.priority.unblock(streamID) self.streams[streamID].windowUpdated() else: # Update strictly applies to all streams. for stream in self.streams.values(): stream.windowUpdated() # If we still have data to send for this stream, unblock it. if self._outboundStreamQueues.get(stream.streamID): self.priority.unblock(stream.streamID) def getPeer(self): """ Get the remote address of this connection. Treat this method with caution. It is the unfortunate result of the CGI and Jabber standards, but should not be considered reliable for the usual host of reasons; port forwarding, proxying, firewalls, IP masquerading, etc. @return: An L{IAddress} provider. """ return self.transport.getPeer() def getHost(self): """ Similar to getPeer, but returns an address describing this side of the connection. @return: An L{IAddress} provider. """ return self.transport.getHost() def openStreamWindow(self, streamID, increment): """ Open the stream window by a given increment. @param streamID: The ID of the stream whose window needs to be opened. @type streamID: L{int} @param increment: The amount by which the stream window must be incremented. @type increment: L{int} """ self.conn.acknowledge_received_data(increment, streamID) data = self.conn.data_to_send() if data: self.transport.write(data) def _isSecure(self): """ Returns L{True} if this channel is using a secure transport. @returns: L{True} if this channel is secure. @rtype: L{bool} """ # A channel is secure if its transport is ISSLTransport. return ISSLTransport(self.transport, None) is not None def _send100Continue(self, streamID): """ Sends a 100 Continue response, used to signal to clients that further processing will be performed. @param streamID: The ID of the stream that needs the 100 Continue response @type streamID: L{int} """ headers = [(b':status', b'100')] self.conn.send_headers(headers=headers, stream_id=streamID) self.transport.write(self.conn.data_to_send()) def _respondToBadRequestAndDisconnect(self, streamID): """ This is a quick and dirty way of responding to bad requests. As described by HTTP standard we should be patient and accept the whole request from the client before sending a polite bad request response, even in the case when clients send tons of data. Unlike in the HTTP/1.1 case, this does not actually disconnect the underlying transport: there's no need. This instead just sends a 400 response and terminates the stream. @param streamID: The ID of the stream that needs the 100 Continue response @type streamID: L{int} """ headers = [(b':status', b'400')] self.conn.send_headers(headers=headers, stream_id=streamID, end_stream=True) self.transport.write(self.conn.data_to_send()) stream = self.streams[streamID] stream.connectionLost("Stream reset") self._requestDone(streamID) def _streamIsActive(self, streamID): """ Checks whether Twisted has still got state for a given stream and so can process events for that stream. @param streamID: The ID of the stream that needs processing. @type streamID: L{int} @return: Whether the stream still has state allocated. @rtype: L{bool} """ return streamID in self.streams
"""" Base class for all canarydrop channels. """ import datetime import simplejson import settings from exception import DuplicateChannel from twisted.logger import Logger log = Logger() class Channel(object): CHANNEL = 'Base' def __init__(self, switchboard=None, name=None): self.switchboard = switchboard self.name = name or self.CHANNEL log.info('Started channel {name}'.format(name=self.name)) class InputChannel(Channel): CHANNEL = 'InputChannel' def __init__(self, switchboard=None, name=None, unique_channel=False): super(InputChannel, self).__init__(switchboard=switchboard, name=name) try: self.register_input_channel() except DuplicateChannel as e:
def __init__( self, domains: Set, network_middleware: RestMiddleware = __DEFAULT_MIDDLEWARE_CLASS(), start_learning_now: bool = False, learn_on_same_thread: bool = False, known_nodes: tuple = None, seed_nodes: Tuple[tuple] = None, node_storage=None, save_metadata: bool = False, abort_on_learning_error: bool = False, lonely: bool = False, ) -> None: self.log = Logger("learning-loop") # type: Logger self.learning_domains = domains self.network_middleware = network_middleware self.save_metadata = save_metadata self.start_learning_now = start_learning_now self.learn_on_same_thread = learn_on_same_thread self._abort_on_learning_error = abort_on_learning_error self._learning_listeners = defaultdict(list) self._node_ids_to_learn_about_immediately = set() self.__known_nodes = self.tracker_class() self.lonely = lonely self.done_seeding = False # Read if node_storage is None: node_storage = self.__DEFAULT_NODE_STORAGE( federated_only=self.federated_only, # TODO: remove federated_only character_class=self.__class__) self.node_storage = node_storage if save_metadata and node_storage is NO_STORAGE_AVAILIBLE: raise ValueError( "Cannot save nodes without a configured node storage") known_nodes = known_nodes or tuple() self.unresponsive_startup_nodes = list( ) # TODO: Attempt to use these again later for node in known_nodes: try: self.remember_node( node ) # TODO: Need to test this better - do we ever init an Ursula-Learner with Node Storage? except self.UnresponsiveTeacher: self.unresponsive_startup_nodes.append(node) self.teacher_nodes = deque() self._current_teacher_node = None # type: Teacher self._learning_task = task.LoopingCall(self.keep_learning_about_nodes) self._learning_round = 0 # type: int self._rounds_without_new_nodes = 0 # type: int self._seed_nodes = seed_nodes or [] self.unresponsive_seed_nodes = set() if self.start_learning_now: self.start_learning_loop(now=self.learn_on_same_thread)
class Teacher: TEACHER_VERSION = LEARNING_LOOP_VERSION verified_stamp = False verified_interface = False _verified_node = False _interface_info_splitter = (int, 4, {'byteorder': 'big'}) log = Logger("teacher") __DEFAULT_MIN_SEED_STAKE = 0 def __init__( self, domains: Set, certificate: Certificate, certificate_filepath: str, interface_signature=NOT_SIGNED.bool_value(False), timestamp=NOT_SIGNED, identity_evidence=NOT_SIGNED, substantiate_immediately=False, passphrase=None, ) -> None: self.serving_domains = domains self.certificate = certificate self.certificate_filepath = certificate_filepath self._interface_signature_object = interface_signature self._timestamp = timestamp self.last_seen = NEVER_SEEN("Haven't connected to this node yet.") self.fleet_state_checksum = None self.fleet_state_updated = None self._evidence_of_decentralized_identity = constant_or_bytes( identity_evidence) if substantiate_immediately: self.substantiate_stamp( password=passphrase) # TODO: Derive from keyring class InvalidNode(SuspiciousActivity): """ Raised when a node has an invalid characteristic - stamp, interface, or address. """ class WrongMode(TypeError): """ Raised when a Character tries to use another Character as decentralized when the latter is federated_only. """ class IsFromTheFuture(TypeError): """ Raised when deserializing a Character from a future version. """ @classmethod def from_tls_hosting_power(cls, tls_hosting_power: TLSHostingPower, *args, **kwargs) -> 'Teacher': certificate_filepath = tls_hosting_power.keypair.certificate_filepath certificate = tls_hosting_power.keypair.certificate return cls(certificate=certificate, certificate_filepath=certificate_filepath, *args, **kwargs) # # Known Nodes # def seed_node_metadata(self, as_teacher_uri=False): if as_teacher_uri: teacher_uri = f'{self.checksum_public_address}@{self.rest_server.rest_interface.host}:{self.rest_server.rest_interface.port}' return teacher_uri return SeednodeMetadata( self.checksum_public_address, # type: str self.rest_server.rest_interface.host, # type: str self.rest_server.rest_interface.port) # type: int def sorted_nodes(self): nodes_to_consider = list(self.known_nodes.values()) + [self] return sorted(nodes_to_consider, key=lambda n: n.checksum_public_address) def _stamp_has_valid_wallet_signature(self): signature_bytes = self._evidence_of_decentralized_identity if signature_bytes is NOT_SIGNED: return False else: signature = EthSignature(signature_bytes) proper_pubkey = signature.recover_public_key_from_msg(bytes( self.stamp)) proper_address = proper_pubkey.to_checksum_address() return proper_address == self.checksum_public_address def update_snapshot(self, checksum, updated, number_of_known_nodes): # We update the simple snapshot here, but of course if we're dealing with an instance that is also a Learner, it has # its own notion of its FleetState, so we probably need a reckoning of sorts here to manage that. In time. self.fleet_state_nickname, self.fleet_state_nickname_metadata = nickname_from_seed( checksum, number_of_pairs=1) self.fleet_state_checksum = checksum self.fleet_state_updated = updated self.fleet_state_icon = icon_from_checksum( self.fleet_state_checksum, nickname_metadata=self.fleet_state_nickname_metadata, number_of_nodes=number_of_known_nodes) # # Stamp # def _stamp_has_valid_wallet_signature(self): signature_bytes = self._evidence_of_decentralized_identity if signature_bytes is NOT_SIGNED: return False else: signature = EthSignature(signature_bytes) proper_pubkey = signature.recover_public_key_from_msg(bytes( self.stamp)) proper_address = proper_pubkey.to_checksum_address() return proper_address == self.checksum_public_address def stamp_is_valid(self): """ :return: """ signature = self._evidence_of_decentralized_identity if self._stamp_has_valid_wallet_signature(): self.verified_stamp = True return True elif self.federated_only and signature is NOT_SIGNED: message = "This node can't be verified in this manner, " \ "but is OK to use in federated mode if you" \ " have reason to believe it is trustworthy." raise self.WrongMode(message) else: raise self.InvalidNode def verify_id(self, ursula_id, digest_factory=bytes): self.verify() if not ursula_id == digest_factory(self.canonical_public_address): raise self.InvalidNode def validate_metadata(self, accept_federated_only=False): if not self.verified_interface: self.interface_is_valid() if not self.verified_stamp: try: self.stamp_is_valid() except self.WrongMode: if not accept_federated_only: raise def verify_node(self, network_middleware, certificate_filepath: str = None, accept_federated_only: bool = False, force: bool = False) -> bool: """ Three things happening here: * Verify that the stamp matches the address (raises InvalidNode is it's not valid, or WrongMode if it's a federated mode and being verified as a decentralized node) * Verify the interface signature (raises InvalidNode if not valid) * Connect to the node, make sure that it's up, and that the signature and address we checked are the same ones this node is using now. (raises InvalidNode if not valid; also emits a specific warning depending on which check failed). """ if not force: if self._verified_node: return True self.validate_metadata(accept_federated_only ) # This is both the stamp and interface check. if not certificate_filepath: if not self.certificate_filepath: raise TypeError( "We haven't saved a certificate for this node yet.") else: certificate_filepath = self.certificate_filepath # The node's metadata is valid; let's be sure the interface is in order. response_data = network_middleware.node_information( host=self.rest_information()[0].host, port=self.rest_information()[0].port, certificate_filepath=certificate_filepath) version, node_bytes = self.version_splitter(response_data, return_remainder=True) node_details = self.internal_splitter(node_bytes) # TODO check timestamp here. 589 verifying_keys_match = node_details[ 'verifying_key'] == self.public_keys(SigningPower) encrypting_keys_match = node_details[ 'encrypting_key'] == self.public_keys(DecryptingPower) addresses_match = node_details[ 'public_address'] == self.canonical_public_address evidence_matches = node_details[ 'identity_evidence'] == self._evidence_of_decentralized_identity if not all((encrypting_keys_match, verifying_keys_match, addresses_match, evidence_matches)): # TODO: Optional reporting. 355 if not addresses_match: self.log.warn( "Wallet address swapped out. It appears that someone is trying to defraud this node." ) if not verifying_keys_match: self.log.warn( "Verifying key swapped out. It appears that someone is impersonating this node." ) raise self.InvalidNode( "Wrong cryptographic material for this node - something fishy going on." ) else: self._verified_node = True def substantiate_stamp(self, password: str): blockchain_power = self._crypto_power.power_ups(BlockchainPower) blockchain_power.unlock_account(password=password) # TODO: 349 signature = blockchain_power.sign_message(bytes(self.stamp)) self._evidence_of_decentralized_identity = signature # # Interface # def interface_is_valid(self): """ Checks that the interface info is valid for this node's canonical address. """ interface_info_message = self._signable_interface_info_message( ) # Contains canonical address. message = self.timestamp_bytes() + interface_info_message interface_is_valid = self._interface_signature.verify( message, self.public_keys(SigningPower)) self.verified_interface = interface_is_valid if interface_is_valid: return True else: raise self.InvalidNode def _signable_interface_info_message(self): message = self.canonical_public_address + self.rest_information()[0] return message def _sign_and_date_interface_info(self): message = self._signable_interface_info_message() self._timestamp = maya.now() self._interface_signature_object = self.stamp(self.timestamp_bytes() + message) @property def _interface_signature(self): if not self._interface_signature_object: try: self._sign_and_date_interface_info() except NoSigningPower: raise NoSigningPower( "This Ursula is a stranger and cannot be used to verify.") return self._interface_signature_object @property def timestamp(self): if not self._timestamp: try: self._sign_and_date_interface_info() except NoSigningPower: raise NoSigningPower( "This Node is a Stranger; you didn't init with a timestamp, so you can't verify." ) return self._timestamp def timestamp_bytes(self): return self.timestamp.epoch.to_bytes(4, 'big') # # Nicknames # @property def nickname_icon(self): return '{} {}'.format(self.nickname_metadata[0][1], self.nickname_metadata[1][1]) def nickname_icon_html(self): icon_template = """ <div class="nucypher-nickname-icon" style="border-top-color:{first_color}; border-left-color:{first_color}; border-bottom-color:{second_color}; border-right-color:{second_color};"> <span class="small">{node_class} v{version}</span> <div class="symbols"> <span class="single-symbol" style="color: {first_color}">{first_symbol}︎</span> <span class="single-symbol" style="color: {second_color}">{second_symbol}︎</span> </div> <br/> <span class="small-address">{address_first6}</span> </div> """.replace(" ", "").replace('\n', "") return icon_template.format(**self.nickname_icon_details) def nickname_icon_details(self): return dict( node_class=self.__class__.__name__, version=self.TEACHER_VERSION, first_color=self.nickname_metadata[0][0] ['hex'], # TODO: These index lookups are awful. first_symbol=self.nickname_metadata[0][1], second_color=self.nickname_metadata[1][0]['hex'], second_symbol=self.nickname_metadata[1][1], address_first6=self.checksum_public_address[2:8])
class FCMRouter(object): """FCM Router Implementation Note: FCM is a newer branch of GCM. While there's not much change required for the server, there is significant work required for the client. To that end, having a separate router allows the "older" GCM to persist and lets the client determine when they want to use the newer FCM route. """ log = Logger() gcm = None dryRun = 0 collapseKey = "simplepush" MAX_TTL = 2419200 reasonTable = { "MissingRegistration": { "msg": ("'to' or 'registration_id' is blank or" " invalid: {regid}"), "err": 500, "errno": 1, }, "InvalidRegistration": { "msg": "registration_id is invalid: {regid}", "err": 410, "errno": 105, }, "NotRegistered": { "msg": "device has unregistered with FCM: {regid}", "err": 410, "errno": 103, }, "InvalidPackageName": { "msg": "Invalid Package Name specified", "err": 500, "errno": 2, "crit": True, }, "MismatchSenderid": { "msg": "Invalid SenderID used: {senderid}", "err": 410, "errno": 105, "crit": True, }, "MessageTooBig": { "msg": "Message length was too big: {nlen}", "err": 413, "errno": 104, }, "InvalidDataKey": { "msg": ("Payload contains an invalid or restricted " "key value"), "err": 500, "errno": 3, "crit": True, }, "InvalidTtl": { "msg": "Invalid TimeToLive {ttl}", "err": 400, "errno": 111, }, "Unavailable": { "msg": "Message has timed out or device is unavailable", "err": 200, "errno": 0, }, "InternalServerError": { "msg": "FCM internal server error", "err": 500, "errno": 999, }, "DeviceMessageRateExceeded": { "msg": "Too many messages for this device", "err": 503, "errno": 4, }, "TopicsMessageRateExceeded": { "msg": "Too many subscribers for this topic", "err": 503, "errno": 5, "crit": True, }, "Unreported": { "msg": "Error has no reported reason.", "err": 500, "errno": 999, "crit": True, } } def __init__(self, conf, router_conf, metrics): """Create a new FCM router and connect to FCM""" self.conf = conf self.router_conf = router_conf self.metrics = metrics self.min_ttl = router_conf.get("ttl", 60) self.dryRun = router_conf.get("dryrun", False) self.collapseKey = router_conf.get("collapseKey", "webpush") self.clients = {} try: for (sid, creds) in router_conf["creds"].items(): self.clients[sid] = pyfcm.FCMNotification( api_key=creds["auth"]) except Exception as e: self.log.error("Could not instantiate FCM {ex}", ex=e) raise IOError("FCM Bridge not initiated in main") self._base_tags = ["platform:fcm"] self.log.debug("Starting FCM router...") def amend_endpoint_response(self, response, router_data): # type: (JSONDict, JSONDict) -> None response["senderid"] = router_data.get('app_id') def register(self, uaid, router_data, app_id, *args, **kwargs): # type: (str, JSONDict, str, *Any, **Any) -> None """Validate that the FCM Instance Token is in the ``router_data``""" # "token" is the FCM registration id token generated by the client. if "token" not in router_data: raise self._error("connect info missing FCM Instance 'token'", status=401, uri=kwargs.get('uri'), senderid=repr(app_id)) # senderid is the remote client's senderID value. This value is # very difficult for the client to change, and there was a problem # where some clients had an older, invalid senderID. We need to # be able to match senderID to it's corresponding auth key. # If the client has an unexpected or invalid SenderID, # it is impossible for us to reach them. if app_id not in self.clients: raise self._error("Invalid SenderID", status=410, errno=105) router_data["app_id"] = app_id def route_notification(self, notification, uaid_data): """Start the FCM notification routing, returns a deferred""" router_data = uaid_data["router_data"] # Kick the entire notification routing off to a thread return deferToThread(self._route, notification, router_data) def _route(self, notification, router_data): """Blocking FCM call to route the notification""" # THIS MUST MATCH THE CHANNELID GENERATED BY THE REGISTRATION SERVICE # Currently this value is in hex form. data = {"chid": notification.channel_id.hex} if not router_data.get("token"): raise self._error( "No registration token found. " "Rejecting message.", 410, errno=106, log_exception=False) regid = router_data.get("token") # Payload data is optional. The endpoint handler validates that the # correct encryption headers are included with the data. if notification.data: mdata = self.router_conf.get('max_data', 4096) if notification.data_length > mdata: raise self._error("This message is intended for a " + "constrained device and is limited " + "to 3070 bytes. Converted buffer too " + "long by %d bytes" % (notification.data_length - mdata), 413, errno=104, log_exception=False) data['body'] = notification.data data['con'] = notification.headers['encoding'] if 'encryption' in notification.headers: data['enc'] = notification.headers['encryption'] if 'crypto_key' in notification.headers: data['cryptokey'] = notification.headers['crypto_key'] elif 'encryption_key' in notification.headers: data['enckey'] = notification.headers['encryption_key'] try: client = self.clients[router_data["app_id"]] except KeyError: self.log.critical("Missing FCM bridge credentials for {id}", id=router_data['app_id']) raise RouterException("Server error", status_code=500, error=901) # registration_ids are the FCM instance tokens (specified during # registration. router_ttl = min(self.MAX_TTL, max(self.min_ttl, notification.ttl or 0)) try: result = client.notify_single_device( collapse_key=self.collapseKey, data_message=data, dry_run=self.dryRun or ('dryrun' in router_data), registration_id=regid, time_to_live=router_ttl, ) except pyfcm.errors.AuthenticationError as e: self.log.error("Authentication Error: %s" % e) raise RouterException("Server error", status_code=500) except ConnectionError as e: self.metrics.increment("notification.bridge.error", tags=make_tags( self._base_tags, error=502, errno=0, reason="connection_unavailable")) self.log.warn("Could not connect to FCM server: %s" % e) raise RouterException("Server error", status_code=502, log_exception=False) except Exception as e: self.log.error("Unhandled FCM Error: %s" % e) raise RouterException("Server error", status_code=500) return self._process_reply(result, notification, router_data, ttl=router_ttl) def _error(self, err, status, **kwargs): """Error handler that raises the RouterException""" self.log.debug(err, **kwargs) return RouterException(err, status_code=status, response_body=err, **kwargs) def _process_reply(self, reply, notification, router_data, ttl): """Process FCM send reply""" # acks: # for reg_id, msg_id in reply.success.items(): # updates result = reply.get('results', [{}])[0] if reply.get('canonical_ids'): old_id = router_data['token'] new_id = result.get('registration_id') self.log.debug("FCM id changed : {old} => {new}", old=old_id, new=new_id) self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, error=502, errno=0, reason="reregister")) return RouterResponse(status_code=503, response_body="Please try request again.", router_data=dict(token=new_id)) if reply.get('failure'): self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, error=502, errno=0, reason="failure")) reason = result.get('error', "Unreported") err = self.reasonTable.get(reason) if err.get("crit", False): self.log.critical( err['msg'], nlen=notification.data_length, regid=router_data["token"], senderid=router_data.get('token'), ttl=notification.ttl, ) raise RouterException("FCM failure to deliver", status_code=err['err'], response_body="Please try request " "later.", log_exception=False) self.log.debug("{msg} : {info}", msg=err['msg'], info={ "app_id": router_data["app_id"], "reason": reason }) return RouterResponse( status_code=err['err'], errno=err['errno'], response_body=err['msg'], router_data={}, ) self.metrics.increment("notification.bridge.sent", tags=self._base_tags) self.metrics.increment("notification.message_data", notification.data_length, tags=make_tags(self._base_tags, destination="Direct")) location = "%s/m/%s" % (self.conf.endpoint_url, notification.version) return RouterResponse(status_code=201, response_body="", headers={ "TTL": ttl, "Location": location }, logged_status=200)
class ClientService(service.Service): """ A L{ClientService} maintains a single outgoing connection to a client endpoint, reconnecting after a configurable timeout when a connection fails, either before or after connecting. @since: 16.1.0 """ _log = Logger() def __init__( self, endpoint, factory, retryPolicy=None, clock=None, prepareConnection=None ): """ @param endpoint: A L{stream client endpoint <interfaces.IStreamClientEndpoint>} provider which will be used to connect when the service starts. @param factory: A L{protocol factory <interfaces.IProtocolFactory>} which will be used to create clients for the endpoint. @param retryPolicy: A policy configuring how long L{ClientService} will wait between attempts to connect to C{endpoint}. @type retryPolicy: callable taking (the number of failed connection attempts made in a row (L{int})) and returning the number of seconds to wait before making another attempt. @param clock: The clock used to schedule reconnection. It's mainly useful to be parametrized in tests. If the factory is serialized, this attribute will not be serialized, and the default value (the reactor) will be restored when deserialized. @type clock: L{IReactorTime} @param prepareConnection: A single argument L{callable} that may return a L{Deferred}. It will be called once with the L{protocol <interfaces.IProtocol>} each time a new connection is made. It may call methods on the protocol to prepare it for use (e.g. authenticate) or validate it (check its health). The C{prepareConnection} callable may raise an exception or return a L{Deferred} which fails to reject the connection. A rejected connection is not used to fire an L{Deferred} returned by L{whenConnected}. Instead, L{ClientService} handles the failure and continues as if the connection attempt were a failure (incrementing the counter passed to C{retryPolicy}). L{Deferred}s returned by L{whenConnected} will not fire until any L{Deferred} returned by the C{prepareConnection} callable fire. Otherwise its successful return value is consumed, but ignored. Present Since Twisted 18.7.0 @type prepareConnection: L{callable} """ clock = _maybeGlobalReactor(clock) retryPolicy = _defaultPolicy if retryPolicy is None else retryPolicy self._machine = _ClientMachine( endpoint, factory, retryPolicy, clock, prepareConnection=prepareConnection, log=self._log, ) def whenConnected(self, failAfterFailures=None): """ Retrieve the currently-connected L{Protocol}, or the next one to connect. @param failAfterFailures: number of connection failures after which the Deferred will deliver a Failure (None means the Deferred will only fail if/when the service is stopped). Set this to 1 to make the very first connection failure signal an error. Use 2 to allow one failure but signal an error if the subsequent retry then fails. @type failAfterFailures: L{int} or None @return: a Deferred that fires with a protocol produced by the factory passed to C{__init__} @rtype: L{Deferred} that may: - fire with L{IProtocol} - fail with L{CancelledError} when the service is stopped - fail with e.g. L{DNSLookupError<twisted.internet.error.DNSLookupError>} or L{ConnectionRefusedError<twisted.internet.error.ConnectionRefusedError>} when the number of consecutive failed connection attempts equals the value of "failAfterFailures" """ return self._machine.whenConnected(failAfterFailures) def startService(self): """ Start this L{ClientService}, initiating the connection retry loop. """ if self.running: self._log.warn("Duplicate ClientService.startService {log_source}") return super().startService() self._machine.start() def stopService(self): """ Stop attempting to reconnect and close any existing connections. @return: a L{Deferred} that fires when all outstanding connections are closed and all in-progress connection attempts halted. """ super().stopService() return self._machine.stop()
def __init__(self, json_ipc, verbose, quiet, no_logs, console_logs, file_logs, sentry_logs, log_level, debug): self.log = Logger(self.__class__.__name__) # Session Emitter for pre and post character control engagement. if verbose and quiet: raise click.BadOptionUsage( option_name="quiet", message="--verbose and --quiet are mutually exclusive " "and cannot be used at the same time.") if verbose: GroupGeneralConfig.verbosity = 2 elif quiet: GroupGeneralConfig.verbosity = 0 else: GroupGeneralConfig.verbosity = 1 if json_ipc: emitter = JSONRPCStdoutEmitter( verbosity=GroupGeneralConfig.verbosity) else: emitter = StdoutEmitter(verbosity=GroupGeneralConfig.verbosity) self.emitter = emitter if verbose: self.emitter.message("Verbose mode is enabled", color='blue') # Logging if debug and no_logs: raise click.BadOptionUsage( option_name="no-logs", message="--debug and --no-logs cannot be used at the same time." ) # Defaults if file_logs is None: file_logs = self.log_to_file if sentry_logs is None: sentry_logs = self.log_to_sentry if debug: console_logs = True file_logs = True sentry_logs = False log_level = 'debug' if no_logs: console_logs = False file_logs = False sentry_logs = False if json_ipc: console_logs = False GlobalLoggerSettings.set_log_level(log_level_name=log_level) if console_logs: GlobalLoggerSettings.start_console_logging() if file_logs: GlobalLoggerSettings.start_text_file_logging() GlobalLoggerSettings.start_json_file_logging() if sentry_logs: GlobalLoggerSettings.start_sentry_logging(self.sentry_endpoint) if json_ipc: GlobalLoggerSettings.stop_console_logging() # JSON-RPC Protection self.debug = debug self.json_ipc = json_ipc
def main(): """ Run the server. """ parser = argparse.ArgumentParser( description='Resolve DNS queries from Database') parser.add_argument('-c', '--config', dest='config_file', type=str, action='store', default='./config.yml', help='Path to the configuration file' ) parser.add_argument('--port', '-p', dest='port', type=int, action='store', default=10053, help='Port number for the service' ) parser.add_argument('--dry-run', '-d', dest='dry_run', action='store_true', help='Dry run, just check the config file' ) #parser.add_argument('--verbose', '-v', # dest='verbose', # action='store_true', # help='Be verbose' #) params = parser.parse_args() # Log to stdout, as this is intended to run in docker log.startLogging(sys.stdout) # Make new logging style compatible to traditional one def observer(event, log=log): log.msg(event['log_format'].format(**event)) if 'log_failure' in event: log.err(event['log_failure']) logger = Logger(namespace='default', observer=observer) # Read config file config = Config(params.config_file, logger) logger.debug("Running with the following parameters:\n{data}", data=config) # Dry run if params.dry_run: sys.exit(0) # Build a connection lasting the lifetime of the service connection = adbapi.ConnectionPool( config.db_driver, host=config.db_host, port=config.db_port, user=config.db_user, passwd=config.db_passwd, db=config.db_name, cp_reconnect=True ) # Build a global Resolver lasting the lifetime of the service resolver = client.createResolver() customResolver = DynamicResolver(config, connection, resolver, logger) # Factory and protocol services factory = server.DNSServerFactory( caches=[ cache.CacheResolver(), ], # Use "clients" instead of "authorities", so caching works clients=[ hosts.Resolver(file=config.dns_hosts, ttl=config.dns_ttl), customResolver, ] ) protocol = dns.DNSDatagramProtocol(controller=factory) # Start polling loop, to avoid timeouts poller = LoopingCall(customResolver.poll) poller.start(config.poll_time) # Listen TCP and UDP reactor.listenUDP(params.port, protocol) reactor.listenTCP(params.port, factory) reactor.run()
class HTTPAuthSessionWrapper: """ Wrap a portal, enforcing supported header-based authentication schemes. @ivar _portal: The L{Portal} which will be used to retrieve L{IResource} avatars. @ivar _credentialFactories: A list of L{ICredentialFactory} providers which will be used to decode I{Authorization} headers into L{ICredentials} providers. """ isLeaf = False _log = Logger() def __init__(self, portal, credentialFactories): """ Initialize a session wrapper @type portal: C{Portal} @param portal: The portal that will authenticate the remote client @type credentialFactories: C{Iterable} @param credentialFactories: The portal that will authenticate the remote client based on one submitted C{ICredentialFactory} """ self._portal = portal self._credentialFactories = credentialFactories def _authorizedResource(self, request): """ Get the L{IResource} which the given request is authorized to receive. If the proper authorization headers are present, the resource will be requested from the portal. If not, an anonymous login attempt will be made. """ authheader = request.getHeader(b'authorization') if not authheader: return util.DeferredResource(self._login(Anonymous())) factory, respString = self._selectParseHeader(authheader) if factory is None: return UnauthorizedResource(self._credentialFactories) try: credentials = factory.decode(respString, request) except error.LoginFailed: return UnauthorizedResource(self._credentialFactories) except: self._log.failure("Unexpected failure from credentials factory") return ErrorPage(500, None, None) else: return util.DeferredResource(self._login(credentials)) def render(self, request): """ Find the L{IResource} avatar suitable for the given request, if possible, and render it. Otherwise, perhaps render an error page requiring authorization or describing an internal server failure. """ return self._authorizedResource(request).render(request) def getChildWithDefault(self, path, request): """ Inspect the Authorization HTTP header, and return a deferred which, when fired after successful authentication, will return an authorized C{Avatar}. On authentication failure, an C{UnauthorizedResource} will be returned, essentially halting further dispatch on the wrapped resource and all children """ # Don't consume any segments of the request - this class should be # transparent! request.postpath.insert(0, request.prepath.pop()) return self._authorizedResource(request) def _login(self, credentials): """ Get the L{IResource} avatar for the given credentials. @return: A L{Deferred} which will be called back with an L{IResource} avatar or which will errback if authentication fails. """ d = self._portal.login(credentials, None, IResource) d.addCallbacks(self._loginSucceeded, self._loginFailed) return d def _loginSucceeded(self, args): """ Handle login success by wrapping the resulting L{IResource} avatar so that the C{logout} callback will be invoked when rendering is complete. """ interface, avatar, logout = args class ResourceWrapper(proxyForInterface(IResource, 'resource')): """ Wrap an L{IResource} so that whenever it or a child of it completes rendering, the cred logout hook will be invoked. An assumption is made here that exactly one L{IResource} from among C{avatar} and all of its children will be rendered. If more than one is rendered, C{logout} will be invoked multiple times and probably earlier than desired. """ def getChildWithDefault(self, name, request): """ Pass through the lookup to the wrapped resource, wrapping the result in L{ResourceWrapper} to ensure C{logout} is called when rendering of the child is complete. """ return ResourceWrapper( self.resource.getChildWithDefault(name, request)) def render(self, request): """ Hook into response generation so that when rendering has finished completely (with or without error), C{logout} is called. """ request.notifyFinish().addBoth(lambda ign: logout()) return super(ResourceWrapper, self).render(request) return ResourceWrapper(avatar) def _loginFailed(self, result): """ Handle login failure by presenting either another challenge (for expected authentication/authorization-related failures) or a server error page (for anything else). """ if result.check(error.Unauthorized, error.LoginFailed): return UnauthorizedResource(self._credentialFactories) else: self._log.failure( "HTTPAuthSessionWrapper.getChildWithDefault encountered " "unexpected error", failure=result, ) return ErrorPage(500, None, None) def _selectParseHeader(self, header): """ Choose an C{ICredentialFactory} from C{_credentialFactories} suitable to use to decode the given I{Authenticate} header. @return: A two-tuple of a factory and the remaining portion of the header value to be decoded or a two-tuple of L{None} if no factory can decode the header value. """ elements = header.split(b' ') scheme = elements[0].lower() for fact in self._credentialFactories: if fact.scheme == scheme: return (fact, b' '.join(elements[1:])) return (None, None) def putChild(self, path, child): # IResource.putChild raise NotImplementedError()
def __init__(self): self.log = Logger(self.__class__.__name__)
class AppSession(ApplicationSession): log = Logger() @inlineCallbacks def onJoin(self, details): middleware = CachingMiddleware(JSONStorage) db = TinyDB(os.path.join(os.path.dirname(__file__), 'data', 'user_data.json'), storage=middleware) auth_config = {'salt': 'fruitfly', 'iterations': 5000, 'keylen': 32} session_user_map = {} chars = string.ascii_letters + string.digits + "@!^_()%[]{}" pwlen = 12 self.log.info('auth started') def auth(realm, authid, details): q = Query() results = db.search(q.username == authid) if not results: raise ApplicationError( "User does not exist", "could not authenticate session - no such user {}".format( authid)) auth_details = results[0]['auth_details'] auth_details['authid'] = results[0]['user_details']['fname'] if results[0]['user_details']['lname']: auth_details['authid'] += (" " + results[0]['user_details']['lname']) if not auth_details['authid']: del auth_details['authid'] session_user_map[details['session']] = { 'username': authid, 'details': details } return auth_details yield self.register(auth, six.u('ffbo.auth_server.auth')) self.log.info("registered ffbo.auth_server.auth") def get_user(session): if session in session_user_map: return session_user_map[session] return {} yield self.register(get_user, six.u('ffbo.auth_server.get_user')) self.log.info("registered ffbo.auth_server.get_user") def generate_password(): rnd = random.SystemRandom() return "".join([rnd.choice(chars) for _ in range(pwlen)]) def register_user(user_details): if user_exists(user_details['username']): return {"error": "User already exists. Please try again"} try: username = user_details['username'] pw = generate_password() salted_pw = ath.derive_key(pw, auth_config['salt'], auth_config['iterations'], auth_config['keylen']) db_rec = { 'username': username, 'user_details': user_details, 'auth_details': copy.deepcopy(auth_config) } db_rec['auth_details']['secret'] = salted_pw db_rec['auth_details']['role'] = u'user' del db_rec['user_details']['username'] #print "Registered user ", db_rec db.insert(db_rec) except Exception as e: print e return {"error": "Unexpected error occured. Please try again"} print "User added to database" send_email(user_details, pw, username) return { "success": "Successfuly registered. Please check your email for your password." } yield self.register(register_user, six.u('ffbo.auth_server.register_user')) self.log.info("registered ffbo.auth_server.register_user") def send_email(user_details, pw, username): title = "Thank you for registering at NeuroNLP" text = "Hi {fname},\n\n" text += "Here are your login details for NeuroNLP.\n\n" text += "Username: {username}\nPassword: {pw}\n\n" text += "If you have any suggestions or feedback, we would love to hear it!" text += " Please use the feedback button on the top left button of the website to write to us.\n\n" text += "Thank you,\nFruit Fly Brain Observatory" text = text.format(username=username, fname=user_details['fname'], pw=pw) msg = MIMEText(text) msg['Subject'] = title msg['From'] = 'NeuroNLP(Fruit Fly Brain Observatory) <*****@*****.**>' msg['To'] = user_details['email'] sender = msg['From'] try: s = smtplib.SMTP(host='localhost', port=465) except Exception as e: print e print "Failed to start SMTP server on localhost" try: # Use a valid smtp server, otherwise the email notification won't be sent out s.sendmail('*****@*****.**', [user_details['email']], msg.as_string()) middleware.flush() print "Email sent to " + user_details[ 'email'] + " for " + user_details[ 'fname'] + ' ' + user_details['lname'] except Exception as e: print e print "Failed to send out email" def user_exists(username): q = Query() results = db.search(q.username == username) if not results: return False return True yield self.register(user_exists, six.u('ffbo.auth_server.user_exists')) self.log.info("registered ffbo.auth_server.user_exists") def get_auth_config(): return auth_config yield self.register(get_auth_config, six.u('ffbo.auth_server.get_auth_config')) self.log.info("registered ffbo.auth_server.get_auth_config") def change_password(): pass def send_new_password(): pass
class DataStore(DatabaseStore): """ Incident Management System MySQL data store. """ _log = Logger() schemaVersion = 2 schemaBasePath = Path(__file__).parent / "schema" sqlFileExtension = "mysql" query = queries @attrs(frozen=False) class _State(object): """ Internal mutable state for :class:`DataStore`. """ db: Optional[ConnectionPool] = attrib( validator=optional(instance_of(ConnectionPool)), default=None, init=False, ) hostName: str = attrib(validator=instance_of(str)) hostPort: int = attrib(validator=instance_of(int)) database: str = attrib(validator=instance_of(str)) username: str = attrib(validator=instance_of(str)) password: str = attrib(validator=instance_of(str)) _state: _State = attrib(default=Factory(_State), init=False) @property def _db(self) -> ConnectionPool: if self._state.db is None: db = ConnectionPool( "pymysql", host=self.hostName, port=self.hostPort, database=self.database, user=self.username, password=self.password, cursorclass=Cursor, cp_reconnect=True, ) # self._upgradeSchema(db) self._state.db = db return self._state.db async def disconnect(self) -> None: """ See :meth:`DatabaseStore.disconnect`. """ if self._state.db is not None: self._state.db.close() self._state.db = None async def runQuery( self, query: Query, parameters: Optional[Parameters] = None ) -> Rows: if parameters is None: parameters = {} try: return iter(await self._db.runQuery(query.text, parameters)) except MySQLError as e: self._log.critical( "Unable to {description}: {error}", description=query.description, query=query, **parameters, error=e, ) raise StorageError(e) async def runOperation( self, query: Query, parameters: Optional[Parameters] = None ) -> None: if parameters is None: parameters = {} try: await self._db.runOperation(query.text, parameters) except MySQLError as e: self._log.critical( "Unable to {description}: {error}", description=query.description, error=e, ) raise StorageError(e) async def runInteraction( self, interaction: Callable, *args: Any, **kwargs: Any ) -> Any: try: return await self._db.runInteraction(interaction, *args, **kwargs) except MySQLError as e: self._log.critical( "Interaction {interaction} failed: {error}", interaction=interaction, error=e, ) raise StorageError(e) async def dbSchemaVersion(self) -> int: """ See `meth:DatabaseStore.dbSchemaVersion`. """ try: for row in await self._db.runQuery(self.query.schemaVersion.text): return cast(int, row["VERSION"]) else: raise StorageError("Invalid schema: no version") except MySQLError as e: message = e.args[1] if ( message.startswith("Table '") and message.endswith(".SCHEMA_INFO' doesn't exist") ): return 0 self._log.critical( "Unable to {description}: {error}", description=self.query.schemaVersion.description, error=e, ) raise StorageError(e) async def applySchema(self, sql: str) -> None: """ See :meth:`IMSDataStore.applySchema`. """ def applySchema(txn: Transaction) -> None: txn.executescript(sql) try: await self.runInteraction(applySchema) except StorageError as e: self._log.critical( "Unable to apply schema: {error}", sql=sql, error=e ) raise StorageError(f"Unable to apply schema: {e}")
def setup_structured_logging( hs, config, log_config: dict, logBeginner: LogBeginner, redirect_stdlib_logging: bool = True, ) -> LogPublisher: """ Set up Twisted's structured logging system. Args: hs: The homeserver to use. config (HomeserverConfig): The configuration of the Synapse homeserver. log_config (dict): The log configuration to use. """ if config.no_redirect_stdio: raise ConfigError( "no_redirect_stdio cannot be defined using structured logging.") logger = Logger() if "drains" not in log_config: raise ConfigError( "The logging configuration requires a list of drains.") observers = [] # type: List[ILogObserver] for observer in parse_drain_configs(log_config["drains"]): # Pipe drains if observer.type == DrainType.CONSOLE: logger.debug("Starting up the {name} console logger drain", name=observer.name) observers.append(SynapseFileLogObserver(observer.location)) elif observer.type == DrainType.CONSOLE_JSON: logger.debug("Starting up the {name} JSON console logger drain", name=observer.name) observers.append(jsonFileLogObserver(observer.location)) elif observer.type == DrainType.CONSOLE_JSON_TERSE: logger.debug( "Starting up the {name} terse JSON console logger drain", name=observer.name, ) observers.append( TerseJSONToConsoleLogObserver(observer.location, metadata={})) # File drains elif observer.type == DrainType.FILE: logger.debug("Starting up the {name} file logger drain", name=observer.name) log_file = open(observer.location, "at", buffering=1, encoding="utf8") observers.append(SynapseFileLogObserver(log_file)) elif observer.type == DrainType.FILE_JSON: logger.debug("Starting up the {name} JSON file logger drain", name=observer.name) log_file = open(observer.location, "at", buffering=1, encoding="utf8") observers.append(jsonFileLogObserver(log_file)) elif observer.type == DrainType.NETWORK_JSON_TERSE: metadata = {"server_name": hs.config.server_name} log_observer = TerseJSONToTCPLogObserver( hs=hs, host=observer.location[0], port=observer.location[1], metadata=metadata, maximum_buffer=observer.options.maximum_buffer, ) log_observer.start() observers.append(log_observer) else: # We should never get here, but, just in case, throw an error. raise ConfigError("%s drain type cannot be configured" % (observer.type, )) publisher = LogPublisher(*observers) log_filter = LogLevelFilterPredicate() for namespace, namespace_config in log_config.get("loggers", DEFAULT_LOGGERS).items(): # Set the log level for twisted.logger.Logger namespaces log_filter.setLogLevelForNamespace( namespace, stdlib_log_level_to_twisted(namespace_config.get("level", "INFO")), ) # Also set the log levels for the stdlib logger namespaces, to prevent # them getting to PythonStdlibToTwistedLogger and having to be formatted if "level" in namespace_config: logging.getLogger(namespace).setLevel( namespace_config.get("level")) f = FilteringLogObserver(publisher, [log_filter]) lco = LogContextObserver(f) if redirect_stdlib_logging: stuff_into_twisted = PythonStdlibToTwistedLogger(lco) stdliblogger = logging.getLogger() stdliblogger.addHandler(stuff_into_twisted) # Always redirect standard I/O, otherwise other logging outputs might miss # it. logBeginner.beginLoggingTo([lco], redirectStandardIO=True) return publisher
class GCMRouter(object): """GCM Router Implementation""" log = Logger() dryRun = 0 collapseKey = "simplepush" MAX_TTL = 2419200 def __init__(self, conf, router_conf, metrics): """Create a new GCM router and connect to GCM""" self.conf = conf self.router_conf = router_conf self.metrics = metrics self.min_ttl = router_conf.get("ttl", 60) self.dryRun = router_conf.get("dryrun", False) self.collapseKey = router_conf.get("collapseKey", "simplepush") self.gcm = {} self.senderIDs = {} # Flatten the SenderID list from human readable and init gcmclient if not router_conf.get("senderIDs"): raise IOError("SenderIDs not configured.") for sid in router_conf.get("senderIDs"): auth = router_conf.get("senderIDs").get(sid).get("auth") self.senderIDs[sid] = auth self.gcm[sid] = gcmclient.GCM(auth) self._base_tags = ["platform:gcm"] self.log.debug("Starting GCM router...") def amend_endpoint_response(self, response, router_data): # type: (JSONDict, JSONDict) -> None response["senderid"] = router_data.get('creds', {}).get('senderID') def register(self, uaid, router_data, app_id, *args, **kwargs): # type: (str, JSONDict, str, *Any, **Any) -> None """Validate that the GCM Instance Token is in the ``router_data``""" # "token" is the GCM registration id token generated by the client. if "token" not in router_data: raise self._error("connect info missing GCM Instance 'token'", status=401) # senderid is the remote client's senderID value. This value is # very difficult for the client to change, and there was a problem # where some clients had an older, invalid senderID. We need to # be able to match senderID to it's corresponding auth key. # If the client has an unexpected or invalid SenderID, # it is impossible for us to reach them. senderid = app_id if senderid not in self.senderIDs: raise self._error("Invalid SenderID", status=410, errno=105, uri=kwargs.get('uri'), senderid=senderid) # Assign a senderid router_data["creds"] = { "senderID": senderid, "auth": self.senderIDs[senderid] } def route_notification(self, notification, uaid_data): """Start the GCM notification routing, returns a deferred""" # Kick the entire notification routing off to a thread return deferToThread(self._route, notification, uaid_data) def _route(self, notification, uaid_data): """Blocking GCM call to route the notification""" router_data = uaid_data["router_data"] # THIS MUST MATCH THE CHANNELID GENERATED BY THE REGISTRATION SERVICE # Currently this value is in hex form. data = {"chid": notification.channel_id.hex} # Payload data is optional. The endpoint handler validates that the # correct encryption headers are included with the data. if notification.data: mdata = self.router_conf.get('max_data', 4096) if notification.data_length > mdata: raise self._error("This message is intended for a " + "constrained device and is limited " + "to 3070 bytes. Converted buffer too " + "long by %d bytes" % (notification.data_length - mdata), 413, errno=104, log_exception=False) data['body'] = notification.data data['con'] = notification.headers['encoding'] if 'encryption' in notification.headers: data['enc'] = notification.headers.get('encryption') if 'crypto_key' in notification.headers: data['cryptokey'] = notification.headers['crypto_key'] elif 'encryption_key' in notification.headers: data['enckey'] = notification.headers['encryption_key'] # registration_ids are the GCM instance tokens (specified during # registration. router_ttl = min(self.MAX_TTL, max(notification.ttl or 0, self.min_ttl)) payload = gcmclient.JSONMessage( registration_ids=[router_data.get("token")], collapse_key=self.collapseKey, time_to_live=router_ttl, dry_run=self.dryRun or ("dryrun" in router_data), data=data, ) try: gcm = self.gcm[router_data['creds']['senderID']] result = gcm.send(payload) except RouterException: raise # pragma nocover except KeyError: self.log.critical("Missing GCM bridge credentials") raise RouterException("Server error", status_code=500, errno=900) except gcmclient.GCMAuthenticationError as e: self.log.error("GCM Authentication Error: %s" % e) raise RouterException("Server error", status_code=500, errno=901) except ConnectionError as e: self.log.warn("GCM Unavailable: %s" % e) self.metrics.increment("notification.bridge.error", tags=make_tags( self._base_tags, reason="connection_unavailable")) raise RouterException("Server error", status_code=502, errno=902, log_exception=False) except Exception as e: self.log.error("Unhandled exception in GCM Routing: %s" % e) raise RouterException("Server error", status_code=500) return self._process_reply(result, uaid_data, ttl=router_ttl, notification=notification) def _error(self, err, status, **kwargs): """Error handler that raises the RouterException""" self.log.debug(err, **kwargs) return RouterException(err, status_code=status, response_body=err, **kwargs) def _process_reply(self, reply, uaid_data, ttl, notification): """Process GCM send reply""" # acks: # for reg_id, msg_id in reply.success.items(): # updates for old_id, new_id in reply.canonicals.items(): self.log.debug("GCM id changed : {old} => {new}", old=old_id, new=new_id) self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="reregister")) return RouterResponse(status_code=503, response_body="Please try request again.", router_data=dict(token=new_id)) # naks: # uninstall: for reg_id in reply.not_registered: self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="unregistered")) self.log.debug("GCM no longer registered: %s" % reg_id) return RouterResponse( status_code=410, response_body="Endpoint requires client update", router_data={}, ) # for reg_id, err_code in reply.failed.items(): if len(reply.failed.items()) > 0: self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="failure")) self.log.debug("GCM failures: {failed()}", failed=lambda: repr(reply.failed.items())) raise RouterException( "GCM unable to deliver", status_code=410, response_body="GCM recipient not available.", log_exception=False, ) # retries: if reply.retry_after: self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="retry")) self.log.warn("GCM retry requested: {failed()}", failed=lambda: repr(reply.failed.items())) raise RouterException("GCM failure to deliver, retry", status_code=503, headers={"Retry-After": reply.retry_after}, response_body="Please try request " "in {} seconds.".format(reply.retry_after), log_exception=False) self.metrics.increment("notification.bridge.sent", tags=self._base_tags) self.metrics.increment("notification.message_data", notification.data_length, tags=make_tags(self._base_tags, destination='Direct')) location = "%s/m/%s" % (self.conf.endpoint_url, notification.version) return RouterResponse(status_code=201, response_body="", headers={ "TTL": ttl, "Location": location }, logged_status=200)
""" NO_BLOCKCHAIN_CONNECTION.bool_value(False) CHARACTER_DESTRUCTION = ''' Delete all {name} character files including: - Private and Public Keys ({keystore}) - Known Nodes ({nodestore}) - Node Configuration File ({config}) - Database ({database}) Are you sure?''' SUCCESSFUL_DESTRUCTION = "Successfully destroyed NuCypher configuration" LOG = Logger('cli.actions') class UnknownIPAddress(RuntimeError): pass def get_password_from_prompt(prompt: str = "Enter password", envvar: str = '', confirm: bool = False) -> str: password = os.environ.get(envvar, NO_PASSWORD) if password is NO_PASSWORD: # Collect password, prefer env var password = click.prompt(prompt, confirmation_prompt=confirm, hide_input=True) return password
class Worker: """High-level view of a worker. Protocol-level details are not dealt with.""" name = None kind = None protocol = None persistent = None rate_est = None log = Logger() def __init__(self, protocol, name, kind): self.name = name self.kind = kind self.protocol = protocol self.persistent = WorkerDB().get(name, kind) self.rate = RateMeter() reactor.callLater(0.5, self._get_to_work) def __str__(self): return "{}/{} @ {}".format(self.name, self.kind, self.protocol.peer) def submit(self): """invoked by the protocol when a share is submitted""" self.rate.mark() self.persistent.submit() def _get_to_work(self): """A miner goes through the following states 1) hashrate estimation 2) optimal difficulty search 3) production """ if self.persistent.optimal_difficulty: self._production() elif self.persistent.maximum_hashrate: self._find_optimal_difficulty() else: self._estimate_hashrate() #### auto-tuning machinery #### def _rate_estimation(self, difficulty, callback, timeout=80, args={}): """Set the difficulty to `difficulty`, send work and wait for `timeout` seconds. Then fire `callback(difficulty, rate, **args)`. If the rate could not be computed, it is None. """ self.protocol.set_difficulty(difficulty) self.protocol.notify() self.rate = RateMeter() reactor.callLater(timeout, self._rate_estimation_end, difficulty, callback, args) def _rate_estimation_end(self, difficulty, callback, args): if self.rate.one_minute_rate() == 0: self.protocol.log.info( "rate estimation failed for {log_source} at difficulty {difficulty}", difficulty=difficulty) rate = None else: rate = self.rate.mean_rate() self.protocol.log.info( "Est. rate={rate:.1f}/s at D={difficulty} [{hashrate:0.1f}Ghash/s] for {log_source} [{count} in {elapsed:.1f}s]", rate=rate, difficulty=difficulty, count=self.rate.count, elapsed=self.rate.elapsed_time(), hashrate=rate * difficulty * (1 << 32) / 1e9) callback(difficulty, rate, **args) def _estimate_hashrate(self): """Try to estimate the maximum possible hashrate of the worker by setting a high difficulty and measuring the rate of shares. If the observed rate is too low, restart with a lower difficulty.""" def hashrate_callback(difficulty, rate): if rate is None or rate <= HASHRATE_ESTIMATION_MINIMUM: if difficulty == 1: hashrate_continuation( 50e6) # educated guess; it's probably a CPU miner else: # restart with lower difficulty self._rate_estimation(max(1, difficulty // 16), hashrate_callback, timeout=HASHRATE_ESTIMATION_TIMEOUT) else: hashrate_continuation(rate * difficulty * (1 << 32)) def hashrate_continuation(h): self.log.info("Maximum hashrate found: {hashrate} ({log_source})", hashrate=h) self.persistent.set_maximum_hashrate(h) self._find_optimal_difficulty() self.state = "Estimating hashrate" self.log.info("starting maximum hashrate estimation ({log_source})") self._rate_estimation(HASHRATE_ESTIMATION_DIFFICULTY, hashrate_callback, timeout=HASHRATE_ESTIMATION_TIMEOUT) def _find_optimal_difficulty(self): def difficulty_callback(difficulty, rate, best_objective=0, measures={}): if rate is None: # stop search, difficulty too high, exploit previous results difficulty_continuation(measures) return measures[difficulty] = rate hashrate = rate * difficulty * (1 << 32) objective = rate * (difficulty**(1 / 3)) # stop if we are at 95% of full hashrate and objective function is decreasing if hashrate >= 0.95 * self.persistent.maximum_hashrate and objective <= 0.95 * best_objective: difficulty_continuation(measures) return self.log.info( "difficulty search: score={objective:.1f} @ D={D} (best={best:.1f}) ({log_source})", objective=objective, D=difficulty, best=best_objective) best_objective = max(best_objective, objective) self._rate_estimation(difficulty + 4, difficulty_callback, args={ 'best_objective': best_objective, 'measures': measures }, timeout=DIFFICULTY_ESTIMATION_TIMEOUT) def difficulty_continuation(measures): best_objective = 1 best_difficulty = 1 for difficulty, rate in measures.items(): objective = rate * (difficulty**(1 / 3)) if objective > best_objective: best_objective = objective best_difficulty = difficulty self.log.info( "Optimal difficulty: {difficulty}, with objective={objective:.1f} ({log_source})", difficulty=best_difficulty, objective=best_objective) self.persistent.set_optimal_difficulty(best_difficulty) self._production() self.log.info("starting optimal difficulty search ({log_source})") self.state = "Finding optimal difficulty" self._rate_estimation(1, difficulty_callback, args={ 'best_objective': 0, 'measures': {} }, timeout=DIFFICULTY_ESTIMATION_TIMEOUT) def _production(self): self.state = "Production" self.log.info( "going into production ({log_source}) at difficulty {difficulty}", difficulty=self.persistent.optimal_difficulty) self.protocol.set_difficulty(self.persistent.optimal_difficulty) self.protocol.notify() self.rate = RateMeter()
class Session(object): log = Logger() def __init__(self, credentials, api, provider_cert): # TODO check if an anonymous credentials is passed. # TODO move provider_cert to api object. # On creation, it should be able to retrieve all the info it needs # (calling bootstrap). # TODO could get a "provider" object instead. # this provider can have an api attribute, # and a "autoconfig" attribute passed on initialization. # TODO get a file-descriptor for password if not in credentials # TODO merge self._request with config.Provider._http_request ? self.username = credentials.username self.password = credentials.password self._provider_cert = provider_cert self._api = api self._initialize_session() def _initialize_session(self): self._agent = cookieAgentFactory(self._provider_cert) username = self.username or '' password = self.password or '' self._srp_auth = _srp.SRPAuthMechanism(username, password) self._srp_signup = _srp.SRPSignupMechanism() self._srp_password = _srp.SRPPasswordChangeMechanism() self._srp_recovery_code = _srp.SRPRecoveryCodeUpdateMechanism() self._token = None self._uuid = None # Session @property def token(self): return self._token @property def uuid(self): return self._uuid @property def is_authenticated(self): return self._srp_auth.srp_user.authenticated() @defer.inlineCallbacks def authenticate(self): uri = self._api.get_handshake_uri() met = self._api.get_handshake_method() self.log.debug('%s to %s' % (met, uri)) params = self._srp_auth.get_handshake_params() handshake = yield self._request(self._agent, uri, values=params, method=met) self._srp_auth.process_handshake(handshake) uri = self._api.get_authenticate_uri(login=self.username) met = self._api.get_authenticate_method() self.log.debug('%s to %s' % (met, uri)) params = self._srp_auth.get_authentication_params() auth = yield self._request(self._agent, uri, values=params, method=met) uuid, token = self._srp_auth.process_authentication(auth) self._srp_auth.verify_authentication() self._uuid = uuid self._token = token defer.returnValue(OK) @_auth_required @defer.inlineCallbacks def logout(self): uri = self._api.get_logout_uri() met = self._api.get_logout_method() yield self._request(self._agent, uri, method=met) self.username = None self.password = None self._initialize_session() defer.returnValue(OK) @_auth_required @defer.inlineCallbacks def change_password(self, password): uri = self._api.get_update_user_uri(uid=self._uuid) met = self._api.get_update_user_method() params = self._srp_password.get_password_params( self.username, password) update = yield self._request(self._agent, uri, values=params, method=met) self.password = password self._srp_auth = _srp.SRPAuthMechanism(self.username, password) defer.returnValue(OK) @_auth_required @defer.inlineCallbacks def update_recovery_code(self, recovery_code): uri = self._api.get_update_user_uri(uid=self._uuid) met = self._api.get_update_user_method() params = self._srp_recovery_code.get_recovery_code_params( self.username, recovery_code) update = yield self._request(self._agent, uri, values=params, method=met) defer.returnValue(update) # User certificates def get_vpn_cert(self): # TODO pass it to the provider object so that it can save it in the # right path. uri = self._api.get_vpn_cert_uri() met = self._api.get_vpn_cert_method() return self._request(self._agent, uri, method=met) @_auth_required def get_smtp_cert(self): # TODO pass it to the provider object so that it can save it in the # right path. uri = self._api.get_smtp_cert_uri() met = self._api.get_smtp_cert_method() print met, "to", uri return self._request(self._agent, uri, method=met) def _request(self, *args, **kw): kw['token'] = self._token return httpRequest(*args, **kw) # User management @defer.inlineCallbacks def signup(self, username, password, invite=None): # XXX should check that it_IS_NOT_authenticated provider.validate_username(username) uri = self._api.get_signup_uri() met = self._api.get_signup_method() params = self._srp_signup.get_signup_params(username, password, invite) signup = yield self._request(self._agent, uri, values=params, method=met) registered_user = self._srp_signup.process_signup(signup) self.username = username self.password = password defer.returnValue((OK, registered_user)) @_auth_required def update_user_record(self): # FIXME to be implemented pass # Authentication-protected configuration @defer.inlineCallbacks def fetch_provider_configs(self, uri, path): config = yield self._request(self._agent, uri) with open(path, 'w') as cf: cf.write(config) defer.returnValue('ok')
GLOBAL_CFG = "/etc/mailmail" LOCAL_CFG = os.path.expanduser("~/.twisted/mailmail") SMARTHOST = '127.0.0.1' ERROR_FMT = """\ Subject: Failed Message Delivery Message delivery failed. The following occurred: %s -- The Twisted sendmail application. """ _logObserver = textFileLogObserver(sys.stderr) _log = Logger(observer=_logObserver) class Options: """ Store the values of the parsed command-line options to the I{mailmail} script. @type to: L{list} of L{str} @ivar to: The addresses to which to deliver this message. @type sender: L{str} @ivar sender: The address from which this message is being sent. @type body: C{file}
class RTL433Demodulator(gr.hier_block2, ExportedState): __log = Logger() # TODO: log to context/client def __init__(self, mode='433', input_rate=0, context=None): assert input_rate > 0 assert context is not None gr.hier_block2.__init__(self, type(self).__name__, gr.io_signature(1, 1, gr.sizeof_gr_complex), gr.io_signature(0, 0, 0)) # The input bandwidth chosen is not primarily determined by the bandwidth of the input signals, but by the frequency error of the transmitters. Therefore it is not too critical, and we can choose the exact rate to make the filtering easy. if input_rate <= upper_preferred_demod_rate: # Skip having a filter at all. self.__band_filter = None demod_rate = input_rate else: # TODO: This gunk is very similar to the stuff that MultistageChannelFilter does. See if we can share some code. lower_rate = input_rate lower_rate_prev = None while lower_rate > upper_preferred_demod_rate and lower_rate != lower_rate_prev: lower_rate_prev = lower_rate if lower_rate % 5 == 0 and lower_rate > upper_preferred_demod_rate * 3: lower_rate /= 5 elif lower_rate % 2 == 0: lower_rate /= 2 else: # non-integer ratio lower_rate = upper_preferred_demod_rate break demod_rate = lower_rate self.__band_filter = MultistageChannelFilter( input_rate=input_rate, output_rate=demod_rate, cutoff_freq=demod_rate * 0.4, transition_width=demod_rate * 0.2) # Subprocess # using /usr/bin/env because twisted spawnProcess doesn't support path search # pylint: disable=no-member self.__process = the_reactor.spawnProcess( RTL433ProcessProtocol(context.output_message, self.__log), '/usr/bin/env', env=None, # inherit environment # These arguments were last reviewed for rtl_433 18.12-142-g6c3ca9b args=[ b'env', b'rtl_433', b'-F', b'json', # output format b'-r', str(demod_rate) + b'sps:iq:cf32:-', # specify input format and to use stdin b'-M', 'newmodel', ], childFDs={ 0: 'w', 1: 'r', 2: 2 }) sink = make_sink_to_process_stdin(self.__process, itemsize=gr.sizeof_gr_complex) agc = analog.agc2_cc(reference=dB(-4)) agc.set_attack_rate(200 / demod_rate) agc.set_decay_rate(200 / demod_rate) if self.__band_filter: self.connect(self, self.__band_filter, agc) else: self.connect(self, agc) self.connect(agc, sink) def _close(self): # TODO: This never gets called except in tests. Do this better, like by having an explicit life cycle for demodulators. self.__process.loseConnection() @exported_value(type=BandShape, changes='never') def get_band_shape(self): """implements IDemodulator""" if self.__band_filter: return self.__band_filter.get_shape() else: # TODO Reuse UnselectiveAMDemodulator's approach to this return BandShape(stop_low=0, pass_low=0, pass_high=0, stop_high=0, markers={}) def get_output_type(self): """implements IDemodulator""" return no_signal
class WSProtocol(WebSocketServerProtocol): wslog = Logger() #--------------------------------------------------------------------------- def __init__(self, *args, **kwargs): super(WSProtocol, self).__init__(*args, **kwargs) self.actionHandlers = {} self.actionHandlers['STEERING_UPDATE'] = self.steering_update #--------------------------------------------------------------------------- def onOpen(self): pass #--------------------------------------------------------------------------- def onMessage(self, payload, isBinary): #----------------------------------------------------------------------- # Check the message validity #----------------------------------------------------------------------- if isBinary: return #----------------------------------------------------------------------- # Decode the message #----------------------------------------------------------------------- try: payload = payload.decode('utf-8') data = json.loads(payload) except Exception as e: self.wslog.debug('Unable to parse message: {}.'.format(str(e))) return for header in ['type', 'action']: if header not in data: msg = 'Header "{}" is missing.'.format(header) self.wslog.debug(msg) return if data['type'] != 'ACTION_NO_RESP': msg = 'Rejecting non-action message: {}.'.format(data['type']) self.wslog.debug(msg) self.send_error_response(data['id'], msg) return if data['action'] not in self.actionHandlers: msg = 'Unknown action: {}.'.format(data['action']) self.wslog.debug(msg) self.send_error_response(data['id'], msg) return #----------------------------------------------------------------------- # Execute the action #----------------------------------------------------------------------- self.actionHandlers[data['action']](data) #--------------------------------------------------------------------------- def onClose(self, wasClean, code, reason): pass #--------------------------------------------------------------------------- def has_fields(self, data, fields): for field in ['channel', 'value']: if field not in data: msg = 'Message for action {}: field "{}" not specified.' \ .format(data['action'], field) self.wslog.debug(msg) return False return True #--------------------------------------------------------------------------- def steering_update(self, data): if not self.has_fields(data, ['channel', 'value']): return try: self.controller.update_channel(data['channel'], data['value']) except Exception as e: pass
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """ LEAP Session management. """ from twisted.internet import defer, reactor from twisted.logger import Logger from leap.bitmask.bonafide import _srp from leap.bitmask.bonafide import provider from leap.bitmask.bonafide._http import httpRequest, cookieAgentFactory logger = Logger() OK = 'ok' def _auth_required(func): """ Decorate a method so that it will not be called if the instance attribute `is_authenticated` does not evaluate to True. """ def decorated(*args, **kwargs): instance = args[0] allowed = getattr(instance, 'is_authenticated') if not allowed: raise RuntimeError('This method requires authentication') return func(*args, **kwargs)
class FleetStateTracker: """ A representation of a fleet of NuCypher nodes. """ _checksum = NO_KNOWN_NODES.bool_value(False) _nickname = NO_KNOWN_NODES _nickname_metadata = NO_KNOWN_NODES _tracking = False most_recent_node_change = NO_KNOWN_NODES snapshot_splitter = BytestringSplitter(32, 4) log = Logger("Learning") state_template = namedtuple( "FleetState", ("nickname", "metadata", "icon", "nodes", "updated")) def __init__(self): self.additional_nodes_to_track = [] self.updated = maya.now() self._nodes = OrderedDict() self.states = OrderedDict() def __setitem__(self, key, value): self._nodes[key] = value if self._tracking: self.log.info( "Updating fleet state after saving node {}".format(value)) self.record_fleet_state() else: self.log.debug("Not updating fleet state.") def __getitem__(self, item): return self._nodes[item] def __bool__(self): return bool(self._nodes) def __contains__(self, item): return item in self._nodes.keys() or item in self._nodes.values() def __iter__(self): yield from self._nodes.values() def __len__(self): return len(self._nodes) def __eq__(self, other): return self._nodes == other._nodes def __repr__(self): return self._nodes.__repr__() @property def checksum(self): return self._checksum @checksum.setter def checksum(self, checksum_value): self._checksum = checksum_value self._nickname, self._nickname_metadata = nickname_from_seed( checksum_value, number_of_pairs=1) @property def nickname(self): return self._nickname @property def nickname_metadata(self): return self._nickname_metadata @property def icon(self) -> str: if self.nickname_metadata is NO_KNOWN_NODES: return str(NO_KNOWN_NODES) return self.nickname_metadata[0][1] def addresses(self): return self._nodes.keys() def icon_html(self): return icon_from_checksum(checksum=self.checksum, number_of_nodes=str(len(self)), nickname_metadata=self.nickname_metadata) def snapshot(self): fleet_state_checksum_bytes = binascii.unhexlify(self.checksum) fleet_state_updated_bytes = self.updated.epoch.to_bytes( 4, byteorder="big") return fleet_state_checksum_bytes + fleet_state_updated_bytes def record_fleet_state(self, additional_nodes_to_track=None): if additional_nodes_to_track: self.additional_nodes_to_track.extend(additional_nodes_to_track) if not self._nodes: # No news here. return sorted_nodes = self.sorted() sorted_nodes_joined = b"".join(bytes(n) for n in sorted_nodes) checksum = keccak_digest(sorted_nodes_joined).hex() if checksum not in self.states: self.checksum = keccak_digest(b"".join( bytes(n) for n in self.sorted())).hex() self.updated = maya.now() # For now we store the sorted node list. Someday we probably spin this out into # its own class, FleetState, and use it as the basis for partial updates. new_state = self.state_template( nickname=self.nickname, metadata=self.nickname_metadata, nodes=sorted_nodes, icon=self.icon, updated=self.updated, ) self.states[checksum] = new_state return checksum, new_state def start_tracking_state(self, additional_nodes_to_track=None): if additional_nodes_to_track is None: additional_nodes_to_track = list() self.additional_nodes_to_track.extend(additional_nodes_to_track) self._tracking = True self.update_fleet_state() def sorted(self): nodes_to_consider = list( self._nodes.values()) + self.additional_nodes_to_track return sorted(nodes_to_consider, key=lambda n: n.checksum_public_address) def shuffled(self): nodes_we_know_about = list(self._nodes.values()) random.shuffle(nodes_we_know_about) return nodes_we_know_about def abridged_states_dict(self): abridged_states = {} for k, v in self.states.items(): abridged_states[k] = self.abridged_state_details(v) return abridged_states def abridged_nodes_dict(self): abridged_nodes = {} for checksum_address, node in self._nodes.items(): abridged_nodes[checksum_address] = self.abridged_node_details(node) return abridged_nodes @staticmethod def abridged_state_details(state): return { "nickname": state.nickname, "symbol": state.metadata[0][1], "color_hex": state.metadata[0][0]['hex'], "color_name": state.metadata[0][0]['color'], "updated": state.updated.rfc2822() } @staticmethod def abridged_node_details(node): try: last_seen = node.last_seen.iso8601() except AttributeError: # TODO: This logic belongs somewhere - anywhere - else. last_seen = str( node.last_seen) # In case it's the constant NEVER_SEEN return { "icon_details": node.nickname_icon_details(), # TODO: Mix this in better. "rest_url": node.rest_url(), "nickname": node.nickname, "checksum_address": node.checksum_public_address, "timestamp": node.timestamp.iso8601(), "last_seen": last_seen, "fleet_state_icon": node.fleet_state_icon, }
class TestTwistedClient(unittest.TestCase): "Twisted client tests" logger = Logger() def setUp(self): cfg = get_config("ServerConn") self.host, self.port = cfg.BEANSTALKD_HOST, int(cfg.BEANSTALKD_PORT) self.creator = protocol.ClientCreator(reactor, Beanstalk) def tearDown(self): self.conn.transport.loseConnection() @deferred() # required for nose integration def test_00_Job(self): "Twisted beanstalk Job instantiation succeeds" def onconn(conn): self.conn = conn rawjob = { 'jid': 1, 'bytes': 13, 'state': 'ok', 'data': 'Look! A job!' } rawjob.update(conn=conn) j = Job(**rawjob) assert j["data"] == 'Look! A job!' self.logger.debug("Job created ok") connector = self.creator.connectTCP(self.host, self.port) return connector.addCallback(onconn) @deferred() # required for nose integration def test_01_Stats(self): "Twisted client correctly gets stats from server" def onconn(conn): self.conn = conn statist = conn.stats() # def onstats(stats): self.assertEqual(stats["state"], "ok") self.logger.debug("got stats ok") # statist.addCallback(onstats) connector = self.creator.connectTCP(self.host, self.port) return connector.addCallback(onconn) @deferred() # required for nose integration def test_02_Put_Reserve(self): "putting and reserving works" def check_reserve_ok(): reserver = self.conn.reserve() # def on_ok(job, ): expected = { 'jid': 1, 'bytes': 13, 'state': 'ok', 'data': 'Look! A job!' } self.assertEqual(job, expected) self.logger.debug("job reservation succeeded") reserver.addCallback(on_ok) # def on_fail(failure): raise Exception(failure) reserver.addErrback(on_fail) return reserver def check_put_ok(): putter = self.conn.put('Look! A job!', 8192, 0, 300) # def on_ok(job): self.assertEqual(job, {'jid': 1, 'state': 'ok'}) self.logger.debug("job putting succeeded") putter.addCallback(on_ok) # def on_fail(failure): raise Exception(failure) putter.addErrback(on_fail) return putter def onconn(conn): "start both putting and reserving" self.conn = conn p = check_put_ok() r = check_reserve_ok() # set up the deferred returned to nose defer.DeferredList((p, r)).chainDeferred(self.r) connector = self.creator.connectTCP(self.host, self.port) self.r = defer.Deferred() connector.addCallback(onconn) return self.r # chained after completion of putting and reserving
class FilePasswordDB: """ A file-based, text-based username/password database. Records in the datafile for this class are delimited by a particular string. The username appears in a fixed field of the columns delimited by this string, as does the password. Both fields are specifiable. If the passwords are not stored plaintext, a hash function must be supplied to convert plaintext passwords to the form stored on disk and this CredentialsChecker will only be able to check IUsernamePassword credentials. If the passwords are stored plaintext, IUsernameHashedPassword credentials will be checkable as well. """ cache = False _credCache = None _cacheTimestamp = 0 _log = Logger() def __init__(self, filename, delim=b':', usernameField=0, passwordField=1, caseSensitive=True, hash=None, cache=False): """ @type filename: C{str} @param filename: The name of the file from which to read username and password information. @type delim: C{str} @param delim: The field delimiter used in the file. @type usernameField: C{int} @param usernameField: The index of the username after splitting a line on the delimiter. @type passwordField: C{int} @param passwordField: The index of the password after splitting a line on the delimiter. @type caseSensitive: C{bool} @param caseSensitive: If true, consider the case of the username when performing a lookup. Ignore it otherwise. @type hash: Three-argument callable or L{None} @param hash: A function used to transform the plaintext password received over the network to a format suitable for comparison against the version stored on disk. The arguments to the callable are the username, the network-supplied password, and the in-file version of the password. If the return value compares equal to the version stored on disk, the credentials are accepted. @type cache: C{bool} @param cache: If true, maintain an in-memory cache of the contents of the password file. On lookups, the mtime of the file will be checked, and the file will only be re-parsed if the mtime is newer than when the cache was generated. """ self.filename = filename self.delim = delim self.ufield = usernameField self.pfield = passwordField self.caseSensitive = caseSensitive self.hash = hash self.cache = cache if self.hash is None: # The passwords are stored plaintext. We can support both # plaintext and hashed passwords received over the network. self.credentialInterfaces = (credentials.IUsernamePassword, credentials.IUsernameHashedPassword) else: # The passwords are hashed on disk. We can support only # plaintext passwords received over the network. self.credentialInterfaces = (credentials.IUsernamePassword, ) def __getstate__(self): d = dict(vars(self)) for k in '_credCache', '_cacheTimestamp': try: del d[k] except KeyError: pass return d def _cbPasswordMatch(self, matched, username): if matched: return username else: return failure.Failure(error.UnauthorizedLogin()) def _loadCredentials(self): """ Loads the credentials from the configured file. @return: An iterable of C{username, password} couples. @rtype: C{iterable} @raise UnauthorizedLogin: when failing to read the credentials from the file. """ try: with open(self.filename, "rb") as f: for line in f: line = line.rstrip() parts = line.split(self.delim) if self.ufield >= len(parts) or self.pfield >= len(parts): continue if self.caseSensitive: yield parts[self.ufield], parts[self.pfield] else: yield parts[self.ufield].lower(), parts[self.pfield] except IOError as e: self._log.error("Unable to load credentials db: {e!r}", e=e) raise error.UnauthorizedLogin() def getUser(self, username): if not self.caseSensitive: username = username.lower() if self.cache: if self._credCache is None or os.path.getmtime( self.filename) > self._cacheTimestamp: self._cacheTimestamp = os.path.getmtime(self.filename) self._credCache = dict(self._loadCredentials()) return username, self._credCache[username] else: for u, p in self._loadCredentials(): if u == username: return u, p raise KeyError(username) def requestAvatarId(self, c): try: u, p = self.getUser(c.username) except KeyError: return defer.fail(error.UnauthorizedLogin()) else: up = credentials.IUsernamePassword(c, None) if self.hash: if up is not None: h = self.hash(up.username, up.password, p) if h == p: return defer.succeed(u) return defer.fail(error.UnauthorizedLogin()) else: return defer.maybeDeferred(c.checkPassword, p).addCallback( self._cbPasswordMatch, u)
class SSHUserAuthServer(service.SSHService): """ A service implementing the server side of the 'ssh-userauth' service. It is used to authenticate the user on the other side as being able to access this server. @ivar name: the name of this service: 'ssh-userauth' @type name: L{bytes} @ivar authenticatedWith: a list of authentication methods that have already been used. @type authenticatedWith: L{list} @ivar loginTimeout: the number of seconds we wait before disconnecting the user for taking too long to authenticate @type loginTimeout: L{int} @ivar attemptsBeforeDisconnect: the number of failed login attempts we allow before disconnecting. @type attemptsBeforeDisconnect: L{int} @ivar loginAttempts: the number of login attempts that have been made @type loginAttempts: L{int} @ivar passwordDelay: the number of seconds to delay when the user gives an incorrect password @type passwordDelay: L{int} @ivar interfaceToMethod: a L{dict} mapping credential interfaces to authentication methods. The server checks to see which of the cred interfaces have checkers and tells the client that those methods are valid for authentication. @type interfaceToMethod: L{dict} @ivar supportedAuthentications: A list of the supported authentication methods. @type supportedAuthentications: L{list} of L{bytes} @ivar user: the last username the client tried to authenticate with @type user: L{bytes} @ivar method: the current authentication method @type method: L{bytes} @ivar nextService: the service the user wants started after authentication has been completed. @type nextService: L{bytes} @ivar portal: the L{twisted.cred.portal.Portal} we are using for authentication @type portal: L{twisted.cred.portal.Portal} @ivar clock: an object with a callLater method. Stubbed out for testing. """ name = b"ssh-userauth" loginTimeout = 10 * 60 * 60 # 10 minutes before we disconnect them attemptsBeforeDisconnect = 20 # 20 login attempts before a disconnect passwordDelay = 1 # number of seconds to delay on a failed password clock = reactor interfaceToMethod = { credentials.ISSHPrivateKey: b"publickey", credentials.IUsernamePassword: b"password", } _log = Logger() def serviceStarted(self): """ Called when the userauth service is started. Set up instance variables, check if we should allow password authentication (only allow if the outgoing connection is encrypted) and set up a login timeout. """ self.authenticatedWith = [] self.loginAttempts = 0 self.user = None self.nextService = None self.portal = self.transport.factory.portal self.supportedAuthentications = [] for i in self.portal.listCredentialsInterfaces(): if i in self.interfaceToMethod: self.supportedAuthentications.append(self.interfaceToMethod[i]) if not self.transport.isEncrypted("in"): # don't let us transport password in plaintext if b"password" in self.supportedAuthentications: self.supportedAuthentications.remove(b"password") self._cancelLoginTimeout = self.clock.callLater( self.loginTimeout, self.timeoutAuthentication ) def serviceStopped(self): """ Called when the userauth service is stopped. Cancel the login timeout if it's still going. """ if self._cancelLoginTimeout: self._cancelLoginTimeout.cancel() self._cancelLoginTimeout = None def timeoutAuthentication(self): """ Called when the user has timed out on authentication. Disconnect with a DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE message. """ self._cancelLoginTimeout = None self.transport.sendDisconnect( transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, b"you took too long" ) def tryAuth(self, kind, user, data): """ Try to authenticate the user with the given method. Dispatches to a auth_* method. @param kind: the authentication method to try. @type kind: L{bytes} @param user: the username the client is authenticating with. @type user: L{bytes} @param data: authentication specific data sent by the client. @type data: L{bytes} @return: A Deferred called back if the method succeeded, or erred back if it failed. @rtype: C{defer.Deferred} """ self._log.debug("{user!r} trying auth {kind!r}", user=user, kind=kind) if kind not in self.supportedAuthentications: return defer.fail(error.ConchError("unsupported authentication, failing")) kind = nativeString(kind.replace(b"-", b"_")) f = getattr(self, "auth_{}".format(kind), None) if f: ret = f(data) if not ret: return defer.fail( error.ConchError( "{} return None instead of a Deferred".format(kind) ) ) else: return ret return defer.fail(error.ConchError("bad auth type: {}".format(kind))) def ssh_USERAUTH_REQUEST(self, packet): """ The client has requested authentication. Payload:: string user string next service string method <authentication specific data> @type packet: L{bytes} """ user, nextService, method, rest = getNS(packet, 3) if user != self.user or nextService != self.nextService: self.authenticatedWith = [] # clear auth state self.user = user self.nextService = nextService self.method = method d = self.tryAuth(method, user, rest) if not d: self._ebBadAuth(failure.Failure(error.ConchError("auth returned none"))) return d.addCallback(self._cbFinishedAuth) d.addErrback(self._ebMaybeBadAuth) d.addErrback(self._ebBadAuth) return d def _cbFinishedAuth(self, result): """ The callback when user has successfully been authenticated. For a description of the arguments, see L{twisted.cred.portal.Portal.login}. We start the service requested by the user. """ (interface, avatar, logout) = result self.transport.avatar = avatar self.transport.logoutFunction = logout service = self.transport.factory.getService(self.transport, self.nextService) if not service: raise error.ConchError( "could not get next service: {}".format(self.nextService) ) self._log.debug( "{user!r} authenticated with {method!r}", user=self.user, method=self.method ) self.transport.sendPacket(MSG_USERAUTH_SUCCESS, b"") self.transport.setService(service()) def _ebMaybeBadAuth(self, reason): """ An intermediate errback. If the reason is error.NotEnoughAuthentication, we send a MSG_USERAUTH_FAILURE, but with the partial success indicator set. @type reason: L{twisted.python.failure.Failure} """ reason.trap(error.NotEnoughAuthentication) self.transport.sendPacket( MSG_USERAUTH_FAILURE, NS(b",".join(self.supportedAuthentications)) + b"\xff" ) def _ebBadAuth(self, reason): """ The final errback in the authentication chain. If the reason is error.IgnoreAuthentication, we simply return; the authentication method has sent its own response. Otherwise, send a failure message and (if the method is not 'none') increment the number of login attempts. @type reason: L{twisted.python.failure.Failure} """ if reason.check(error.IgnoreAuthentication): return if self.method != b"none": self._log.debug( "{user!r} failed auth {method!r}", user=self.user, method=self.method ) if reason.check(UnauthorizedLogin): self._log.debug( "unauthorized login: {message}", message=reason.getErrorMessage() ) elif reason.check(error.ConchError): self._log.debug("reason: {reason}", reason=reason.getErrorMessage()) else: self._log.failure( "Error checking auth for user {user}", failure=reason, user=self.user, ) self.loginAttempts += 1 if self.loginAttempts > self.attemptsBeforeDisconnect: self.transport.sendDisconnect( transport.DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE, b"too many bad auths", ) return self.transport.sendPacket( MSG_USERAUTH_FAILURE, NS(b",".join(self.supportedAuthentications)) + b"\x00" ) def auth_publickey(self, packet): """ Public key authentication. Payload:: byte has signature string algorithm name string key blob [string signature] (if has signature is True) Create a SSHPublicKey credential and verify it using our portal. """ hasSig = ord(packet[0:1]) algName, blob, rest = getNS(packet[1:], 2) try: pubKey = keys.Key.fromString(blob) except keys.BadKeyError: error = "Unsupported key type {} or bad key".format(algName.decode("ascii")) self._log.error(error) return defer.fail(UnauthorizedLogin(error)) signature = hasSig and getNS(rest)[0] or None if hasSig: b = ( NS(self.transport.sessionID) + bytes((MSG_USERAUTH_REQUEST,)) + NS(self.user) + NS(self.nextService) + NS(b"publickey") + bytes((hasSig,)) + NS(pubKey.sshType()) + NS(blob) ) c = credentials.SSHPrivateKey(self.user, algName, blob, b, signature) return self.portal.login(c, None, interfaces.IConchUser) else: c = credentials.SSHPrivateKey(self.user, algName, blob, None, None) return self.portal.login(c, None, interfaces.IConchUser).addErrback( self._ebCheckKey, packet[1:] ) def _ebCheckKey(self, reason, packet): """ Called back if the user did not sent a signature. If reason is error.ValidPublicKey then this key is valid for the user to authenticate with. Send MSG_USERAUTH_PK_OK. """ reason.trap(error.ValidPublicKey) # if we make it here, it means that the publickey is valid self.transport.sendPacket(MSG_USERAUTH_PK_OK, packet) return failure.Failure(error.IgnoreAuthentication()) def auth_password(self, packet): """ Password authentication. Payload:: string password Make a UsernamePassword credential and verify it with our portal. """ password = getNS(packet[1:])[0] c = credentials.UsernamePassword(self.user, password) return self.portal.login(c, None, interfaces.IConchUser).addErrback( self._ebPassword ) def _ebPassword(self, f): """ If the password is invalid, wait before sending the failure in order to delay brute-force password guessing. """ d = defer.Deferred() self.clock.callLater(self.passwordDelay, d.callback, f) return d
class XMLRPC(resource.Resource): """ A resource that implements XML-RPC. You probably want to connect this to '/RPC2'. Methods published can return XML-RPC serializable results, Faults, Binary, Boolean, DateTime, Deferreds, or Handler instances. By default methods beginning with 'xmlrpc_' are published. Sub-handlers for prefixed methods (e.g., system.listMethods) can be added with putSubHandler. By default, prefixes are separated with a '.'. Override self.separator to change this. @ivar allowNone: Permit XML translating of Python constant None. @type allowNone: C{bool} @ivar useDateTime: Present C{datetime} values as C{datetime.datetime} objects? @type useDateTime: C{bool} """ # Error codes for Twisted, if they conflict with yours then # modify them at runtime. NOT_FOUND = 8001 FAILURE = 8002 isLeaf = 1 separator = "." allowedMethods = (b"POST", ) _log = Logger() def __init__(self, allowNone=False, useDateTime=False): resource.Resource.__init__(self) self.subHandlers = {} self.allowNone = allowNone self.useDateTime = useDateTime def __setattr__(self, name, value): self.__dict__[name] = value def putSubHandler(self, prefix, handler): self.subHandlers[prefix] = handler def getSubHandler(self, prefix): return self.subHandlers.get(prefix, None) def getSubHandlerPrefixes(self): return list(self.subHandlers.keys()) def render_POST(self, request): request.content.seek(0, 0) request.setHeader(b"content-type", b"text/xml; charset=utf-8") try: args, functionPath = xmlrpclib.loads(request.content.read(), use_datetime=self.useDateTime) except Exception as e: f = Fault(self.FAILURE, "Can't deserialize input: %s" % (e, )) self._cbRender(f, request) else: try: function = self.lookupProcedure(functionPath) except Fault as f: self._cbRender(f, request) else: # Use this list to track whether the response has failed or not. # This will be used later on to decide if the result of the # Deferred should be written out and Request.finish called. responseFailed = [] request.notifyFinish().addErrback(responseFailed.append) if getattr(function, "withRequest", False): d = defer.maybeDeferred(function, request, *args) else: d = defer.maybeDeferred(function, *args) d.addErrback(self._ebRender) d.addCallback(self._cbRender, request, responseFailed) return server.NOT_DONE_YET def _cbRender(self, result, request, responseFailed=None): if responseFailed: return if isinstance(result, Handler): result = result.result if not isinstance(result, Fault): result = (result, ) try: try: content = xmlrpclib.dumps(result, methodresponse=True, allow_none=self.allowNone) except Exception as e: f = Fault(self.FAILURE, "Can't serialize output: %s" % (e, )) content = xmlrpclib.dumps(f, methodresponse=True, allow_none=self.allowNone) if isinstance(content, str): content = content.encode("utf8") request.setHeader(b"content-length", b"%d" % (len(content), )) request.write(content) except Exception: self._log.failure("") request.finish() def _ebRender(self, failure): if isinstance(failure.value, Fault): return failure.value self._log.failure("", failure) return Fault(self.FAILURE, "error") def lookupProcedure(self, procedurePath): """ Given a string naming a procedure, return a callable object for that procedure or raise NoSuchFunction. The returned object will be called, and should return the result of the procedure, a Deferred, or a Fault instance. Override in subclasses if you want your own policy. The base implementation that given C{'foo'}, C{self.xmlrpc_foo} will be returned. If C{procedurePath} contains C{self.separator}, the sub-handler for the initial prefix is used to search for the remaining path. If you override C{lookupProcedure}, you may also want to override C{listProcedures} to accurately report the procedures supported by your resource, so that clients using the I{system.listMethods} procedure receive accurate results. @since: 11.1 """ if procedurePath.find(self.separator) != -1: prefix, procedurePath = procedurePath.split(self.separator, 1) handler = self.getSubHandler(prefix) if handler is None: raise NoSuchFunction(self.NOT_FOUND, "no such subHandler %s" % prefix) return handler.lookupProcedure(procedurePath) f = getattr(self, "xmlrpc_%s" % procedurePath, None) if not f: raise NoSuchFunction(self.NOT_FOUND, "procedure %s not found" % procedurePath) elif not callable(f): raise NoSuchFunction(self.NOT_FOUND, "procedure %s not callable" % procedurePath) else: return f def listProcedures(self): """ Return a list of the names of all xmlrpc procedures. @since: 11.1 """ return reflect.prefixedMethodNames(self.__class__, "xmlrpc_")
else: print(vote["time"],\ "DOWNVOTE",\ vote["voter"],"=>",vote_event["author"],\ vote["rshares"],"(",\ start_rshares , "->" , start_rshares + float(vote["rshares"]) , ")") #Update the total rshares recorded before our downvote start_rshares = start_rshares + float(vote["rshares"]) if found == False: print("vote not found, possibly to old.",vote_event["voter"],"=>",vote_event["author"],vote_event["permlink"]) #Set the above closure as callback. opp.on_result(process_content) #This is a bit fiddly at this low level, start nextblock a bit higer than where we start out nextblock = 19933100 obs = textFileLogObserver(sys.stdout) log = Logger(observer=obs,namespace="jsonrpc_test") #Create our JSON-RPC RpcClient rpcclient = RpcClient(reactor,log,nodelist="stage") #Count the number of active block queries active_block_queries = 0 sync_block = None #Function for fetching a block and its operations. def get_block(blk): """Request a single block asynchonously.""" global active_block_queries #This one is for processing the results from get_block def process_block(event, client): """Process the result from block getting request.""" global active_block_queries global nextblock global sync_block
class Request(Copyable, http.Request, components.Componentized): """ An HTTP request. @ivar defaultContentType: A C{bytes} giving the default I{Content-Type} value to send in responses if no other value is set. L{None} disables the default. @ivar _insecureSession: The L{Session} object representing state that will be transmitted over plain-text HTTP. @ivar _secureSession: The L{Session} object representing the state that will be transmitted only over HTTPS. """ defaultContentType = b"text/html" site = None appRootURL = None __pychecker__ = 'unusednames=issuer' _inFakeHead = False _encoder = None _log = Logger() def __init__(self, *args, **kw): http.Request.__init__(self, *args, **kw) components.Componentized.__init__(self) def getStateToCopyFor(self, issuer): x = self.__dict__.copy() del x['transport'] # XXX refactor this attribute out; it's from protocol # del x['server'] del x['channel'] del x['content'] del x['site'] self.content.seek(0, 0) x['content_data'] = self.content.read() x['remote'] = ViewPoint(issuer, self) # Address objects aren't jellyable x['host'] = _addressToTuple(x['host']) x['client'] = _addressToTuple(x['client']) # Header objects also aren't jellyable. x['requestHeaders'] = list(x['requestHeaders'].getAllRawHeaders()) return x # HTML generation helpers def sibLink(self, name): """ Return the text that links to a sibling of the requested resource. """ if self.postpath: return (len(self.postpath) * b"../") + name else: return name def childLink(self, name): """ Return the text that links to a child of the requested resource. """ lpp = len(self.postpath) if lpp > 1: return ((lpp - 1) * b"../") + name elif lpp == 1: return name else: # lpp == 0 if len(self.prepath) and self.prepath[-1]: return self.prepath[-1] + b'/' + name else: return name def process(self): """ Process a request. """ # get site from channel self.site = self.channel.site # set various default headers self.setHeader(b'server', version) self.setHeader(b'date', http.datetimeToString()) # Resource Identification self.prepath = [] self.postpath = list(map(unquote, self.path[1:].split(b'/'))) # Short-circuit for requests whose path is '*'. if self.path == b'*': self._handleStar() return try: resrc = self.site.getResourceFor(self) if resource._IEncodingResource.providedBy(resrc): encoder = resrc.getEncoder(self) if encoder is not None: self._encoder = encoder self.render(resrc) except: self.processingFailed(failure.Failure()) def write(self, data): """ Write data to the transport (if not responding to a HEAD request). @param data: A string to write to the response. """ if not self.startedWriting: # Before doing the first write, check to see if a default # Content-Type header should be supplied. We omit it on # NOT_MODIFIED and NO_CONTENT responses. We also omit it if there # is a Content-Length header set to 0, as empty bodies don't need # a content-type. needsCT = self.code not in (http.NOT_MODIFIED, http.NO_CONTENT) contentType = self.responseHeaders.getRawHeaders(b'content-type') contentLength = self.responseHeaders.getRawHeaders( b'content-length') contentLengthZero = contentLength and (contentLength[0] == b'0') if (needsCT and contentType is None and self.defaultContentType is not None and not contentLengthZero): self.responseHeaders.setRawHeaders(b'content-type', [self.defaultContentType]) # Only let the write happen if we're not generating a HEAD response by # faking out the request method. Note, if we are doing that, # startedWriting will never be true, and the above logic may run # multiple times. It will only actually change the responseHeaders # once though, so it's still okay. if not self._inFakeHead: if self._encoder: data = self._encoder.encode(data) http.Request.write(self, data) def finish(self): """ Override C{http.Request.finish} for possible encoding. """ if self._encoder: data = self._encoder.finish() if data: http.Request.write(self, data) return http.Request.finish(self) def render(self, resrc): """ Ask a resource to render itself. @param resrc: a L{twisted.web.resource.IResource}. """ try: body = resrc.render(self) except UnsupportedMethod as e: allowedMethods = e.allowedMethods if (self.method == b"HEAD") and (b"GET" in allowedMethods): # We must support HEAD (RFC 2616, 5.1.1). If the # resource doesn't, fake it by giving the resource # a 'GET' request and then return only the headers, # not the body. self._log.info("Using GET to fake a HEAD request for {resrc}", resrc=resrc) self.method = b"GET" self._inFakeHead = True body = resrc.render(self) if body is NOT_DONE_YET: self._log.info( "Tried to fake a HEAD request for {resrc}, but " "it got away from me.", resrc=resrc) # Oh well, I guess we won't include the content length. else: self.setHeader(b'content-length', intToBytes(len(body))) self._inFakeHead = False self.method = b"HEAD" self.write(b'') self.finish() return if self.method in (supportedMethods): # We MUST include an Allow header # (RFC 2616, 10.4.6 and 14.7) self.setHeader(b'Allow', b', '.join(allowedMethods)) s = ('''Your browser approached me (at %(URI)s) with''' ''' the method "%(method)s". I only allow''' ''' the method%(plural)s %(allowed)s here.''' % { 'URI': escape(nativeString(self.uri)), 'method': nativeString(self.method), 'plural': ((len(allowedMethods) > 1) and 's') or '', 'allowed': ', '.join([nativeString(x) for x in allowedMethods]) }) epage = resource.ErrorPage(http.NOT_ALLOWED, "Method Not Allowed", s) body = epage.render(self) else: epage = resource.ErrorPage( http.NOT_IMPLEMENTED, "Huh?", "I don't know how to treat a %s request." % (escape(self.method.decode("charmap")), )) body = epage.render(self) # end except UnsupportedMethod if body == NOT_DONE_YET: return if not isinstance(body, bytes): body = resource.ErrorPage( http.INTERNAL_SERVER_ERROR, "Request did not return bytes", "Request: " + util._PRE(reflect.safe_repr(self)) + "<br />" + "Resource: " + util._PRE(reflect.safe_repr(resrc)) + "<br />" + "Value: " + util._PRE(reflect.safe_repr(body))).render(self) if self.method == b"HEAD": if len(body) > 0: # This is a Bad Thing (RFC 2616, 9.4) self._log.info( "Warning: HEAD request {slf} for resource {resrc} is" " returning a message body. I think I'll eat it.", slf=self, resrc=resrc) self.setHeader(b'content-length', intToBytes(len(body))) self.write(b'') else: self.setHeader(b'content-length', intToBytes(len(body))) self.write(body) self.finish() def processingFailed(self, reason): """ Finish this request with an indication that processing failed and possibly display a traceback. @param reason: Reason this request has failed. @type reason: L{twisted.python.failure.Failure} @return: The reason passed to this method. @rtype: L{twisted.python.failure.Failure} """ self._log.failure('', failure=reason) if self.site.displayTracebacks: body = (b"<html><head><title>web.Server Traceback" b" (most recent call last)</title></head>" b"<body><b>web.Server Traceback" b" (most recent call last):</b>\n\n" + util.formatFailure(reason) + b"\n\n</body></html>\n") else: body = (b"<html><head><title>Processing Failed" b"</title></head><body>" b"<b>Processing Failed</b></body></html>") self.setResponseCode(http.INTERNAL_SERVER_ERROR) self.setHeader(b'content-type', b"text/html") self.setHeader(b'content-length', intToBytes(len(body))) self.write(body) self.finish() return reason def view_write(self, issuer, data): """Remote version of write; same interface. """ self.write(data) def view_finish(self, issuer): """Remote version of finish; same interface. """ self.finish() def view_addCookie(self, issuer, k, v, **kwargs): """Remote version of addCookie; same interface. """ self.addCookie(k, v, **kwargs) def view_setHeader(self, issuer, k, v): """Remote version of setHeader; same interface. """ self.setHeader(k, v) def view_setLastModified(self, issuer, when): """Remote version of setLastModified; same interface. """ self.setLastModified(when) def view_setETag(self, issuer, tag): """Remote version of setETag; same interface. """ self.setETag(tag) def view_setResponseCode(self, issuer, code, message=None): """ Remote version of setResponseCode; same interface. """ self.setResponseCode(code, message) def view_registerProducer(self, issuer, producer, streaming): """Remote version of registerProducer; same interface. (requires a remote producer.) """ self.registerProducer(_RemoteProducerWrapper(producer), streaming) def view_unregisterProducer(self, issuer): self.unregisterProducer() ### these calls remain local _secureSession = None _insecureSession = None @property def session(self): """ If a session has already been created or looked up with L{Request.getSession}, this will return that object. (This will always be the session that matches the security of the request; so if C{forceNotSecure} is used on a secure request, this will not return that session.) @return: the session attribute @rtype: L{Session} or L{None} """ if self.isSecure(): return self._secureSession else: return self._insecureSession def getSession(self, sessionInterface=None, forceNotSecure=False): """ Check if there is a session cookie, and if not, create it. By default, the cookie with be secure for HTTPS requests and not secure for HTTP requests. If for some reason you need access to the insecure cookie from a secure request you can set C{forceNotSecure = True}. @param forceNotSecure: Should we retrieve a session that will be transmitted over HTTP, even if this L{Request} was delivered over HTTPS? @type forceNotSecure: L{bool} """ # Make sure we aren't creating a secure session on a non-secure page secure = self.isSecure() and not forceNotSecure if not secure: cookieString = b"TWISTED_SESSION" sessionAttribute = "_insecureSession" else: cookieString = b"TWISTED_SECURE_SESSION" sessionAttribute = "_secureSession" session = getattr(self, sessionAttribute) if session is not None: # We have a previously created session. try: # Refresh the session, to keep it alive. session.touch() except (AlreadyCalled, AlreadyCancelled): # Session has already expired. session = None if session is None: # No session was created yet for this request. cookiename = b"_".join([cookieString] + self.sitepath) sessionCookie = self.getCookie(cookiename) if sessionCookie: try: session = self.site.getSession(sessionCookie) except KeyError: pass # if it still hasn't been set, fix it up. if not session: session = self.site.makeSession() self.addCookie(cookiename, session.uid, path=b"/", secure=secure) setattr(self, sessionAttribute, session) if sessionInterface: return session.getComponent(sessionInterface) return session def _prePathURL(self, prepath): port = self.getHost().port if self.isSecure(): default = 443 else: default = 80 if port == default: hostport = '' else: hostport = ':%d' % port prefix = networkString( 'http%s://%s%s/' % (self.isSecure() and 's' or '', nativeString(self.getRequestHostname()), hostport)) path = b'/'.join([quote(segment, safe=b'') for segment in prepath]) return prefix + path def prePathURL(self): return self._prePathURL(self.prepath) def URLPath(self): from twisted.python import urlpath return urlpath.URLPath.fromRequest(self) def rememberRootURL(self): """ Remember the currently-processed part of the URL for later recalling. """ url = self._prePathURL(self.prepath[:-1]) self.appRootURL = url def getRootURL(self): """ Get a previously-remembered URL. """ return self.appRootURL def _handleStar(self): """ Handle receiving a request whose path is '*'. RFC 7231 defines an OPTIONS * request as being something that a client can send as a low-effort way to probe server capabilities or readiness. Rather than bother the user with this, we simply fast-path it back to an empty 200 OK. Any non-OPTIONS verb gets a 405 Method Not Allowed telling the client they can only use OPTIONS. """ if self.method == b'OPTIONS': self.setResponseCode(http.OK) else: self.setResponseCode(http.NOT_ALLOWED) self.setHeader(b'Allow', b'OPTIONS') # RFC 7231 says we MUST set content-length 0 when responding to this # with no body. self.setHeader(b'Content-Length', b'0') self.finish()
class _WSGIResponse: """ Helper for L{WSGIResource} which drives the WSGI application using a threadpool and hooks it up to the L{http.Request}. @ivar started: A L{bool} indicating whether or not the response status and headers have been written to the request yet. This may only be read or written in the WSGI application thread. @ivar reactor: An L{IReactorThreads} provider which is used to call methods on the request in the I/O thread. @ivar threadpool: A L{ThreadPool} which is used to call the WSGI application object in a non-I/O thread. @ivar application: The WSGI application object. @ivar request: The L{http.Request} upon which the WSGI environment is based and to which the application's output will be sent. @ivar environ: The WSGI environment L{dict}. @ivar status: The HTTP response status L{str} supplied to the WSGI I{start_response} callable by the application. @ivar headers: A list of HTTP response headers supplied to the WSGI I{start_response} callable by the application. @ivar _requestFinished: A flag which indicates whether it is possible to generate more response data or not. This is L{False} until L{http.Request.notifyFinish} tells us the request is done, then L{True}. """ _requestFinished = False _log = Logger() def __init__(self, reactor, threadpool, application, request): self.started = False self.reactor = reactor self.threadpool = threadpool self.application = application self.request = request self.request.notifyFinish().addBoth(self._finished) if request.prepath: scriptName = b'/' + b'/'.join(request.prepath) else: scriptName = b'' if request.postpath: pathInfo = b'/' + b'/'.join(request.postpath) else: pathInfo = b'' parts = request.uri.split(b'?', 1) if len(parts) == 1: queryString = b'' else: queryString = parts[1] # All keys and values need to be native strings, i.e. of type str in # *both* Python 2 and Python 3, so says PEP-3333. self.environ = { 'REQUEST_METHOD': _wsgiString(request.method), 'REMOTE_ADDR': _wsgiString(request.getClientIP()), 'SCRIPT_NAME': _wsgiString(scriptName), 'PATH_INFO': _wsgiString(pathInfo), 'QUERY_STRING': _wsgiString(queryString), 'CONTENT_TYPE': _wsgiString(request.getHeader(b'content-type') or ''), 'CONTENT_LENGTH': _wsgiString(request.getHeader(b'content-length') or ''), 'SERVER_NAME': _wsgiString(request.getRequestHostname()), 'SERVER_PORT': _wsgiString(str(request.getHost().port)), 'SERVER_PROTOCOL': _wsgiString(request.clientproto) } # The application object is entirely in control of response headers; # disable the default Content-Type value normally provided by # twisted.web.server.Request. self.request.defaultContentType = None for name, values in request.requestHeaders.getAllRawHeaders(): name = 'HTTP_' + _wsgiString(name).upper().replace('-', '_') # It might be preferable for http.HTTPChannel to clear out # newlines. self.environ[name] = ','.join(_wsgiString(v) for v in values).replace('\n', ' ') self.environ.update({ 'wsgi.version': (1, 0), 'wsgi.url_scheme': request.isSecure() and 'https' or 'http', 'wsgi.run_once': False, 'wsgi.multithread': True, 'wsgi.multiprocess': False, 'wsgi.errors': _ErrorStream(), # Attend: request.content was owned by the I/O thread up until # this point. By wrapping it and putting the result into the # environment dictionary, it is effectively being given to # another thread. This means that whatever it is, it has to be # safe to access it from two different threads. The access # *should* all be serialized (first the I/O thread writes to # it, then the WSGI thread reads from it, then the I/O thread # closes it). However, since the request is made available to # arbitrary application code during resource traversal, it's # possible that some other code might decide to use it in the # I/O thread concurrently with its use in the WSGI thread. # More likely than not, this will break. This seems like an # unlikely possibility to me, but if it is to be allowed, # something here needs to change. -exarkun 'wsgi.input': _InputStream(request.content) }) def _finished(self, ignored): """ Record the end of the response generation for the request being serviced. """ self._requestFinished = True def startResponse(self, status, headers, excInfo=None): """ The WSGI I{start_response} callable. The given values are saved until they are needed to generate the response. This will be called in a non-I/O thread. """ if self.started and excInfo is not None: reraise(excInfo[1], excInfo[2]) # PEP-3333 mandates that status should be a native string. In practice # this is mandated by Twisted's HTTP implementation too, so we enforce # on both Python 2 and Python 3. if not isinstance(status, str): raise TypeError("status must be str, not %r (%s)" % (status, type(status).__name__)) # PEP-3333 mandates that headers should be a plain list, but in # practice we work with any sequence type and only warn when it's not # a plain list. if isinstance(headers, list): pass # This is okay. elif isinstance(headers, Sequence): warn("headers should be a list, not %r (%s)" % (headers, type(headers).__name__), category=RuntimeWarning) else: raise TypeError("headers must be a list, not %r (%s)" % (headers, type(headers).__name__)) # PEP-3333 mandates that each header should be a (str, str) tuple, but # in practice we work with any sequence type and only warn when it's # not a plain list. for header in headers: if isinstance(header, tuple): pass # This is okay. elif isinstance(header, Sequence): warn("header should be a (str, str) tuple, not %r (%s)" % (header, type(header).__name__), category=RuntimeWarning) else: raise TypeError( "header must be a (str, str) tuple, not %r (%s)" % (header, type(header).__name__)) # However, the sequence MUST contain only 2 elements. if len(header) != 2: raise TypeError("header must be a (str, str) tuple, not %r" % (header, )) # Both elements MUST be native strings. Non-native strings will be # rejected by the underlying HTTP machinery in any case, but we # reject them here in order to provide a more informative error. for elem in header: if not isinstance(elem, str): raise TypeError("header must be (str, str) tuple, not %r" % (header, )) self.status = status self.headers = headers return self.write def write(self, data): """ The WSGI I{write} callable returned by the I{start_response} callable. The given bytes will be written to the response body, possibly flushing the status and headers first. This will be called in a non-I/O thread. """ # PEP-3333 states: # # The server or gateway must transmit the yielded bytestrings to the # client in an unbuffered fashion, completing the transmission of # each bytestring before requesting another one. # # This write() method is used for the imperative and (indirectly) for # the more familiar iterable-of-bytestrings WSGI mechanism. It uses # C{blockingCallFromThread} to schedule writes. This allows exceptions # to propagate up from the underlying HTTP implementation. However, # that underlying implementation does not, as yet, provide any way to # know if the written data has been transmitted, so this method # violates the above part of PEP-3333. # # PEP-3333 also says that a server may: # # Use a different thread to ensure that the block continues to be # transmitted while the application produces the next block. # # Which suggests that this is actually compliant with PEP-3333, # because writes are done in the reactor thread. # # However, providing some back-pressure may nevertheless be a Good # Thing at some point in the future. def wsgiWrite(started): if not started: self._sendResponseHeaders() self.request.write(data) try: return blockingCallFromThread(self.reactor, wsgiWrite, self.started) finally: self.started = True def _sendResponseHeaders(self): """ Set the response code and response headers on the request object, but do not flush them. The caller is responsible for doing a write in order for anything to actually be written out in response to the request. This must be called in the I/O thread. """ code, message = self.status.split(None, 1) code = int(code) self.request.setResponseCode(code, _wsgiStringToBytes(message)) for name, value in self.headers: # Don't allow the application to control these required headers. if name.lower() not in ('server', 'date'): self.request.responseHeaders.addRawHeader( _wsgiStringToBytes(name), _wsgiStringToBytes(value)) def start(self): """ Start the WSGI application in the threadpool. This must be called in the I/O thread. """ self.threadpool.callInThread(self.run) def run(self): """ Call the WSGI application object, iterate it, and handle its output. This must be called in a non-I/O thread (ie, a WSGI application thread). """ try: appIterator = self.application(self.environ, self.startResponse) for elem in appIterator: if elem: self.write(elem) if self._requestFinished: break close = getattr(appIterator, 'close', None) if close is not None: close() except: def wsgiError(started, type, value, traceback): self._log.failure("WSGI application error", failure=Failure(value, type, traceback)) if started: self.request.loseConnection() else: self.request.setResponseCode(INTERNAL_SERVER_ERROR) self.request.finish() self.reactor.callFromThread(wsgiError, self.started, *exc_info()) else: def wsgiFinish(started): if not self._requestFinished: if not started: self._sendResponseHeaders() self.request.finish() self.reactor.callFromThread(wsgiFinish, self.started) self.started = True