def __try_to_connect_to_address(self, connect_info: TCPConnectInfo): address = connect_info.socket_addresses[0].address port = connect_info.socket_addresses[0].port logger.debug("Connection to host %r: %r", address, port) use_ipv6 = connect_info.socket_addresses[0].ipv6 use_hostname = connect_info.socket_addresses[0].hostname if use_ipv6: endpoint = TCP6ClientEndpoint(self.reactor, address, port, self.timeout) elif use_hostname: endpoint = HostnameEndpoint(self.reactor, address, port, self.timeout) else: endpoint = TCP4ClientEndpoint(self.reactor, address, port, self.timeout) defer = endpoint.connect(self.outgoing_protocol_factory) defer.addCallback(self.__connection_established, self.__connection_to_address_established, connect_info) defer.addErrback(self.__connection_failure, self.__connection_to_address_failure, connect_info)
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port try: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory)) return result except Exception as e: logger.info("Failed to connect to %s:%i: %s", host.decode("ascii"), port, e) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc, ))
def connect(self, host): point = HostnameEndpoint(self._reactor, host, SCPI_PORT) d = point.connect(SCPIClientFactory()) @d.addCallback def connect_vrt(scpi): self._scpi = scpi point = HostnameEndpoint(self._reactor, host, VRT_PORT) return point.connect(VRTClientFactory(self._vrt_callback)) @d.addCallback def save_vrt(vrt): self._vrt = vrt return d
def _asynchronous_check(self, item): logger.debug('checking...') from twisted.internet import reactor # must here import endpoint = HostnameEndpoint(reactor, item[0], int(item[1])) agent = ProxyAgent(endpoint) # headers = {'User-Agent': ['Mozilla/5.0']} headers = Headers(headers) # headers wrapper cd = agent.request(b'GET', b'https://www.baidu.com/', headers=headers) # ? cd._connectTimeout = 3 def check_code(response, **kwargs): if response.code < 300: logger.info('valid ip!') return kwargs.pop('item', None) else: raise Exception('invalid') def err(f): logger.debug(f) return f cd.addCallbacks(check_code, err, callbackKeywords={'item': item}) # cd.addErrback(err) return cd
def __init__( self, reactor, contextFactory=BrowserLikePolicyForHTTPS(), connectTimeout=None, bindAddress=None, pool=None, proxy_url_str: Optional[str] = None, ): _AgentBase.__init__(self, reactor, pool) self._endpoint_kwargs = {} if connectTimeout is not None: self._endpoint_kwargs["timeout"] = connectTimeout if bindAddress is not None: self._endpoint_kwargs["bindAddress"] = bindAddress if proxy_url_str is not None: parsed_url = decompose_http_proxy_url(proxy_url_str) self._proxy_auth = parsed_url.credentials self.proxy_endpoint = HostnameEndpoint(reactor, parsed_url.hostname, parsed_url.port, **self._endpoint_kwargs) else: self.proxy_endpoint = None self._policy_for_https = contextFactory self._reactor = reactor
def download_request(self, request, spider): bindaddress = request.meta.get('bindaddress') timeout = request.meta.get('download_timeout') maxsize = getattr(spider, 'download_maxsize', self.default_maxsize) warnsize = getattr(spider, 'download_warnsize', self.default_warnsize) parts = urlparse(request.url) remote_host = bindaddress or parts.hostname remote_port = parts.port or 1965 hostname = HostnameEndpoint(reactor, remote_host, remote_port) # The recommended helper method for this (optionsForClientTLS) does not # allow setting up a client context that accepts unverified certificates. # So we are forced to use the private ClientTLSOptions method instead. options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext()) # noinspection PyTypeChecker endpoint = wrapClientTLS(options, hostname) logger.debug(f"Creating download request for {request.url}") protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout) # If the connection fails (DNS lookup, etc.) propagate the error so # that scrapy knows the request has completed. connected = connectProtocol(endpoint, protocol) connected.addErrback(protocol.finished.errback) return protocol.finished
def inner(reactor): print('Creating client service') from twisted.internet import reactor endpoint = HostnameEndpoint(reactor, '127.0.0.1', self.server.port) client = ClientSessionService(endpoint, 'test', 'secret') client.subscribe('test-chan') print('Starting client service') client.startService() # Wait till client connected print('Waiting to be connected') yield client.whenConnected print('Publishing test message') client.publish('test-chan', b'test message') print('Waiting for read()') payload = yield client.read() assert ('test', 'test-chan', b'test message') == payload print('Stopping client') yield client.stopService() print('Stopping server for reals') yield self.server.close()
def getMapUpdaterComponentService(self, url, realm, mapUpdater): def create(config): try: session = MapUpdaterComponent(mapUpdater, config) except Exception as e: # the app component could not be created .. fatal print(e) else: session.debug_app = True return session sessionFactory = ApplicationSessionFactory( ComponentConfig(realm, None)) sessionFactory.session = create transportFactory = WampWebSocketClientFactory( sessionFactory, url=url) transportFactory.noisy = False transportFactory.autoPingInterval = 30 transportFactory.autoPingTimeout = 30 isSecure, host, port, resource, path, params = parseWsUrl(url) endpoint = HostnameEndpoint(reactor, host.encode('utf8'), port) if isSecure: contextFactory = optionsForClientTLS(hostname=host) endpoint = wrapClientTLS(contextFactory, endpoint) return ClientService(endpoint, transportFactory)
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port endpoint: IStreamClientEndpoint try: if self.https_proxy_endpoint and not proxy_bypass(host.decode()): logger.debug( "Connecting to %s:%i via %s", host.decode("ascii"), port, self.https_proxy_endpoint, ) connect_headers = Headers() # Determine whether we need to set Proxy-Authorization headers if self.https_proxy_creds: # Set a Proxy-Authorization header connect_headers.addRawHeader( b"Proxy-Authorization", self.https_proxy_creds.as_proxy_authorization_value(), ) endpoint = HTTPConnectProxyEndpoint( self._reactor, self.https_proxy_endpoint, host, port, headers=connect_headers, ) else: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) # not using a proxy endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory) ) return result except Exception as e: logger.info( "Failed to connect to %s:%i: %s", host.decode("ascii"), port, e ) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,))
async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port should_skip_proxy = False if self.no_proxy is not None: should_skip_proxy = proxy_bypass_environment( host.decode(), proxies={"no": self.no_proxy}, ) endpoint: IStreamClientEndpoint try: if self._https_proxy_endpoint and not should_skip_proxy: logger.debug( "Connecting to %s:%i via %s", host.decode("ascii"), port, self._https_proxy_endpoint, ) endpoint = HTTPConnectProxyEndpoint( self._reactor, self._https_proxy_endpoint, host, port, proxy_creds=self._https_proxy_creds, ) else: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) # not using a proxy endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory)) return result except Exception as e: logger.info("Failed to connect to %s:%i: %s", host.decode("ascii"), port, e) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc, ))
def create_proxy_client(host, port, proxy_spec, instance): """ host: the host of the final destination port: the port number of the final destination proxy_spec: the address of the proxy server as a urlparse.SplitResult instance: is the instance to be associated with the endpoint Returns a deferred that will fire when the connection to the SOCKS server has been established. """ # Inline import so that txsocksx is an optional dependency. from twisted.internet.endpoints import HostnameEndpoint from txsocksx.client import SOCKS4ClientEndpoint, SOCKS5ClientEndpoint from obfsproxy.network.http import HTTPConnectClientEndpoint TCPPoint = HostnameEndpoint(reactor, proxy_spec.hostname, proxy_spec.port) username = proxy_spec.username password = proxy_spec.password # Do some logging log.debug("Connecting via %s proxy %s:%d", proxy_spec.scheme, log.safe_addr_str(proxy_spec.hostname), proxy_spec.port) if username or password: log.debug("Using %s:%s as the proxy credentials", log.safe_addr_str(username), log.safe_addr_str(password)) if proxy_spec.scheme in ["socks4a", "socks5"]: if proxy_spec.scheme == "socks4a": if username: assert(password == None) SOCKSPoint = SOCKS4ClientEndpoint(host, port, TCPPoint, user=username) else: SOCKSPoint = SOCKS4ClientEndpoint(host, port, TCPPoint) elif proxy_spec.scheme == "socks5": if username and password: SOCKSPoint = SOCKS5ClientEndpoint(host, port, TCPPoint, methods={'login': (username, password)}) else: assert(username == None and password == None) SOCKSPoint = SOCKS5ClientEndpoint(host, port, TCPPoint) d = SOCKSPoint.connect(instance) return d elif proxy_spec.scheme == "http": if username and password: HTTPPoint = HTTPConnectClientEndpoint(host, port, TCPPoint, username, password) else: assert(username == None and password == None) HTTPPoint = HTTPConnectClientEndpoint(host, port, TCPPoint) d = HTTPPoint.connect(instance) return d else: # Should *NEVER* happen raise RuntimeError("Invalid proxy scheme %s" % proxy_spec.scheme)
def hint_to_endpoint(self, hint, reactor, update_status): # Return (endpoint, hostname), where "hostname" is what we pass to the # HTTP "Host:" header so a dumb HTTP server can be used to redirect us. mo = NEW_STYLE_HINT_RE.search(hint) if not mo: raise InvalidHintError("unrecognized TCP hint") host, port = mo.group(1), int(mo.group(2)) host = host.lstrip("[").rstrip("]") return HostnameEndpoint(reactor, host, port), host
def channelOpen(self, specificData): """ See: L{channel.SSHChannel} """ log.msg("connecting to %s:%i" % self.hostport) ep = HostnameEndpoint(self._reactor, self.hostport[0], self.hostport[1]) d = connectProtocol(ep, SSHForwardingClient(self)) d.addCallbacks(self._setClient, self._close) self._channelOpenDeferred = d
def endpointForURI(self, uri): # noqa: N802 """Create an endpoint for URI.""" endpoint = HostnameEndpoint(self.reactor, self.address, uri.port) if uri.scheme == b'http': return endpoint elif uri.scheme == b'https': connection_creator = self.context_factory.creatorForNetloc( uri.host, uri.port) return wrapClientTLS(connection_creator, endpoint)
class LoggingHostnameEndpoint(object): """A wrapper for HostnameEndpint which logs when it connects""" def __init__(self, reactor, host, port, *args, **kwargs): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) logger.info("Endpoint created with %s:%d", host, port) def connect(self, protocol_factory): logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port) return self.ep.connect(protocol_factory)
class LoggingHostnameEndpoint(object): """A wrapper for HostnameEndpint which logs when it connects""" def __init__(self, reactor, host, port, *args, **kwargs): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) def connect(self, protocol_factory): logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port) return self.ep.connect(protocol_factory)
def channelOpen(self, specificData): """ See: L{channel.SSHChannel} """ self._log.info("connecting to {host}:{port}", host=self.hostport[0], port=self.hostport[1]) ep = HostnameEndpoint(self._reactor, self.hostport[0], self.hostport[1]) d = connectProtocol(ep, SSHForwardingClient(self)) d.addCallbacks(self._setClient, self._close) self._channelOpenDeferred = d
def endpoint_from_hint_obj(hint, tor, reactor): if tor: if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)): # this Tor object will throw ValueError for non-public IPv4 # addresses and any IPv6 address try: return tor.stream_via(hint.hostname, hint.port) except ValueError: return None return None if isinstance(hint, DirectTCPV1Hint): return HostnameEndpoint(reactor, hint.hostname, hint.port) return None
def main(reactor): from twisted.python import log log.startLogging(sys.stdout) tahoe_dir = "testgrid/alice" cfg = read_config(tahoe_dir, "portnum") token = cfg.get_private_config("api_auth_token").strip() webport = cfg.get_config("node", "web.port") if webport.startswith("tcp:"): port = webport.split(':')[1] else: port = webport factory = WebSocketClientFactory( url=u"ws://127.0.0.1:{}/private/logs/v1".format(port), headers={ "Authorization": "tahoe-lafs {}".format(token), } ) factory.on_open = Deferred() factory.on_close = Deferred() factory.protocol = TahoeLogProtocol endpoint = HostnameEndpoint(reactor, "127.0.0.1", int(port)) try: port = yield endpoint.connect(factory) except ConnectError as e: print("Connection failed: {}".format(e)) return print("port: {}".format(port)) yield factory.on_open print("opened") yield factory.on_close print("closed")
def _new_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: self._pending_requests[key] = deque() conn_lost_deferred = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) d = Deferred() self._pending_requests[key].append(d) return d
class LoggingHostnameEndpoint: """A wrapper for HostnameEndpint which logs when it connects""" def __init__(self, reactor: IReactorTime, host: bytes, port: int, *args: Any, **kwargs: Any): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) logger.info("Endpoint created with %s:%d", host, port) def connect( self, protocol_factory: IProtocolFactory) -> "defer.Deferred[IProtocol]": logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port) return self.ep.connect(protocol_factory)
def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], **kwargs: object, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy Args: proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>] This currently supports http:// and https:// proxies. A hostname without scheme is assumed to be http. reactor: reactor to be used to connect to the proxy tls_options_factory: the TLS options to use when connecting through a https proxy kwargs: other args to be passed to HostnameEndpoint Returns: a tuple of endpoint to use to connect to the proxy, or None ProxyCredentials or if no credentials were found, or None Raise: ValueError if proxy has no hostname or unsupported scheme. RuntimeError if no tls_options_factory is given for a https connection """ if proxy is None: return None, None # Note: urlsplit/urlparse cannot be used here as that does not work (for Python # 3.9+) on scheme-less proxies, e.g. host:port. scheme, host, port, credentials = parse_proxy(proxy) proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs) if scheme == b"https": if tls_options_factory: tls_options = tls_options_factory.creatorForNetloc(host, port) proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) else: raise RuntimeError( f"No TLS options for a https connection via proxy {proxy!s}" ) return proxy_endpoint, credentials
def endpoint_from_hint_obj(hint, tor, reactor): if tor: if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)): # this Tor object will throw ValueError for non-public IPv4 # addresses and any IPv6 address try: return tor.stream_via(hint.hostname, hint.port) except ValueError: return None return None if isinstance(hint, DirectTCPV1Hint): # avoid DNS lookup unless necessary if isIPAddress(hint.hostname): return TCP4ClientEndpoint(reactor, hint.hostname, hint.port) if isIPv6Address(hint.hostname): return TCP6ClientEndpoint(reactor, hint.hostname, hint.port) return HostnameEndpoint(reactor, hint.hostname, hint.port) return None
def __init__( self, host: str, port: int, maximum_buffer: int = 1000, level=logging.NOTSET, _reactor=None, ): super().__init__(level=level) self.host = host self.port = port self.maximum_buffer = maximum_buffer self._buffer = deque() # type: Deque[logging.LogRecord] self._connection_waiter = None # type: Optional[Deferred] self._producer = None # type: Optional[LogProducer] # Connect without DNS lookups if it's a direct IP. if _reactor is None: from twisted.internet import reactor _reactor = reactor try: ip = ip_address(self.host) if isinstance(ip, IPv4Address): endpoint = TCP4ClientEndpoint( _reactor, self.host, self.port) # type: IStreamClientEndpoint elif isinstance(ip, IPv6Address): endpoint = TCP6ClientEndpoint(_reactor, self.host, self.port) else: raise ValueError("Unknown IP address provided: %s" % (self.host, )) except ValueError: endpoint = HostnameEndpoint(_reactor, self.host, self.port) factory = Factory.forProtocol(Protocol) self._service = ClientService(endpoint, factory, clock=_reactor) self._service.startService() self._stopping = False self._connect()
async def connect(self, host, port, connect_timeout, source_address=None, socket_options=None): # HostnameEndpoint only supports setting source host, not source port if source_address is not None: raise NotImplementedError( "twisted backend doesn't support setting source_address") # factory = protocol.Factory.forProtocol(TwistedSocketProtocol) endpoint = HostnameEndpoint(self._reactor, host, port) d = connectProtocol(endpoint, TwistedSocketProtocol()) # XX d.addTimeout(...) protocol = await d if socket_options is not None: for opt in socket_options: if opt[:2] == (socket.IPPROTO_TCP, socket.TCP_NODELAY): protocol.transport.setTcpNoDelay(opt[2]) else: raise NotImplementedError( "unrecognized socket option for twisted backend") return TwistedSocket(protocol)
def start(self) -> None: # Connect without DNS lookups if it's a direct IP. try: ip = ip_address(self.host) if isinstance(ip, IPv4Address): endpoint = TCP4ClientEndpoint( self.hs.get_reactor(), self.host, self.port ) elif isinstance(ip, IPv6Address): endpoint = TCP6ClientEndpoint( self.hs.get_reactor(), self.host, self.port ) except ValueError: endpoint = HostnameEndpoint(self.hs.get_reactor(), self.host, self.port) factory = Factory.forProtocol(Protocol) self._service = ClientService(endpoint, factory, clock=self.hs.get_reactor()) self._service.startService() self._connect()
def runClient(connect, rate): http_proxy = os.getenv('HTTP_PROXY') if http_proxy: http_proxy = furl(http_proxy) ep = HostnameEndpoint(reactor, http_proxy.host, http_proxy.port) ua = ProxyAgent(ep) else: ua = Agent(reactor) client = Client(connect, ua) looper = task.LoopingCall(client.request_GET) # register signal handler to stop the looping call def signal_handler(signal, frame): looper.stop() reactor.runUntilCurrent() reactor.stop() signal.signal(signal.SIGINT, signal_handler) looper.start(1 / rate)
def _http_proxy_endpoint(proxy, reactor, **kwargs): """Parses an http proxy setting and returns an endpoint for the proxy Args: proxy (bytes|None): the proxy setting reactor: reactor to be used to connect to the proxy kwargs: other args to be passed to HostnameEndpoint Returns: interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy, or None """ if proxy is None: return None # currently we only support hostname:port. Some apps also support # protocol://<host>[:port], which allows a way of requiring a TLS connection to the # proxy. host, port = parse_host_port(proxy, default_port=1080) return HostnameEndpoint(reactor, host, port, **kwargs)
def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs): """Parses an http proxy setting and returns an endpoint for the proxy Args: proxy: the proxy setting in the form: [<username>:<password>@]<host>[:<port>] Note that compared to other apps, this function currently lacks support for specifying a protocol schema (i.e. protocol://...). reactor: reactor to be used to connect to the proxy kwargs: other args to be passed to HostnameEndpoint Returns: interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy, or None """ if proxy is None: return None # Parse the connection string host, port = parse_host_port(proxy, default_port=1080) return HostnameEndpoint(reactor, host, port, **kwargs)
def connect_vrt(scpi): self._scpi = scpi point = HostnameEndpoint(self._reactor, host, VRT_PORT) return point.connect(VRTClientFactory(self._vrt_callback))
def __init__(self, reactor, host, port, *args, **kwargs): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs) logger.info("Endpoint created with %s:%d", host, port)
def __init__(self, reactor, host, port, *args, **kwargs): self.host = host self.port = port self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
def transport_endpoint(reactor, host, port, timeout): return wrapClientTLS( ssl_context_factory, HostnameEndpoint(reactor, host, port, timeout=timeout))