def endpointForURI(_uri): ep = LoggingHostnameEndpoint( self._reactor, res.target_host, res.target_port, ) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port try: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory)) return result except Exception as e: logger.info("Failed to connect to %s:%i: %s", host.decode("ascii"), port, e) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc, ))
def main(): hostname = raw_input('IMAP4 Server Hostname: ') port = raw_input('IMAP4 Server Port (the default is 143, 993 uses SSL): ') username = raw_input('IMAP4 Username: '******'IMAP4 Password: '******'utf-8')) endpoint = endpoints.wrapClientTLS(contextFactory, endpoint) endpoint.connect(factory) reactor.run()
def download_request(self, request, spider): bindaddress = request.meta.get('bindaddress') timeout = request.meta.get('download_timeout') maxsize = getattr(spider, 'download_maxsize', self.default_maxsize) warnsize = getattr(spider, 'download_warnsize', self.default_warnsize) parts = urlparse(request.url) remote_host = bindaddress or parts.hostname remote_port = parts.port or 1965 hostname = HostnameEndpoint(reactor, remote_host, remote_port) # The recommended helper method for this (optionsForClientTLS) does not # allow setting up a client context that accepts unverified certificates. # So we are forced to use the private ClientTLSOptions method instead. options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext()) # noinspection PyTypeChecker endpoint = wrapClientTLS(options, hostname) logger.debug(f"Creating download request for {request.url}") protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout) # If the connection fails (DNS lookup, etc.) propagate the error so # that scrapy knows the request has completed. connected = connectProtocol(endpoint, protocol) connected.addErrback(protocol.finished.errback) return protocol.finished
def main(): hostname = raw_input("IMAP4 Server Hostname: ") port = raw_input("IMAP4 Server Port (the default is 143, 993 uses SSL): ") # Usernames are bytes. username = raw_input("IMAP4 Username: "******"ascii") # Passwords are bytes. password = util.getPassword("IMAP4 Password: "******"ascii") onConn = (defer.Deferred().addCallback( cbServerGreeting, username, password).addErrback(ebConnection).addBoth(cbClose)) factory = SimpleIMAP4ClientFactory(username, onConn) if not port: port = 143 else: port = int(port) from twisted.internet import reactor endpoint = endpoints.HostnameEndpoint(reactor, hostname, port) if port == 993: if isinstance(hostname, bytes): # This is python 2 hostname = hostname.decode("utf-8") contextFactory = ssl.optionsForClientTLS(hostname=hostname, ) endpoint = endpoints.wrapClientTLS(contextFactory, endpoint) endpoint.connect(factory) reactor.run()
def connect(self): """Connect and authenticate with the IMAP server """ if self.connecting: defer.returnValue(None) self.connecting = True endpoint = endpoints.HostnameEndpoint(reactor, self.host, self.port) if self.ssl: contextFactory = ssl.optionsForClientTLS( hostname=self.host.decode('utf-8')) endpoint = endpoints.wrapClientTLS(contextFactory, endpoint) de = defer.Deferred() factory = IMAP4ClientFactory(self.user, de) factory.debug = self.debug yield endpoint.connect(factory) self.proto = yield de yield self.proto.authenticate(self.password) self.connected = True self.connecting = False
def _get_new(self): """ Get new mails from IMAP server. This will define the `main loop` of the IMAP service. """ onConn = defer.Deferred().addCallback( self.cb_server_greeting ).addErrback( self.eb_server_greeting ) # Connect with endpoints. Connect and disconnect from time to time # (defined by the service's interval) instead of establishing a # persistent connection factory = SimpleIMAP4ClientFactory(self.username, onConn) from twisted.internet import reactor endpoint = endpoints.HostnameEndpoint(reactor, self.host, self.port) contextFactory = ssl.optionsForClientTLS( hostname=self.host.decode('utf-8') ) endpoint = endpoints.wrapClientTLS(contextFactory, endpoint) log.debug("IMAP:: Connecting to Google's IMAP servers.") endpoint.connect(factory)
def endpointForURI(_uri): ep = LoggingHostnameEndpoint( self._reactor, res.target_host, res.target_port, ) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep
def getMapUpdaterComponentService(self, url, realm, mapUpdater): def create(config): try: session = MapUpdaterComponent(mapUpdater, config) except Exception as e: # the app component could not be created .. fatal print(e) else: session.debug_app = True return session sessionFactory = ApplicationSessionFactory( ComponentConfig(realm, None)) sessionFactory.session = create transportFactory = WampWebSocketClientFactory( sessionFactory, url=url) transportFactory.noisy = False transportFactory.autoPingInterval = 30 transportFactory.autoPingTimeout = 30 isSecure, host, port, resource, path, params = parseWsUrl(url) endpoint = HostnameEndpoint(reactor, host.encode('utf8'), port) if isSecure: contextFactory = optionsForClientTLS(hostname=host) endpoint = wrapClientTLS(contextFactory, endpoint) return ClientService(endpoint, transportFactory)
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port endpoint: IStreamClientEndpoint try: if self.https_proxy_endpoint and not proxy_bypass(host.decode()): logger.debug( "Connecting to %s:%i via %s", host.decode("ascii"), port, self.https_proxy_endpoint, ) connect_headers = Headers() # Determine whether we need to set Proxy-Authorization headers if self.https_proxy_creds: # Set a Proxy-Authorization header connect_headers.addRawHeader( b"Proxy-Authorization", self.https_proxy_creds.as_proxy_authorization_value(), ) endpoint = HTTPConnectProxyEndpoint( self._reactor, self.https_proxy_endpoint, host, port, headers=connect_headers, ) else: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) # not using a proxy endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory) ) return result except Exception as e: logger.info( "Failed to connect to %s:%i: %s", host.decode("ascii"), port, e ) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,))
async def _do_connect(self, protocol_factory: IProtocolFactory) -> IProtocol: first_exception = None server_list = await self._resolve_server() for server in server_list: host = server.host port = server.port should_skip_proxy = False if self.no_proxy is not None: should_skip_proxy = proxy_bypass_environment( host.decode(), proxies={"no": self.no_proxy}, ) endpoint: IStreamClientEndpoint try: if self._https_proxy_endpoint and not should_skip_proxy: logger.debug( "Connecting to %s:%i via %s", host.decode("ascii"), port, self._https_proxy_endpoint, ) endpoint = HTTPConnectProxyEndpoint( self._reactor, self._https_proxy_endpoint, host, port, proxy_creds=self._https_proxy_creds, ) else: logger.debug("Connecting to %s:%i", host.decode("ascii"), port) # not using a proxy endpoint = HostnameEndpoint(self._reactor, host, port) if self._tls_options: endpoint = wrapClientTLS(self._tls_options, endpoint) result = await make_deferred_yieldable( endpoint.connect(protocol_factory)) return result except Exception as e: logger.info("Failed to connect to %s:%i: %s", host.decode("ascii"), port, e) if not first_exception: first_exception = e # We return the first failure because that's probably the most interesting. if first_exception: raise first_exception # This shouldn't happen as we should always have at least one host/port # to try and if that doesn't work then we'll have an exception. raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc, ))
def endpointForURI(_uri: URI) -> IStreamClientEndpoint: ep: IStreamClientEndpoint = LoggingHostnameEndpoint( self._reactor, routing.target_host, routing.target_port, ) if tls_options is not None: ep = wrapClientTLS(tls_options, ep) return ep
def endpointForURI(self, uri): # noqa: N802 """Create an endpoint for URI.""" endpoint = HostnameEndpoint(self.reactor, self.address, uri.port) if uri.scheme == b'http': return endpoint elif uri.scheme == b'https': connection_creator = self.context_factory.creatorForNetloc( uri.host, uri.port) return wrapClientTLS(connection_creator, endpoint)
def connect_client(listening_port): port_number = listening_port.getHost().port client = endpoints.TCP4ClientEndpoint(reactor, '127.0.0.1', port_number) options = optionsForClientTLS(hostname=hostname, trustRoot=PEM_ROOT) client = endpoints.wrapClientTLS(options, client) connectDeferred = client.connect(client_factory) def aggregate(client_proto): return (client_proto, listening_port) connectDeferred.addCallback(aggregate) return connectDeferred
def http_proxy_endpoint( proxy: Optional[bytes], reactor: IReactorCore, tls_options_factory: Optional[IPolicyForHTTPS], **kwargs: object, ) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]: """Parses an http proxy setting and returns an endpoint for the proxy Args: proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>] This currently supports http:// and https:// proxies. A hostname without scheme is assumed to be http. reactor: reactor to be used to connect to the proxy tls_options_factory: the TLS options to use when connecting through a https proxy kwargs: other args to be passed to HostnameEndpoint Returns: a tuple of endpoint to use to connect to the proxy, or None ProxyCredentials or if no credentials were found, or None Raise: ValueError if proxy has no hostname or unsupported scheme. RuntimeError if no tls_options_factory is given for a https connection """ if proxy is None: return None, None # Note: urlsplit/urlparse cannot be used here as that does not work (for Python # 3.9+) on scheme-less proxies, e.g. host:port. scheme, host, port, credentials = parse_proxy(proxy) proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs) if scheme == b"https": if tls_options_factory: tls_options = tls_options_factory.creatorForNetloc(host, port) proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint) else: raise RuntimeError( f"No TLS options for a https connection via proxy {proxy!s}" ) return proxy_endpoint, credentials
def connect_client(listening_port): port_number = listening_port.getHost().port client = endpoints.TCP4ClientEndpoint(reactor, '127.0.0.1', port_number) maybe_alpn = {} if acceptable_protocols is not None: maybe_alpn['acceptableProtocols'] = acceptable_protocols options = optionsForClientTLS(hostname=hostname, trustRoot=PEM_ROOT, **maybe_alpn) client = endpoints.wrapClientTLS(options, client) connectDeferred = client.connect(client_factory) def aggregate(client_proto): return (client_proto, listening_port) connectDeferred.addCallback(aggregate) return connectDeferred
def connect_client(listening_port): port_number = listening_port.getHost().port client = endpoints.TCP4ClientEndpoint( reactor, '127.0.0.1', port_number ) maybe_alpn = {} if acceptable_protocols is not None: maybe_alpn['acceptableProtocols'] = acceptable_protocols options = optionsForClientTLS( hostname=hostname, trustRoot=PEM_ROOT, **maybe_alpn ) client = endpoints.wrapClientTLS(options, client) connectDeferred = client.connect(client_factory) def aggregate(client_proto): return (client_proto, listening_port) connectDeferred.addCallback(aggregate) return connectDeferred
def transport_endpoint(reactor, host, port, timeout): return wrapClientTLS( tls_client_options_factory.get_options(host), HostnameEndpoint(reactor, host, port, timeout=timeout))
def transport_endpoint(reactor, host, port, timeout): return wrapClientTLS( ssl_context_factory, HostnameEndpoint(reactor, host, port, timeout=timeout))
def endpoint_factory(reactor, host, port, **kw): return wrapClientTLS( tlsCreator, HostnameEndpoint(reactor, host, port, **kw))
def transport_endpoint(reactor, host, port, timeout): return wrapClientTLS( ssl_context_factory, HostnameEndpoint(reactor, host, port, timeout=timeout))
def __init__(self, buildmaster_host, port, name, passwd, basedir, keepalive, usePTY=None, keepaliveTimeout=None, umask=None, maxdelay=None, numcpus=None, unicode_encoding=None, protocol='pb', useTls=None, allow_shutdown=None, maxRetries=None, connection_string=None, delete_leftover_dirs=False, proxy_connection_string=None): assert usePTY is None, "worker-side usePTY is not supported anymore" assert (connection_string is None or (buildmaster_host, port) == (None, None)), ( "If you want to supply a connection string, " "then set host and port to None") if protocol == 'pb': bot_class = BotPb elif protocol == 'msgpack_experimental_v1': if sys.version_info.major < 3: raise NotImplementedError( 'Msgpack protocol is not supported in Python2') bot_class = BotMsgpack else: raise ValueError('Unknown protocol {}'.format(protocol)) WorkerBase.__init__(self, name, basedir, bot_class, umask=umask, unicode_encoding=unicode_encoding, delete_leftover_dirs=delete_leftover_dirs) if keepalive == 0: keepalive = None name = unicode2bytes(name, self.bot.unicode_encoding) self.numcpus = numcpus self.shutdown_loop = None if allow_shutdown == 'signal': if not hasattr(signal, 'SIGHUP'): raise ValueError("Can't install signal handler") elif allow_shutdown == 'file': self.shutdown_file = os.path.join(basedir, 'shutdown.stamp') self.shutdown_mtime = 0 self.allow_shutdown = allow_shutdown if protocol == 'pb': passwd = unicode2bytes(passwd, self.bot.unicode_encoding) bf = self.bf = BotFactory(buildmaster_host, port, keepalive, maxdelay) bf.startLogin(credentials.UsernamePassword(name, passwd), client=self.bot) elif protocol == 'msgpack_experimental_v1': if connection_string is None: ws_conn_string = "ws://{}:{}".format(buildmaster_host, port) else: from urllib.parse import urlparse parsed_url = urlparse(connection_string) ws_conn_string = "ws://{}:{}".format(parsed_url.hostname, parsed_url.port) bf = self.bf = BuildbotWebSocketClientFactory(ws_conn_string) bf.protocol = BuildbotWebSocketClientProtocol self.bf.buildbot_bot = self.bot self.bf.name = name self.bf.password = passwd else: raise ValueError('Unknown protocol {}'.format(protocol)) def get_connection_string(host, port): if useTls: connection_type = 'tls' else: connection_type = 'tcp' return '{}:host={}:port={}'.format( connection_type, host.replace(':', r'\:'), # escape ipv6 addresses port) assert not (proxy_connection_string and connection_string), ( "If you want to use HTTP tunneling, then supply build master " "host and port rather than a connection string") if proxy_connection_string: log.msg("Using HTTP tunnel to connect through proxy") proxy_endpoint = clientFromString(reactor, proxy_connection_string) endpoint = HTTPTunnelEndpoint(buildmaster_host, port, proxy_endpoint) if useTls: from twisted.internet.endpoints import wrapClientTLS from twisted.internet.ssl import optionsForClientTLS contextFactory = optionsForClientTLS(hostname=buildmaster_host) endpoint = wrapClientTLS(contextFactory, endpoint) else: if connection_string is None: connection_string = get_connection_string( buildmaster_host, port) endpoint = clientFromString(reactor, connection_string) def policy(attempt): if maxRetries and attempt >= maxRetries: reactor.stop() return backoffPolicy()(attempt) pb_service = ClientService(endpoint, bf, retryPolicy=policy) self.addService(pb_service)
def request(self, method, uri, headers=None, bodyProducer=None): """ Issue a request to the server indicated by the given uri. Supports `http` and `https` schemes. An existing connection from the connection pool may be used or a new one may be created. See also: twisted.web.iweb.IAgent.request Args: method (bytes): The request method to use, such as `GET`, `POST`, etc uri (bytes): The location of the resource to request. headers (Headers|None): Extra headers to send with the request bodyProducer (IBodyProducer|None): An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or, None if the request is to have no body. Returns: Deferred[IResponse]: completes when the header of the response has been received (regardless of the response status code). Can fail with: SchemeNotSupported: if the uri is not http or https twisted.internet.error.TimeoutError if the server we are connecting to (proxy or destination) does not accept a connection before connectTimeout. ... other things too. """ uri = uri.strip() if not _VALID_URI.match(uri): raise ValueError("Invalid URI {!r}".format(uri)) parsed_uri = URI.fromBytes(uri) pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port) request_path = parsed_uri.originForm should_skip_proxy = False if self.no_proxy is not None: should_skip_proxy = proxy_bypass_environment( parsed_uri.host.decode(), proxies={"no": self.no_proxy}, ) if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint and not should_skip_proxy): # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: pool_key = ("http-proxy", self.http_proxy_endpoint) endpoint = self.http_proxy_endpoint request_path = uri elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint and not should_skip_proxy): connect_headers = Headers() # Determine whether we need to set Proxy-Authorization headers if self.https_proxy_creds: # Set a Proxy-Authorization header connect_headers.addRawHeader( b"Proxy-Authorization", self.https_proxy_creds.as_proxy_authorization_value(), ) endpoint = HTTPConnectProxyEndpoint( self.proxy_reactor, self.https_proxy_endpoint, parsed_uri.host, parsed_uri.port, headers=connect_headers, ) else: # not using a proxy endpoint = HostnameEndpoint(self._reactor, parsed_uri.host, parsed_uri.port, **self._endpoint_kwargs) logger.debug("Requesting %s via %s", uri, endpoint) if parsed_uri.scheme == b"https": tls_connection_creator = self._policy_for_https.creatorForNetloc( parsed_uri.host, parsed_uri.port) endpoint = wrapClientTLS(tls_connection_creator, endpoint) elif parsed_uri.scheme == b"http": pass else: return defer.fail( Failure( SchemeNotSupported("Unsupported scheme: %r" % (parsed_uri.scheme, )))) return self._requestWithEndpoint(pool_key, endpoint, method, parsed_uri, headers, bodyProducer, request_path)
def transport_endpoint(reactor, host, port, timeout): return wrapClientTLS( tls_options, HostnameEndpoint(reactor, host, port, timeout=timeout), )
def request(self, method, uri, headers=None, bodyProducer=None): """ Issue a request to the server indicated by the given uri. Supports `http` and `https` schemes. An existing connection from the connection pool may be used or a new one may be created. See also: twisted.web.iweb.IAgent.request Args: method (bytes): The request method to use, such as `GET`, `POST`, etc uri (bytes): The location of the resource to request. headers (Headers|None): Extra headers to send with the request bodyProducer (IBodyProducer|None): An object which can generate bytes to make up the body of this request (for example, the properly encoded contents of a file for a file upload). Or, None if the request is to have no body. Returns: Deferred[IResponse]: completes when the header of the response has been received (regardless of the response status code). """ uri = uri.strip() if not _VALID_URI.match(uri): raise ValueError("Invalid URI {!r}".format(uri)) parsed_uri = URI.fromBytes(uri) pool_key: tuple = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port) request_path = parsed_uri.originForm if parsed_uri.scheme == b"http" and self.proxy_endpoint: # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: pool_key = ("http-proxy", self.proxy_endpoint) endpoint = self.proxy_endpoint request_path = uri elif parsed_uri.scheme == b"https" and self.proxy_endpoint: endpoint = HTTPConnectProxyEndpoint( self._reactor, self.proxy_endpoint, parsed_uri.host, parsed_uri.port, self._proxy_auth, ) else: # not using a proxy endpoint = HostnameEndpoint(self._reactor, parsed_uri.host, parsed_uri.port, **self._endpoint_kwargs) logger.debug("Requesting %s via %s", uri, endpoint) if parsed_uri.scheme == b"https": tls_connection_creator = self._policy_for_https.creatorForNetloc( parsed_uri.host, parsed_uri.port) endpoint = wrapClientTLS(tls_connection_creator, endpoint) elif parsed_uri.scheme == b"http": pass else: return defer.fail( Failure( SchemeNotSupported("Unsupported scheme: %r" % (parsed_uri.scheme, )))) return self._requestWithEndpoint(pool_key, endpoint, method, parsed_uri, headers, bodyProducer, request_path)