def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(
self._reactor, res.target_host, res.target_port,
)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None:
first_exception = None
server_list = await self._resolve_server()
for server in server_list:
host = server.host
port = server.port
try:
logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
endpoint = HostnameEndpoint(self._reactor, host, port)
if self._tls_options:
endpoint = wrapClientTLS(self._tls_options, endpoint)
result = await make_deferred_yieldable(
endpoint.connect(protocol_factory))
return result
except Exception as e:
logger.info("Failed to connect to %s:%i: %s",
host.decode("ascii"), port, e)
if not first_exception:
first_exception = e
# We return the first failure because that's probably the most interesting.
if first_exception:
raise first_exception
# This shouldn't happen as we should always have at least one host/port
# to try and if that doesn't work then we'll have an exception.
raise Exception("Failed to resolve server %r" %
(self._parsed_uri.netloc, ))
def main():
hostname = raw_input('IMAP4 Server Hostname: ')
port = raw_input('IMAP4 Server Port (the default is 143, 993 uses SSL): ')
username = raw_input('IMAP4 Username: '******'IMAP4 Password: '******'utf-8'))
endpoint = endpoints.wrapClientTLS(contextFactory, endpoint)
endpoint.connect(factory)
reactor.run()
def download_request(self, request, spider):
bindaddress = request.meta.get('bindaddress')
timeout = request.meta.get('download_timeout')
maxsize = getattr(spider, 'download_maxsize', self.default_maxsize)
warnsize = getattr(spider, 'download_warnsize', self.default_warnsize)
parts = urlparse(request.url)
remote_host = bindaddress or parts.hostname
remote_port = parts.port or 1965
hostname = HostnameEndpoint(reactor, remote_host, remote_port)
# The recommended helper method for this (optionsForClientTLS) does not
# allow setting up a client context that accepts unverified certificates.
# So we are forced to use the private ClientTLSOptions method instead.
options = ScrapyClientTLSOptions(remote_host, self.context_factory.getContext())
# noinspection PyTypeChecker
endpoint = wrapClientTLS(options, hostname)
logger.debug(f"Creating download request for {request.url}")
protocol = GeminiClientProtocol(request, maxsize, warnsize, timeout)
# If the connection fails (DNS lookup, etc.) propagate the error so
# that scrapy knows the request has completed.
connected = connectProtocol(endpoint, protocol)
connected.addErrback(protocol.finished.errback)
return protocol.finished
def main():
hostname = raw_input("IMAP4 Server Hostname: ")
port = raw_input("IMAP4 Server Port (the default is 143, 993 uses SSL): ")
# Usernames are bytes.
username = raw_input("IMAP4 Username: "******"ascii")
# Passwords are bytes.
password = util.getPassword("IMAP4 Password: "******"ascii")
onConn = (defer.Deferred().addCallback(
cbServerGreeting, username,
password).addErrback(ebConnection).addBoth(cbClose))
factory = SimpleIMAP4ClientFactory(username, onConn)
if not port:
port = 143
else:
port = int(port)
from twisted.internet import reactor
endpoint = endpoints.HostnameEndpoint(reactor, hostname, port)
if port == 993:
if isinstance(hostname, bytes):
# This is python 2
hostname = hostname.decode("utf-8")
contextFactory = ssl.optionsForClientTLS(hostname=hostname, )
endpoint = endpoints.wrapClientTLS(contextFactory, endpoint)
endpoint.connect(factory)
reactor.run()
def connect(self):
"""Connect and authenticate with the IMAP server
"""
if self.connecting:
defer.returnValue(None)
self.connecting = True
endpoint = endpoints.HostnameEndpoint(reactor, self.host, self.port)
if self.ssl:
contextFactory = ssl.optionsForClientTLS(
hostname=self.host.decode('utf-8'))
endpoint = endpoints.wrapClientTLS(contextFactory, endpoint)
de = defer.Deferred()
factory = IMAP4ClientFactory(self.user, de)
factory.debug = self.debug
yield endpoint.connect(factory)
self.proto = yield de
yield self.proto.authenticate(self.password)
self.connected = True
self.connecting = False
def _get_new(self):
"""
Get new mails from IMAP server. This will define the `main loop` of
the IMAP service.
"""
onConn = defer.Deferred().addCallback(
self.cb_server_greeting
).addErrback(
self.eb_server_greeting
)
# Connect with endpoints. Connect and disconnect from time to time
# (defined by the service's interval) instead of establishing a
# persistent connection
factory = SimpleIMAP4ClientFactory(self.username, onConn)
from twisted.internet import reactor
endpoint = endpoints.HostnameEndpoint(reactor, self.host, self.port)
contextFactory = ssl.optionsForClientTLS(
hostname=self.host.decode('utf-8')
)
endpoint = endpoints.wrapClientTLS(contextFactory, endpoint)
log.debug("IMAP:: Connecting to Google's IMAP servers.")
endpoint.connect(factory)
def endpointForURI(_uri):
ep = LoggingHostnameEndpoint(
self._reactor, res.target_host, res.target_port,
)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
def getMapUpdaterComponentService(self, url, realm, mapUpdater):
def create(config):
try:
session = MapUpdaterComponent(mapUpdater, config)
except Exception as e:
# the app component could not be created .. fatal
print(e)
else:
session.debug_app = True
return session
sessionFactory = ApplicationSessionFactory(
ComponentConfig(realm, None))
sessionFactory.session = create
transportFactory = WampWebSocketClientFactory(
sessionFactory, url=url)
transportFactory.noisy = False
transportFactory.autoPingInterval = 30
transportFactory.autoPingTimeout = 30
isSecure, host, port, resource, path, params = parseWsUrl(url)
endpoint = HostnameEndpoint(reactor, host.encode('utf8'), port)
if isSecure:
contextFactory = optionsForClientTLS(hostname=host)
endpoint = wrapClientTLS(contextFactory, endpoint)
return ClientService(endpoint, transportFactory)
async def _do_connect(self, protocol_factory: IProtocolFactory) -> None:
first_exception = None
server_list = await self._resolve_server()
for server in server_list:
host = server.host
port = server.port
endpoint: IStreamClientEndpoint
try:
if self.https_proxy_endpoint and not proxy_bypass(host.decode()):
logger.debug(
"Connecting to %s:%i via %s",
host.decode("ascii"),
port,
self.https_proxy_endpoint,
)
connect_headers = Headers()
# Determine whether we need to set Proxy-Authorization headers
if self.https_proxy_creds:
# Set a Proxy-Authorization header
connect_headers.addRawHeader(
b"Proxy-Authorization",
self.https_proxy_creds.as_proxy_authorization_value(),
)
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self.https_proxy_endpoint,
host,
port,
headers=connect_headers,
)
else:
logger.debug("Connecting to %s:%i", host.decode("ascii"), port)
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, host, port)
if self._tls_options:
endpoint = wrapClientTLS(self._tls_options, endpoint)
result = await make_deferred_yieldable(
endpoint.connect(protocol_factory)
)
return result
except Exception as e:
logger.info(
"Failed to connect to %s:%i: %s", host.decode("ascii"), port, e
)
if not first_exception:
first_exception = e
# We return the first failure because that's probably the most interesting.
if first_exception:
raise first_exception
# This shouldn't happen as we should always have at least one host/port
# to try and if that doesn't work then we'll have an exception.
raise Exception("Failed to resolve server %r" % (self._parsed_uri.netloc,))
async def _do_connect(self,
protocol_factory: IProtocolFactory) -> IProtocol:
first_exception = None
server_list = await self._resolve_server()
for server in server_list:
host = server.host
port = server.port
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
host.decode(),
proxies={"no": self.no_proxy},
)
endpoint: IStreamClientEndpoint
try:
if self._https_proxy_endpoint and not should_skip_proxy:
logger.debug(
"Connecting to %s:%i via %s",
host.decode("ascii"),
port,
self._https_proxy_endpoint,
)
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self._https_proxy_endpoint,
host,
port,
proxy_creds=self._https_proxy_creds,
)
else:
logger.debug("Connecting to %s:%i", host.decode("ascii"),
port)
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, host, port)
if self._tls_options:
endpoint = wrapClientTLS(self._tls_options, endpoint)
result = await make_deferred_yieldable(
endpoint.connect(protocol_factory))
return result
except Exception as e:
logger.info("Failed to connect to %s:%i: %s",
host.decode("ascii"), port, e)
if not first_exception:
first_exception = e
# We return the first failure because that's probably the most interesting.
if first_exception:
raise first_exception
# This shouldn't happen as we should always have at least one host/port
# to try and if that doesn't work then we'll have an exception.
raise Exception("Failed to resolve server %r" %
(self._parsed_uri.netloc, ))
def endpointForURI(_uri: URI) -> IStreamClientEndpoint:
ep: IStreamClientEndpoint = LoggingHostnameEndpoint(
self._reactor,
routing.target_host,
routing.target_port,
)
if tls_options is not None:
ep = wrapClientTLS(tls_options, ep)
return ep
def endpointForURI(self, uri): # noqa: N802
"""Create an endpoint for URI."""
endpoint = HostnameEndpoint(self.reactor, self.address, uri.port)
if uri.scheme == b'http':
return endpoint
elif uri.scheme == b'https':
connection_creator = self.context_factory.creatorForNetloc(
uri.host, uri.port)
return wrapClientTLS(connection_creator, endpoint)
def connect_client(listening_port):
port_number = listening_port.getHost().port
client = endpoints.TCP4ClientEndpoint(reactor, '127.0.0.1',
port_number)
options = optionsForClientTLS(hostname=hostname, trustRoot=PEM_ROOT)
client = endpoints.wrapClientTLS(options, client)
connectDeferred = client.connect(client_factory)
def aggregate(client_proto):
return (client_proto, listening_port)
connectDeferred.addCallback(aggregate)
return connectDeferred
def http_proxy_endpoint(
proxy: Optional[bytes],
reactor: IReactorCore,
tls_options_factory: Optional[IPolicyForHTTPS],
**kwargs: object,
) -> Tuple[Optional[IStreamClientEndpoint], Optional[ProxyCredentials]]:
"""Parses an http proxy setting and returns an endpoint for the proxy
Args:
proxy: the proxy setting in the form: [scheme://][<username>:<password>@]<host>[:<port>]
This currently supports http:// and https:// proxies.
A hostname without scheme is assumed to be http.
reactor: reactor to be used to connect to the proxy
tls_options_factory: the TLS options to use when connecting through a https proxy
kwargs: other args to be passed to HostnameEndpoint
Returns:
a tuple of
endpoint to use to connect to the proxy, or None
ProxyCredentials or if no credentials were found, or None
Raise:
ValueError if proxy has no hostname or unsupported scheme.
RuntimeError if no tls_options_factory is given for a https connection
"""
if proxy is None:
return None, None
# Note: urlsplit/urlparse cannot be used here as that does not work (for Python
# 3.9+) on scheme-less proxies, e.g. host:port.
scheme, host, port, credentials = parse_proxy(proxy)
proxy_endpoint = HostnameEndpoint(reactor, host, port, **kwargs)
if scheme == b"https":
if tls_options_factory:
tls_options = tls_options_factory.creatorForNetloc(host, port)
proxy_endpoint = wrapClientTLS(tls_options, proxy_endpoint)
else:
raise RuntimeError(
f"No TLS options for a https connection via proxy {proxy!s}"
)
return proxy_endpoint, credentials
def connect_client(listening_port):
port_number = listening_port.getHost().port
client = endpoints.TCP4ClientEndpoint(reactor, '127.0.0.1',
port_number)
maybe_alpn = {}
if acceptable_protocols is not None:
maybe_alpn['acceptableProtocols'] = acceptable_protocols
options = optionsForClientTLS(hostname=hostname,
trustRoot=PEM_ROOT,
**maybe_alpn)
client = endpoints.wrapClientTLS(options, client)
connectDeferred = client.connect(client_factory)
def aggregate(client_proto):
return (client_proto, listening_port)
connectDeferred.addCallback(aggregate)
return connectDeferred
def connect_client(listening_port):
port_number = listening_port.getHost().port
client = endpoints.TCP4ClientEndpoint(
reactor, '127.0.0.1', port_number
)
maybe_alpn = {}
if acceptable_protocols is not None:
maybe_alpn['acceptableProtocols'] = acceptable_protocols
options = optionsForClientTLS(
hostname=hostname,
trustRoot=PEM_ROOT,
**maybe_alpn
)
client = endpoints.wrapClientTLS(options, client)
connectDeferred = client.connect(client_factory)
def aggregate(client_proto):
return (client_proto, listening_port)
connectDeferred.addCallback(aggregate)
return connectDeferred
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
tls_client_options_factory.get_options(host),
HostnameEndpoint(reactor, host, port, timeout=timeout))
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
ssl_context_factory,
HostnameEndpoint(reactor, host, port, timeout=timeout))
def endpoint_factory(reactor, host, port, **kw):
return wrapClientTLS(
tlsCreator, HostnameEndpoint(reactor, host, port, **kw))
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
ssl_context_factory,
HostnameEndpoint(reactor, host, port, timeout=timeout))
def __init__(self,
buildmaster_host,
port,
name,
passwd,
basedir,
keepalive,
usePTY=None,
keepaliveTimeout=None,
umask=None,
maxdelay=None,
numcpus=None,
unicode_encoding=None,
protocol='pb',
useTls=None,
allow_shutdown=None,
maxRetries=None,
connection_string=None,
delete_leftover_dirs=False,
proxy_connection_string=None):
assert usePTY is None, "worker-side usePTY is not supported anymore"
assert (connection_string is None
or (buildmaster_host, port) == (None, None)), (
"If you want to supply a connection string, "
"then set host and port to None")
if protocol == 'pb':
bot_class = BotPb
elif protocol == 'msgpack_experimental_v1':
if sys.version_info.major < 3:
raise NotImplementedError(
'Msgpack protocol is not supported in Python2')
bot_class = BotMsgpack
else:
raise ValueError('Unknown protocol {}'.format(protocol))
WorkerBase.__init__(self,
name,
basedir,
bot_class,
umask=umask,
unicode_encoding=unicode_encoding,
delete_leftover_dirs=delete_leftover_dirs)
if keepalive == 0:
keepalive = None
name = unicode2bytes(name, self.bot.unicode_encoding)
self.numcpus = numcpus
self.shutdown_loop = None
if allow_shutdown == 'signal':
if not hasattr(signal, 'SIGHUP'):
raise ValueError("Can't install signal handler")
elif allow_shutdown == 'file':
self.shutdown_file = os.path.join(basedir, 'shutdown.stamp')
self.shutdown_mtime = 0
self.allow_shutdown = allow_shutdown
if protocol == 'pb':
passwd = unicode2bytes(passwd, self.bot.unicode_encoding)
bf = self.bf = BotFactory(buildmaster_host, port, keepalive,
maxdelay)
bf.startLogin(credentials.UsernamePassword(name, passwd),
client=self.bot)
elif protocol == 'msgpack_experimental_v1':
if connection_string is None:
ws_conn_string = "ws://{}:{}".format(buildmaster_host, port)
else:
from urllib.parse import urlparse
parsed_url = urlparse(connection_string)
ws_conn_string = "ws://{}:{}".format(parsed_url.hostname,
parsed_url.port)
bf = self.bf = BuildbotWebSocketClientFactory(ws_conn_string)
bf.protocol = BuildbotWebSocketClientProtocol
self.bf.buildbot_bot = self.bot
self.bf.name = name
self.bf.password = passwd
else:
raise ValueError('Unknown protocol {}'.format(protocol))
def get_connection_string(host, port):
if useTls:
connection_type = 'tls'
else:
connection_type = 'tcp'
return '{}:host={}:port={}'.format(
connection_type,
host.replace(':', r'\:'), # escape ipv6 addresses
port)
assert not (proxy_connection_string and connection_string), (
"If you want to use HTTP tunneling, then supply build master "
"host and port rather than a connection string")
if proxy_connection_string:
log.msg("Using HTTP tunnel to connect through proxy")
proxy_endpoint = clientFromString(reactor, proxy_connection_string)
endpoint = HTTPTunnelEndpoint(buildmaster_host, port,
proxy_endpoint)
if useTls:
from twisted.internet.endpoints import wrapClientTLS
from twisted.internet.ssl import optionsForClientTLS
contextFactory = optionsForClientTLS(hostname=buildmaster_host)
endpoint = wrapClientTLS(contextFactory, endpoint)
else:
if connection_string is None:
connection_string = get_connection_string(
buildmaster_host, port)
endpoint = clientFromString(reactor, connection_string)
def policy(attempt):
if maxRetries and attempt >= maxRetries:
reactor.stop()
return backoffPolicy()(attempt)
pb_service = ClientService(endpoint, bf, retryPolicy=policy)
self.addService(pb_service)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
Can fail with:
SchemeNotSupported: if the uri is not http or https
twisted.internet.error.TimeoutError if the server we are connecting
to (proxy or destination) does not accept a connection before
connectTimeout.
... other things too.
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
parsed_uri.host.decode(),
proxies={"no": self.no_proxy},
)
if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
and not should_skip_proxy):
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.http_proxy_endpoint)
endpoint = self.http_proxy_endpoint
request_path = uri
elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
and not should_skip_proxy):
connect_headers = Headers()
# Determine whether we need to set Proxy-Authorization headers
if self.https_proxy_creds:
# Set a Proxy-Authorization header
connect_headers.addRawHeader(
b"Proxy-Authorization",
self.https_proxy_creds.as_proxy_authorization_value(),
)
endpoint = HTTPConnectProxyEndpoint(
self.proxy_reactor,
self.https_proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
headers=connect_headers,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
tls_options,
HostnameEndpoint(reactor, host, port, timeout=timeout),
)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key: tuple = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
if parsed_uri.scheme == b"http" and self.proxy_endpoint:
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.proxy_endpoint)
endpoint = self.proxy_endpoint
request_path = uri
elif parsed_uri.scheme == b"https" and self.proxy_endpoint:
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self.proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
self._proxy_auth,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
