def endpoint_from_hint_obj(hint, tor, reactor):
if tor:
if isinstance(hint, (DirectTCPV1Hint, TorTCPV1Hint)):
# this Tor object will throw ValueError for non-public IPv4
# addresses and any IPv6 address
try:
return tor.stream_via(hint.hostname, hint.port)
except ValueError:
return None
return None
if isinstance(hint, DirectTCPV1Hint):
# avoid DNS lookup unless necessary
if isIPAddress(hint.hostname):
return TCP4ClientEndpoint(reactor, hint.hostname, hint.port)
if isIPv6Address(hint.hostname):
return TCP6ClientEndpoint(reactor, hint.hostname, hint.port)
return HostnameEndpoint(reactor, hint.hostname, hint.port)
return None
def __init__(
self,
host: str,
port: int,
maximum_buffer: int = 1000,
level=logging.NOTSET,
_reactor=None,
):
super().__init__(level=level)
self.host = host
self.port = port
self.maximum_buffer = maximum_buffer
self._buffer = deque() # type: Deque[logging.LogRecord]
self._connection_waiter = None # type: Optional[Deferred]
self._producer = None # type: Optional[LogProducer]
# Connect without DNS lookups if it's a direct IP.
if _reactor is None:
from twisted.internet import reactor
_reactor = reactor
try:
ip = ip_address(self.host)
if isinstance(ip, IPv4Address):
endpoint = TCP4ClientEndpoint(
_reactor, self.host,
self.port) # type: IStreamClientEndpoint
elif isinstance(ip, IPv6Address):
endpoint = TCP6ClientEndpoint(_reactor, self.host, self.port)
else:
raise ValueError("Unknown IP address provided: %s" %
(self.host, ))
except ValueError:
endpoint = HostnameEndpoint(_reactor, self.host, self.port)
factory = Factory.forProtocol(Protocol)
self._service = ClientService(endpoint, factory, clock=_reactor)
self._service.startService()
self._stopping = False
self._connect()
def start(self) -> None:
# Connect without DNS lookups if it's a direct IP.
try:
ip = ip_address(self.host)
if isinstance(ip, IPv4Address):
endpoint = TCP4ClientEndpoint(
self.hs.get_reactor(), self.host, self.port
)
elif isinstance(ip, IPv6Address):
endpoint = TCP6ClientEndpoint(
self.hs.get_reactor(), self.host, self.port
)
except ValueError:
endpoint = HostnameEndpoint(self.hs.get_reactor(), self.host, self.port)
factory = Factory.forProtocol(Protocol)
self._service = ClientService(endpoint, factory, clock=self.hs.get_reactor())
self._service.startService()
self._connect()
async def connect(self, host, port, connect_timeout,
source_address=None, socket_options=None):
# HostnameEndpoint only supports setting source host, not source port
if source_address is not None:
raise NotImplementedError(
"twisted backend doesn't support setting source_address")
# factory = protocol.Factory.forProtocol(TwistedSocketProtocol)
endpoint = HostnameEndpoint(self._reactor, host, port)
d = connectProtocol(endpoint, TwistedSocketProtocol())
# XX d.addTimeout(...)
protocol = await d
if socket_options is not None:
for opt in socket_options:
if opt[:2] == (socket.IPPROTO_TCP, socket.TCP_NODELAY):
protocol.transport.setTcpNoDelay(opt[2])
else:
raise NotImplementedError(
"unrecognized socket option for twisted backend")
return TwistedSocket(protocol)
def runClient(connect, rate):
http_proxy = os.getenv('HTTP_PROXY')
if http_proxy:
http_proxy = furl(http_proxy)
ep = HostnameEndpoint(reactor, http_proxy.host, http_proxy.port)
ua = ProxyAgent(ep)
else:
ua = Agent(reactor)
client = Client(connect, ua)
looper = task.LoopingCall(client.request_GET)
# register signal handler to stop the looping call
def signal_handler(signal, frame):
looper.stop()
reactor.runUntilCurrent()
reactor.stop()
signal.signal(signal.SIGINT, signal_handler)
looper.start(1 / rate)
def _http_proxy_endpoint(proxy, reactor, **kwargs):
"""Parses an http proxy setting and returns an endpoint for the proxy
Args:
proxy (bytes|None): the proxy setting
reactor: reactor to be used to connect to the proxy
kwargs: other args to be passed to HostnameEndpoint
Returns:
interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy,
or None
"""
if proxy is None:
return None
# currently we only support hostname:port. Some apps also support
# protocol://<host>[:port], which allows a way of requiring a TLS connection to the
# proxy.
host, port = parse_host_port(proxy, default_port=1080)
return HostnameEndpoint(reactor, host, port, **kwargs)
def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs):
"""Parses an http proxy setting and returns an endpoint for the proxy
Args:
proxy: the proxy setting in the form: [<username>:<password>@]<host>[:<port>]
Note that compared to other apps, this function currently lacks support
for specifying a protocol schema (i.e. protocol://...).
reactor: reactor to be used to connect to the proxy
kwargs: other args to be passed to HostnameEndpoint
Returns:
interfaces.IStreamClientEndpoint|None: endpoint to use to connect to the proxy,
or None
"""
if proxy is None:
return None
# Parse the connection string
host, port = parse_host_port(proxy, default_port=1080)
return HostnameEndpoint(reactor, host, port, **kwargs)
def main(reactor):
from twisted.python import log
log.startLogging(sys.stdout)
tahoe_dir = "testgrid/alice"
cfg = read_config(tahoe_dir, "portnum")
token = cfg.get_private_config("api_auth_token").strip()
webport = cfg.get_config("node", "web.port")
if webport.startswith("tcp:"):
port = webport.split(':')[1]
else:
port = webport
factory = WebSocketClientFactory(
url=u"ws://127.0.0.1:{}/private/logs/v1".format(port),
headers={
"Authorization": "tahoe-lafs {}".format(token),
}
)
factory.on_open = Deferred()
factory.on_close = Deferred()
factory.protocol = TahoeLogProtocol
endpoint = HostnameEndpoint(reactor, "127.0.0.1", int(port))
try:
port = yield endpoint.connect(factory)
except ConnectError as e:
print("Connection failed: {}".format(e))
return
print("port: {}".format(port))
yield factory.on_open
print("opened")
yield factory.on_close
print("closed")
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
Can fail with:
SchemeNotSupported: if the uri is not http or https
twisted.internet.error.TimeoutError if the server we are connecting
to (proxy or destination) does not accept a connection before
connectTimeout.
... other things too.
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
parsed_uri.host.decode(),
proxies={"no": self.no_proxy},
)
if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
and not should_skip_proxy):
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.http_proxy_endpoint)
endpoint = self.http_proxy_endpoint
request_path = uri
elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
and not should_skip_proxy):
connect_headers = Headers()
# Determine whether we need to set Proxy-Authorization headers
if self.https_proxy_creds:
# Set a Proxy-Authorization header
connect_headers.addRawHeader(
b"Proxy-Authorization",
self.https_proxy_creds.as_proxy_authorization_value(),
)
endpoint = HTTPConnectProxyEndpoint(
self.proxy_reactor,
self.https_proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
headers=connect_headers,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key: tuple = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
if parsed_uri.scheme == b"http" and self.proxy_endpoint:
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.proxy_endpoint)
endpoint = self.proxy_endpoint
request_path = uri
elif parsed_uri.scheme == b"https" and self.proxy_endpoint:
endpoint = HTTPConnectProxyEndpoint(
self._reactor,
self.proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
self._proxy_auth,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
tls_options,
HostnameEndpoint(reactor, host, port, timeout=timeout),
)
def __init__(self, reactor, host, port, *args, **kwargs):
self.host = host
self.port = port
self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
def __init__(self, address: str, port: int, destination: str,
is_app_hosting: bool):
self._address = address
self._port = port
self._destination = destination
self._is_app_hosting = is_app_hosting
self._zmq_factory = ZmqFactory()
# if the ZMQ app is binding and hosting the server, we need to connect to that instead
if is_app_hosting:
zmq_socket_class = ZmqDealerConnection
zmq_endpoint = ZmqEndpoint(ZmqEndpointType.connect,
"tcp://%s:%d" % (address, port))
LOG.info("Configured txZMQ for connecting to application "
"- connected to tcp://%s:%d" % (address, port))
else:
# otherwise, bind to the address/port and have them connect to us
zmq_socket_class = ZmqRouterConnection
zmq_endpoint = ZmqEndpoint(ZmqEndpointType.bind,
"tcp://%s:%d" % (address, port))
LOG.info("Configured txZMQ for application connecting to us "
"- socket bound to tcp://%s:%d" % (address, port))
self._zmq_socket = zmq_socket_class(self._zmq_factory, zmq_endpoint)
# store the socket identity of the client; we need it to send data back to the local ZMQ app
self._zmq_socket_identity = None
LOG.debug("Initializing socket and agent")
# check if we want to use an HTTPS proxy; useful for Fiddler
if USE_HTTPS_PROXY:
self._twisted_agent = ProxyAgent(
HostnameEndpoint(reactor, PROXY_HOST, PROXY_PORT), reactor)
LOG.warning("Agent is using HTTP proxy for outbound work!")
else:
# otherwise, use the standard Agent with a nulled SSL verification process, since self-signed certificates
# fail the connection process entirely
self._twisted_agent = Agent(
reactor, contextFactory=DisableSSLVerificationFactory())
# setup auto-POST method for our socket
def post_data(*zmq_data_recv):
self._zmq_socket_identity = zmq_data_recv[0]
data = zmq_data_recv[-1]
LOG.debug("Received %d bytes of data" % len(data))
# hash and base64 our data for validation and transportation
data_hash = hashlib.sha256(data).hexdigest()
b64_data = base64.b64encode(data)
# POST it to the remote server
request = self._twisted_agent.request(
b'POST', (destination + "/zmq").encode(),
Headers({
'User-Agent': ['ZMQ-HTTP-Bridge-Agent'],
'X-Verify-Hash': [data_hash]
}),
bodyProducer=StringProducer(b64_data))
def handle_twisted_error(fail):
# print out _all_ errors, since Twisted doesn't provide all exceptions
for error in fail.value.reasons:
LOG.error("%s", str(error))
request.addErrback(handle_twisted_error)
request.addCallback(
lambda ignored: LOG.debug("Request completed."))
LOG.info("Forwarded data to destination (hash preview: %s)" %
data_hash[0:8])
self._zmq_socket.gotMessage = post_data
def endpoint_factory(reactor, host, port, **kw):
return wrapClientTLS(
tlsCreator, HostnameEndpoint(reactor, host, port, **kw))
def connect_vrt(scpi):
self._scpi = scpi
point = HostnameEndpoint(self._reactor, host, VRT_PORT)
return point.connect(VRTClientFactory(self._vrt_callback))
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
tls_client_options_factory.get_options(host),
HostnameEndpoint(reactor, host, port, timeout=timeout))
def __init__(self, reactor, host, port, *args, **kwargs):
self.host = host
self.port = port
self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
logger.info("Endpoint created with %s:%d", host, port)
from twisted.internet.defer import inlineCallbacks
from twisted.internet.endpoints import HostnameEndpoint, clientFromString
from foolscap.api import Referenceable, Tub
tub = Tub()
which = sys.argv[1] if len(sys.argv) > 1 else None
if which == "tcp":
furl = "pb://%s@tcp:%s:%d/calculator" % (TUBID, HOSTNAME, LOCALPORT)
elif which == "socks":
# "slogin -D 8013 HOSTNAME" starts a SOCKS server on localhost 8013, for
# which connections will emerge from the other end. Check the server logs
# to see the peer address of each addObserver call to verify that it is
# coming from 127.0.0.1 rather than the client host.
from foolscap.connections import socks
h = socks.socks_endpoint(HostnameEndpoint(reactor, "localhost", 8013))
tub.removeAllConnectionHintHandlers()
tub.addConnectionHintHandler("tcp", h)
furl = "pb://%s@tcp:localhost:%d/calculator" % (TUBID, LOCALPORT)
elif which in ("tor-default", "tor-socks", "tor-control", "tor-launch"):
from foolscap.connections import tor
if which == "tor-default":
h = tor.default_socks()
elif which == "tor-socks":
h = tor.socks_port(int(sys.argv[2]))
elif which == "tor-control":
control_ep = clientFromString(reactor, sys.argv[2])
h = tor.control_endpoint(control_ep)
elif which == "tor-launch":
data_directory = None
if len(sys.argv) > 2:
def fetch(self):
endpoint = HostnameEndpoint(reactor, arguments.host, arguments.port)
endpoint.connect(self)
self.reactor.run()
return self.tcp.cert
def _url_to_endpoint(reactor, url):
netloc = urlsplit(url).netloc
host, port = netloc.split(":")
return HostnameEndpoint(reactor, host, int(port))
def transport_endpoint(reactor, host, port, timeout):
return wrapClientTLS(
ssl_context_factory,
HostnameEndpoint(reactor, host, port, timeout=timeout))
def __init__(self, reactor: IReactorTime, host: bytes, port: int,
*args: Any, **kwargs: Any):
self.host = host
self.port = port
self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
logger.info("Endpoint created with %s:%d", host, port)
