def _get_agent(self, request, timeout):
bindAddress = request.meta.get('bindaddress') or self._bindAddress
# this needs http_proxy environment variable or proxy middleware,
# otherwise it will be none
# proxy = request.meta.get('proxy')
proxy = settings.get('SOCKSPROXY', '127.0.0.1:9050')
logger.debug('downloader agent proxy: %s' % proxy)
if proxy:
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
_, _, host, port, proxyParams = _parse(request.url)
proxyEndpoint = TCP4ClientEndpoint(reactor,
proxyHost,
proxyPort,
timeout=timeout,
bindAddress=bindAddress)
newcirc = settings.get('NEWCIRC', False)
if newcirc:
username = hex(random.randint(0, 2**32))
password = hex(random.randint(0, 2**32))
agent = SOCKS5Agent(
reactor,
proxyEndpoint=proxyEndpoint,
endpointArgs=dict(methods=dict(login=(username,
password))))
else:
agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
return agent
return self._Agent(reactor,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindAddress,
pool=self._pool)
def download(self, name, url, path):
# keep track of current download
self.current_download_path = path
self.current_download_url = url
mirror_url = url.format(self.common.settings['mirror'])
# convert mirror_url from unicode to string, if needed (#205)
if isinstance(mirror_url, unicode):
mirror_url = unicodedata.normalize('NFKD', mirror_url).encode(
'ascii', 'ignore')
# initialize the progress bar
self.progressbar.set_fraction(0)
self.progressbar.set_text(_('Downloading {0}').format(name))
self.progressbar.show()
self.refresh_gtk()
if self.common.settings['download_over_tor']:
from twisted.internet.endpoints import TCP4ClientEndpoint
from txsocksx.http import SOCKS5Agent
torEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050)
# default mirror gets certificate pinning, only for requests that use the mirror
if self.common.settings[
'mirror'] == self.common.default_mirror and '{0}' in url:
agent = SOCKS5Agent(reactor,
TorProjectPolicyForHTTPS(
self.common.paths['torproject_pem']),
proxyEndpoint=torEndpoint)
else:
agent = SOCKS5Agent(reactor, proxyEndpoint=torEndpoint)
else:
if self.common.settings[
'mirror'] == self.common.default_mirror and '{0}' in url:
agent = Agent(
reactor,
TorProjectPolicyForHTTPS(
self.common.paths['torproject_pem']))
else:
agent = Agent(reactor)
# actually, agent needs to follow redirect
agent = RedirectAgent(agent)
# start the request
d = agent.request('GET', mirror_url,
Headers({'User-Agent': ['torbrowser-launcher']}),
None)
self.file_download = open(path, 'w')
d.addCallback(self.response_received).addErrback(self.download_error)
if not reactor.running:
reactor.run()
def main(reactor):
torEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050)
agent = SOCKS5Agent(reactor, proxyEndpoint=torEndpoint)
d = agent.request('GET', 'http://api.externalip.net/ip/')
d.addCallback(readBody)
d.addCallback(print)
return d
def get_tor_agent(socks_host='127.0.0.1', socks_port=9050):
'''An HTTP agent that uses SOCKS5 to proxy all requests through the socks_port
It is implicitly understood that the socks_port points to the locally configured
tor daemon'''
torServerEndpoint = TCP4ClientEndpoint(reactor, socks_host, socks_port)
agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint)
return agent
def download(self, name, url, path):
# keep track of current download
self.current_download_path = path
self.current_download_url = url
# initialize the progress bar
mirror_url = url.format(self.common.settings['mirror'])
self.progressbar.set_fraction(0)
self.progressbar.set_text(_('Downloading {0}').format(name))
self.progressbar.show()
self.refresh_gtk()
if self.common.settings['update_over_tor']:
print _('Updating over Tor')
from twisted.internet.endpoints import TCP4ClientEndpoint
from txsocksx.http import SOCKS5Agent
torEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1', 9050)
# default mirror gets certificate pinning, only for requests that use the mirror
if self.common.settings['mirror'] == self.common.default_mirror and '{0}' in url:
agent = SOCKS5Agent(reactor, VerifyTorProjectCert(self.common.paths['torproject_pem']), proxyEndpoint=torEndpoint)
else:
agent = SOCKS5Agent(reactor, proxyEndpoint=torEndpoint)
else:
if self.common.settings['mirror'] == self.common.default_mirror and '{0}' in url:
agent = Agent(reactor, VerifyTorProjectCert(self.common.paths['torproject_pem']))
else:
agent = Agent(reactor)
# actually, agent needs to follow redirect
agent = RedirectAgent(agent)
# start the request
d = agent.request('GET', mirror_url,
Headers({'User-Agent': ['torbrowser-launcher']}),
None)
self.file_download = open(path, 'w')
d.addCallback(self.response_received).addErrback(self.download_error)
if not reactor.running:
reactor.run()
def _get_agent(self, request, timeout):
bindAddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
_, _, host, port, proxyParams = _parse(request.url)
proxyEndpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
timeout=timeout, bindAddress=bindAddress)
agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
return agent
return self._Agent(reactor, contextFactory=self._contextFactory,
connectTimeout=timeout, bindAddress=bindAddress, pool=self._pool)
def _get_agent(self, request, timeout):
# bindAddress = request.meta.get('bindaddress') or self._bindAddress
# proxy = request.meta.get('proxy')
# if proxy:
# _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
# _, _, host, port, proxyParams = _parse(request.url)
torServerEndpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort)
# proxyEndpoint = SOCKS5ClientEndpoint(proxyHost, proxyPort,proxyEndpoint=torServerEndpoint, methods={'login': (proxyUser, proxyPass)})
agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint,
endpointArgs=dict(methods={'login': (proxyUser, proxyPass)}))
return agent
def __init__(self, base_url="", socks_proxy=None):
"""
Args:
socks_proxy: if set will use the specified SOCKS5 proxy for
performing HTTP requests. (ex. "127.0.0.1:9050")
"""
self.agent = client.Agent(reactor)
self.base_url = base_url
if socks_proxy:
socks_host, socks_port = socks_proxy.split(':')
from txsocksx.http import SOCKS5Agent
torEndpoint = TCP4ClientEndpoint(reactor, socks_host,
int(socks_port))
self.agent = SOCKS5Agent(reactor, proxyEndpoint=torEndpoint)
def set_port(self, circuit, port):
e = self.exitaddr
cdrsp = self.circuits[circuit.id]
cdrsp.port = port
self.ports[port] = cdrsp
bindAddress = ("localhost", port)
sockspoint = TCP4ClientEndpoint(e.reactor,
"localhost",
e.options.socks_port,
bindAddress=bindAddress)
agent = SOCKS5Agent(e.reactor, proxyEndpoint=sockspoint)
# check.torproject.org/api/ip
url = "http%s://ipinfo.io/ip" % ("s" if cdrsp.dest == 443 else "")
d = agent.request("GET", url)
d.addCallback(readBody)
d.addCallback(functools.partial(self.print_body, cdrsp))
canceler = e.reactor.callLater(15, d.cancel)
d.addBoth(functools.partial(cancelCanceler, canceler))
d.addErrback(
functools.partial(self.failed, cdrsp.router, "socks5agent"))
d.addErrback(log.err)
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
url = self.collectorAddress + '/report'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'probe_cc': self.testDetails['probe_cc'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'test_start_time': self.testDetails['test_start_time'],
'input_hashes': self.testDetails['input_hashes'],
'data_format_version': self.testDetails['data_format_version'],
'format': 'json'
}
# import values from the environment
request.update([(k.lower(), v) for (k, v) in os.environ.iteritems()
if k.startswith('PROBE_')])
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def _get_agent(self, request, timeout):
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
proxyScheme, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
if proxyScheme.startswith(b'http'):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if scheme == b'https' and not omitConnectTunnel:
proxyConf = (proxyHost, proxyPort,
request.headers.get(b'Proxy-Authorization',
None))
return self._TunnelingAgent(
reactor,
proxyConf,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
else:
return self._ProxyAgent(reactor,
proxyURI=to_bytes(
proxy, encoding='ascii'),
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
elif proxyScheme == b'socks4':
proxyEndPoint = TCP4ClientEndpoint(reactor,
proxyHost,
proxyPort,
timeout=timeout,
bindAddress=bindaddress)
agent = SOCKS4Agent(reactor,
proxyEndPoint=proxyEndPoint,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
return agent
elif proxyScheme == b'socks5':
proxyEndPoint = TCP4ClientEndpoint(reactor,
proxyHost,
proxyPort,
timeout=timeout,
bindAddress=bindaddress)
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
if proxyAuth:
proxyUser, proxyPassword = b64decode(
proxyAuth.split()[-1]).split(b':')
agent = SOCKS5Agent(
reactor,
proxyEndpoint=proxyEndPoint,
endpointArgs=dict(
methods={'login': (proxyUser, proxyPassword)}),
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
else:
agent = SOCKS5Agent(reactor,
proxyEndpoint=proxyEndPoint,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
return agent
return self._Agent(reactor,
contextFactory=self._contextFactory,
connectTimeout=timeout,
bindAddress=bindaddress,
pool=self._pool)
def ensure_agent(self):
if self.agent is None:
torServerEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
self.cfg['tor_proxy_port'])
self.agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint)
def get_tor_agent(socks_host='127.0.0.1', socks_port=9050):
torServerEndpoint = TCP4ClientEndpoint(reactor, socks_host, socks_port)
agent = SOCKS5Agent(reactor, proxyEndpoint=torServerEndpoint)
return agent
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
from txsocksx.http import SOCKS5Agent
from twisted.internet import reactor
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
elif self.collectorAddress.startswith('https://'):
# XXX add support for securely reporting to HTTPS collectors.
log.err("HTTPS based collectors are currently not supported.")
url = self.collectorAddress + '/report'
content = '---\n'
content += safe_dump(self.testDetails)
content += '...\n'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'input_hashes': self.testDetails['input_hashes'],
# XXX there is a bunch of redundancy in the arguments getting sent
# to the backend. This may need to get changed in the client and
# the backend.
'content': content
}
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST", url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError