def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
url = self.collectorAddress + '/report/' + self.reportID
if "json" in self.supportedFormats:
serialisation_format = 'json'
else:
serialisation_format = 'yaml'
request = {
'format': serialisation_format,
'content': self.serializeEntry(entry, serialisation_format)
}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
try:
yield self.agent.request("POST",
str(url),
bodyProducer=bodyProducer)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
content = '---\n'
content += safe_dump(entry)
content += '...\n'
url = self.collectorAddress + '/report'
request = {'report_id': self.reportID,
'content': content}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
try:
response = yield self.agent.request("PUT", url,
bodyProducer=bodyProducer)
except:
# XXX we must trap this in the runner and make sure to report the
# data later.
log.err("Error in writing report entry")
raise errors.OONIBReportUpdateError
def queryBackend(self, method, urn, query=None, retries=3):
bodyProducer = None
if query:
bodyProducer = StringProducer(json.dumps(query))
def genReceiver(finished, content_length):
def process_response(s):
# If empty string then don't parse it.
if not s:
return
try:
response = json.loads(s)
except ValueError:
raise e.get_error(None)
if 'error' in response:
log.debug("Got this backend error message %s" % response)
raise e.get_error(response['error'])
return response
return BodyReceiver(finished, content_length, process_response)
return self._request(method, urn, genReceiver, bodyProducer, retries)
class OONIBReporter(OReporter):
def __init__(self, test_details, collector_address):
self.collectorAddress = collector_address
self.validateCollectorAddress()
self.reportID = None
OReporter.__init__(self, test_details)
def validateCollectorAddress(self):
"""
Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
if the oonib reporter is not valid.
"""
regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
if not re.match(regexp, self.collectorAddress) or \
len(self.collectorAddress) < 30:
raise errors.InvalidOONIBCollectorAddress
@defer.inlineCallbacks
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
content = '---\n'
content += safe_dump(entry)
content += '...\n'
url = self.collectorAddress + '/report'
request = {'report_id': self.reportID,
'content': content}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
try:
response = yield self.agent.request("PUT", url,
bodyProducer=bodyProducer)
except:
# XXX we must trap this in the runner and make sure to report the
# data later.
log.err("Error in writing report entry")
raise errors.OONIBReportUpdateError
@defer.inlineCallbacks
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
from ooni.utils.txagentwithsocks import Agent
from twisted.internet import reactor
try:
self.agent = Agent(reactor, sockshost="127.0.0.1",
socksport=int(config.tor.socks_port))
except Exception, e:
log.exception(e)
url = self.collectorAddress + '/report'
content = '---\n'
content += safe_dump(self.testDetails)
content += '...\n'
request = {'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
# XXX there is a bunch of redundancy in the arguments getting sent
# to the backend. This may need to get changed in the client and the
# backend.
'content': content
}
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST", url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed (ConnectionRefusedError)")
#yield defer.fail(OONIBReportCreationError())
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def doRequest(self,
url,
method="GET",
headers={},
body=None,
headers_processor=None,
body_processor=None,
use_tor=False):
"""
Perform an HTTP request with the specified method and headers.
Args:
url (str): the full URL of the request. The scheme may be either
http, https, or httpo for http over Tor Hidden Service.
Kwargs:
method (str): the HTTP method name to use for the request
headers (dict): the request headers to send
body (str): the request body
headers_processor : a function to be used for processing the HTTP
header responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a dict.
body_processory: a function to be used for processing the HTTP
response body (defaults to self.processResponseBody). This
function takes the response body as an argument.
use_tor (bool): specify if the HTTP request should be done over Tor
or not.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using Tor for the request to %s" % url)
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" %
(self.localOptions['socksproxy']))
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
request['tor'] = {'exit_ip': None, 'exit_name': None}
if use_tor:
request['tor']['is_tor'] = True
else:
request['tor']['is_tor'] = False
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
self.report['requests'] = self.report.get('requests', [])
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure, request):
if request['tor']['is_tor']:
log.err("Error performing torified HTTP request: %s" %
request['url'])
else:
log.err("Error performing HTTP request: %s" % request['url'])
failure_string = handleAllFailures(failure)
previous_response = None
if getattr(failure, "previousResponse", None):
previous_response = failure.previousResponse
if getattr(failure, "requestLocation", None):
request['url'] = failure.requestLocation
self.addToReport(request,
failure_string=failure_string,
previous_response=previous_response)
return failure
if use_tor:
state = config.tor_state
if state:
state.add_stream_listener(StreamListener(request))
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback, request)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
return d
def doRequest(self,
url,
method="GET",
headers={},
body=None,
headers_processor=None,
body_processor=None,
use_tor=False):
"""
Perform an HTTP request with the specified method.
url: the full url path of the request
method: the HTTP Method to be used
headers: the request headers to be sent as a dict
body: the request body
headers_processor: a function to be used for processing the HTTP header
responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a
dict.
body_processory: a function to be used for processing the HTTP response
body (defaults to self.processResponseBody).
This function takes the response body as an argument.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using control agent for the request")
url = 's' + url
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" %
(self.localOptions['socksproxy']))
url = 's' + url
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
log.debug("Writing to report the request")
if 'requests' not in self.report:
self.report['requests'] = []
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure):
failure.trap(ConnectionRefusedError, SOCKSError)
if type(failure.value) is ConnectionRefusedError:
log.err("Connection refused. The backend may be down")
else:
log.err("Sock error. The SOCK proxy may be down")
self.report["failure"] = str(failure.value)
def finished(data):
return
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
d.addCallback(finished)
return d
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
from ooni.utils.hacks import SOCKS5Agent
from twisted.internet import reactor
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
elif self.collectorAddress.startswith('https://'):
# XXX add support for securely reporting to HTTPS collectors.
log.err("HTTPS based collectors are currently not supported.")
url = self.collectorAddress + '/report'
content = '---\n'
content += safe_dump(self.testDetails)
content += '...\n'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'input_hashes': self.testDetails['input_hashes'],
# XXX there is a bunch of redundancy in the arguments getting sent
# to the backend. This may need to get changed in the client and
# the backend.
'content': content
}
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
url = self.collectorAddress + '/report'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'probe_cc': self.testDetails['probe_cc'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'test_start_time': self.testDetails['test_start_time'],
'input_hashes': self.testDetails['input_hashes'],
'data_format_version': self.testDetails['data_format_version'],
'format': 'json'
}
# import values from the environment
request.update([(k.lower(), v) for (k, v) in os.environ.iteritems()
if k.startswith('PROBE_')])
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
def doRequest(self, url, method="GET",
headers={}, body=None, headers_processor=None,
body_processor=None, use_tor=False):
"""
Perform an HTTP request with the specified method and headers.
Args:
url (str): the full URL of the request. The scheme may be either
http, https, or httpo for http over Tor Hidden Service.
Kwargs:
method (str): the HTTP method name to use for the request
headers (dict): the request headers to send
body (str): the request body
headers_processor : a function to be used for processing the HTTP
header responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a dict.
body_processory: a function to be used for processing the HTTP
response body (defaults to self.processResponseBody). This
function takes the response body as an argument.
use_tor (bool): specify if the HTTP request should be done over Tor
or not.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using Tor for the request to %s" % url)
url = 's'+url
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy']))
url = 's'+url
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
if 'requests' not in self.report:
self.report['requests'] = []
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure, request):
failure_string = handleAllFailures(failure)
log.err("Error performing %s" % request)
HTTPTest.addToReport(self, request, failure_string=failure_string)
return failure
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
d.addErrback(errback, request)
return d