def _cbResponse(self, response, request, headers_processor,
body_processor):
if not response:
log.err("Got no response")
return
else:
log.debug("Got response %s" % response)
if str(response.code).startswith('3'):
self.processRedirect(response.headers.getRawHeaders('Location')[0])
# [!] We are passing to the headers_processor the headers dict and
# not the Headers() object
response_headers_dict = list(response.headers.getAllRawHeaders())
if headers_processor:
headers_processor(response_headers_dict)
else:
self.processResponseHeaders(response_headers_dict)
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished))
finished.addCallback(self._processResponseBody, request, response,
body_processor)
return finished
def _response(self, response):
content_length = response.headers.getRawHeaders('content-length')
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished, content_length))
finished.addCallback(self.parseResponse)
return finished
def _cbResponse(self, response, request, headers_processor,
body_processor):
"""
This callback is fired once we have gotten a response for our request.
If we are using a RedirectAgent then this will fire once we have
reached the end of the redirect chain.
Args:
response (:twisted.web.iweb.IResponse:): a provider for getting our response
request (dict): the dict containing our response (XXX this should be dropped)
header_processor (func): a function to be called with argument a
dict containing the response headers. This will lead
self.headerProcessor to not be called.
body_processor (func): a function to be called with as argument the
body of the response. This will lead self.bodyProcessor to not
be called.
"""
if not response:
log.err("Got no response for request %s" % request)
HTTPTest.addToReport(self, request, response)
return
else:
log.debug("Got response %s" % response)
if str(response.code).startswith('3'):
self.processRedirect(response.headers.getRawHeaders('Location')[0])
# [!] We are passing to the headers_processor the headers dict and
# not the Headers() object
response_headers_dict = list(response.headers.getAllRawHeaders())
if headers_processor:
headers_processor(response_headers_dict)
else:
self.processResponseHeaders(response_headers_dict)
try:
content_length = int(
response.headers.getRawHeaders('content-length')[0])
except Exception:
content_length = None
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished, content_length))
finished.addCallback(self._processResponseBody, request, response,
body_processor)
finished.addErrback(self._processResponseBodyFail, request, response)
return finished
def genReceiver(finished, content_length):
def process_response(s):
# If empty string then don't parse it.
if not s:
return
try:
response = json.loads(s)
except ValueError:
raise e.get_error(None)
if 'error' in response:
log.debug("Got this backend error message %s" % response)
raise e.get_error(response['error'])
return response
return BodyReceiver(finished, content_length, process_response)
#yield defer.fail(OONIBReportCreationError())
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
# This is a little trix to allow us to unspool the response. We create
# a deferred and call yield on it.
response_body = defer.Deferred()
response.deliverBody(BodyReceiver(response_body))
backend_response = yield response_body
try:
parsed_response = json.loads(backend_response)
except Exception, e:
log.err("Failed to parse collector response")
log.exception(e)
raise errors.OONIBReportCreationError
self.reportID = parsed_response['report_id']
self.backendVersion = parsed_response['backend_version']
log.debug("Created report with id %s" % parsed_response['report_id'])
class ReportClosed(Exception):
class OONIBReporter(OReporter):
def __init__(self, test_details, collector_address):
self.collectorAddress = collector_address
self.validateCollectorAddress()
self.reportID = None
OReporter.__init__(self, test_details)
def validateCollectorAddress(self):
"""
Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
if the oonib reporter is not valid.
"""
regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
if not re.match(regexp, self.collectorAddress):
raise errors.InvalidOONIBCollectorAddress
@defer.inlineCallbacks
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
content = '---\n'
if isinstance(entry, Measurement):
content += safe_dump(entry.testInstance.report)
elif isinstance(entry, Failure):
content += entry.value
elif isinstance(entry, dict):
content += safe_dump(entry)
content += '...\n'
url = self.collectorAddress + '/report'
request = {'report_id': self.reportID, 'content': content}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
try:
yield self.agent.request("PUT", url, bodyProducer=bodyProducer)
except:
# XXX we must trap this in the runner and make sure to report the
# data later.
log.err("Error in writing report entry")
raise errors.OONIBReportUpdateError
@defer.inlineCallbacks
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
from ooni.utils.hacks import SOCKS5Agent
from twisted.internet import reactor
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
elif self.collectorAddress.startswith('https://'):
# XXX add support for securely reporting to HTTPS collectors.
log.err("HTTPS based collectors are currently not supported.")
url = self.collectorAddress + '/report'
content = '---\n'
content += safe_dump(self.testDetails)
content += '...\n'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'input_hashes': self.testDetails['input_hashes'],
# XXX there is a bunch of redundancy in the arguments getting sent
# to the backend. This may need to get changed in the client and
# the backend.
'content': content
}
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(json.dumps(request))
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
# This is a little trix to allow us to unspool the response. We create
# a deferred and call yield on it.
response_body = defer.Deferred()
response.deliverBody(BodyReceiver(response_body))
backend_response = yield response_body
try:
parsed_response = json.loads(backend_response)
except Exception, e:
log.err("Failed to parse collector response %s" % backend_response)
log.exception(e)
raise errors.OONIBReportCreationError
class OONIBReporter(OReporter):
def __init__(self, test_details, collector_address):
self.collectorAddress = collector_address
self.validateCollectorAddress()
self.reportID = None
self.supportedFormats = ["yaml"]
if self.collectorAddress.startswith('https://'):
# not sure if there's something else it needs. Seems to work.
# Very difficult to get it to work with self-signed certs.
self.agent = Agent(reactor)
elif self.collectorAddress.startswith('http://'):
log.msg("Warning using unencrypted collector")
self.agent = Agent(reactor)
OReporter.__init__(self, test_details)
def validateCollectorAddress(self):
"""
Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
if the oonib reporter is not valid.
"""
regexp = '^(http|https|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
if not re.match(regexp, self.collectorAddress):
raise errors.InvalidOONIBCollectorAddress
def serializeEntry(self, entry, serialisation_format="yaml"):
if serialisation_format == "json":
if isinstance(entry, Measurement):
report_entry = {
'input':
entry.testInstance.report.pop('input', None),
'id':
str(uuid.uuid4()),
'test_start_time':
entry.testInstance.report.pop('test_start_time', None),
'measurement_start_time':
entry.testInstance.report.pop('measurement_start_time',
None),
'test_runtime':
entry.testInstance.report.pop('test_runtime', None),
'test_keys':
entry.testInstance.report
}
elif isinstance(entry, dict):
report_entry = {
'input':
entry.pop('input', None),
'id':
str(uuid.uuid4()),
'test_start_time':
entry.pop('test_start_time', None),
'measurement_start_time':
entry.pop('measurement_start_time', None),
'test_runtime':
entry.pop('test_runtime', None),
'test_keys':
entry
}
else:
raise Exception("Failed to serialise entry")
report_entry.update(self.testDetails)
return report_entry
else:
content = '---\n'
if isinstance(entry, Measurement):
report_entry = entry.testInstance.report
elif isinstance(entry, dict):
report_entry = entry
else:
raise Exception("Failed to serialise entry")
content += safe_dump(report_entry)
content += '...\n'
return content
@defer.inlineCallbacks
def writeReportEntry(self, entry):
log.debug("Writing report with OONIB reporter")
url = self.collectorAddress + '/report/' + self.reportID
if "json" in self.supportedFormats:
serialisation_format = 'json'
else:
serialisation_format = 'yaml'
request = {
'format': serialisation_format,
'content': self.serializeEntry(entry, serialisation_format)
}
log.debug("Updating report with id %s (%s)" % (self.reportID, url))
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
try:
yield self.agent.request("POST",
str(url),
bodyProducer=bodyProducer)
except Exception as exc:
log.err("Error in writing report entry")
log.exception(exc)
raise errors.OONIBReportUpdateError
@defer.inlineCallbacks
def createReport(self):
"""
Creates a report on the oonib collector.
"""
# XXX we should probably be setting this inside of the constructor,
# however config.tor.socks_port is not set until Tor is started and the
# reporter is instantiated before Tor is started. We probably want to
# do this with some deferred kung foo or instantiate the reporter after
# tor is started.
if self.collectorAddress.startswith('httpo://'):
self.collectorAddress = \
self.collectorAddress.replace('httpo://', 'http://')
proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.socks_port)
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
url = self.collectorAddress + '/report'
request = {
'software_name': self.testDetails['software_name'],
'software_version': self.testDetails['software_version'],
'probe_asn': self.testDetails['probe_asn'],
'probe_cc': self.testDetails['probe_cc'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
'test_start_time': self.testDetails['test_start_time'],
'input_hashes': self.testDetails['input_hashes'],
'data_format_version': self.testDetails['data_format_version'],
'format': 'json'
}
# import values from the environment
request.update([(k.lower(), v) for (k, v) in os.environ.iteritems()
if k.startswith('PROBE_')])
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
bodyProducer = StringProducer(request_json)
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
try:
response = yield self.agent.request("POST",
url,
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed "
"(ConnectionRefusedError)")
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
log.err("Host is not reachable (HostUnreachable error")
raise errors.OONIBReportCreationError
except Exception, e:
log.err("Failed to connect to reporter backend")
log.exception(e)
raise errors.OONIBReportCreationError
# This is a little trix to allow us to unspool the response. We create
# a deferred and call yield on it.
response_body = defer.Deferred()
response.deliverBody(BodyReceiver(response_body))
backend_response = yield response_body
try:
parsed_response = json.loads(backend_response)
except Exception, e:
log.err("Failed to parse collector response %s" % backend_response)
log.exception(e)
raise errors.OONIBReportCreationError