def write_error(self, status_code, exception=None, **kw):
self.set_status(status_code)
if hasattr(exception, 'log_message') and exception.log_message is not None:
self.write({'error': exception.log_message})
elif 400 <= status_code < 600:
self.write({'error': status_code})
else:
log.exception(exception)
self.write({'error': 'error'})
def write_error(self, status_code, exception=None, **kw):
self.set_status(status_code)
if hasattr(exception,
'log_message') and exception.log_message is not None:
self.write({'error': exception.log_message})
elif 400 <= status_code < 600:
self.write({'error': status_code})
else:
log.exception(exception)
self.write({'error': 'error'})
def post(self):
try:
request = json.loads(self.request.body)
self.validate_request(request)
include_http_responses = request.get("include_http_responses",
False)
self.control_measurement(
str(request['http_request']),
request['tcp_connect'],
include_http_responses
)
except HTTPError:
raise
except Exception as exc:
log.msg("Got invalid request")
log.exception(exc)
raise HTTPError(400, 'invalid request')
def post(self):
try:
request = json.loads(self.request.body)
# Here we fix inconsistencies in the tcp_connect field. Due to:
# https://github.com/TheTorProject/ooni-probe/issues/727 ooniprobe
# was sending hostnames as part of the tcp_connect key as well as
# IP addresses.
# If we find something that isn't an IP address we return it with
# the key value None to support backward compatibility with older
# bugged clients.
tcp_connect = []
invalid_sockets = []
for socket in request['tcp_connect']:
if SOCKET_REGEXP.match(socket):
tcp_connect.append(socket)
else:
invalid_sockets.append(socket)
request['tcp_connect'] = tcp_connect
self.validate_request(request)
include_http_responses = request.get(
"include_http_responses",
False
)
# We convert headers to str so twisted is happy (unicode triggers
# errors)
http_request_headers = {}
for k, v in request.get('http_request_headers', {}).iteritems():
http_request_headers[str(k)] = map(str, v)
self.control_measurement(
http_url=str(request['http_request']),
include_http_responses=include_http_responses,
http_request_headers=http_request_headers,
socket_list=request['tcp_connect'],
invalid_sockets=invalid_sockets
)
except HTTPError:
raise
except Exception as exc:
log.msg("Got invalid request")
log.exception(exc)
raise HTTPError(400, 'invalid request')
def get(self, inputID):
bn = os.path.basename(inputID) + ".desc"
try:
with open(os.path.join(config.main.input_dir, bn)) as f:
response = {}
inputDesc = yaml.safe_load(f)
for k in ["name", "description", "version", "author", "date"]:
response[k] = inputDesc[k]
response["id"] = inputID
self.write(response)
except IOError:
log.err("No Input Descriptor found for id %s" % inputID)
self.set_status(404)
self.write({"error": "missing-input"})
except Exception, e:
log.exception(e)
log.err("Invalid Input Descriptor found for id %s" % inputID)
self.set_status(500)
self.write({"error": "invalid-input-descriptor"})
def get(self, inputID):
bn = os.path.basename(inputID) + ".desc"
try:
with open(os.path.join(config.main.input_dir, bn)) as f:
response = {}
inputDesc = yaml.safe_load(f)
for k in ['name', 'description', 'version', 'author', 'date']:
response[k] = inputDesc[k]
response['id'] = inputID
self.write(response)
except IOError:
log.err("No Input Descriptor found for id %s" % inputID)
self.set_status(404)
self.write({'error': 'missing-input'})
except Exception, e:
log.exception(e)
log.err("Invalid Input Descriptor found for id %s" % inputID)
self.set_status(500)
self.write({'error': 'invalid-input-descriptor'})
def get(self, inputID):
bn = os.path.basename(inputID) + ".desc"
try:
with open(os.path.join(config.main.input_dir, bn)) as f:
response = {}
inputDesc = yaml.safe_load(f)
for k in ['name', 'description', 'version', 'author', 'date']:
response[k] = inputDesc[k]
response['id'] = inputID
self.write(response)
except IOError:
log.err("No Input Descriptor found for id %s" % inputID)
self.set_status(404)
self.write({'error': 'missing-input'})
except Exception, e:
log.exception(e)
log.err("Invalid Input Descriptor found for id %s" % inputID)
self.set_status(500)
self.write({'error': 'invalid-input-descriptor'})
def get(self, inputID):
bn = os.path.basename(inputID) + ".desc"
try:
f = open(os.path.join(config.main.input_dir, bn))
except IOError:
log.err("No Input Descriptor found for id %s" % inputID)
self.set_status(404)
self.write({'error': 'missing-input'})
return
with f:
inputDesc = yaml.safe_load(f)
response = {'id': inputID}
for k in ['name', 'description', 'version', 'author', 'date']:
try:
response[k] = inputDesc[k]
except Exception, e: # XXX this should probably be KeyError
log.exception(e)
log.err("Invalid Input Descriptor found for id %s" % inputID)
self.set_status(500)
self.write({'error': 'invalid-input-descriptor'})
return
def txSetupFailed(self, failure):
log.err("Setup failed")
log.exception(failure)
def txSetupFailed(self, failure):
log.err("Setup failed")
log.exception(failure)
def http_request(self, url, http_request_headers, include_http_responses=False):
key = url + json.dumps(http_request_headers)
cached_value = yield self.lookup('http_request', key)
if cached_value is not None:
if include_http_responses is not True:
cached_value.pop('responses', None)
defer.returnValue(cached_value)
page_info = {
'body_length': -1,
'status_code': -1,
'headers': {},
'failure': None
}
agent = ContentDecoderAgent(
FixedRedirectAgent(TrueHeadersAgent(reactor),
ignorePrivateRedirects=True),
[('gzip', GzipDecoder)]
)
try:
retries = 0
while True:
try:
response = yield agent.request('GET', url,
TrueHeaders(http_request_headers))
headers = {}
for name, value in response.headers.getAllRawHeaders():
headers[name] = unicode(value[0], errors='ignore')
body_length = -1
body = None
try:
body = yield readBody(response)
body_length = len(body)
except PartialDownloadError as pde:
if pde.response:
body_length = len(pde.response)
body = pde.response
page_info['body_length'] = body_length
page_info['status_code'] = response.code
page_info['headers'] = headers
page_info['title'] = extractTitle(body)
response.body = body
page_info['responses'] = encodeResponses(response)
break
except:
if retries > self.http_retries:
raise
retries += 1
except DNSLookupError:
page_info['failure'] = 'dns_lookup_error'
except TimeoutError:
page_info['failure'] = 'generic_timeout_error'
except ConnectionRefusedError:
page_info['failure'] = 'connection_refused_error'
except ConnectError:
page_info['failure'] = 'connect_error'
except Exception as exc:
# XXX map more failures
page_info['failure'] = 'unknown_error'
log.err("Unknown error occurred")
log.exception(exc)
yield self.cache_value('http_request', key, page_info)
if include_http_responses is not True:
page_info.pop('responses', None)
defer.returnValue(page_info)
