def write_error(self, status_code, exception=None, **kw): self.set_status(status_code) if hasattr(exception, 'log_message') and exception.log_message is not None: self.write({'error': exception.log_message}) elif 400 <= status_code < 600: self.write({'error': status_code}) else: log.exception(exception) self.write({'error': 'error'})
def post(self): try: request = json.loads(self.request.body) self.validate_request(request) include_http_responses = request.get("include_http_responses", False) self.control_measurement( str(request['http_request']), request['tcp_connect'], include_http_responses ) except HTTPError: raise except Exception as exc: log.msg("Got invalid request") log.exception(exc) raise HTTPError(400, 'invalid request')
def post(self): try: request = json.loads(self.request.body) # Here we fix inconsistencies in the tcp_connect field. Due to: # https://github.com/TheTorProject/ooni-probe/issues/727 ooniprobe # was sending hostnames as part of the tcp_connect key as well as # IP addresses. # If we find something that isn't an IP address we return it with # the key value None to support backward compatibility with older # bugged clients. tcp_connect = [] invalid_sockets = [] for socket in request['tcp_connect']: if SOCKET_REGEXP.match(socket): tcp_connect.append(socket) else: invalid_sockets.append(socket) request['tcp_connect'] = tcp_connect self.validate_request(request) include_http_responses = request.get( "include_http_responses", False ) # We convert headers to str so twisted is happy (unicode triggers # errors) http_request_headers = {} for k, v in request.get('http_request_headers', {}).iteritems(): http_request_headers[str(k)] = map(str, v) self.control_measurement( http_url=str(request['http_request']), include_http_responses=include_http_responses, http_request_headers=http_request_headers, socket_list=request['tcp_connect'], invalid_sockets=invalid_sockets ) except HTTPError: raise except Exception as exc: log.msg("Got invalid request") log.exception(exc) raise HTTPError(400, 'invalid request')
def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: with open(os.path.join(config.main.input_dir, bn)) as f: response = {} inputDesc = yaml.safe_load(f) for k in ["name", "description", "version", "author", "date"]: response[k] = inputDesc[k] response["id"] = inputID self.write(response) except IOError: log.err("No Input Descriptor found for id %s" % inputID) self.set_status(404) self.write({"error": "missing-input"}) except Exception, e: log.exception(e) log.err("Invalid Input Descriptor found for id %s" % inputID) self.set_status(500) self.write({"error": "invalid-input-descriptor"})
def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: with open(os.path.join(config.main.input_dir, bn)) as f: response = {} inputDesc = yaml.safe_load(f) for k in ['name', 'description', 'version', 'author', 'date']: response[k] = inputDesc[k] response['id'] = inputID self.write(response) except IOError: log.err("No Input Descriptor found for id %s" % inputID) self.set_status(404) self.write({'error': 'missing-input'}) except Exception, e: log.exception(e) log.err("Invalid Input Descriptor found for id %s" % inputID) self.set_status(500) self.write({'error': 'invalid-input-descriptor'})
def get(self, inputID): bn = os.path.basename(inputID) + ".desc" try: f = open(os.path.join(config.main.input_dir, bn)) except IOError: log.err("No Input Descriptor found for id %s" % inputID) self.set_status(404) self.write({'error': 'missing-input'}) return with f: inputDesc = yaml.safe_load(f) response = {'id': inputID} for k in ['name', 'description', 'version', 'author', 'date']: try: response[k] = inputDesc[k] except Exception, e: # XXX this should probably be KeyError log.exception(e) log.err("Invalid Input Descriptor found for id %s" % inputID) self.set_status(500) self.write({'error': 'invalid-input-descriptor'}) return
def txSetupFailed(self, failure): log.err("Setup failed") log.exception(failure)
def http_request(self, url, http_request_headers, include_http_responses=False): key = url + json.dumps(http_request_headers) cached_value = yield self.lookup('http_request', key) if cached_value is not None: if include_http_responses is not True: cached_value.pop('responses', None) defer.returnValue(cached_value) page_info = { 'body_length': -1, 'status_code': -1, 'headers': {}, 'failure': None } agent = ContentDecoderAgent( FixedRedirectAgent(TrueHeadersAgent(reactor), ignorePrivateRedirects=True), [('gzip', GzipDecoder)] ) try: retries = 0 while True: try: response = yield agent.request('GET', url, TrueHeaders(http_request_headers)) headers = {} for name, value in response.headers.getAllRawHeaders(): headers[name] = unicode(value[0], errors='ignore') body_length = -1 body = None try: body = yield readBody(response) body_length = len(body) except PartialDownloadError as pde: if pde.response: body_length = len(pde.response) body = pde.response page_info['body_length'] = body_length page_info['status_code'] = response.code page_info['headers'] = headers page_info['title'] = extractTitle(body) response.body = body page_info['responses'] = encodeResponses(response) break except: if retries > self.http_retries: raise retries += 1 except DNSLookupError: page_info['failure'] = 'dns_lookup_error' except TimeoutError: page_info['failure'] = 'generic_timeout_error' except ConnectionRefusedError: page_info['failure'] = 'connection_refused_error' except ConnectError: page_info['failure'] = 'connect_error' except Exception as exc: # XXX map more failures page_info['failure'] = 'unknown_error' log.err("Unknown error occurred") log.exception(exc) yield self.cache_value('http_request', key, page_info) if include_http_responses is not True: page_info.pop('responses', None) defer.returnValue(page_info)