示例#1
0
 def write_error(self, status_code, exception=None, **kw):
     self.set_status(status_code)
     if hasattr(exception, 'log_message') and exception.log_message is not None:
         self.write({'error': exception.log_message})
     elif 400 <= status_code < 600:
         self.write({'error': status_code})
     else:
         log.exception(exception)
         self.write({'error': 'error'})
示例#2
0
 def write_error(self, status_code, exception=None, **kw):
     self.set_status(status_code)
     if hasattr(exception,
                'log_message') and exception.log_message is not None:
         self.write({'error': exception.log_message})
     elif 400 <= status_code < 600:
         self.write({'error': status_code})
     else:
         log.exception(exception)
         self.write({'error': 'error'})
示例#3
0
 def post(self):
     try:
         request = json.loads(self.request.body)
         self.validate_request(request)
         include_http_responses = request.get("include_http_responses",
                                              False)
         self.control_measurement(
             str(request['http_request']),
             request['tcp_connect'],
             include_http_responses
         )
     except HTTPError:
         raise
     except Exception as exc:
         log.msg("Got invalid request")
         log.exception(exc)
         raise HTTPError(400, 'invalid request')
示例#4
0
    def post(self):
        try:
            request = json.loads(self.request.body)

            # Here we fix inconsistencies in the tcp_connect field.  Due to:
            # https://github.com/TheTorProject/ooni-probe/issues/727 ooniprobe
            # was sending hostnames as part of the tcp_connect key as well as
            # IP addresses.
            # If we find something that isn't an IP address we return it with
            # the key value None to support backward compatibility with older
            # bugged clients.
            tcp_connect = []
            invalid_sockets = []
            for socket in request['tcp_connect']:
                if SOCKET_REGEXP.match(socket):
                    tcp_connect.append(socket)
                else:
                    invalid_sockets.append(socket)
            request['tcp_connect'] = tcp_connect

            self.validate_request(request)
            include_http_responses = request.get(
                    "include_http_responses",
                    False
            )

            # We convert headers to str so twisted is happy (unicode triggers
            # errors)
            http_request_headers = {}
            for k, v in request.get('http_request_headers', {}).iteritems():
                http_request_headers[str(k)] = map(str, v)
            self.control_measurement(
                http_url=str(request['http_request']),
                include_http_responses=include_http_responses,
                http_request_headers=http_request_headers,
                socket_list=request['tcp_connect'],
                invalid_sockets=invalid_sockets
            )
        except HTTPError:
            raise
        except Exception as exc:
            log.msg("Got invalid request")
            log.exception(exc)
            raise HTTPError(400, 'invalid request')
示例#5
0
    def get(self, inputID):
        bn = os.path.basename(inputID) + ".desc"
        try:
            with open(os.path.join(config.main.input_dir, bn)) as f:
                response = {}
                inputDesc = yaml.safe_load(f)
                for k in ["name", "description", "version", "author", "date"]:
                    response[k] = inputDesc[k]
                response["id"] = inputID
            self.write(response)
        except IOError:
            log.err("No Input Descriptor found for id %s" % inputID)
            self.set_status(404)
            self.write({"error": "missing-input"})

        except Exception, e:
            log.exception(e)
            log.err("Invalid Input Descriptor found for id %s" % inputID)
            self.set_status(500)
            self.write({"error": "invalid-input-descriptor"})
    def get(self, inputID):
        bn = os.path.basename(inputID) + ".desc"
        try:
            with open(os.path.join(config.main.input_dir, bn)) as f:
                response = {}
                inputDesc = yaml.safe_load(f)
                for k in ['name', 'description', 'version', 'author', 'date']:
                    response[k] = inputDesc[k]
                response['id'] = inputID
            self.write(response)
        except IOError:
            log.err("No Input Descriptor found for id %s" % inputID) 
            self.set_status(404)
            self.write({'error': 'missing-input'})
 
        except Exception, e:
            log.exception(e)
            log.err("Invalid Input Descriptor found for id %s" % inputID) 
            self.set_status(500)
            self.write({'error': 'invalid-input-descriptor'})
示例#7
0
    def get(self, inputID):
        bn = os.path.basename(inputID) + ".desc"
        try:
            with open(os.path.join(config.main.input_dir, bn)) as f:
                response = {}
                inputDesc = yaml.safe_load(f)
                for k in ['name', 'description', 'version', 'author', 'date']:
                    response[k] = inputDesc[k]
                response['id'] = inputID
            self.write(response)
        except IOError:
            log.err("No Input Descriptor found for id %s" % inputID) 
            self.set_status(404)
            self.write({'error': 'missing-input'})
 
        except Exception, e:
            log.exception(e)
            log.err("Invalid Input Descriptor found for id %s" % inputID) 
            self.set_status(500)
            self.write({'error': 'invalid-input-descriptor'})
示例#8
0
    def get(self, inputID):
        bn = os.path.basename(inputID) + ".desc"
        try:
            f = open(os.path.join(config.main.input_dir, bn))
        except IOError:
            log.err("No Input Descriptor found for id %s" % inputID) 
            self.set_status(404)
            self.write({'error': 'missing-input'})
            return
        with f:
            inputDesc = yaml.safe_load(f)
 
        response = {'id': inputID}
        for k in ['name', 'description', 'version', 'author', 'date']:
            try:
                response[k] = inputDesc[k]
            except Exception, e: # XXX this should probably be KeyError
                log.exception(e)
                log.err("Invalid Input Descriptor found for id %s" % inputID) 
                self.set_status(500)
                self.write({'error': 'invalid-input-descriptor'})
                return
示例#9
0
 def txSetupFailed(self, failure):
     log.err("Setup failed")
     log.exception(failure)
示例#10
0
 def txSetupFailed(self, failure):
     log.err("Setup failed")
     log.exception(failure)
示例#11
0
    def http_request(self, url, http_request_headers, include_http_responses=False):
        key = url + json.dumps(http_request_headers)
        cached_value = yield self.lookup('http_request', key)
        if cached_value is not None:
            if include_http_responses is not True:
                cached_value.pop('responses', None)
            defer.returnValue(cached_value)

        page_info = {
            'body_length': -1,
            'status_code': -1,
            'headers': {},
            'failure': None
        }

        agent = ContentDecoderAgent(
            FixedRedirectAgent(TrueHeadersAgent(reactor),
                               ignorePrivateRedirects=True),
            [('gzip', GzipDecoder)]
        )
        try:
            retries = 0
            while True:
                try:
                    response = yield agent.request('GET', url,
                                                   TrueHeaders(http_request_headers))
                    headers = {}
                    for name, value in response.headers.getAllRawHeaders():
                        headers[name] = unicode(value[0], errors='ignore')
                    body_length = -1
                    body = None
                    try:
                        body = yield readBody(response)
                        body_length = len(body)
                    except PartialDownloadError as pde:
                        if pde.response:
                            body_length = len(pde.response)
                            body = pde.response
                    page_info['body_length'] = body_length
                    page_info['status_code'] = response.code
                    page_info['headers'] = headers
                    page_info['title'] = extractTitle(body)
                    response.body = body
                    page_info['responses'] = encodeResponses(response)
                    break
                except:
                    if retries > self.http_retries:
                        raise
                    retries += 1
        except DNSLookupError:
            page_info['failure'] = 'dns_lookup_error'
        except TimeoutError:
            page_info['failure'] = 'generic_timeout_error'
        except ConnectionRefusedError:
            page_info['failure'] = 'connection_refused_error'
        except ConnectError:
            page_info['failure'] = 'connect_error'
        except Exception as exc:
            # XXX map more failures
            page_info['failure'] = 'unknown_error'
            log.err("Unknown error occurred")
            log.exception(exc)

        yield self.cache_value('http_request', key, page_info)
        if include_http_responses is not True:
            page_info.pop('responses', None)
        defer.returnValue(page_info)