Exemple #1
0
    def user_queue(self, request):
        """ Return queue info for the user, creating if necessary 
            (queues hold subscriptions) 
        
        @method POST

        @param request Request object containing "Credentials"
            where credentials meet the proper authorization
            module requirements. (e.g. for basic auth, use
            'username' & 'password', for BrowserID pass

        Users have Queues. 
        Queues hold subscriptions.
        Subscriptions hold messages. 

        If a username already has a queue, return that information, otherwise
        create the new queue and return the new info.

        """
        self._init(self.app.config)
        username = self._get_uid(request)

        #TODO: Authenticate the user using the backend auth.
        try:
            result = self.msg_backend.create_client_queue(username)
            return json_response(result)
        except Exception, e:
            logger.error("Error creating client queue %s" % str(e))
            raise HTTPInternalServerError
 def get_quota(self, request):
     user_id = request.user['userid']
     used = self._get_storage(request).get_total_size(user_id)
     if not self._get_storage(request).use_quota:
         limit = None
     else:
         limit = self._get_storage(request).quota_size
     return json_response((used, limit))
    def delete_storage(self, request):
        """Deletes all records for the user.

        Will return a precondition error unless an X-Confirm-Delete header
        is included.
        """
        if 'X-Confirm-Delete' not in request.headers:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WRITE)
        user_id = request.user['userid']
        self._get_storage(request).delete_storage(user_id)  # XXX failures ?
        return json_response(request.server_time)
    def get_item(self, request, full=True):  # always full
        """Returns a single WBO object."""
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']
        user_id = request.user['userid']
        fields = _WBO_FIELDS
        storage = self._get_storage(request)
        res = storage.get_item(user_id, collection_name, item_id,
                               fields=fields)
        if res is None:
            raise HTTPNotFound()

        return json_response(res)
Exemple #5
0
    def new_token(self, request):
        """ Create an return a new valid token 
        
        @method GET

        @params None

        Returns a string containing the new token.
        """
        self._init(self.app.config)
        try:
            token = new_token()
            return json_response(token)
        except Exception, e:
            logger.error("Error generating token %s" % str(e))
            raise HTTPInternalServerError
    def set_item(self, request):
        """Sets a single WBO object."""
        storage = self._get_storage(request)
        if storage.use_quota:
            left = self._check_quota(request)
        else:
            left = 0.

        user_id = request.user['userid']
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']

        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        try:
            data = json.loads(request.body)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)

        try:
            wbo = WBO(data)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

        consistent, msg = wbo.validate()
        if not consistent:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

        if self._has_modifiers(wbo):
            wbo['modified'] = request.server_time

        try:
            res = storage.set_item(user_id, collection_name, item_id,
                                   storage_time=request.server_time, **wbo)
        except StorageConflictError:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WRITE)
        response = json_response(res)
        if storage.use_quota and left <= _ONE_MEG:
            response.headers['X-Weave-Quota-Remaining'] = str(left)
        return response
    def delete_collection(self, request, **kw):
        """Deletes the collection and all contents.

        Additional request parameters may modify the selection of which
        items to delete.
        """
        kw = self._convert_args(kw)
        collection_name = request.sync_info['collection']
        user_id = request.user['userid']
        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        self._get_storage(request).delete_items(user_id,
                                        collection_name,
                                        kw.get('ids'), kw['filters'],
                                        limit=kw.get('limit'),
                                        offset=kw.get('offset'),
                                        sort=kw.get('sort'),
                                        storage_time=request.server_time)

        return json_response(request.server_time)
    def delete_item(self, request):
        """Deletes a single WBO object."""
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']

        user_id = request.user['userid']
        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        self._get_storage(request).delete_item(user_id, collection_name,
                                              item_id,
                                              storage_time=request.server_time)

        # Not logging this event for now. Infrasec may want it again in future
        #
        #if collection_name == 'crypto' and item_id == 'keys':
        #    msg = 'Crypto keys deleted'
        #    username = request.user['username']
        #    log_cef(msg, 5, request.environ, self.app.config, username)

        return json_response(request.server_time)
    def test_response_conversions(self):
        data = {'some': 'data'}
        resp = text_response(data)
        self.assertEquals(resp.body, "{'some': 'data'}")
        self.assertEquals(resp.content_type, 'text/plain')

        data = "abc"
        resp = whoisi_response(data)
        self.assertEquals(
            resp.body,
            '\x00\x00\x00\x03"a"\x00\x00\x00\x03"b"\x00\x00\x00\x03"c"')
        self.assertEquals(resp.content_type, 'application/whoisi')

        request = Request({})
        request.accept = 'application/whoisi'
        resp = convert_response(request, data)
        self.assertEquals(
            resp.body,
            '\x00\x00\x00\x03"a"\x00\x00\x00\x03"b"\x00\x00\x00\x03"c"')
        self.assertEquals(resp.content_type, 'application/whoisi')

        resp = newlines_response(data)
        self.assertEquals(resp.body, '"a"\n"b"\n"c"\n')
        self.assertEquals(resp.content_type, 'application/newlines')

        request = Request({})
        request.accept = 'application/newlines'
        resp = convert_response(request, data)
        self.assertEquals(resp.body, '"a"\n"b"\n"c"\n')
        self.assertEquals(resp.content_type, 'application/newlines')

        data = {'some': 'data'}
        resp = json_response(data)
        self.assertEquals(resp.body, '{"some": "data"}')
        self.assertEquals(resp.content_type, 'application/json')

        request = Request({})
        resp = convert_response(request, data)
        self.assertEquals(resp.body, '{"some": "data"}')
        self.assertEquals(resp.content_type, 'application/json')
    def test_response_conversions(self):
        data = {'some': 'data'}
        resp = text_response(data)
        self.assertEquals(resp.body, "{'some': 'data'}")
        self.assertEquals(resp.content_type, 'text/plain')

        data = "abc"
        resp = whoisi_response(data)
        self.assertEquals(resp.body,
                '\x00\x00\x00\x03"a"\x00\x00\x00\x03"b"\x00\x00\x00\x03"c"')
        self.assertEquals(resp.content_type, 'application/whoisi')

        request = Request({})
        request.accept = 'application/whoisi'
        resp = convert_response(request, data)
        self.assertEquals(resp.body,
                '\x00\x00\x00\x03"a"\x00\x00\x00\x03"b"\x00\x00\x00\x03"c"')
        self.assertEquals(resp.content_type, 'application/whoisi')

        resp = newlines_response(data)
        self.assertEquals(resp.body, '"a"\n"b"\n"c"\n')
        self.assertEquals(resp.content_type, 'application/newlines')

        request = Request({})
        request.accept = 'application/newlines'
        resp = convert_response(request, data)
        self.assertEquals(resp.body, '"a"\n"b"\n"c"\n')
        self.assertEquals(resp.content_type, 'application/newlines')

        data = {'some': 'data'}
        resp = json_response(data)
        self.assertEquals(resp.body, '{"some": "data"}')
        self.assertEquals(resp.content_type, 'application/json')

        request = Request({})
        resp = convert_response(request, data)
        self.assertEquals(resp.body, '{"some": "data"}')
        self.assertEquals(resp.content_type, 'application/json')
 def get_user_details(self, request, **kw):
     """Returns a JSON blob of account details for the current user."""
     return json_response({
         'userid': request.user['userid'],
         'syncNode': request.user['syncNode'],
     })
Exemple #12
0
            return text_response(syncnode)

        # Assign a new node using the backend.
        # This assumes it will remember the assignment, and optimally that
        # it will cache it in the user's syncNode property for our reference.
        try:
            new_node = self.nodes.get_best_node('sync', user=request.user)
        except NodeAlreadyWrittenError, e:
            new_node = e.node
        except Exception:
            self.logger.error(traceback.format_exc())
            self.logger.error("Node assignment failed")
            return json_response(None)

        if new_node is None:
            return json_response(None)

        if new_node != "null":
            if not new_node.startswith('http'):
                new_node = 'https://' + new_node
            if not new_node.endswith('/'):
                new_node = new_node + '/'

        return text_response(new_node)

    def password_reset(self, request, **data):
        """Sends an e-mail for a password reset request."""
        if self.reset is None:
            self.logger.debug('reset attempted, but no resetcode library '
                              'installed')
            raise HTTPServiceUnavailable()
 def return_fallback(self):
     if self.fallback_node is None:
         return json_response(None)
     return self.fallback_node
 def get_user_details(self, request, **kw):
     """Returns a JSON blob of account details for the current user."""
     return json_response({
         'userid': request.user['userid'],
         'syncNode': request.user['syncNode'],
     })
 def get_collection_usage(self, request):
     user_id = request.user['userid']
     storage = self._get_storage(request)
     return json_response(storage.get_collection_sizes(user_id))
class StorageController(object):

    def __init__(self, app):
        self.app = app
        self.logger = self.app.logger
        self.batch_size = app.config.get('storage.batch_size', 100)
        self.batch_max_count = app.config.get('storage.batch_max_count',
                                              100)
        self.batch_max_bytes = app.config.get('storage.batch_max_bytes',
                                              1024 * 1024)

    def _has_modifiers(self, data):
        return 'payload' in data

    def _get_storage(self, request):
        return self.app.get_storage(request)

    def _was_modified(self, request, user_id, collection_name):
        """Checks the X-If-Unmodified-Since header."""
        unmodified = request.headers.get('X-If-Unmodified-Since')
        if unmodified is None:
            return False
        unmodified = round_time(unmodified)
        storage = self._get_storage(request)
        max = storage.get_collection_max_timestamp(user_id,
                                                   collection_name)
        if max is None:
            return False

        return max > unmodified

    def get_storage(self, request):
        # XXX returns a 400 if the root is called
        raise HTTPBadRequest()

    def get_collections(self, request, **metrics):
        """Returns a hash of collections associated with the account,
        Along with the last modified timestamp for each collection
        """
        # metrics are additional parameters used by the various clients
        # to mark the logs for stats. We also want to send a CEF log when
        # this happens
        #
        # Disabled for now, since CEF doesn't need them at the moment
        #
        #if metrics != {}:
        #    username = request.user['username']
        #    values = ['%s=%s' % (key, value)
        #              for key, value in metrics.items()]
        #    log_cef('Daily metric call', 5, request.environ, self.app.config,
        #            username=username, msg=','.join(values))

        user_id = request.user['userid']
        storage = self._get_storage(request)
        collections = storage.get_collection_timestamps(user_id)
        response = convert_response(request, collections)
        response.headers['X-Weave-Records'] = str(len(collections))
        return response

    def get_collection_counts(self, request):
        """Returns a hash of collections associated with the account,
        Along with the total number of items for each collection.
        """
        user_id = request.user['userid']
        counts = self._get_storage(request).get_collection_counts(user_id)
        response = convert_response(request, counts)
        response.headers['X-Weave-Records'] = str(len(counts))
        return response

    def get_quota(self, request):
        user_id = request.user['userid']
        used = self._get_storage(request).get_total_size(user_id)
        if not self._get_storage(request).use_quota:
            limit = None
        else:
            limit = self._get_storage(request).quota_size
        return json_response((used, limit))

    def get_collection_usage(self, request):
        user_id = request.user['userid']
        storage = self._get_storage(request)
        return json_response(storage.get_collection_sizes(user_id))

    def _convert_args(self, kw):
        """Converts incoming arguments for GET and DELETE on collections.

        This function will also raise a 400 on bad args.
        Unknown args are just dropped.
        XXX see if we want to raise a 400 in that case
        """
        args = {}
        filters = {}
        convert_name = {'older': 'modified',
                        'newer': 'modified',
                        'index_above': 'sortindex',
                        'index_below': 'sortindex'}

        for arg in ('older', 'newer', 'index_above', 'index_below'):
            value = kw.get(arg)
            if value is None:
                continue
            try:
                if arg in ('older', 'newer'):
                    value = round_time(value)
                else:
                    value = float(value)
            except ValueError:
                raise HTTPBadRequest('Invalid value for "%s"' % arg)
            if arg in ('older', 'index_below'):
                filters[convert_name[arg]] = '<', value
            else:
                filters[convert_name[arg]] = '>', value

        # convert limit and offset
        limit = offset = None
        for arg in ('limit', 'offset'):
            value = kw.get(arg)
            if value is None:
                continue
            try:
                value = int(value)
            except ValueError:
                raise HTTPBadRequest('Invalid value for "%s"' % arg)
            if arg == 'limit':
                limit = value
            else:
                offset = value

        # we can't have offset without limit
        if limit is not None:
            args['limit'] = limit

        if offset is not None and limit is not None:
            args['offset'] = offset

        for arg in ('predecessorid', 'parentid'):
            value = kw.get(arg)
            if value is None:
                continue
            filters[arg] = '=', value

        # XXX should we control id lengths ?
        for arg in ('ids',):
            value = kw.get(arg)
            if value is None:
                continue
            filters['id'] = 'in', value.split(',')

        sort = kw.get('sort')
        if sort in ('oldest', 'newest', 'index'):
            args['sort'] = sort
        args['full'] = kw.get('full', False)
        args['filters'] = filters
        return args

    def get_collection(self, request, **kw):
        """Returns a list of the WBO ids contained in a collection."""
        kw = self._convert_args(kw)
        collection_name = request.sync_info['collection']
        user_id = request.user['userid']
        full = kw['full']

        if not full:
            fields = ['id']
        else:
            fields = _WBO_FIELDS

        storage = self._get_storage(request)
        res = storage.get_items(user_id, collection_name, fields,
                                kw['filters'],
                                kw.get('limit'), kw.get('offset'),
                                kw.get('sort'))
        if not full:
            res = [line['id'] for line in res]

        response = convert_response(request, res)
        response.headers['X-Weave-Records'] = str(len(res))
        return response

    def get_item(self, request, full=True):  # always full
        """Returns a single WBO object."""
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']
        user_id = request.user['userid']
        fields = _WBO_FIELDS
        storage = self._get_storage(request)
        res = storage.get_item(user_id, collection_name, item_id,
                               fields=fields)
        if res is None:
            raise HTTPNotFound()

        return json_response(res)

    def _check_quota(self, request):
        """Checks the quota.

        If under the treshold, adds a header
        If the quota is reached, issues a 400
        """
        user_id = request.user['userid']
        storage = self._get_storage(request)
        left = storage.get_size_left(user_id)
        if left < _ONE_MEG:
            left = storage.get_size_left(user_id, recalculate=True)
        if left <= 0.:  # no space left
            raise HTTPJsonBadRequest(WEAVE_OVER_QUOTA)
        return left

    def set_item(self, request):
        """Sets a single WBO object."""
        storage = self._get_storage(request)
        if storage.use_quota:
            left = self._check_quota(request)
        else:
            left = 0.

        user_id = request.user['userid']
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']

        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        try:
            data = json.loads(request.body)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)

        try:
            wbo = WBO(data)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

        consistent, msg = wbo.validate()
        if not consistent:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

        if self._has_modifiers(wbo):
            wbo['modified'] = request.server_time

        try:
            res = storage.set_item(user_id, collection_name, item_id,
                                   storage_time=request.server_time, **wbo)
        except StorageConflictError:
            raise HTTPJsonBadRequest(WEAVE_INVALID_WRITE)
        response = json_response(res)
        if storage.use_quota and left <= _ONE_MEG:
            response.headers['X-Weave-Quota-Remaining'] = str(left)
        return response

    def delete_item(self, request):
        """Deletes a single WBO object."""
        collection_name = request.sync_info['collection']
        item_id = request.sync_info['item']

        user_id = request.user['userid']
        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        self._get_storage(request).delete_item(user_id, collection_name,
                                              item_id,
                                              storage_time=request.server_time)

        # Not logging this event for now. Infrasec may want it again in future
        #
        #if collection_name == 'crypto' and item_id == 'keys':
        #    msg = 'Crypto keys deleted'
        #    username = request.user['username']
        #    log_cef(msg, 5, request.environ, self.app.config, username)

        return json_response(request.server_time)

    def set_collection(self, request):
        """Sets a batch of WBO objects into a collection."""

        user_id = request.user['userid']
        collection_name = request.sync_info['collection']

        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        try:
            wbos = json.loads(request.body)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)

        if not isinstance(wbos, (tuple, list)):
            # thats a batch of one
            try:
                id_ = str(wbos['id'])
            except (KeyError, TypeError):
                raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
            if '/' in id_:
                raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

            request.sync_info['item'] = id_
            return self.set_item(request)

        res = {'success': [], 'failed': {}}

        # Sanity-check each of the WBOs.
        # Limit the batch based on both count and payload size.
        kept_wbos = []
        total_bytes = 0
        for count, wbo in enumerate(wbos):
            try:
                wbo = WBO(wbo)
            except ValueError:
                res['failed'][''] = ['invalid wbo']
                continue

            if 'id' not in wbo:
                res['failed'][''] = ['invalid id']
                continue

            consistent, msg = wbo.validate()
            item_id = wbo['id']
            if not consistent:
                res['failed'][item_id] = [msg]
                continue

            if count >= self.batch_max_count:
                res['failed'][item_id] = ['retry wbo']
                continue
            if 'payload' in wbo:
                total_bytes += len(wbo['payload'])
            if total_bytes >= self.batch_max_bytes:
                res['failed'][item_id] = ['retry bytes']
                continue

            if self._has_modifiers(wbo):
                wbo['modified'] = request.server_time

            kept_wbos.append(wbo)

        storage = self._get_storage(request)
        if storage.use_quota:
            left = self._check_quota(request)
        else:
            left = 0.

        storage_time = request.server_time

        for wbos in batch(kept_wbos, size=self.batch_size):
            wbos = list(wbos)   # to avoid exhaustion
            try:
                storage.set_items(user_id, collection_name,
                                  wbos, storage_time=storage_time)

            except Exception, e:   # we want to swallow the 503 in that case
                # something went wrong
                self.logger.error('Could not set items')
                self.logger.error(str(e))
                for wbo in wbos:
                    res['failed'][wbo['id']] = "db error"
            else:
                res['success'].extend([wbo['id'] for wbo in wbos])

        res['modified'] = storage_time
        response = json_response(res)
        if storage.use_quota and left <= 1024:
            response.headers['X-Weave-Quota-Remaining'] = str(left)
        return response
Exemple #17
0
 def return_fallback(self):
     if self.fallback_node is None:
         return json_response(None)
     return self.fallback_node