def test_batch(self):
     self.assertEquals(len(list(batch(range(250)))), 3)
     self.assertEquals(len(list(batch(range(190)))), 2)
     self.assertEquals(len(list(batch(range(24, 25)))), 1)
Ejemplo n.º 2
0
    def set_collection(self, request):
        """Sets a batch of WBO objects into a collection."""

        user_id = request.user['userid']
        collection_name = request.sync_info['collection']

        if self._was_modified(request, user_id, collection_name):
            raise HTTPPreconditionFailed(collection_name)

        try:
            wbos = json.loads(request.body)
        except ValueError:
            raise HTTPJsonBadRequest(WEAVE_MALFORMED_JSON)

        if not isinstance(wbos, (tuple, list)):
            # thats a batch of one
            try:
                id_ = str(wbos['id'])
            except (KeyError, TypeError):
                raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)
            if '/' in id_:
                raise HTTPJsonBadRequest(WEAVE_INVALID_WBO)

            request.sync_info['item'] = id_
            return self.set_item(request)

        res = {'success': [], 'failed': {}}

        # Sanity-check each of the WBOs.
        # Limit the batch based on both count and payload size.
        kept_wbos = []
        total_bytes = 0
        for count, wbo in enumerate(wbos):
            try:
                wbo = WBO(wbo)
            except ValueError:
                res['failed'][''] = ['invalid wbo']
                continue

            if 'id' not in wbo:
                res['failed'][''] = ['invalid id']
                continue

            consistent, msg = wbo.validate()
            item_id = wbo['id']
            if not consistent:
                res['failed'][item_id] = [msg]
                continue

            if count >= self.batch_max_count:
                res['failed'][item_id] = ['retry wbo']
                continue
            if 'payload' in wbo:
                total_bytes += len(wbo['payload'])
            if total_bytes >= self.batch_max_bytes:
                res['failed'][item_id] = ['retry bytes']
                continue

            if self._has_modifiers(wbo):
                wbo['modified'] = request.server_time

            kept_wbos.append(wbo)

        storage = self._get_storage(request)
        if storage.use_quota:
            left = self._check_quota(request)
        else:
            left = 0.

        storage_time = request.server_time

        for wbos in batch(kept_wbos, size=self.batch_size):
            wbos = list(wbos)   # to avoid exhaustion
            try:
                storage.set_items(user_id, collection_name,
                                  wbos, storage_time=storage_time)

            except Exception, e:   # we want to swallow the 503 in that case
                # something went wrong
                self.logger.error('Could not set items')
                self.logger.error(str(e))
                for wbo in wbos:
                    res['failed'][wbo['id']] = "db error"
            else:
                res['success'].extend([wbo['id'] for wbo in wbos])