Esempio n. 1
0
    def flush(self):
        batch_data = {
            self.table.table_name: [
                # We'll insert data here shortly.
            ],
        }

        for put in self._to_put:
            item = Item(self.table, data=put)
            batch_data[self.table.table_name].append(
                {'PutRequest': {
                    'Item': item.prepare_full(),
                }})

        for delete in self._to_delete:
            batch_data[self.table.table_name].append(
                {'DeleteRequest': {
                    'Key': self.table._encode_keys(delete),
                }})

        resp = self.table.connection.batch_write_item(batch_data)
        self.handle_unprocessed(resp)

        self._to_put = []
        self._to_delete = []
        return True
Esempio n. 2
0
    def flush(self):
        batch_data = {
            self.table.table_name: [
                # We'll insert data here shortly.
            ],
        }

        for put in self._to_put:
            item = Item(self.table, data=put)
            batch_data[self.table.table_name].append({
                'PutRequest': {
                    'Item': item.prepare_full(),
                }
            })

        for delete in self._to_delete:
            batch_data[self.table.table_name].append({
                'DeleteRequest': {
                    'Key': self.table._encode_keys(delete),
                }
            })

        resp = self.table.connection.batch_write_item(batch_data)
        self.handle_unprocessed(resp)

        self._to_put = []
        self._to_delete = []
        return True
Esempio n. 3
0
    def _put_with_retries(self, boto_table, table_name, data, overwrite,
                          condition, vars, retry):
        boto_item = BotoItem(boto_table, data)

        # Use internal boto method to access to full AWS Dynamo capabilities
        final_data = boto_item.prepare_full()

        def try_function():
            expected = boto_item.build_expects(
            ) if overwrite is False else None
            return boto_table.connection.put_item(
                table_name,
                final_data,
                expected=expected,  # Don't overwrite
                condition_expression=condition,
                expression_attribute_values=vars)

        try:
            ret = self._attempt_throttled_operation(
                try_function,
                retry,
                boto_table,
                increased_throughput=get_double_writes(boto_table))
        except ConditionalCheckFailedException as e:
            raise self.ClariDynamoConditionCheckFailedException(
                str(e) + ' - ' + 'This could be due to a duplicate insertion.')
        return ret
Esempio n. 4
0
    def save(self, items, overwrite=None):
        """
        Save models to dynamo

        Parameters
        ----------
        items : list or :class:`~flywheel.models.Model`
        overwrite : bool, optional
            If False, raise exception if item already exists (default set by
            :attr:`.default_conflict`)

        Raises
        ------
        exc : :class:`boto.dynamodb2.exceptions.ConditionalCheckFailedException`
            If overwrite is False and an item already exists in the database

        Notes
        -----
        Overwrite will replace the *entire* item with the new one, not just
        different fields. After calling save(overwrite=True) you are guaranteed
        that the item in the database is exactly the item you saved.

        Due to the structure of the AWS API, saving with overwrite=True is much
        faster because the requests can be batched.

        """
        if overwrite is None:
            overwrite = self.default_conflict in ('update', 'overwrite')
        if isinstance(items, Model):
            items = [items]
        if not items:
            return
        tables = defaultdict(list)
        for item in items:
            tables[item.meta_.ddb_tablename].append(item)
        for tablename, items in tables.iteritems():
            table = Table(tablename, connection=self.dynamo)
            if overwrite:
                with table.batch_write() as batch:
                    for item in items:
                        item.pre_save_(self)
                        batch.put_item(data=item.ddb_dump_())
                        item.post_save_()
            else:
                for item in items:
                    expected = {}
                    for name in item.meta_.fields:
                        expected[name] = {
                            'Exists': False,
                        }
                    item.pre_save_(self)
                    boto_item = Item(table, data=item.ddb_dump_())
                    self.dynamo.put_item(tablename, boto_item.prepare_full(),
                                         expected=expected)
                    item.post_save_()
    def flush(self):
        batch_data = {
            self.table.table_name: [
                # We'll insert data here shortly.
            ],
        }

        for put in self._to_put:
            item = Item(self.table, data=put)
            batch_data[self.table.table_name].append({
                'PutRequest': {
                    'Item': item.prepare_full(),
                }
            })

        for delete in self._to_delete:
            batch_data[self.table.table_name].append({
                'DeleteRequest': {
                    'Key': self.table._encode_keys(delete),
                }
            })
            
        
        response = self.table.connection.batch_write_item(batch_data)
        self._to_put = []
        self._to_delete = []
        
        # handle unprocessed items
        unprocessed = response.get('UnprocessedItems', None)
        if unprocessed:
            unprocessed_list = unprocessed[self.table.table_name]
            item = Item(self)
            for u in unprocessed_list:
                if u.has_key("PutRequest"):
                    item.load({
                        'Item': u['PutRequest']['Item'],
                    })
                    self._to_put.append(item._data)
                elif u.has_key("DeleteRequest"):
                    item.load({
                        'Item': u['DeleteRequest']['Key'],
                    })
                    self._to_delete.append(item._data)
                else:
                    raise Exception("Error respond")
        return True
Esempio n. 6
0
    def flush(self):
        batch_data = {
            self.table.table_name: [
                # We'll insert data here shortly.
            ]
        }

        for put in self._to_put:
            item = Item(self.table, data=put)
            batch_data[self.table.table_name].append({"PutRequest": {"Item": item.prepare_full()}})

        for delete in self._to_delete:
            batch_data[self.table.table_name].append({"DeleteRequest": {"Key": self.table._encode_keys(delete)}})

        self.table.connection.batch_write_item(batch_data)
        self._to_put = []
        self._to_delete = []
        return True