def flush(self):
batch_data = {
self.table.table_name: [
# We'll insert data here shortly.
],
}
for put in self._to_put:
item = Item(self.table, data=put)
batch_data[self.table.table_name].append(
{'PutRequest': {
'Item': item.prepare_full(),
}})
for delete in self._to_delete:
batch_data[self.table.table_name].append(
{'DeleteRequest': {
'Key': self.table._encode_keys(delete),
}})
resp = self.table.connection.batch_write_item(batch_data)
self.handle_unprocessed(resp)
self._to_put = []
self._to_delete = []
return True
def flush(self):
batch_data = {
self.table.table_name: [
# We'll insert data here shortly.
],
}
for put in self._to_put:
item = Item(self.table, data=put)
batch_data[self.table.table_name].append({
'PutRequest': {
'Item': item.prepare_full(),
}
})
for delete in self._to_delete:
batch_data[self.table.table_name].append({
'DeleteRequest': {
'Key': self.table._encode_keys(delete),
}
})
resp = self.table.connection.batch_write_item(batch_data)
self.handle_unprocessed(resp)
self._to_put = []
self._to_delete = []
return True
def _put_with_retries(self, boto_table, table_name, data, overwrite,
condition, vars, retry):
boto_item = BotoItem(boto_table, data)
# Use internal boto method to access to full AWS Dynamo capabilities
final_data = boto_item.prepare_full()
def try_function():
expected = boto_item.build_expects(
) if overwrite is False else None
return boto_table.connection.put_item(
table_name,
final_data,
expected=expected, # Don't overwrite
condition_expression=condition,
expression_attribute_values=vars)
try:
ret = self._attempt_throttled_operation(
try_function,
retry,
boto_table,
increased_throughput=get_double_writes(boto_table))
except ConditionalCheckFailedException as e:
raise self.ClariDynamoConditionCheckFailedException(
str(e) + ' - ' + 'This could be due to a duplicate insertion.')
return ret
def save(self, items, overwrite=None):
"""
Save models to dynamo
Parameters
----------
items : list or :class:`~flywheel.models.Model`
overwrite : bool, optional
If False, raise exception if item already exists (default set by
:attr:`.default_conflict`)
Raises
------
exc : :class:`boto.dynamodb2.exceptions.ConditionalCheckFailedException`
If overwrite is False and an item already exists in the database
Notes
-----
Overwrite will replace the *entire* item with the new one, not just
different fields. After calling save(overwrite=True) you are guaranteed
that the item in the database is exactly the item you saved.
Due to the structure of the AWS API, saving with overwrite=True is much
faster because the requests can be batched.
"""
if overwrite is None:
overwrite = self.default_conflict in ('update', 'overwrite')
if isinstance(items, Model):
items = [items]
if not items:
return
tables = defaultdict(list)
for item in items:
tables[item.meta_.ddb_tablename].append(item)
for tablename, items in tables.iteritems():
table = Table(tablename, connection=self.dynamo)
if overwrite:
with table.batch_write() as batch:
for item in items:
item.pre_save_(self)
batch.put_item(data=item.ddb_dump_())
item.post_save_()
else:
for item in items:
expected = {}
for name in item.meta_.fields:
expected[name] = {
'Exists': False,
}
item.pre_save_(self)
boto_item = Item(table, data=item.ddb_dump_())
self.dynamo.put_item(tablename, boto_item.prepare_full(),
expected=expected)
item.post_save_()
def flush(self):
batch_data = {
self.table.table_name: [
# We'll insert data here shortly.
],
}
for put in self._to_put:
item = Item(self.table, data=put)
batch_data[self.table.table_name].append({
'PutRequest': {
'Item': item.prepare_full(),
}
})
for delete in self._to_delete:
batch_data[self.table.table_name].append({
'DeleteRequest': {
'Key': self.table._encode_keys(delete),
}
})
response = self.table.connection.batch_write_item(batch_data)
self._to_put = []
self._to_delete = []
# handle unprocessed items
unprocessed = response.get('UnprocessedItems', None)
if unprocessed:
unprocessed_list = unprocessed[self.table.table_name]
item = Item(self)
for u in unprocessed_list:
if u.has_key("PutRequest"):
item.load({
'Item': u['PutRequest']['Item'],
})
self._to_put.append(item._data)
elif u.has_key("DeleteRequest"):
item.load({
'Item': u['DeleteRequest']['Key'],
})
self._to_delete.append(item._data)
else:
raise Exception("Error respond")
return True
def flush(self):
batch_data = {
self.table.table_name: [
# We'll insert data here shortly.
]
}
for put in self._to_put:
item = Item(self.table, data=put)
batch_data[self.table.table_name].append({"PutRequest": {"Item": item.prepare_full()}})
for delete in self._to_delete:
batch_data[self.table.table_name].append({"DeleteRequest": {"Key": self.table._encode_keys(delete)}})
self.table.connection.batch_write_item(batch_data)
self._to_put = []
self._to_delete = []
return True