def batch_get(self,
data_table: DataTable[_RecordType],
consistent_read: bool = False) -> DataTable[_RecordType]:
"""
Get multuple records as a DataTable from DB.
`data_table` must have all columns to calculate table keys.
Example:
```python
# UserTable is a subclass of a DynamoTable
user_table = UserTable()
# we should provide table keys or fields to calculate them
# in our case, PK is calculated from `email` field.
users_table = DataTable[UserRecord]().add_record(
{
"email": "*****@*****.**",
},
{
"email": "*****@*****.**",
},
)
user_records = user_table.batch_get(users_table)
for user_record in user_records:
# print found records
# if record was not found - it will still be returned
# but only with the data you provided
print(user_record)
```
Arguments:
data_table -- Request data table.
consistent_read -- `ConsistentRead` boto3 parameter.
Returns:
DataTable with existing records.
"""
if not data_table:
return data_table.copy()
get_data_table = DataTable()
for record in data_table.get_records():
record = self._convert_record(record)
record = self.normalize_record(record)
record.update(self._get_record_keys(record))
get_data_table.add_record(record)
results: DataTable[Any] = (
self.dynamo_query_class.build_batch_get_item(
consistent_read=consistent_read, logger=self._logger).table(
table_keys=self.table_keys,
table=self.table).execute(data_table=get_data_table))
return DataTable(record_class=self.record_class).add_table(results)
def cached_batch_get(
self,
data_table: DataTable[_RecordType]) -> DataTable[_RecordType]:
"""
Get multuple records as a DataTable from DB with caching.
`data_table` must have all columns to calculate table keys.
Can be used instead of `batch_get` method.
Arguments:
data_table -- Request data table.
Returns:
DataTable with existing records.
"""
if not data_table:
return data_table.copy()
result = DataTable(record_class=self.record_class)
non_cached_data_table = DataTable(record_class=self.record_class)
cached_data_table = DataTable(record_class=self.record_class)
for record in data_table.get_records():
record = self._convert_record(record)
record = self.normalize_record(record)
record_keys = self._get_record_keys(record)
cached_record = self._get_cached_record(record_keys)
if cached_record is self.NO_RECORD:
non_cached_data_table.add_record(record)
continue
if cached_record and not isinstance(cached_record, SentinelValue):
cached_data_table.add_record(cached_record)
non_cached_results = self.batch_get(non_cached_data_table)
for record in non_cached_results:
record_keys = self._get_record_keys(record)
self._cache_record(record_keys, record)
result.add_table(cached_data_table)
result.add_table(non_cached_results)
return result
def test_add_record() -> None:
data_table = DataTable({"a": [1], "b": [3]})
result = data_table.add_record({"a": 5, "c": 4}, {"c": 5})
assert result is data_table
assert data_table == {
"a": [1, 5, data_table.NOT_SET],
"b": [3, data_table.NOT_SET, data_table.NOT_SET],
"c": [data_table.NOT_SET, 4, 5],
}
with pytest.raises(DataTableError):
DataTable({"a": [1], "b": []}).add_record({"a": 1})
def test_custom_record(self):
data_table = DataTable(record_class=UserRecord)
data_table.add_record({"name": "Jon"})
data_table.add_record(UserRecord(name="test", age=12))
assert isinstance(data_table.get_record(0), UserRecord)
assert list(data_table.get_records()) == [
UserRecord(name="Jon"),
UserRecord(name="test", age=12),
]
with pytest.raises(ValueError):
data_table.add_record({"unknown": "Jon"})
with pytest.raises(ValueError):
data_table.add_record({})
def main() -> None:
users_table = DataTable(record_class=UserRecord)
users_table.add_record(
{
"email": "*****@*****.**",
"name": "John",
"age": 34
},
{
"email": "*****@*****.**",
"company": "CiscoSystems",
"name": "Mary",
"age": 34
},
)
print("Get John's record:")
print(users_table.get_record(0))
print("All records as a list:")
print(list(users_table.get_records()))
print("Find record with name=Mary:")
print(users_table.filter_records({"name": "Mary"}).get_record(0))
def batch_upsert(
self,
data_table: DataTable[_RecordType],
set_if_not_exists_keys: Iterable[str] = (),
) -> DataTable[_RecordType]:
"""
Upsert multuple records as a DataTable to DB.
`data_table` must have all columns to calculate table keys.
Sets `dt_created` field equal to current UTC datetime if a record was created.
Sets `dt_modified` field equal to current UTC datetime.
Example:
```python
# UserTable is a subclass of a DynamoTable
user_table = UserTable()
# we should provide table keys or fields to calculate them
# in our case, PK is calculated from `email` field.
users_table = DataTable[UserRecord]().add_record(
{
"email": "*****@*****.**",
"name": "Doge Barky",
"age": 20,
},
{
"email": "*****@*****.**",
"name": "Elon Musk",
"age": 5289,
},
)
upserted_records = user_table.batch_upsert(users_table)
for upserted_record in upserted_records:
# print created and updated records
print(upserted_record)
```
Arguments:
data_table -- Request DataTable.
set_if_not_exists_keys -- List of keys to set only if they no do exist in DB.
Returns:
A DataTable with upserted results.
"""
if not data_table:
return DataTable[_RecordType](record_class=self.record_class)
set_if_not_exists = set(set_if_not_exists_keys)
existing_records = self.batch_get(data_table)
now = datetime.datetime.utcnow()
now_str = now.isoformat()
update_data_table = DataTable(record_class=self.record_class)
for record_index, record in enumerate(existing_records.get_records()):
updated_record = self._convert_record(
data_table.get_record(record_index))
preserve_keys_record: Dict[str, Any] = {}
for key in set_if_not_exists:
if key in record:
preserve_keys_record[key] = record[key]
new_record = self._convert_record({
**record,
**updated_record,
**preserve_keys_record,
"dt_created":
record.get("dt_created") or now_str,
"dt_modified":
now_str,
})
normalized_record = self.normalize_record(new_record)
self.validate_record_attributes(normalized_record)
update_data_table.add_record(dict(normalized_record))
results: DataTable[Any] = (
self.dynamo_query_class.build_batch_update_item(
logger=self._logger).table(
table_keys=self.table_keys,
table=self.table,
).execute(update_data_table))
results.record_class = self.record_class
return results