def save(self, **kwargs):
"""
Save global object
To force split field, add field names to `Table._fat_fields`
"""
in_place = kwargs.pop('in_place', False)
obj = self if in_place else deepcopy(self)
# Zip fat fields
list(map(lambda ff: setattr(obj, ff, json_zip(\
getattr(self, ff, None))), type(self)._fat_fields))
obj.updated_ts = int(datetime.now().timestamp())
# Put all slices for one item
_ = [delattr(obj, f) for f in obj.__slots__ if None == getattr(obj, f, None)]
get_brk(type(self).ID).put(obj())
def rebuild(cls, **kwargs):
"""
Rebuild objects by attributes
Args:
source_unique: A `string` indicates `source_unique` value for query
id: A `string` indecates item identifier
filter_expr: ComparisonCondition object defined in [https://boto3.readthedocs.io/en/latest/_modules/boto3/dynamodb/conditions.html]
"""
brk = get_brk(cls.ID)
source_unique = kwargs.pop('source_unique', None)
if source_unique:
kwargs['cond'] = Key('source_unique').eq(source_unique)
kwargs['ind'] = GSI.SOURCE_UNIQUE.value
iid = kwargs.pop('id', None)
if iid:
kwargs['cond'] = Key(cls.ID.value).eq(iid)
#cls.filter_data_type(kwargs)
rsp = brk._query(**kwargs)
items = rsp.get('Items')
if not items:
return [], []
return cls.batch_get(items)
def selftest_general():
now = datetime.now()
item = {
'global_id': 'selftest',
'source_unique': 'zzzex',
'update_ts': int(now.timestamp()),
'created_ts': int(now.timestamp()),
'slice_type': 'beyond basic',
'data': json_zip({'name': 'Amazing Project'}),
}
t = get_brk()
rsp = t.put(item)
print("++ [put] {}".format(rsp))
rsp = t._query(Key('global_id').eq('selftest'))
print("++ [query] {}".format(rsp))
rsp = t._query(Key('source_unique').eq('zzzex'), 'source_unique_index')
print("++ [query] {}".format(rsp))
rsp = t._update({
'global_id': 'selftest',
'slice_type': 'beyond_basic',
}, {'data': 'super amazing project'.encode()})
print("++ [update] {}".format(rsp))
t.batch_delete([{'global_id': 'selftest', 'slice_type': 'beyond basic'}])
print("++ [delete] {}".format('done'))
items = [{
'global_id': str(uuid.uuid1()),
'source_unique': 'zzzex',
'update_ts': int(now.timestamp()),
'created_ts': int(now.timestamp()),
'slice_type': 'beyond basic',
'data': 'Amazing Project'.encode(),
} for _ in range(10)]
def foreach_item():
yield from items
t.batch_put(foreach_item())
print("++ [batch-put] {}".format('done'))
rsp = t._scan(Attr('slice_type').eq('beyond basic'))
print("++ [scan] {}".format(rsp))
assert (len(items) == rsp.get('Count'))
for item in rsp['Items']:
ret = t._get({
'global_id': item['global_id'],
'slice_type': item['slice_type']
})
print("++ [get] {}".format(ret))
ret = t._delete({
'global_id': item['global_id'],
'slice_type': item['slice_type']
})
print("++ [delete] {}".format(ret))
def iscan(cls, **kwargs):
brk = get_brk(cls.ID)
filter_expr = kwargs.pop('filter_expr', None)
if filter_expr:
filter_expr = Attr('data_type').eq(cls.DT.value) & filter_expr
else:
filter_expr = Attr('data_type').eq(cls.DT.value)
for chunk in brk._iscan(filter_expr=filter_expr, **kwargs):
yield cls.batch_build(chunk['Items'])
def selftest_iscan():
t = get_brk()
expr = Attr('data_type').eq('journal_article') & Attr(
'slice_type').begins_with('basic')
for i in t._iscan(expr, chunksize=100):
print(i)
if not i.get('Items'):
break
print('=' * 30)
print(len(i.get('Items')), i['Items'][-1].get('data_type'),
i['Items'][-1].get('slice_type'))
for item in i.get('Items'):
article = json_unzip(item['data'].value)
if not article.get('abstract'):
break
print(article['abstract'])
def batch_get(cls, items, **kwargs):
"""
Batch build object from given `dict` items
Args:
items: List of `dict` contains information to rebuild objects
@return A list of `Table` objects
A list of tuple indicates error items and correspoinding error
"""
keys = cls.extract_key(items)
rsp = get_brk(cls.ID)._batch_get(keys)
if not rsp:
return [], []
items = rsp.get('Responses', {}).get(TableBroker.ITEMID_TABLETYPE[cls.ID].value)
if not items:
return [], []
return cls.batch_build(items)
def selftest_iquery():
t = get_brk()
expr = Attr('slice_type').begins_with('basic')
for i in t._iquery(Key('data_type').eq('journal_article'),
filter_expr=expr,
chunksize=100):
if not i.get('Items'):
break
print('=' * 30)
print(len(i.get('Items')), i['Items'][-1].get('data_type'),
i['Items'][-1].get('slice_type'))
for item in i.get('Items'):
rsp = t._get({
'global_id': item['global_id'],
'slice_type': item['slice_type']
})
print(rsp)
def delete(cls, **kwargs):
"""
Delete by attributes
Args:
id:
batch: Wether perform batch delete, by default `True`
Reference to `Broker._delete` and `Broker.batch_delete`
Notes:
- Either `id` must be provided
"""
brk = get_brk(cls.ID)
batch = kwargs.pop('batch', True)
iid = kwargs.pop('id', None)
if iid:
brk._delete(iid)
return
filter_expr = kwargs.pop('filter_expr', None)
source_unique = kwargs.pop('source_unique', None)
if source_unique:
kwargs['cond'] = Key('source_unique').eq(source_unique)
kwargs['ind'] = GSI.SOURCE_UNIQUE.value
cls.filter_data_type(kwargs)
rsp = brk._query(**kwargs)
items = rsp.get('Items')
if not items:
return
items = cls.extract_key(items)
if batch:
return brk.batch_delete(items)
return list(map(brk._delete, items))
def iquery(cls, **kwargs):
"""
Query chunk by chunk
Args:
verbose: If `True`, perform further `batch_get` for details,
By default, False, return raw items in query results
cond: Condition for query
ind: Index name for query
Reference to `Broker._query`
"""
brk = get_brk(cls.ID)
verbose = kwargs.pop('verbose', False)
if kwargs.get('cond') is None:
kwargs['cond'] = Key('data_type').eq(cls.DT)
for chunk in brk._iquery(**kwargs):
if verbose:
yield cls.batch_get(chunk['Items'])
else:
yield chunk['Items']