def set(self, key: object, *, text: bool = False):
key = get_ref_id(key)
path = _get_cache_path(self.path, key)
path.parent.mkdir(parents=True, exist_ok=True)
mode = 'wt' if text else 'wb'
encoding = 'utf-8' if text else None
with path.open(mode, encoding=encoding) as f:
yield f
def pull_model(self, model, source, dependency):
command, args = next(iter(source.items()), None)
rows = self.run(model, {command: {**args, 'dependency': dependency}})
for row in rows:
data = {'type': f'{model.name}/:source/{self.obj.name}'}
for prop in model.properties.values():
if isinstance(prop.source, list):
data[prop.name] = [
self.get_value_from_source(prop, command, prop_source,
row, dependency)
for prop_source in prop.source
]
elif prop.source:
data[prop.name] = self.get_value_from_source(
prop, command, prop.source, row, dependency)
if prop.ref and prop.name in data:
data[prop.name] = get_ref_id(data[prop.name])
if self.check_key(data.get('id')):
yield data
def _pull(context: Context, model: Model, source, dependency):
dataset = model.parent
rows = source(context, model, dependency=dependency)
for row in rows:
data = {'type': f'{model.name}/:source/{dataset.name}'}
for prop in model.properties.values():
if isinstance(prop.source, CommandList):
data[prop.name] = [
_get_value_from_source(context, prop, prop_source, row,
dependency)
for prop_source in prop.source.commands
]
elif prop.source:
data[prop.name] = _get_value_from_source(
context, prop, prop.source, row, dependency)
if prop.ref and prop.name in data:
data[prop.name] = get_ref_id(data[prop.name])
if _check_key(data.get('id')):
yield data
def execute(self):
transaction = self.args.transaction
connection = transaction.connection
table = _get_table(self)
data = self.serialize(self.args.data)
key = get_ref_id(data.pop('id'))
values = {
'data': data,
'transaction_id': transaction.id,
}
row = self.backend.get(
connection,
[table.main.c.data, table.main.c.transaction_id],
table.main.c.id == key,
default=None,
)
action = None
# Insert.
if row is None:
action = INSERT_ACTION
result = connection.execute(table.main.insert().values({
'id':
key,
'created':
utcnow(),
**values,
}))
changes = data
# Update.
else:
changes = _get_patch_changes(row[table.main.c.data], data)
if changes:
action = UPDATE_ACTION
result = connection.execute(
table.main.update().where(table.main.c.id == key).where(
table.main.c.transaction_id == row[
table.main.c.transaction_id]).values({
**values,
'updated':
utcnow(),
}))
# TODO: Retries are needed if result.rowcount is 0, if such
# situation happens, that means a concurrent transaction
# changed the data and we need to reread it.
#
# And assumption is made here, than in the same
# transaction there are no concurrent updates, if this
# assumption is false, then we need to check against
# change_id instead of transaction_id.
else:
# Nothing to update.
return None
# Track changes.
connection.execute(
table.changes.insert().values(
transaction_id=transaction.id,
id=key,
datetime=utcnow(),
action=action,
change=changes,
), )
# Sanity check, is primary key was really what we tell it to be?
assert action != INSERT_ACTION or result.inserted_primary_key[
0] == key, f'{result.inserted_primary_key[0]} == {key}'
# Sanity check, do we really updated just one row?
assert action != UPDATE_ACTION or result.rowcount == 1, result.rowcount
return key
def get(self, key: object):
key = get_ref_id(key)
path = _get_cache_path(self.path, key)
if path.exists():
return path