def _add_virtual_columns(self, this_name, data):
'''
DATA contains only values stored in table NAME.
We need to fill relationship fields based on other tables.
I.e. if we have parent-child relationship probably child table has
'parent_id', and parent has no 'children' column,
so if NAME == 'parent' we need to add 'children' key in data,
based on relationship fields.
This operation should reverse _remove_virtual_columns.
'''
# Determine IDs
pkey_name = dm().object(this_name).pkey_field().name
ids = [d[pkey_name] for d in data]
for field in dm().object(this_name).fields():
name = field.name
if field.rel and name not in data[0]:
other_name = field.stores.name
other_field_name = field.other.name
all_related = self._select_objects(other_name, {other_field_name: ids})
related_pkey_name = dm().object(other_name).pkey_field().name
for el in data:
this_related = [x for x in all_related if x[other_field_name] == el[pkey_name]]
related_ids = [x[related_pkey_name] for x in this_related]
if field.multi:
el[name] = related_ids
else:
el[name] = related_ids[0] if related_ids else None
return data
def selected_ids(self, this_name, wr, sort, limit):
'''
Return IDs from table NAME matching WR.
SORT and LIMIT are ignored (storages are allwed to ignore those parameters, they are applied
later in Enigne.get).
HOW IT SHOULD BE DONE
1. WR is interpreted as WHERE
2. SORT becomes ORDER BY
3. LIMIT becomes LIMIT
and everything is processed in a single query.
That would be easy if we assumed that all REL fields have information stored in THIS_NAME table
but unfortunately REL field could be stored on any table, so instead
of WHEREs we might get some JOINS and this becomes more complicated.
HOW IT IS CURRENLY DONE
1. WR is split into two parts:
* one select for THIS_NAME table with all possible WHEREs
* one select for each joined table with REL field stored on the other side
2. Intersection of IDs from all selects is returned
3. SORT and LIMIT are ignored. SORT is ignored because there is no way of implementing it
different from both:
* HOW IT SHOULD BE DONE above
* sorting in Engine.get
and LIMIT is ignored because SORTing first is necesary.
'''
model = dm().object(this_name)
# First, split to parts
this_table_wr = {}
other_selects = []
for key, val in wr.items():
if key in self._q().table_columns(this_name):
this_table_wr[key] = val
else:
field = model.field(key)
other_name = field.stores.name
other_field_name = field.other.name
other_pkey_name = dm().object(other_name).pkey_field().name
other_selects.append((other_name, other_field_name, {other_pkey_name: val}))
# List of sets of ids, to be intersected later
sets_of_ids = []
# This table ids
this_table_objects = self._select_objects(this_name, this_table_wr)
this_pkey_name = model.pkey_field().name
sets_of_ids.append(set([x[this_pkey_name] for x in this_table_objects]))
# Other tables ids
for other_name, other_fk_name, other_table_wr in other_selects:
other_table_objects = self._select_objects(other_name, other_table_wr)
sets_of_ids.append(set([x[other_fk_name] for x in other_table_objects]))
# Final ids
return sorted(set.intersection(*sets_of_ids))
def test_multi_default_1(init_world, get_client):
'''
friends - always childred 1 & 2
'''
init_world(family.dm)
client = get_client()
# 1. Add best_friend field.
child = dm().object('child')
child.add_field(Rel('friends', stores=child, multi=True, default=[1, 2]))
# If this is PG storage, we need to update database schema
if issubclass(type(world().storage), PGStorage):
conn = world().storage._conn
conn.cursor().execute('ALTER TABLE child ADD COLUMN friends integer[]')
conn.commit()
# 2. Post a child without friends
data, status, headers = client.post('child', {'name': 'c'})
assert status == 201
assert data['friends'] == [1, 2]
# 3. Post a child with friends
data, status, headers = client.post('child', {
'name': 'c',
'friends': [2, 3]
})
assert status == 201
assert data['friends'] == [2, 3]
def test_ext_name_put(init_world, get_client):
# 1. Init
init_world(cookies.dm)
client = get_client()
# 2. Set an ext_name
dm().object('cookie').field('type').ext_name = 'Cookie Type'
# 3. Post data
data, status, headers = client.put('cookie', 4, {'Cookie Type': 'tasty'})
# 4. Check
assert status == 201
# 5. Get created data
data, status, headers = client.get('cookie', 4)
# 6. Check
assert status == 200
assert data['Cookie Type'] == 'tasty'
# 7. Put bad data
data, status, headers = client.put('cookie', 5, {'type': 'tasty'})
# 8. Check
assert status == 400
def test_api_expected(init_world, get_client):
init_world(cookies.dm)
client = get_client()
# Check before change
data, status, headers = client.post('cookie', {})
assert status == 201
assert 'type' not in data
data, status, headers = client.post('cookie', {'type': 'muffin'})
assert status == 201
assert data['type'] == 'muffin'
# Set default
dm().object('cookie').field('type').default = 'donut'
# Check after change
data, status, headers = client.post('cookie', {})
assert status == 201
assert data['type'] == 'donut'
data, status, headers = client.post('cookie', {'type': 'muffin'})
assert status == 201
assert data['type'] == 'muffin'
data, status, headers = client.post('cookie', {'type': None})
assert status == 201
assert 'type' not in data
def default_create_function():
from importlib import import_module
from blargh.engine import dm
dm_name = dm().name
module_name = 'example.{}.create'.format(dm_name)
create = import_module(module_name)
return create.__all__[0]
def test_single_default_1(init_world, get_client):
'''
Best Friend - always child 1
'''
init_world(family.dm)
client = get_client()
# 1. Add best_friend field.
child = dm().object('child')
child.add_field(Rel('best_friend', stores=child, multi=False, default=1))
# If this is PG storage, we need to update database schema
if issubclass(type(world().storage), PGStorage):
conn = world().storage._conn
conn.cursor().execute(
'ALTER TABLE child ADD COLUMN best_friend integer REFERENCES child(id) DEFERRABLE'
)
conn.commit()
# 2. Post a child without best friend
data, status, headers = client.post('child', {'name': 'c'})
assert status == 201
assert data['best_friend'] == 1
# 3. Post a child with best friend
data, status, headers = client.post('child', {
'name': 'c',
'best_friend': 2
})
assert status == 201
assert data['best_friend'] == 2
def add_cookie_cnt_field():
def getter(instance):
field = instance.model.field('cookies')
return len(instance.get_val(field).repr(0))
def setter(instance, new_cookie_cnt):
# 1. Current state
cookies_field = instance.model.field('cookies')
current_cookies = instance.get_val(cookies_field).repr(0)
current_cookie_cnt = len(current_cookies)
if new_cookie_cnt == current_cookie_cnt:
# 2A - no changes
new_cookies = current_cookies
if new_cookie_cnt < current_cookie_cnt:
# 2B - less cookies
new_cookies = current_cookies[:new_cookie_cnt]
else:
# 2C - more cookies
fresh_cookies = [{} for i in range(current_cookie_cnt, new_cookie_cnt)]
new_cookies = current_cookies + fresh_cookies
# 3. Return value
return {'cookies': new_cookies}
jar_obj = dm().object('jar')
jar_obj.add_field(Calc('cookie_cnt', getter=getter, setter=setter))
def world_data():
'''
Returns expected data for current world, based on
* world's datamodel name
* example.*.world_data() function
'''
d = eval(engine.dm().name)
return d.world_data(type(engine.world().storage).__name__)
def test_male_children(init_world, get_client, status, method, args):
init_world(family.dm)
client = get_client()
# Modify datamodel
dm().object('male').field('children')._writable = False
assert getattr(client, method)(*args)[1] == status
def _hide_cookie_jar():
def getter(instance):
jar_field = instance.model.field('jar')
return bool(instance.get_val(jar_field).stored())
cookie = dm().object('cookie')
cookie.field('jar').hidden = True
cookie.add_field(Calc('in_jar', getter=getter))
def set_writable_1():
'''
TEST 1 - type changes only when not in the jar
'''
def not_in_jar(instance):
return instance.get_val(instance.model.field('jar')).repr() is None
cookie_obj = dm().object('cookie')
cookie_obj.field('type')._writable = not_in_jar
def set_writable_2():
'''
TEST 2 - no removing from the jar
'''
def not_in_jar(instance):
return instance.get_val(instance.model.field('jar')).repr() is None
cookie_obj = dm().object('cookie')
cookie_obj.field('jar')._writable = not_in_jar
def add_default_blargh_resources(self, base):
for name, model in dm().objects().items():
# Create class inheriting from Resource, with model
cls = type(name, (Resource, ), {'model': model})
# Resource URIs - collection and single element
collection_url = path.join(base, name)
object_url = path.join(collection_url, '<id_>')
# Add resource
self.add_resource(cls, object_url, collection_url)
def test_sort_and_limit(init_world, get_client, ids, kwargs):
init_world(family.dm)
client = get_client()
# sorting uses external names so it would be nice to have at least one different from internal name
dm().object('child').field('father').ext_name = 'dad'
data, status_code, _ = client.get('child', depth=0, **kwargs)
assert status_code == 200
assert data == ids
def _hide_type_add_is_shortbread():
def getter(instance):
type_field = instance.model.field('type')
return instance.get_val(type_field).stored() == 'shortbread'
def setter(instance, is_shortbread):
new_type = 'shortbread' if is_shortbread else 'not_shortbread'
return {'type': new_type}
cookie = dm().object('cookie')
cookie.field('type').hidden = True
cookie.add_field(Calc('is_shortbread', getter=getter, setter=setter))
def test_single_multi_2(init_world, get_client):
'''
Best Friend - new female named BLARGH (note: new anonymous child creates infinite recursion)
'''
init_world(family.dm)
client = get_client()
# 1. Add best_friend field. New child's best friend is always child no 1.
child = dm().object('child')
child.add_field(
Rel('best_friend',
stores=dm().object('female'),
multi=False,
default={'name': 'BLARGH'}))
# If this is PG storage, we need to update database schema
if issubclass(type(world().storage), PGStorage):
conn = world().storage._conn
conn.cursor().execute(
'ALTER TABLE child ADD COLUMN best_friend integer REFERENCES child(id) DEFERRABLE'
)
conn.commit()
# 2. Post a child without best friend
data, status, headers = client.post('child', {'name': 'c'})
assert status == 201
assert data['best_friend'] == 3
# 3. Check if name was saved
data, status, headers = client.get('female', 3)
assert status == 200
assert data['name'] == 'BLARGH'
# 4. Post a child with best friend
data, status, headers = client.post('child', {
'name': 'c',
'best_friend': 2
})
assert status == 201
assert data['best_friend'] == 2
def test_ext_name_get(init_world, get_client):
# 1. Init
init_world(cookies.dm)
client = get_client()
# 2. Set an ext_name
dm().object('cookie').field('type').ext_name = 'Cookie Type'
# 3. Get data
data, status, headers = client.get('cookie', 1)
# 4. Check
assert 'Cookie Type' in data
assert 'type' not in data
def test_multi_default_2(init_world, get_client):
'''
Friends - females named BLARGH and BLERGH
'''
init_world(family.dm)
client = get_client()
# 1. Add friends field - two fresh females, BLARGH and BLERGH
child = dm().object('child')
child.add_field(
Rel('friends',
stores=dm().object('female'),
multi=True,
default=[{
'name': 'BLARGH'
}, {
'name': 'BLERGH'
}]))
# If this is PG storage, we need to update database schema
if issubclass(type(world().storage), PGStorage):
conn = world().storage._conn
conn.cursor().execute('ALTER TABLE child ADD COLUMN friends integer[]')
conn.commit()
# 2. Post a child without best friend
data, status, headers = client.post('child', {'name': 'c'})
assert status == 201
assert data['friends'] == [3, 4]
# 3. Post a child with best friend
data, status, headers = client.post('child', {
'name': 'c',
'friends': [1, 2]
})
assert status == 201
assert data['friends'] == [1, 2]
def load_many(self, name, ids):
if not ids:
return []
# Determine column name
pkey_name = dm().object(name).pkey_field().name
stored_data = self._select_objects(name, {pkey_name: ids})
if len(stored_data) != len(ids):
got_ids = [d[pkey_name] for d in stored_data]
missing_ids = [id_ for id_ in ids if id_ not in got_ids]
raise exceptions.e404(object_name=name, object_id=missing_ids[0])
full_data = self._add_virtual_columns(name, stored_data)
return full_data
def test_cascade_father(init_world, get_client, cascades, method, args, deleted):
# Init
init_world(family.dm)
client = get_client()
for obj_name, field_name in cascades:
dm().object(obj_name).field(field_name).cascade = True
# Perform action (delete, probably)
assert str(getattr(client, method)(*args)[1]).startswith('2')
# Check
ids_map = {'male': [1, 2], 'female': [1, 2], 'child': [1, 2, 3]}
for name, ids in ids_map.items():
for id_ in ids:
print(name, id_)
status = client.get(name, id_)[1]
expected_status = 404 if id_ in deleted.get(name, []) else 200
assert status == expected_status
def test_put_patch_diff(init_world, resource, id_, arg_data):
init_world(family.dm)
client = BaseApiClient()
# 1. Expected PATCH result - GET with depth == 1 updated with arg_data
patch_data = client.get(resource, id_, depth=1)[0]
patch_data.update(arg_data)
# 2. Expected PUT result:
# * arg_data
# * pkey column
# * all calc fields
# * empty list for all multi rel fields
put_data = arg_data.copy()
put_data[engine.dm().object(resource).pkey_field().ext_name] = id_
world = engine.world()
world.begin()
instance = world.get_instance(resource, id_)
world.rollback()
for field, val in instance.field_values():
if type(field) is fields.Calc:
repr_val = val.repr()
if repr_val is not None:
put_data[field.ext_name] = repr_val
elif field.rel and field.multi:
put_data[field.ext_name] = []
# 3. Test if this test makes any sense (just to be sure, this would indicate serious problems)
assert patch_data != put_data
# 4. Test PATCH (twice - second PUT should change nothing)
for i in (0, 1):
for data, status, headers in (client.patch(resource, id_, arg_data),
client.get(resource, id_)):
assert status == 200
assert data == patch_data
# 5. Test PUT (twice - second PUT should change nothing)
for i in (0, 1):
for data, status, headers in (client.put(resource, id_, arg_data),
client.get(resource, id_)):
assert str(status).startswith('2')
assert data == put_data
def test_male_children(init_world, get_client, resource, data):
init_world(cookies.dm)
client = get_client()
# Check before changes
assert client.post(resource, data)[1] == 201
assert client.put(resource, 1, data)[1] == 201
assert client.put(resource, 7, data)[1] == 201
assert client.patch(resource, 1, data)[1] == 200
assert client.patch(resource, 7, data)[1] == 200
# Modify datamodel
field_name = list(data.keys())[0]
dm().object(resource).field(field_name).readonly = True
# Check after changes
assert client.post(resource, data)[1] == 400
assert client.put(resource, 1, data)[1] == 400
assert client.put(resource, 7, data)[1] == 400
assert client.patch(resource, 1, data)[1] == 400
def test_ext_name_filter(init_world, get_client):
# 1. Init
init_world(cookies.dm)
client = get_client()
# 2. Set an ext_name
dm().object('cookie').field('type').ext_name = 'Cookie Type'
# 3. Get data
data, status, headers = client.get('cookie',
filter_={'Cookie Type': 'biscuit'})
# 4. Check
assert status == 200
assert type(data) is list
assert len(data) is 1
# 5. Check "bad" filter
data, status, headers = client.get('cookie', filter_={'type': 'biscuit'})
assert status == 400
def add_resource(self, resource, *urls, **kwargs):
super().add_resource(resource, *urls, **kwargs)
# TODO
# This is extremaly ugly - just a POC
def get_url(instance):
import re
from flask import request
url = request.url_root[:-1] + urls[0]
url = re.sub('<.+$', '', url)
url = path.join(url, str(instance.id()))
return url
# Operation performed only for blargh.api.flask.Resources.
# Simple flask_restful.Resource is also accepted here.
if issubclass(resource, Resource):
obj_name = resource.model.name
url_field = dm().object(obj_name).field('url')
if url_field is not None:
url_field._getter = get_url
def test_ext_name_storage(init_world, get_client):
'''
ext_name should have no influence on stored data
'''
# 1. Init
init_world(cookies.dm)
client = get_client()
# 2. Post
data, status, headers = client.post('cookie', {'type': 'tasty'})
id_ = data['id']
stored_before = world().data()
# 4. Set an ext_name
dm().object('cookie').field('type').ext_name = 'Cookie Type'
# 5. Put the same data, but using ext_name
data, status, headers = client.put('cookie', id_, {'Cookie Type': 'tasty'})
# 6. Check
assert stored_before == world().data()
def next_id(self, name):
'''
If NAME primary key column has default value, it is returned.
This works well with
* nextval(sequence)
* any simmilar user-defined function
If there is no default, an exception is raised. This might change and one
day we'll look for the biggest current ID and add 1.
NOTE: Any value returned by any generator might be already taken, if client set it
in an explicit way (probably via PUT). Generator is called repeatedly, until we
find a non-duplicated value. This might take long, if there were many PUT's,
but next time, it will probably be fast (if nextval(sequence) is used).
Also:
* if generator returns twice the same value, exception is raised
* maybe this could be done better? Note - we want to handle also other than nextval() defaults,
i.e. dependant on now().
'''
pkey_name = dm().object(name).pkey_field().name
default_expr = self._q().default_pkey_expr(name, pkey_name)
if default_expr is None:
raise exceptions.ProgrammingError("Unknown default pkey value for {}".format(name))
old_val = None
while True:
cursor = self._conn.cursor()
cursor.execute("SELECT {}".format(default_expr))
val = cursor.fetchone()[0]
if self._select_objects(name, {pkey_name: val}):
if old_val == val:
raise exceptions.ProgrammingError('Pkey value generator returned twice the same value. \
Table: {}, val: {}'.format(name, val))
else:
old_val = val
else:
return val
def _q(self):
if self._query is None:
self._query = self._query_cls(self._conn, self._schema,
{o.name: o.pkey_field().name for o in dm().objects().values()})
return self._query
def data(self):
d = {}
for name, obj in dm().objects().items():
d[name] = self._q().dump_table(name, obj.pkey_field().name)
return d
def new_cookies_dm(order_id):
def add_field_closed(dm):
jar = dm.object('jar')
jar.add_field(Scalar('closed', default=False))
# Add database column if PGStorage
if issubclass(type(engine.world().storage), engine.PGStorage):
engine.world().storage._conn.cursor().execute('''
ALTER TABLE jar ADD COLUMN closed boolean;
''')
def add_field_cookie_cnt(dm):
def getter(instance):
field = instance.model.field('cookies')
return len(instance.get_val(field).repr(0))
def setter(instance, new_cookie_cnt):
# 1. Current state
cookies_field = instance.model.field('cookies')
current_cookies = instance.get_val(cookies_field).repr(0)
current_cookie_cnt = len(current_cookies)
if new_cookie_cnt == current_cookie_cnt:
# 2A - no changes
new_cookies = current_cookies
if new_cookie_cnt < current_cookie_cnt:
# 2B - less cookies
new_cookies = current_cookies[:new_cookie_cnt]
else:
# 2C - more cookies
fresh_cookies = [
{} for i in range(current_cookie_cnt, new_cookie_cnt)
]
new_cookies = current_cookies + fresh_cookies
# 3. Return value
return {'cookies': new_cookies}
jar = dm.object('jar')
jar.add_field(Calc('cookie_cnt', getter=getter, setter=setter))
def set_cookies_writable(dm):
def open_jar(instance):
return not instance.get_val(instance.model.field('closed')).repr()
jar = dm.object('jar')
jar.field('cookies')._writable = open_jar
def set_fields_order(dm, field_names):
jar = dm.object('jar')
jar._fields = [jar.field(x) for x in field_names]
dm = engine.dm()
add_field_closed(dm)
add_field_cookie_cnt(dm)
set_cookies_writable(dm)
orders = {
1: ('id', 'cookies', 'cookie_cnt', 'closed'),
2: ('id', 'closed', 'cookie_cnt', 'cookies'),
}
set_fields_order(dm, orders[order_id])
