def _add_virtual_columns(self, this_name, data): ''' DATA contains only values stored in table NAME. We need to fill relationship fields based on other tables. I.e. if we have parent-child relationship probably child table has 'parent_id', and parent has no 'children' column, so if NAME == 'parent' we need to add 'children' key in data, based on relationship fields. This operation should reverse _remove_virtual_columns. ''' # Determine IDs pkey_name = dm().object(this_name).pkey_field().name ids = [d[pkey_name] for d in data] for field in dm().object(this_name).fields(): name = field.name if field.rel and name not in data[0]: other_name = field.stores.name other_field_name = field.other.name all_related = self._select_objects(other_name, {other_field_name: ids}) related_pkey_name = dm().object(other_name).pkey_field().name for el in data: this_related = [x for x in all_related if x[other_field_name] == el[pkey_name]] related_ids = [x[related_pkey_name] for x in this_related] if field.multi: el[name] = related_ids else: el[name] = related_ids[0] if related_ids else None return data
def selected_ids(self, this_name, wr, sort, limit): ''' Return IDs from table NAME matching WR. SORT and LIMIT are ignored (storages are allwed to ignore those parameters, they are applied later in Enigne.get). HOW IT SHOULD BE DONE 1. WR is interpreted as WHERE 2. SORT becomes ORDER BY 3. LIMIT becomes LIMIT and everything is processed in a single query. That would be easy if we assumed that all REL fields have information stored in THIS_NAME table but unfortunately REL field could be stored on any table, so instead of WHEREs we might get some JOINS and this becomes more complicated. HOW IT IS CURRENLY DONE 1. WR is split into two parts: * one select for THIS_NAME table with all possible WHEREs * one select for each joined table with REL field stored on the other side 2. Intersection of IDs from all selects is returned 3. SORT and LIMIT are ignored. SORT is ignored because there is no way of implementing it different from both: * HOW IT SHOULD BE DONE above * sorting in Engine.get and LIMIT is ignored because SORTing first is necesary. ''' model = dm().object(this_name) # First, split to parts this_table_wr = {} other_selects = [] for key, val in wr.items(): if key in self._q().table_columns(this_name): this_table_wr[key] = val else: field = model.field(key) other_name = field.stores.name other_field_name = field.other.name other_pkey_name = dm().object(other_name).pkey_field().name other_selects.append((other_name, other_field_name, {other_pkey_name: val})) # List of sets of ids, to be intersected later sets_of_ids = [] # This table ids this_table_objects = self._select_objects(this_name, this_table_wr) this_pkey_name = model.pkey_field().name sets_of_ids.append(set([x[this_pkey_name] for x in this_table_objects])) # Other tables ids for other_name, other_fk_name, other_table_wr in other_selects: other_table_objects = self._select_objects(other_name, other_table_wr) sets_of_ids.append(set([x[other_fk_name] for x in other_table_objects])) # Final ids return sorted(set.intersection(*sets_of_ids))
def test_multi_default_1(init_world, get_client): ''' friends - always childred 1 & 2 ''' init_world(family.dm) client = get_client() # 1. Add best_friend field. child = dm().object('child') child.add_field(Rel('friends', stores=child, multi=True, default=[1, 2])) # If this is PG storage, we need to update database schema if issubclass(type(world().storage), PGStorage): conn = world().storage._conn conn.cursor().execute('ALTER TABLE child ADD COLUMN friends integer[]') conn.commit() # 2. Post a child without friends data, status, headers = client.post('child', {'name': 'c'}) assert status == 201 assert data['friends'] == [1, 2] # 3. Post a child with friends data, status, headers = client.post('child', { 'name': 'c', 'friends': [2, 3] }) assert status == 201 assert data['friends'] == [2, 3]
def test_ext_name_put(init_world, get_client): # 1. Init init_world(cookies.dm) client = get_client() # 2. Set an ext_name dm().object('cookie').field('type').ext_name = 'Cookie Type' # 3. Post data data, status, headers = client.put('cookie', 4, {'Cookie Type': 'tasty'}) # 4. Check assert status == 201 # 5. Get created data data, status, headers = client.get('cookie', 4) # 6. Check assert status == 200 assert data['Cookie Type'] == 'tasty' # 7. Put bad data data, status, headers = client.put('cookie', 5, {'type': 'tasty'}) # 8. Check assert status == 400
def test_api_expected(init_world, get_client): init_world(cookies.dm) client = get_client() # Check before change data, status, headers = client.post('cookie', {}) assert status == 201 assert 'type' not in data data, status, headers = client.post('cookie', {'type': 'muffin'}) assert status == 201 assert data['type'] == 'muffin' # Set default dm().object('cookie').field('type').default = 'donut' # Check after change data, status, headers = client.post('cookie', {}) assert status == 201 assert data['type'] == 'donut' data, status, headers = client.post('cookie', {'type': 'muffin'}) assert status == 201 assert data['type'] == 'muffin' data, status, headers = client.post('cookie', {'type': None}) assert status == 201 assert 'type' not in data
def default_create_function(): from importlib import import_module from blargh.engine import dm dm_name = dm().name module_name = 'example.{}.create'.format(dm_name) create = import_module(module_name) return create.__all__[0]
def test_single_default_1(init_world, get_client): ''' Best Friend - always child 1 ''' init_world(family.dm) client = get_client() # 1. Add best_friend field. child = dm().object('child') child.add_field(Rel('best_friend', stores=child, multi=False, default=1)) # If this is PG storage, we need to update database schema if issubclass(type(world().storage), PGStorage): conn = world().storage._conn conn.cursor().execute( 'ALTER TABLE child ADD COLUMN best_friend integer REFERENCES child(id) DEFERRABLE' ) conn.commit() # 2. Post a child without best friend data, status, headers = client.post('child', {'name': 'c'}) assert status == 201 assert data['best_friend'] == 1 # 3. Post a child with best friend data, status, headers = client.post('child', { 'name': 'c', 'best_friend': 2 }) assert status == 201 assert data['best_friend'] == 2
def add_cookie_cnt_field(): def getter(instance): field = instance.model.field('cookies') return len(instance.get_val(field).repr(0)) def setter(instance, new_cookie_cnt): # 1. Current state cookies_field = instance.model.field('cookies') current_cookies = instance.get_val(cookies_field).repr(0) current_cookie_cnt = len(current_cookies) if new_cookie_cnt == current_cookie_cnt: # 2A - no changes new_cookies = current_cookies if new_cookie_cnt < current_cookie_cnt: # 2B - less cookies new_cookies = current_cookies[:new_cookie_cnt] else: # 2C - more cookies fresh_cookies = [{} for i in range(current_cookie_cnt, new_cookie_cnt)] new_cookies = current_cookies + fresh_cookies # 3. Return value return {'cookies': new_cookies} jar_obj = dm().object('jar') jar_obj.add_field(Calc('cookie_cnt', getter=getter, setter=setter))
def world_data(): ''' Returns expected data for current world, based on * world's datamodel name * example.*.world_data() function ''' d = eval(engine.dm().name) return d.world_data(type(engine.world().storage).__name__)
def test_male_children(init_world, get_client, status, method, args): init_world(family.dm) client = get_client() # Modify datamodel dm().object('male').field('children')._writable = False assert getattr(client, method)(*args)[1] == status
def _hide_cookie_jar(): def getter(instance): jar_field = instance.model.field('jar') return bool(instance.get_val(jar_field).stored()) cookie = dm().object('cookie') cookie.field('jar').hidden = True cookie.add_field(Calc('in_jar', getter=getter))
def set_writable_1(): ''' TEST 1 - type changes only when not in the jar ''' def not_in_jar(instance): return instance.get_val(instance.model.field('jar')).repr() is None cookie_obj = dm().object('cookie') cookie_obj.field('type')._writable = not_in_jar
def set_writable_2(): ''' TEST 2 - no removing from the jar ''' def not_in_jar(instance): return instance.get_val(instance.model.field('jar')).repr() is None cookie_obj = dm().object('cookie') cookie_obj.field('jar')._writable = not_in_jar
def add_default_blargh_resources(self, base): for name, model in dm().objects().items(): # Create class inheriting from Resource, with model cls = type(name, (Resource, ), {'model': model}) # Resource URIs - collection and single element collection_url = path.join(base, name) object_url = path.join(collection_url, '<id_>') # Add resource self.add_resource(cls, object_url, collection_url)
def test_sort_and_limit(init_world, get_client, ids, kwargs): init_world(family.dm) client = get_client() # sorting uses external names so it would be nice to have at least one different from internal name dm().object('child').field('father').ext_name = 'dad' data, status_code, _ = client.get('child', depth=0, **kwargs) assert status_code == 200 assert data == ids
def _hide_type_add_is_shortbread(): def getter(instance): type_field = instance.model.field('type') return instance.get_val(type_field).stored() == 'shortbread' def setter(instance, is_shortbread): new_type = 'shortbread' if is_shortbread else 'not_shortbread' return {'type': new_type} cookie = dm().object('cookie') cookie.field('type').hidden = True cookie.add_field(Calc('is_shortbread', getter=getter, setter=setter))
def test_single_multi_2(init_world, get_client): ''' Best Friend - new female named BLARGH (note: new anonymous child creates infinite recursion) ''' init_world(family.dm) client = get_client() # 1. Add best_friend field. New child's best friend is always child no 1. child = dm().object('child') child.add_field( Rel('best_friend', stores=dm().object('female'), multi=False, default={'name': 'BLARGH'})) # If this is PG storage, we need to update database schema if issubclass(type(world().storage), PGStorage): conn = world().storage._conn conn.cursor().execute( 'ALTER TABLE child ADD COLUMN best_friend integer REFERENCES child(id) DEFERRABLE' ) conn.commit() # 2. Post a child without best friend data, status, headers = client.post('child', {'name': 'c'}) assert status == 201 assert data['best_friend'] == 3 # 3. Check if name was saved data, status, headers = client.get('female', 3) assert status == 200 assert data['name'] == 'BLARGH' # 4. Post a child with best friend data, status, headers = client.post('child', { 'name': 'c', 'best_friend': 2 }) assert status == 201 assert data['best_friend'] == 2
def test_ext_name_get(init_world, get_client): # 1. Init init_world(cookies.dm) client = get_client() # 2. Set an ext_name dm().object('cookie').field('type').ext_name = 'Cookie Type' # 3. Get data data, status, headers = client.get('cookie', 1) # 4. Check assert 'Cookie Type' in data assert 'type' not in data
def test_multi_default_2(init_world, get_client): ''' Friends - females named BLARGH and BLERGH ''' init_world(family.dm) client = get_client() # 1. Add friends field - two fresh females, BLARGH and BLERGH child = dm().object('child') child.add_field( Rel('friends', stores=dm().object('female'), multi=True, default=[{ 'name': 'BLARGH' }, { 'name': 'BLERGH' }])) # If this is PG storage, we need to update database schema if issubclass(type(world().storage), PGStorage): conn = world().storage._conn conn.cursor().execute('ALTER TABLE child ADD COLUMN friends integer[]') conn.commit() # 2. Post a child without best friend data, status, headers = client.post('child', {'name': 'c'}) assert status == 201 assert data['friends'] == [3, 4] # 3. Post a child with best friend data, status, headers = client.post('child', { 'name': 'c', 'friends': [1, 2] }) assert status == 201 assert data['friends'] == [1, 2]
def load_many(self, name, ids): if not ids: return [] # Determine column name pkey_name = dm().object(name).pkey_field().name stored_data = self._select_objects(name, {pkey_name: ids}) if len(stored_data) != len(ids): got_ids = [d[pkey_name] for d in stored_data] missing_ids = [id_ for id_ in ids if id_ not in got_ids] raise exceptions.e404(object_name=name, object_id=missing_ids[0]) full_data = self._add_virtual_columns(name, stored_data) return full_data
def test_cascade_father(init_world, get_client, cascades, method, args, deleted): # Init init_world(family.dm) client = get_client() for obj_name, field_name in cascades: dm().object(obj_name).field(field_name).cascade = True # Perform action (delete, probably) assert str(getattr(client, method)(*args)[1]).startswith('2') # Check ids_map = {'male': [1, 2], 'female': [1, 2], 'child': [1, 2, 3]} for name, ids in ids_map.items(): for id_ in ids: print(name, id_) status = client.get(name, id_)[1] expected_status = 404 if id_ in deleted.get(name, []) else 200 assert status == expected_status
def test_put_patch_diff(init_world, resource, id_, arg_data): init_world(family.dm) client = BaseApiClient() # 1. Expected PATCH result - GET with depth == 1 updated with arg_data patch_data = client.get(resource, id_, depth=1)[0] patch_data.update(arg_data) # 2. Expected PUT result: # * arg_data # * pkey column # * all calc fields # * empty list for all multi rel fields put_data = arg_data.copy() put_data[engine.dm().object(resource).pkey_field().ext_name] = id_ world = engine.world() world.begin() instance = world.get_instance(resource, id_) world.rollback() for field, val in instance.field_values(): if type(field) is fields.Calc: repr_val = val.repr() if repr_val is not None: put_data[field.ext_name] = repr_val elif field.rel and field.multi: put_data[field.ext_name] = [] # 3. Test if this test makes any sense (just to be sure, this would indicate serious problems) assert patch_data != put_data # 4. Test PATCH (twice - second PUT should change nothing) for i in (0, 1): for data, status, headers in (client.patch(resource, id_, arg_data), client.get(resource, id_)): assert status == 200 assert data == patch_data # 5. Test PUT (twice - second PUT should change nothing) for i in (0, 1): for data, status, headers in (client.put(resource, id_, arg_data), client.get(resource, id_)): assert str(status).startswith('2') assert data == put_data
def test_male_children(init_world, get_client, resource, data): init_world(cookies.dm) client = get_client() # Check before changes assert client.post(resource, data)[1] == 201 assert client.put(resource, 1, data)[1] == 201 assert client.put(resource, 7, data)[1] == 201 assert client.patch(resource, 1, data)[1] == 200 assert client.patch(resource, 7, data)[1] == 200 # Modify datamodel field_name = list(data.keys())[0] dm().object(resource).field(field_name).readonly = True # Check after changes assert client.post(resource, data)[1] == 400 assert client.put(resource, 1, data)[1] == 400 assert client.put(resource, 7, data)[1] == 400 assert client.patch(resource, 1, data)[1] == 400
def test_ext_name_filter(init_world, get_client): # 1. Init init_world(cookies.dm) client = get_client() # 2. Set an ext_name dm().object('cookie').field('type').ext_name = 'Cookie Type' # 3. Get data data, status, headers = client.get('cookie', filter_={'Cookie Type': 'biscuit'}) # 4. Check assert status == 200 assert type(data) is list assert len(data) is 1 # 5. Check "bad" filter data, status, headers = client.get('cookie', filter_={'type': 'biscuit'}) assert status == 400
def add_resource(self, resource, *urls, **kwargs): super().add_resource(resource, *urls, **kwargs) # TODO # This is extremaly ugly - just a POC def get_url(instance): import re from flask import request url = request.url_root[:-1] + urls[0] url = re.sub('<.+$', '', url) url = path.join(url, str(instance.id())) return url # Operation performed only for blargh.api.flask.Resources. # Simple flask_restful.Resource is also accepted here. if issubclass(resource, Resource): obj_name = resource.model.name url_field = dm().object(obj_name).field('url') if url_field is not None: url_field._getter = get_url
def test_ext_name_storage(init_world, get_client): ''' ext_name should have no influence on stored data ''' # 1. Init init_world(cookies.dm) client = get_client() # 2. Post data, status, headers = client.post('cookie', {'type': 'tasty'}) id_ = data['id'] stored_before = world().data() # 4. Set an ext_name dm().object('cookie').field('type').ext_name = 'Cookie Type' # 5. Put the same data, but using ext_name data, status, headers = client.put('cookie', id_, {'Cookie Type': 'tasty'}) # 6. Check assert stored_before == world().data()
def next_id(self, name): ''' If NAME primary key column has default value, it is returned. This works well with * nextval(sequence) * any simmilar user-defined function If there is no default, an exception is raised. This might change and one day we'll look for the biggest current ID and add 1. NOTE: Any value returned by any generator might be already taken, if client set it in an explicit way (probably via PUT). Generator is called repeatedly, until we find a non-duplicated value. This might take long, if there were many PUT's, but next time, it will probably be fast (if nextval(sequence) is used). Also: * if generator returns twice the same value, exception is raised * maybe this could be done better? Note - we want to handle also other than nextval() defaults, i.e. dependant on now(). ''' pkey_name = dm().object(name).pkey_field().name default_expr = self._q().default_pkey_expr(name, pkey_name) if default_expr is None: raise exceptions.ProgrammingError("Unknown default pkey value for {}".format(name)) old_val = None while True: cursor = self._conn.cursor() cursor.execute("SELECT {}".format(default_expr)) val = cursor.fetchone()[0] if self._select_objects(name, {pkey_name: val}): if old_val == val: raise exceptions.ProgrammingError('Pkey value generator returned twice the same value. \ Table: {}, val: {}'.format(name, val)) else: old_val = val else: return val
def _q(self): if self._query is None: self._query = self._query_cls(self._conn, self._schema, {o.name: o.pkey_field().name for o in dm().objects().values()}) return self._query
def data(self): d = {} for name, obj in dm().objects().items(): d[name] = self._q().dump_table(name, obj.pkey_field().name) return d
def new_cookies_dm(order_id): def add_field_closed(dm): jar = dm.object('jar') jar.add_field(Scalar('closed', default=False)) # Add database column if PGStorage if issubclass(type(engine.world().storage), engine.PGStorage): engine.world().storage._conn.cursor().execute(''' ALTER TABLE jar ADD COLUMN closed boolean; ''') def add_field_cookie_cnt(dm): def getter(instance): field = instance.model.field('cookies') return len(instance.get_val(field).repr(0)) def setter(instance, new_cookie_cnt): # 1. Current state cookies_field = instance.model.field('cookies') current_cookies = instance.get_val(cookies_field).repr(0) current_cookie_cnt = len(current_cookies) if new_cookie_cnt == current_cookie_cnt: # 2A - no changes new_cookies = current_cookies if new_cookie_cnt < current_cookie_cnt: # 2B - less cookies new_cookies = current_cookies[:new_cookie_cnt] else: # 2C - more cookies fresh_cookies = [ {} for i in range(current_cookie_cnt, new_cookie_cnt) ] new_cookies = current_cookies + fresh_cookies # 3. Return value return {'cookies': new_cookies} jar = dm.object('jar') jar.add_field(Calc('cookie_cnt', getter=getter, setter=setter)) def set_cookies_writable(dm): def open_jar(instance): return not instance.get_val(instance.model.field('closed')).repr() jar = dm.object('jar') jar.field('cookies')._writable = open_jar def set_fields_order(dm, field_names): jar = dm.object('jar') jar._fields = [jar.field(x) for x in field_names] dm = engine.dm() add_field_closed(dm) add_field_cookie_cnt(dm) set_cookies_writable(dm) orders = { 1: ('id', 'cookies', 'cookie_cnt', 'closed'), 2: ('id', 'closed', 'cookie_cnt', 'cookies'), } set_fields_order(dm, orders[order_id])