def set_top_resource_path(self, item_type, path):
'''Set the type of resource items we operate on, and its path.'''
self._path = path
self._listener_table = qvarn.table_name(
resource_type=item_type, auxtable=u'listener')
self._notification_table = qvarn.table_name(
resource_type=item_type, auxtable=u'notification')
def set_top_resource_path(self, item_type, path):
'''Set the type of resource items we operate on, and its path.'''
self._path = path
self._listener_table = qvarn.table_name(
resource_type=item_type, auxtable=u'listener')
self._notification_table = qvarn.table_name(
resource_type=item_type, auxtable=u'notification')
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
strings = item[field][pos][str_list_field]
self._insert_str_list(table_name, str_list_field, self._item_id, pos,
strings)
def visit_inner_dict_list(self, item, outer_field, inner_field,
column_names):
if not self._main_field_ok(outer_field): # pragma: no cover
return
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=outer_field,
subdict_list_field=inner_field)
column_names = [u'list_pos', u'dict_list_pos'] + column_names
match = ('=', table_name, u'id', self._item_id)
rows = self._transaction.select(table_name, column_names, match)
def get_pos(row):
return row[u'dict_list_pos'], row[u'list_pos']
in_order = list(sorted(rows, key=get_pos))
for outer_dict in item[outer_field]:
if inner_field not in outer_dict:
outer_dict[inner_field] = []
for row in self._make_dicts_from_rows(in_order, column_names):
i = row.pop(u'dict_list_pos')
j = row.pop(u'list_pos')
inner_list = item[outer_field][i][inner_field]
assert j == len(inner_list), '{} != {}'.format(j, len(inner_list))
inner_list.append(row)
def visit_inner_dict_list(self, item, outer_field, inner_field,
column_names):
if not self._main_field_ok(outer_field): # pragma: no cover
return
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=outer_field,
subdict_list_field=inner_field)
column_names = [u'list_pos', u'dict_list_pos'] + column_names
match = ('=', table_name, u'id', self._item_id)
rows = self._transaction.select(table_name, column_names, match)
def get_pos(row):
return row[u'dict_list_pos'], row[u'list_pos']
in_order = list(sorted(rows, key=get_pos))
for outer_dict in item[outer_field]:
if inner_field not in outer_dict:
outer_dict[inner_field] = []
for row in self._make_dicts_from_rows(in_order, column_names):
i = row.pop(u'dict_list_pos')
j = row.pop(u'list_pos')
inner_list = item[outer_field][i][inner_field]
assert j == len(inner_list), '{} != {}'.format(j, len(inner_list))
inner_list.append(row)
def _insert_subitem_into_database(self, transaction, item_id, subitem_name,
subitem):
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
table_name = qvarn.table_name(resource_type=self._item_type,
subpath=subitem_name)
ww = WriteWalker(transaction, table_name, item_id)
ww.walk_item(subitem, prototype)
def _update_revision(self, transaction, item_id, new_revision):
table_name = qvarn.table_name(resource_type=self._item_type)
match_columns = ('=', table_name, u'id', item_id)
values = {
u'revision': new_revision,
}
transaction.update(table_name, match_columns, values)
def _insert_subitem_into_database(self, transaction, item_id,
subitem_name, subitem):
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
table_name = qvarn.table_name(
resource_type=self._item_type, subpath=subitem_name)
ww = WriteWalker(transaction, table_name, item_id)
ww.walk_item(subitem, prototype)
def _kludge(self, transaction, schema, search_params): # pragma: no cover
sql = getattr(transaction, '_sql')
main_table = qvarn.table_name(resource_type=self._item_type)
with self._m.new('build param conditions'):
values = {}
tables_used = [main_table]
conds = [
self._kludge_conds(
sql, schema, param, values, main_table, tables_used)
for param in search_params]
with self._m.new('build full sql query'):
query = u'SELECT DISTINCT {1}.id FROM {0} AS {1}'.format(
sql.quote(main_table), u't0')
for idx, table_name in enumerate(tables_used):
if table_name != main_table:
table_alias = u't' + str(idx)
query += (u' LEFT JOIN {1} AS {2} ON {0}.id = {2}.id'
.format(u't0', sql.quote(table_name),
sql.quote(table_alias)))
query += u' WHERE ' + u' AND '.join(
u'({})'.format(c) for c in conds)
self._m.note(query=query, values=values)
return self._kludge_execute(sql, query, values)
def _get_current_revision(self, transaction, item_id):
table_name = qvarn.table_name(resource_type=self._item_type)
column_names = [u'revision']
match_columns = ('=', table_name, u'id', item_id)
rows = transaction.select(table_name, column_names, match_columns)
for row in rows:
return row[u'revision']
def _delete_subitem_in_transaction(self, transaction, item_id,
subitem_name):
table_name = qvarn.table_name(
resource_type=self._item_type, subpath=subitem_name)
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
dw = DeleteWalker(transaction, table_name, item_id)
dw.walk_item(prototype, prototype)
def _delete_subitem_in_transaction(self, transaction, item_id,
subitem_name):
table_name = qvarn.table_name(resource_type=self._item_type,
subpath=subitem_name)
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
dw = DeleteWalker(transaction, table_name, item_id)
dw.walk_item(prototype, prototype)
def _update_revision(self, transaction, item_id, new_revision):
table_name = qvarn.table_name(resource_type=self._item_type)
match_columns = ('=', table_name, u'id', item_id)
values = {
u'revision': new_revision,
}
transaction.update(table_name, match_columns, values)
def _get_current_revision(self, transaction, item_id):
table_name = qvarn.table_name(resource_type=self._item_type)
column_names = [u'revision']
match_columns = ('=', table_name, u'id', item_id)
rows = transaction.select(table_name, column_names, match_columns)
for row in rows:
return row[u'revision']
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
strings = item[field][pos][str_list_field]
self._insert_str_list(table_name, str_list_field, self._item_id,
pos, strings)
def visit_main_dict_list(self, item, field, column_names):
table_name = qvarn.table_name(list_field=field,
**self._table_name_kwargs)
self._add_column(table_name, u'id', unicode)
self._add_column(table_name, u'list_pos', int)
for column_name in column_names:
column_type = type(item[field][0][column_name])
self._add_column(table_name, column_name, column_type)
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
table_name = qvarn.table_name(list_field=field,
subdict_list_field=str_list_field,
**self._table_name_kwargs)
self._add_column(table_name, u'id', unicode)
self._add_column(table_name, u'dict_list_pos', int)
self._add_column(table_name, u'list_pos', int)
self._add_column(table_name, str_list_field, unicode)
def test_updates_data_for_each_version(self):
prototype_v1 = {
u'type': u'',
u'id': u'',
u'foo': u'',
}
prototype_v2 = {
u'type': u'',
u'id': u'',
u'bar': u'', # note that foo is dropped
}
prototype_v3 = {
u'type': u'',
u'id': u'',
u'bar': u'',
u'foobar': u'', # added field, thus bar should be copied
}
called = []
resource_type = u'resource'
table_name = qvarn.table_name(resource_type=resource_type)
def callback_v1(t, temp_tables):
# Insert a row with an id. It should remain at end.
t.insert(table_name, {u'id': u'foo.id'})
called.append(callback_v1)
def callback_v2(t, temp_tables):
called.append(callback_v2)
def callback_v3(t, temp_tables):
called.append(callback_v3)
vs = qvarn.VersionedStorage()
vs.set_resource_type(resource_type)
vs.start_version(u'v1', callback_v1)
vs.add_prototype(prototype_v1)
vs.start_version(u'v2', callback_v2)
vs.add_prototype(prototype_v2)
vs.start_version(u'v3', callback_v3)
vs.add_prototype(prototype_v3)
sql = qvarn.SqliteAdapter()
dbconn = qvarn.DatabaseConnection()
dbconn.set_sql(sql)
with dbconn.transaction() as t:
vs.prepare_storage(t)
self.assertEqual(called, [callback_v1, callback_v2, callback_v3])
rows = t.select(table_name, [u'bar', u'foobar', u'id'], None)
self.assertEqual(
rows,
[{u'id': u'foo.id', u'bar': None, u'foobar': None}])
def _kludge(self,
transaction,
schema,
search_params,
sort_params=None,
limit=None,
offset=None): # pragma: no cover
sql = getattr(transaction, '_sql')
main_table = qvarn.table_name(resource_type=self._item_type)
tables_used = [main_table]
with self._m.new('build param conditions'):
values = {}
conds = [
self._kludge_conds(sql, schema, param, values, main_table,
tables_used) for param in search_params
]
with self._m.new('build order by fields'):
join_conditions = {}
sort_params = sort_params or []
order_by_fields = [
self._kludge_order_by_fields(sql, schema, key, main_table,
tables_used, join_conditions)
for key in sort_params
]
with self._m.new('build full sql query'):
main_table_alias = u't0'
# With `SELECT DISTINCT` PostgreSQL requires all ORDER BY fields to
# be included in select list too.
select_list = [main_table_alias + u'.id'] + order_by_fields
query = (u'SELECT DISTINCT {select_list} '
u'FROM {main_table} AS {main_table_alias}').format(
select_list=u', '.join(select_list),
main_table=sql.quote(main_table),
main_table_alias=main_table_alias,
)
for idx, table_name in enumerate(tables_used):
if table_name != main_table:
table_alias = u't' + str(idx)
query += (
u' LEFT JOIN {1} AS {2} ON {0}.id = {2}.id'.format(
u't0', sql.quote(table_name),
sql.quote(table_alias)))
if idx in join_conditions:
query += ' AND ' + join_conditions[idx]
if conds:
query += u' WHERE ' + u' AND '.join(u'({})'.format(c)
for c in conds)
if order_by_fields:
query += u' ORDER BY ' + u', '.join(order_by_fields)
if limit is not None or offset is not None:
query += u' ' + sql.format_limit(limit, offset)
self._m.note(query=query, values=values)
return self._kludge_execute(sql, query, values)
def get_subitem(self, transaction, item_id, subitem_name):
'''Get a specific subitem.'''
subitem = {}
table_name = qvarn.table_name(
resource_type=self._item_type, subpath=subitem_name)
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
rw = ReadWalker(transaction, table_name, item_id)
rw.walk_item(subitem, prototype)
return subitem
def get_subitem(self, transaction, item_id, subitem_name):
'''Get a specific subitem.'''
subitem = {}
table_name = qvarn.table_name(resource_type=self._item_type,
subpath=subitem_name)
prototype = self._subitem_prototypes.get(self._item_type, subitem_name)
rw = ReadWalker(transaction, table_name, item_id)
rw.walk_item(subitem, prototype)
return subitem
def visit_inner_dict_list(self, item, field, inner_field, simple_columns):
table_name = qvarn.table_name(list_field=field,
subdict_list_field=inner_field,
**self._table_name_kwargs)
self._add_column(table_name, u'id', unicode)
self._add_column(table_name, u'dict_list_pos', int)
self._add_column(table_name, u'list_pos', int)
for column_name in simple_columns:
column_type = type(item[field][0][inner_field][0][column_name])
self._add_column(table_name, column_name, column_type)
def visit_dict_in_list(self, item, field, pos, column_names):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field)
columns = {
u'id': self._item_id,
u'list_pos': pos,
}
for column_name in column_names:
columns[column_name] = item[field][pos][column_name]
self._transaction.insert(table_name, columns)
def visit_dict_in_list(self, item, field, pos, column_names):
table_name = qvarn.table_name(
resource_type=self._item_type, list_field=field)
columns = {
u'id': self._item_id,
u'list_pos': pos,
}
for column_name in column_names:
columns[column_name] = item[field][pos][column_name]
self._transaction.insert(table_name, columns)
def test_gives_correct_schema_from_prototype_for_subresource(self):
prototype = {
u'foo': u'',
}
schema = qvarn.schema_from_prototype(prototype,
resource_type='big',
subpath=u'secret')
table_name = qvarn.table_name(resource_type=u'big', subpath=u'secret')
self.assertEqual(
sorted(schema),
sorted([
(table_name, u'id', unicode),
(table_name, u'foo', unicode),
]))
def visit_dict_in_inner_list(self, item, field, outer_pos, inner_field,
inner_pos, column_names):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field,
subdict_list_field=inner_field)
columns = {
u'id': self._item_id,
u'dict_list_pos': outer_pos,
u'list_pos': inner_pos,
}
inner_dict = item[field][outer_pos][inner_field][inner_pos]
for column_name in column_names:
columns[column_name] = inner_dict[column_name]
self._transaction.insert(table_name, columns)
def test_gives_correct_schema_from_prototype_for_subresource(self):
prototype = {
u'foo': u'',
}
schema = qvarn.schema_from_prototype(
prototype, resource_type='big', subpath=u'secret')
table_name = qvarn.table_name(
resource_type=u'big', subpath=u'secret')
self.assertEqual(
sorted(schema),
sorted([
(table_name, u'id', unicode),
(table_name, u'foo', unicode),
]))
def visit_dict_in_inner_list_str_list(self, item, outer_field,
outer_pos, inner_field,
inner_pos,
str_list_field):
table_name = qvarn.table_name(
list_field=outer_field,
subdict_list_field=inner_field,
inner_dict_list_field=str_list_field,
**self._table_name_kwargs)
self._add_column(table_name, u'id', six.text_type)
self._add_column(table_name, u'dict_list_pos', int)
self._add_column(table_name, u'list_pos', int)
self._add_column(table_name, u'str_list_pos', int)
self._add_column(table_name, str_list_field, six.text_type)
def wipe(self, resource_names):
for resource_name in resource_names:
resource_type, spec = self._specs[resource_name]
dw = DeleteWalker(self._db, resource_type)
dw.walk_item(spec, spec)
files = spec.get(u'files', [])
subpaths = spec.get('subpaths', [])
for subpath in subpaths:
if subpath not in files:
proto = subpaths[subpath][u'prototype']
table_name = qvarn.table_name(resource_type=resource_type,
subpath=subpath)
dw = DeleteWalker(self._db, table_name)
dw.walk_item(proto, proto)
def visit_dict_in_inner_list(self, item, field, outer_pos, inner_field,
inner_pos, column_names):
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=field,
subdict_list_field=inner_field)
columns = {
u'id': self._item_id,
u'dict_list_pos': outer_pos,
u'list_pos': inner_pos,
}
inner_dict = item[field][outer_pos][inner_field][inner_pos]
for column_name in column_names:
columns[column_name] = inner_dict[column_name]
self._transaction.insert(table_name, columns)
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
if not self._main_field_ok(field): # pragma: no cover
return
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
match = ('AND', ('=', table_name, u'id', self._item_id),
('=', table_name, u'dict_list_pos', unicode(pos)))
rows = self._transaction.select(table_name,
[u'list_pos', str_list_field], match)
in_order = self._sort_rows(rows)
result = [row[str_list_field] for row in in_order]
item[field][pos][str_list_field] = result
def test_updates_data_for_each_version(self):
prototype_v1 = {u"type": u"", u"id": u"", u"foo": u""}
prototype_v2 = {u"type": u"", u"id": u"", u"bar": u""} # note that foo is dropped
prototype_v3 = {u"type": u"", u"id": u"", u"bar": u"", u"foobar": u""} # added field, thus bar should be copied
called = []
resource_type = u"resource"
table_name = qvarn.table_name(resource_type=resource_type)
def callback_v1(t, temp_tables):
# Insert a row with an id. It should remain at end.
t.insert(table_name, {u"id": u"foo.id"})
called.append(callback_v1)
def callback_v2(t, temp_tables):
called.append(callback_v2)
def callback_v3(t, temp_tables):
called.append(callback_v3)
vs = qvarn.VersionedStorage()
vs.set_resource_type(resource_type)
vs.start_version(u"v1", callback_v1)
vs.add_prototype(prototype_v1)
vs.start_version(u"v2", callback_v2)
vs.add_prototype(prototype_v2)
vs.start_version(u"v3", callback_v3)
vs.add_prototype(prototype_v3)
sql = qvarn.SqliteAdapter()
dbconn = qvarn.DatabaseConnection()
dbconn.set_sql(sql)
with dbconn.transaction() as t:
vs.prepare_storage(t)
self.assertEqual(called, [callback_v1, callback_v2, callback_v3])
rows = t.select(table_name, [u"bar", u"foobar", u"id"], None)
self.assertEqual(rows, [{u"id": u"foo.id", u"bar": None, u"foobar": None}])
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
if not self._main_field_ok(field): # pragma: no cover
return
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
match = (
'AND',
('=', table_name, u'id', self._item_id),
('=', table_name, u'dict_list_pos', unicode(pos))
)
rows = self._transaction.select(
table_name, [u'list_pos', str_list_field], match)
in_order = self._sort_rows(rows)
result = [row[str_list_field] for row in in_order]
item[field][pos][str_list_field] = result
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
self._delete_rows(table_name, self._item_id)
def test_fails_if_both_auxtable_and_subdict_list_field(self):
with self.assertRaises(qvarn.ComplicatedTableNameError):
qvarn.table_name(
resource_type=u'foo', auxtable=u'aux',
list_field='yo', subdict_list_field=u'bar')
def visit_main_str_list(self, item, field):
table_name = qvarn.table_name(
resource_type=self._item_type, list_field=field)
self._delete_rows(table_name, self._item_id)
def test_returns_name_for_string_list_in_inner_dict_list(self):
name = qvarn.table_name(
resource_type=u'foo', list_field=u'bar', subdict_list_field=u'yo',
inner_dict_list_field='blah')
self.assertEqual(name, u'foo_bar_yo_blah')
def visit_inner_dict_list(self, item, field, inner_field, column_names):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field,
subdict_list_field=inner_field)
self._delete_rows(table_name, self._item_id)
def visit_main_str_list(self, item, field):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field)
self._delete_rows(table_name, self._item_id)
def visit_main_dict_list(self, item, field, column_names):
if self._main_field_ok(field):
table_name = qvarn.table_name(
resource_type=self._item_type, list_field=field)
item[field] = self._get_list(
table_name, self._item_id, column_names)
def visit_main_str_list(self, item, field):
table_name = qvarn.table_name(list_field=field,
**self._table_name_kwargs)
self._add_column(table_name, u'id', unicode)
self._add_column(table_name, u'list_pos', int)
self._add_column(table_name, field, unicode)
def visit_main_dict(self, item, column_names):
table_name = qvarn.table_name(**self._table_name_kwargs)
for name in column_names:
self._add_column(table_name, name, type(item[name]))
def test_fails_without_resource_type(self):
with self.assertRaises(qvarn.ComplicatedTableNameError):
qvarn.table_name()
def test_fails_if_subdict_list_field_without_list_field(self):
with self.assertRaises(qvarn.ComplicatedTableNameError):
qvarn.table_name(
resource_type=u'foo', subdict_list_field=u'bar')
def test_returns_name_for_auxiliary_table_list_field(self):
name = qvarn.table_name(
resource_type=u'foo', auxtable=u'listeners', list_field=u'bar')
self.assertEqual(name, u'foo__aux_listeners_bar')
def test_returns_name_for_subresource_list_in_subdict(self):
name = qvarn.table_name(
resource_type=u'foo', subpath=u'bar', list_field=u'yo',
subdict_list_field=u'ugh')
self.assertEqual(name, u'foo__path_bar_yo_ugh')
def test_returns_name_for_subresource(self):
name = qvarn.table_name(
resource_type=u'foo', subpath=u'bar')
self.assertEqual(name, u'foo__path_bar')
def visit_inner_dict_list(self, item, field, inner_field, column_names):
table_name = qvarn.table_name(
resource_type=self._item_type,
list_field=field,
subdict_list_field=inner_field)
self._delete_rows(table_name, self._item_id)
def test_returns_correct_name_for_just_resource_type(self):
name = qvarn.table_name(resource_type=u'foo')
self.assertEqual(name, u'foo')
def visit_dict_in_list_str_list(self, item, field, pos, str_list_field):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field,
subdict_list_field=str_list_field)
self._delete_rows(table_name, self._item_id)
def visit_main_str_list(self, item, field):
table_name = qvarn.table_name(
resource_type=self._item_type, list_field=field)
self._insert_str_list(table_name, field, self._item_id, None,
item[field])
def visit_main_str_list(self, item, field):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field)
self._insert_str_list(table_name, field, self._item_id, None,
item[field])
def visit_main_dict_list(self, item, field, column_names):
if self._main_field_ok(field):
table_name = qvarn.table_name(resource_type=self._item_type,
list_field=field)
item[field] = self._get_list(table_name, self._item_id,
column_names)
def _versions_table_name(self):
return qvarn.table_name(resource_type=self._resource_type,
auxtable=u'versions')
def test_returns_name_for_list_field(self):
name = qvarn.table_name(resource_type=u'foo', list_field=u'bar')
self.assertEqual(name, u'foo_bar')
def test_returns_name_for_subresource(self):
name = qvarn.table_name(resource_type=u'foo', subpath=u'bar')
self.assertEqual(name, u'foo__path_bar')
def test_fails_if_both_auxtable_and_subpath(self):
with self.assertRaises(qvarn.ComplicatedTableNameError):
qvarn.table_name(
resource_type=u'foo', auxtable=u'aux', subpath=u'path')
def test_returns_name_for_string_list_in_inner_dict_list(self):
name = qvarn.table_name(resource_type=u'foo',
list_field=u'bar',
subdict_list_field=u'yo',
inner_dict_list_field='blah')
self.assertEqual(name, u'foo_bar_yo_blah')
def test_returns_name_for_subresource_list_in_subdict(self):
name = qvarn.table_name(resource_type=u'foo',
subpath=u'bar',
list_field=u'yo',
subdict_list_field=u'ugh')
self.assertEqual(name, u'foo__path_bar_yo_ugh')