def test_migrations(self):
tdset = table_data_set.TableDataSet()
tdset.apply_doc_actions(schema_version0())
migration_actions = migrations.create_migrations(tdset.all_tables)
tdset.apply_doc_actions(migration_actions)
# Compare schema derived from migrations to the current schema.
migrated_schema = tdset.get_schema()
current_schema = {a.table_id: {c['id']: c for c in a.columns}
for a in schema.schema_create_actions()}
# pylint: disable=too-many-nested-blocks
if migrated_schema != current_schema:
# Figure out the version of new migration to suggest, and whether to update SCHEMA_VERSION.
new_version = max(schema.SCHEMA_VERSION, migrations.get_last_migration_version() + 1)
# Figure out the missing actions.
doc_actions = []
for table_id in sorted(six.viewkeys(current_schema) | six.viewkeys(migrated_schema)):
if table_id not in migrated_schema:
doc_actions.append(actions.AddTable(table_id, current_schema[table_id].values()))
elif table_id not in current_schema:
doc_actions.append(actions.RemoveTable(table_id))
else:
current_cols = current_schema[table_id]
migrated_cols = migrated_schema[table_id]
for col_id in sorted(six.viewkeys(current_cols) | six.viewkeys(migrated_cols)):
if col_id not in migrated_cols:
doc_actions.append(actions.AddColumn(table_id, col_id, current_cols[col_id]))
elif col_id not in current_cols:
doc_actions.append(actions.RemoveColumn(table_id, col_id))
else:
current_info = current_cols[col_id]
migrated_info = migrated_cols[col_id]
delta = {k: v for k, v in six.iteritems(current_info) if v != migrated_info.get(k)}
if delta:
doc_actions.append(actions.ModifyColumn(table_id, col_id, delta))
suggested_migration = (
"----------------------------------------------------------------------\n" +
"*** migrations.py ***\n" +
"----------------------------------------------------------------------\n" +
"@migration(schema_version=%s)\n" % new_version +
"def migration%s(tdset):\n" % new_version +
" return tdset.apply_doc_actions([\n" +
"".join(stringify(a) + ",\n" for a in doc_actions) +
" ])\n"
)
if new_version != schema.SCHEMA_VERSION:
suggested_schema_update = (
"----------------------------------------------------------------------\n" +
"*** schema.py ***\n" +
"----------------------------------------------------------------------\n" +
"SCHEMA_VERSION = %s\n" % new_version
)
else:
suggested_schema_update = ""
self.fail("Migrations are incomplete. Suggested migration to add:\n" +
suggested_schema_update + suggested_migration)
def alist():
return [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
]
def RemoveColumn(self, table_id, col_id):
table = self._engine.tables[table_id]
assert table.has_column(
col_id), "Column %s not in table %s" % (col_id, table_id)
# Generate (if needed) the undo action to restore the data.
undo_action = None
column = table.get_column(col_id)
if not column.is_private():
default = column.getdefault()
# Add to undo a BulkUpdateRecord for non-default values in the column being removed.
undo_values = [(r, column.raw_get(r)) for r in table.row_ids
if not strict_equal(column.raw_get(r), default)]
# Remove the specified column from the schema object.
colinfo = self._engine.schema[table_id].columns.pop(col_id)
self._engine.rebuild_usercode()
# Generate the undo action(s); if for a formula column, add them to the calc summary.
if undo_values:
if column.is_formula():
changes = [(r, v, default) for (r, v) in undo_values]
self._engine.out_actions.summary.add_changes(
table_id, col_id, changes)
else:
row_ids = [r for (r, v) in undo_values]
values = [v for (r, v) in undo_values]
undo_action = actions.BulkUpdateRecord(table_id, row_ids, {
col_id: values
}).simplify()
self._engine.out_actions.undo.append(undo_action)
self._engine.out_actions.undo.append(
actions.AddColumn(table_id, col_id,
schema.col_to_dict(colinfo, include_id=False)))
self._engine.out_actions.summary.remove_column(table_id, col_id)
def test_add_remove_lookup(self):
# Verify that when we add or remove a lookup formula, we get appropriate changes.
self.load_sample(testsamples.sample_students)
# Add another lookup formula.
out_actions = self.add_column(
"Schools",
"lastNames",
formula=(
"','.join(Students.lookupRecords(schoolName=$name).lastName)"))
self.assertPartialOutActions(
out_actions, {
"stored": [
actions.AddColumn(
"Schools", "lastNames", {
"formula":
"','.join(Students.lookupRecords(schoolName=$name).lastName)",
"isFormula": True,
"type": "Any"
}),
actions.AddRecord(
"_grist_Tables_column", 22, {
"colId": "lastNames",
"formula":
"','.join(Students.lookupRecords(schoolName=$name).lastName)",
"isFormula": True,
"label": "lastNames",
"parentId": 2,
"parentPos": 6.0,
"type": "Any",
"widgetOptions": ""
}),
_bulk_update(
"Schools", ["id", "lastNames"],
[[1, "Obama,Clinton"], [2, "Obama,Clinton"],
[3, "Bush,Bush,Ford"], [4, "Bush,Bush,Ford"]]),
],
"calls": {
"Schools": {
"lastNames": 4
},
"Students": {
"#lookup#schoolName": 6
}
},
})
# Make sure it responds to changes.
out_actions = self.update_record("Students", 5, schoolName="Columbia")
self.assertPartialOutActions(
out_actions, {
"stored": [
actions.UpdateRecord("Students", 5,
{"schoolName": "Columbia"}),
_bulk_update("Schools", ["id", "lastNames"],
[[1, "Obama,Clinton,Reagan"],
[2, "Obama,Clinton,Reagan"]]),
actions.UpdateRecord(
"Students", 5, {"schoolCities": "New York:Colombia"}),
actions.UpdateRecord("Students", 5, {"schoolIds": "1:2"}),
],
"calls": {
"Students": {
'schoolCities': 1,
'schoolIds': 1,
'#lookup#schoolName': 1
},
"Schools": {
'lastNames': 2
}
},
})
# Modify the column: in the process, the LookupMapColumn on Students.schoolName becomes unused
# while the old formula column is removed, but used again when it's added. It should not have
# to be rebuilt (so there should be no calls to recalculate the LookupMapColumn.
out_actions = self.modify_column(
"Schools",
"lastNames",
formula=(
"','.join(Students.lookupRecords(schoolName=$name).firstName)"
))
self.assertPartialOutActions(
out_actions, {
"stored": [
actions.ModifyColumn(
"Schools", "lastNames", {
"formula":
"','.join(Students.lookupRecords(schoolName=$name).firstName)"
}),
actions.UpdateRecord(
"_grist_Tables_column", 22, {
"formula":
"','.join(Students.lookupRecords(schoolName=$name).firstName)"
}),
_bulk_update(
"Schools", ["id", "lastNames"],
[[1, "Barack,Bill,Ronald"], [2, "Barack,Bill,Ronald"],
[3, "George W,George H,Gerald"],
[4, "George W,George H,Gerald"]])
],
"calls": {
"Schools": {
"lastNames": 4
}
}
})
# Remove the new lookup formula.
out_actions = self.remove_column("Schools", "lastNames")
self.assertPartialOutActions(out_actions, {}) # No calc actions
# Make sure that changes still work without errors.
out_actions = self.update_record("Students", 5, schoolName="Eureka")
self.assertPartialOutActions(
out_actions,
{
"stored": [
actions.UpdateRecord("Students", 5,
{"schoolName": "Eureka"}),
actions.UpdateRecord("Students", 5, {"schoolCities": ""}),
actions.UpdateRecord("Students", 5, {"schoolIds": ""}),
],
# This should NOT have '#lookup#schoolName' recalculation because there are no longer any
# formulas which do such a lookup.
"calls": {
"Students": {
'schoolCities': 1,
'schoolIds': 1
}
}
})
def add_column(table_id, col_id, col_type, *args, **kwargs):
return actions.AddColumn(
table_id, col_id, schema.make_column(col_id, col_type, *args,
**kwargs))
def test_prune_actions(self):
# prune_actions is in-place, so we make a new list every time.
def alist():
return [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
]
def prune(table_id, col_id):
a = alist()
actions.prune_actions(a, table_id, col_id)
return a
self.assertEqual(
prune('Table1', 'Foo'),
[
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
# It doesn't do anything with column renames; it can be addressed if needed.
actions.RenameColumn("Table1", "Foo", "Bar"),
# It doesn't do anything with AddTable, which is expected.
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
])
self.assertEqual(prune('Table2', 'Foo'), [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3],
{'Bar': ['a', 'b', 'c']}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {'Bar': 'a'}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {'Bar': 'a'}),
actions.ReplaceTableData("Table2", 18, {'Bar': 'a'}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
])