def test_migrations(self):
tdset = table_data_set.TableDataSet()
tdset.apply_doc_actions(schema_version0())
migration_actions = migrations.create_migrations(tdset.all_tables)
tdset.apply_doc_actions(migration_actions)
# Compare schema derived from migrations to the current schema.
migrated_schema = tdset.get_schema()
current_schema = {a.table_id: {c['id']: c for c in a.columns}
for a in schema.schema_create_actions()}
# pylint: disable=too-many-nested-blocks
if migrated_schema != current_schema:
# Figure out the version of new migration to suggest, and whether to update SCHEMA_VERSION.
new_version = max(schema.SCHEMA_VERSION, migrations.get_last_migration_version() + 1)
# Figure out the missing actions.
doc_actions = []
for table_id in sorted(six.viewkeys(current_schema) | six.viewkeys(migrated_schema)):
if table_id not in migrated_schema:
doc_actions.append(actions.AddTable(table_id, current_schema[table_id].values()))
elif table_id not in current_schema:
doc_actions.append(actions.RemoveTable(table_id))
else:
current_cols = current_schema[table_id]
migrated_cols = migrated_schema[table_id]
for col_id in sorted(six.viewkeys(current_cols) | six.viewkeys(migrated_cols)):
if col_id not in migrated_cols:
doc_actions.append(actions.AddColumn(table_id, col_id, current_cols[col_id]))
elif col_id not in current_cols:
doc_actions.append(actions.RemoveColumn(table_id, col_id))
else:
current_info = current_cols[col_id]
migrated_info = migrated_cols[col_id]
delta = {k: v for k, v in six.iteritems(current_info) if v != migrated_info.get(k)}
if delta:
doc_actions.append(actions.ModifyColumn(table_id, col_id, delta))
suggested_migration = (
"----------------------------------------------------------------------\n" +
"*** migrations.py ***\n" +
"----------------------------------------------------------------------\n" +
"@migration(schema_version=%s)\n" % new_version +
"def migration%s(tdset):\n" % new_version +
" return tdset.apply_doc_actions([\n" +
"".join(stringify(a) + ",\n" for a in doc_actions) +
" ])\n"
)
if new_version != schema.SCHEMA_VERSION:
suggested_schema_update = (
"----------------------------------------------------------------------\n" +
"*** schema.py ***\n" +
"----------------------------------------------------------------------\n" +
"SCHEMA_VERSION = %s\n" % new_version
)
else:
suggested_schema_update = ""
self.fail("Migrations are incomplete. Suggested migration to add:\n" +
suggested_schema_update + suggested_migration)
def AddColumn(self, table_id, col_id, col_info):
table = self._engine.tables[table_id]
assert not table.has_column(
col_id), "Column %s already exists in %s" % (col_id, table_id)
# Add the new column to the schema object maintained in the engine.
self._engine.schema[table_id].columns[col_id] = schema.dict_to_col(
col_info, col_id=col_id)
self._engine.rebuild_usercode()
self._engine.new_column_name(table)
# Generate the undo action.
self._engine.out_actions.undo.append(
actions.RemoveColumn(table_id, col_id))
self._engine.out_actions.summary.add_column(table_id, col_id)
def alist():
return [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
]
def migration7(tdset):
"""
Add summarySourceTable/summarySourceCol fields to metadata, and adjust existing summary tables
to correspond to the new style.
"""
# Note: this migration has some faults.
# - It doesn't delete viewSectionFields for columns it removes (if a user added some special
# columns manually.
# - It doesn't fix types of Reference columns that refer to old-style summary tables
# (if the user created some such columns manually).
doc_actions = [
action for action in [
maybe_add_column(tdset, '_grist_Tables', 'summarySourceTable',
'Ref:_grist_Tables'),
maybe_add_column(tdset, '_grist_Tables_column', 'summarySourceCol',
'Ref:_grist_Tables_column')
] if action
]
# Maps tableRef to Table object.
tables_map = {
t.id: t
for t in actions.transpose_bulk_action(
tdset.all_tables['_grist_Tables'])
}
# Maps tableName to tableRef
table_name_to_ref = {t.tableId: t.id for t in six.itervalues(tables_map)}
# List of Column objects
columns = list(
actions.transpose_bulk_action(
tdset.all_tables['_grist_Tables_column']))
# Maps columnRef to Column object.
columns_map_by_ref = {c.id: c for c in columns}
# Maps (tableRef, colName) to Column object.
columns_map_by_table_colid = {(c.parentId, c.colId): c for c in columns}
# Set of all tableNames.
table_name_set = set(table_name_to_ref.keys())
remove_cols = [] # List of columns to remove
formula_updates = [] # List of (column, new_table_name, new_formula) pairs
table_renames = [] # List of (table, new_name) pairs
source_tables = [] # List of (table, summarySourceTable) pairs
source_cols = [] # List of (column, summarySourceColumn) pairs
# Summary tables used to be named as "Summary_<SourceName>_<ColRef1>_<ColRef2>". This regular
# expression parses that.
summary_re = re.compile(r'^Summary_(\w+?)((?:_\d+)*)$')
for t in six.itervalues(tables_map):
m = summary_re.match(t.tableId)
if not m or m.group(1) not in table_name_to_ref:
continue
# We have a valid summary table.
source_table_name = m.group(1)
source_table_ref = table_name_to_ref[source_table_name]
groupby_colrefs = [int(x) for x in m.group(2).strip("_").split("_")]
# Prepare a new-style name for the summary table. Be sure not to conflict with existing tables
# or with each other (i.e. don't rename multiple tables to the same name).
new_name = summary.encode_summary_table_name(source_table_name)
new_name = identifiers.pick_table_ident(new_name, avoid=table_name_set)
table_name_set.add(new_name)
log.warn("Upgrading summary table %s for %s(%s) to %s" %
(t.tableId, source_table_name, groupby_colrefs, new_name))
# Remove the "lookupOrAddDerived" column from the source table (which is named using the
# summary table name for its colId).
remove_cols.extend(
c for c in columns
if c.parentId == source_table_ref and c.colId == t.tableId)
# Upgrade the "group" formula in the summary table.
expected_group_formula = "%s.lookupRecords(%s=$id)" % (
source_table_name, t.tableId)
new_formula = "table.getSummarySourceGroup(rec)"
formula_updates.extend((c, new_name, new_formula) for c in columns
if (c.parentId == t.id and c.colId == "group"
and c.formula == expected_group_formula))
# Schedule a rename of the summary table.
table_renames.append((t, new_name))
# Set summarySourceTable fields on the metadata.
source_tables.append((t, source_table_ref))
# Set summarySourceCol fields in the metadata. We need to find the right summary column.
groupby_cols = set()
for col_ref in groupby_colrefs:
src_col = columns_map_by_ref.get(col_ref)
sum_col = columns_map_by_table_colid.get(
(t.id, src_col.colId)) if src_col else None
if sum_col:
groupby_cols.add(sum_col)
source_cols.append((sum_col, src_col.id))
else:
log.warn(
"Upgrading summary table %s: couldn't find column %s" %
(t.tableId, col_ref))
# Finally, we have to remove all non-formula columns that are not groupby-columns (e.g.
# 'manualSort'), because the new approach assumes ALL non-formula columns are for groupby.
remove_cols.extend(c for c in columns if c.parentId == t.id
and c not in groupby_cols and not c.isFormula)
# Create all the doc actions from the arrays we prepared.
# Process remove_cols
doc_actions.extend(
actions.RemoveColumn(tables_map[c.parentId].tableId, c.colId)
for c in remove_cols)
doc_actions.append(
actions.BulkRemoveRecord('_grist_Tables_column',
[c.id for c in remove_cols]))
# Process table_renames
doc_actions.extend(
actions.RenameTable(t.tableId, new) for (t, new) in table_renames)
doc_actions.append(
actions.BulkUpdateRecord(
'_grist_Tables', [t.id for t, new in table_renames],
{'tableId': [new for t, new in table_renames]}))
# Process source_tables and source_cols
doc_actions.append(
actions.BulkUpdateRecord(
'_grist_Tables', [t.id for t, ref in source_tables],
{'summarySourceTable': [ref for t, ref in source_tables]}))
doc_actions.append(
actions.BulkUpdateRecord(
'_grist_Tables_column', [t.id for t, ref in source_cols],
{'summarySourceCol': [ref for t, ref in source_cols]}))
# Process formula_updates. Do this last since recalculation of these may cause new records added
# to summary tables, so we should have all the tables correctly set up by this time.
doc_actions.extend(
actions.ModifyColumn(table_id, c.colId, {'formula': f})
for c, table_id, f in formula_updates)
doc_actions.append(
actions.BulkUpdateRecord(
'_grist_Tables_column', [c.id for c, t, f in formula_updates],
{'formula': [f for c, t, f in formula_updates]}))
return tdset.apply_doc_actions(doc_actions)
def migration6(tdset):
# This undoes the previous migration, since primaryViewTable is now a formula private to the
# sandbox rather than part of the document schema.
return tdset.apply_doc_actions([
actions.RemoveColumn('_grist_Views', 'primaryViewTable'),
])
def test_prune_actions(self):
# prune_actions is in-place, so we make a new list every time.
def alist():
return [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
]
def prune(table_id, col_id):
a = alist()
actions.prune_actions(a, table_id, col_id)
return a
self.assertEqual(
prune('Table1', 'Foo'),
[
actions.BulkUpdateRecord("Table2", [1, 2, 3], {
'Foo': [10, 20, 30],
'Bar': ['a', 'b', 'c']
}),
actions.UpdateRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.BulkAddRecord("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.ReplaceTableData("Table2", 18, {
'Foo': 10,
'Bar': 'a'
}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
# It doesn't do anything with column renames; it can be addressed if needed.
actions.RenameColumn("Table1", "Foo", "Bar"),
# It doesn't do anything with AddTable, which is expected.
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
])
self.assertEqual(prune('Table2', 'Foo'), [
actions.BulkUpdateRecord("Table1", [1, 2, 3],
{'Foo': [10, 20, 30]}),
actions.BulkUpdateRecord("Table2", [1, 2, 3],
{'Bar': ['a', 'b', 'c']}),
actions.UpdateRecord("Table1", 17, {'Foo': 10}),
actions.UpdateRecord("Table2", 18, {'Bar': 'a'}),
actions.AddRecord("Table1", 17, {'Foo': 10}),
actions.BulkAddRecord("Table2", 18, {'Bar': 'a'}),
actions.ReplaceTableData("Table2", 18, {'Bar': 'a'}),
actions.RemoveRecord("Table1", 17),
actions.BulkRemoveRecord("Table2", [17, 18]),
actions.AddColumn("Table1", "Foo", {"type": "Text"}),
actions.RenameColumn("Table1", "Foo", "Bar"),
actions.ModifyColumn("Table1", "Foo", {"type": "Text"}),
actions.RemoveColumn("Table1", "Foo"),
actions.AddTable("THello", [{
"id": "Foo"
}, {
"id": "Bar"
}]),
actions.RemoveTable("THello"),
actions.RenameTable("THello", "TWorld"),
])