def walk_down(self, from_node: Migration, unapplied_only=True, _node_counters=None):
"""
Walks down over migrations graph. Iterates in order as migrations
should be applied.
We're used modified DFS (depth-first search) algorithm to traverse
the graph. Migrations are built into directed graph (digraph)
counted from one root node to the last ones. Commonly DFS tries to
walk to the maximum depth firstly.
But there are some problems:
* Graph can have directed cycles. This happens when some
migration has several dependencies. Therefore we'll walk
over such migration several times
* Another problem arises from the first one. Typically we must walk
over all dependencies before a dependent migration will be
touched. DFS will process only one dependency before get to
a dependent migration
In order to manage it we use counter for each migration
(node in digraph) initially equal to its parents count.
Every time the algorithm gets to node it decrements this counter.
If counter > 0 after that then don't touch this node and
break traversing on this depth and go up. If counter == 0 then
continue traversing.
:param from_node: current node in graph
:param unapplied_only: if True then return only unapplied migrations
or return all migrations otherwise
:param _node_counters:
:raises MigrationGraphError: if graph has a closed cycle
:return: Migration objects generator
"""
# FIXME: may yield nodes not related to target migration if branchy graph
# FIXME: if migration was applied after its dependencies unapplied then it is an error
# FIXME: should have stable migrations order
if _node_counters is None:
_node_counters = {}
if from_node is None:
return ()
_node_counters.setdefault(from_node.name, len(self._parents[from_node.name]) or 1)
_node_counters[from_node.name] -= 1
if _node_counters[from_node.name] > 0:
# Stop on this depth if not all parents has been viewed
return
if _node_counters[from_node.name] < 0:
# A node was already returned and we're reached it again
# This means there is a closed cycle
raise MigrationGraphError(f'Found closed cycle in migration graph, '
f'{from_node.name!r} is repeated twice')
if not (from_node.applied and unapplied_only):
yield from_node
for child in self._children[from_node.name]:
yield from self.walk_down(child, unapplied_only, _node_counters)
def verify(self):
"""
Verify migrations graph to be satisfied to consistency rules
Graph must not have loops, disconnections.
Also it should have single initial migration and (for a while)
single last migration.
:raises MigrationGraphError: if problem in graph was found
:return:
"""
# FIXME: This function is not used anywhere
initials = []
last_children = []
for name, obj in self._migrations.items():
if not self._parents[name]:
initials.append(name)
if not self._children[name]:
last_children.append(name)
if len(obj.dependencies) > len(self._parents[name]):
diff = set(obj.dependencies) - {x.name for x in self._parents[name]}
raise MigrationGraphError(f'Unknown dependencies in migration {name!r}: {diff}')
if name in (x.name for x in self._children[name]):
raise MigrationGraphError(f'Found migration which dependent on itself: {name!r}')
if len(initials) == len(last_children) and len(initials) > 1:
raise MigrationGraphError(f'Migrations graph is disconnected, history segments '
f'started on: {initials!r}, ended on: {last_children!r}')
if len(initials) > 1:
raise MigrationGraphError(f'Several initial migrations found: {initials!r}')
if len(last_children) > 1:
raise MigrationGraphError(f'Several last migrations found: {last_children!r}')
if not initials or not last_children:
raise MigrationGraphError(f'No initial or last children found')
def upgrade(self,
migration_name: str,
graph: Optional[MigrationsGraph] = None):
"""
Upgrade db to the given migration
:param migration_name: target migration name
:param graph: Optional. Migrations graph. If omitted, then it
will be loaded
:return:
"""
if graph is None:
log.debug('Loading migration files...')
graph = self.build_graph()
log.debug('Loading schema from database...')
left_schema = self.load_db_schema()
if migration_name not in graph.migrations:
raise MigrationGraphError(f'Migration {migration_name} not found')
db = self.db
for migration in graph.walk_down(graph.initial, unapplied_only=True):
log.info('Upgrading %s...', migration.name)
for idx, action_object in enumerate(migration.get_actions(),
start=1):
log.debug('> [%d] %s', idx, str(action_object))
if not action_object.dummy_action and not runtime_flags.schema_only:
action_object.prepare(db, left_schema, migration.policy)
action_object.run_forward()
action_object.cleanup()
try:
left_schema = patch(
action_object.to_schema_patch(left_schema),
left_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action_object!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
graph.migrations[migration.name].applied = True
if not runtime_flags.dry_run:
log.debug('Writing db schema and migrations graph...')
self.write_db_schema(left_schema)
self.write_db_migrations_graph(graph)
if migration.name == migration_name:
break # We've reached the target migration
self._verify_schema(left_schema)
def migrate(self, migration_name: str = None):
"""
Migrate db in order to reach a given migration. This process
may require either upgrading or downgrading
:param migration_name: target migration name
:return:
"""
log.debug('Loading migration files...')
graph = self.build_graph()
if not graph.last:
raise MigrationGraphError('No migrations found')
if migration_name is None:
migration_name = graph.last.name
if migration_name not in graph.migrations:
raise MigrationGraphError(f'Migration {migration_name} not found')
migration = graph.migrations[migration_name]
if migration.applied:
self.downgrade(migration_name, graph)
else:
self.upgrade(migration_name, graph)
def walk_up(self,
from_node: Migration,
applied_only=True,
_node_counters=None):
"""
Walks up over migrations graph. Iterates in order as migrations
should be reverted.
We're using modified DFS (depth-first search) algorithm which in
reversed order (see `walk_down`). Instead of looking at node
parents count we're consider children count in order to return
all dependent nodes before dependency.
Because of the migrations graph may have many orphan child nodes
they all should be passed as parameter
:param from_node: last children node we are starting for
:param applied_only: if True then return only applied migrations,
return all migrations otherwise
:param _node_counters:
:raises MigrationGraphError: if graph has a closed cycle
:return: Migration objects generator
"""
# FIXME: may yield nodes not related to reverting if branchy graph
# if migration was unapplied before its dependencies applied then it is an error
if _node_counters is None:
_node_counters = {}
if from_node is None:
return ()
_node_counters.setdefault(from_node.name,
len(self._children[from_node.name]) or 1)
_node_counters[from_node.name] -= 1
if _node_counters[from_node.name] > 0:
# Stop in this depth if not all children has been viewed
return
if _node_counters[from_node.name] < 0:
# A node was already returned and we're reached it again
# This means there is a closed cycle
raise MigrationGraphError(
f'Found closed cycle in migration graph, '
f'{from_node.name!r} is repeated twice')
if from_node.applied or not applied_only:
yield from_node
for child in self._parents[from_node.name]:
yield from self.walk_up(child, applied_only, _node_counters)
def build_graph(self) -> MigrationsGraph:
"""Build migrations graph with all migration modules"""
graph = MigrationsGraph()
for m in self.load_migrations(Path(self.migration_dir)):
graph.add(m)
applied = []
for migration_name in self.get_db_migration_names():
if migration_name not in graph.migrations:
raise MigrationGraphError(
f'Migration {migration_name} was applied, but its python module not found. '
f'You can use schema repair to fix this issue')
graph.migrations[migration_name].applied = True
applied.append(migration_name)
log.debug('> Applied migrations: %s', applied)
if graph.last:
log.debug('> Last migration is: %s', graph.last.name)
else:
log.debug('> Last migration is: %s', 'None')
return graph
def downgrade(self,
migration_name: str,
graph: Optional[MigrationsGraph] = None):
"""
Downgrade db to the given migration
:param migration_name: target migration name
:param graph: Optional. Migrations graph. If omitted, then it
will be loaded
:return:
"""
if graph is None:
log.debug('Loading migration files...')
graph = self.build_graph()
log.debug('Loading schema from database...')
left_schema = self.load_db_schema()
if migration_name not in graph.migrations:
raise MigrationGraphError(f'Migration {migration_name} not found')
log.debug('Precalculating schema diffs...')
# Collect schema diffs across all migrations
migration_diffs = {} # {migration_name: [action1_diff, ...]}
temp_left_schema = Schema()
for migration in graph.walk_down(graph.initial, unapplied_only=False):
migration_diffs[migration.name] = []
for action in migration.get_actions():
forward_patch = action.to_schema_patch(temp_left_schema)
migration_diffs[migration.name].append(forward_patch)
try:
temp_left_schema = patch(forward_patch, temp_left_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
db = self.db
for migration in graph.walk_up(graph.last, applied_only=True):
if migration.name == migration_name:
break # We've reached the target migration
log.info('Downgrading %s...', migration.name)
action_diffs = zip(migration.get_actions(),
migration_diffs[migration.name],
range(1,
len(migration.get_actions()) + 1))
for action_object, action_diff, idx in reversed(
list(action_diffs)):
log.debug('> [%d] %s', idx, str(action_object))
try:
left_schema = patch(list(swap(action_diff)), left_schema)
except (TypeError, ValueError, KeyError) as e:
raise ActionError(
f"Unable to apply schema patch of {action_object!r}. More likely that the "
f"schema is corrupted. You can use schema repair tools to fix this issue"
) from e
if not action_object.dummy_action and not runtime_flags.schema_only:
action_object.prepare(db, left_schema, migration.policy)
action_object.run_backward()
action_object.cleanup()
graph.migrations[migration.name].applied = False
if not runtime_flags.dry_run:
log.debug('Writing db schema and migrations graph...')
self.write_db_schema(left_schema)
self.write_db_migrations_graph(graph)
self._verify_schema(left_schema)