def get_uniongraph(self):
"""
Build a union graph of intermodel dependencies and model local dependencies.
This graph represents the final update cascades triggered by certain field updates.
The union graph is needed to spot cycles introduced by model local dependencies,
that otherwise might went unnoticed, example:
- global dep graph (acyclic): ``A.comp --> B.comp, B.comp2 --> A.comp``
- modelgraph of B (acyclic): ``B.comp --> B.comp2``
Here the resulting union graph is not a DAG anymore, since both subgraphs short-circuit
to a cycle of ``A.comp --> B.comp --> B.comp2 --> A.comp``.
"""
if not self.union:
graph = Graph()
# copy intermodel edges
for edge in self.edges:
graph.add_edge(edge)
# copy modelgraph edges
self.prepare_modelgraphs()
for model, modelgraph in self.modelgraphs.items():
name = modelname(model)
for edge in modelgraph.edges:
graph.add_edge(
Edge(Node((name, edge.left.data)),
Node((name, edge.right.data))))
self.union = graph
return self.union
def _clean_data(self, data):
"""
Converts the global dependency data into an adjacency list tree
to be used with the underlying graph.
"""
cleaned = OrderedDict()
for model, fielddata in data.items():
self.models[modelname(model)] = model
for field, modeldata in fielddata.items():
for depmodel, relations in modeldata.items():
self.models[modelname(depmodel)] = depmodel
for dep in relations:
key = (modelname(depmodel), dep['depends'])
value = (modelname(model), field)
cleaned.setdefault(key, set()).add(value)
return cleaned
def _clean_data(self, data):
"""
Converts the dependency data into an adjacency list tree
to be used with the underlying graph.
"""
cleaned = OrderedDict()
for model, fielddata in data.items():
self.models[modelname(model)] = model
for field, modeldata in fielddata.items():
for depmodel, relations in modeldata.items():
self.models[modelname(depmodel)] = depmodel
for dep in relations:
# normally we refer to the given model field
# if none is given, set it to '#' which assumes
# any chance to the model should trigger the update
# Note: '#' is only triggered for `.save` without
# setting update_fields!
depends = dep.get('depends', '#')
key = (modelname(depmodel), depends)
value = (modelname(model), field)
cleaned.setdefault(key, set()).add(value)
return cleaned
def action_check(self, models, progress, size, json_out):
has_desync = False
for model in models:
qs = model.objects.all()
amount = qs.count()
fields = set(active_resolver.computed_models[model].keys())
qsize = active_resolver.get_querysize(model, fields, size)
self.eprint(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
self.eprint(f' Fields: {", ".join(fields)}')
self.eprint(f' Records: {amount}')
if not amount:
continue
# apply select/prefetch rules
select = active_resolver.get_select_related(model, fields)
prefetch = active_resolver.get_prefetch_related(model, fields)
if select:
qs = qs.select_related(*select)
if prefetch:
qs = qs.prefetch_related(*prefetch)
# check sync state
desync = []
if progress:
with tqdm(total=amount,
desc=' Check',
unit=' rec',
disable=self.silent) as bar:
for obj in slice_iterator(qs, qsize):
if not check_instance(model, fields, obj):
desync.append(obj.pk)
bar.update(1)
else:
for obj in slice_iterator(qs, qsize):
if not check_instance(model, fields, obj):
desync.append(obj.pk)
if not desync:
self.eprint(self.style.SUCCESS(f' Desync: 0 records'))
else:
has_desync = True
self.eprint(
self.style.WARNING(
f' Desync: {len(desync)} records ({percent(len(desync), amount)})'
))
if not self.silent and not self.skip_tainted:
mode, tainted = try_tainted(qs, desync, amount)
if tainted:
self.eprint(
self.style.NOTICE(f' Tainted dependants:'))
for level, submodel, fields, count in tainted:
records = ''
if mode == 'concrete':
records = '~'
elif mode == 'approx':
records = '>>'
records += f'{count} records' if count != -1 else 'records unknown'
self.eprint(
self.style.NOTICE(
' ' * level +
f'└─ {modelname(submodel)}: {", ".join(fields)} ({records})'
))
if len(tainted) >= TAINTED_MAXLENGTH:
self.eprint(
self.style.NOTICE(' (listing shortened...)'))
if json_out:
json_out.write(
dumps({
'model': modelname(model),
'desync': desync
}))
return has_desync