def get_uniongraph(self): """ Build a union graph of intermodel dependencies and model local dependencies. This graph represents the final update cascades triggered by certain field updates. The union graph is needed to spot cycles introduced by model local dependencies, that otherwise might went unnoticed, example: - global dep graph (acyclic): ``A.comp --> B.comp, B.comp2 --> A.comp`` - modelgraph of B (acyclic): ``B.comp --> B.comp2`` Here the resulting union graph is not a DAG anymore, since both subgraphs short-circuit to a cycle of ``A.comp --> B.comp --> B.comp2 --> A.comp``. """ if not self.union: graph = Graph() # copy intermodel edges for edge in self.edges: graph.add_edge(edge) # copy modelgraph edges self.prepare_modelgraphs() for model, modelgraph in self.modelgraphs.items(): name = modelname(model) for edge in modelgraph.edges: graph.add_edge( Edge(Node((name, edge.left.data)), Node((name, edge.right.data)))) self.union = graph return self.union
def _clean_data(self, data): """ Converts the global dependency data into an adjacency list tree to be used with the underlying graph. """ cleaned = OrderedDict() for model, fielddata in data.items(): self.models[modelname(model)] = model for field, modeldata in fielddata.items(): for depmodel, relations in modeldata.items(): self.models[modelname(depmodel)] = depmodel for dep in relations: key = (modelname(depmodel), dep['depends']) value = (modelname(model), field) cleaned.setdefault(key, set()).add(value) return cleaned
def _clean_data(self, data): """ Converts the dependency data into an adjacency list tree to be used with the underlying graph. """ cleaned = OrderedDict() for model, fielddata in data.items(): self.models[modelname(model)] = model for field, modeldata in fielddata.items(): for depmodel, relations in modeldata.items(): self.models[modelname(depmodel)] = depmodel for dep in relations: # normally we refer to the given model field # if none is given, set it to '#' which assumes # any chance to the model should trigger the update # Note: '#' is only triggered for `.save` without # setting update_fields! depends = dep.get('depends', '#') key = (modelname(depmodel), depends) value = (modelname(model), field) cleaned.setdefault(key, set()).add(value) return cleaned
def action_check(self, models, progress, size, json_out): has_desync = False for model in models: qs = model.objects.all() amount = qs.count() fields = set(active_resolver.computed_models[model].keys()) qsize = active_resolver.get_querysize(model, fields, size) self.eprint(f'- {self.style.MIGRATE_LABEL(modelname(model))}') self.eprint(f' Fields: {", ".join(fields)}') self.eprint(f' Records: {amount}') if not amount: continue # apply select/prefetch rules select = active_resolver.get_select_related(model, fields) prefetch = active_resolver.get_prefetch_related(model, fields) if select: qs = qs.select_related(*select) if prefetch: qs = qs.prefetch_related(*prefetch) # check sync state desync = [] if progress: with tqdm(total=amount, desc=' Check', unit=' rec', disable=self.silent) as bar: for obj in slice_iterator(qs, qsize): if not check_instance(model, fields, obj): desync.append(obj.pk) bar.update(1) else: for obj in slice_iterator(qs, qsize): if not check_instance(model, fields, obj): desync.append(obj.pk) if not desync: self.eprint(self.style.SUCCESS(f' Desync: 0 records')) else: has_desync = True self.eprint( self.style.WARNING( f' Desync: {len(desync)} records ({percent(len(desync), amount)})' )) if not self.silent and not self.skip_tainted: mode, tainted = try_tainted(qs, desync, amount) if tainted: self.eprint( self.style.NOTICE(f' Tainted dependants:')) for level, submodel, fields, count in tainted: records = '' if mode == 'concrete': records = '~' elif mode == 'approx': records = '>>' records += f'{count} records' if count != -1 else 'records unknown' self.eprint( self.style.NOTICE( ' ' * level + f'└─ {modelname(submodel)}: {", ".join(fields)} ({records})' )) if len(tainted) >= TAINTED_MAXLENGTH: self.eprint( self.style.NOTICE(' (listing shortened...)')) if json_out: json_out.write( dumps({ 'model': modelname(model), 'desync': desync })) return has_desync