def test_low_override_wins(self): # q1000 wins self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q1000'], 10000), 1000) # override wins self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q1000'], 10), 10)
def test_lowest_in_updates(self): # the lowest local cf value always wins self.assertEqual(active_resolver.get_querysize(Querysize), 1) self.assertEqual(active_resolver.get_querysize(Querysize, None, 10000), 1) # q10 limits self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q10']), 10) self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q10'], 10000), 10) self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q10', 'q100', 'q1000']), 10) # q100 limits self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q100']), 100) self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q100', 'q1000'], 10000), 100) # q1000 limits self.assertEqual( active_resolver.get_querysize(Querysize, ['default', 'q1000'], 10000), 1000)
def action_loop(self, models, size, show_progress): print('Update mode: loop') print(f'Global querysize: {size}') print('Models:') if size != settings.COMPUTEDFIELDS_QUERYSIZE: # patch django settings in case querysize was explicitly given # needed here, as we have no other API to announce the changed value from django.conf import settings as ds ds.COMPUTEDFIELDS_QUERYSIZE = size for model in models: qs = model.objects.all() amount = qs.count() fields = list(active_resolver.computed_models[model].keys()) qsize = active_resolver.get_querysize(model, fields, size) print(f'- {self.style.MIGRATE_LABEL(modelname(model))}') print(f' Fields: {", ".join(fields)}') print(f' Records: {amount}') print(f' Querysize: {qsize}') if not amount: continue # also apply select/prefetch rules select = active_resolver.get_select_related(model, fields) prefetch = active_resolver.get_prefetch_related(model, fields) if select: qs = qs.select_related(*select) if prefetch: qs = qs.prefetch_related(*prefetch) if show_progress: with tqdm(total=amount, desc=' Progress', unit=' rec') as bar: for obj in slice_iterator(qs, qsize): obj.save() bar.update(1) else: for obj in slice_iterator(qs, qsize): obj.save()
def test_default(self): self.assertEqual(active_resolver.get_querysize(Querysize, ['default']), settings.COMPUTEDFIELDS_QUERYSIZE) self.assertEqual(active_resolver.get_querysize(EmailUser), settings.COMPUTEDFIELDS_QUERYSIZE)
def test_chain(self): # c_10_100 can do 100, but is limited by prev q10 mro = active_resolver.get_local_mro(Querysize, ['q10']) self.assertEqual(active_resolver.get_querysize(Querysize, mro, 10000), 1)
def test_default_altered(self): self.assertEqual(settings.COMPUTEDFIELDS_QUERYSIZE, 10000) self.assertEqual( active_resolver.get_querysize(Querysize, ['default'], 10000), settings.COMPUTEDFIELDS_QUERYSIZE)
def action_check(self, models, progress, size, json_out): has_desync = False for model in models: qs = model.objects.all() amount = qs.count() fields = set(active_resolver.computed_models[model].keys()) qsize = active_resolver.get_querysize(model, fields, size) self.eprint(f'- {self.style.MIGRATE_LABEL(modelname(model))}') self.eprint(f' Fields: {", ".join(fields)}') self.eprint(f' Records: {amount}') if not amount: continue # apply select/prefetch rules select = active_resolver.get_select_related(model, fields) prefetch = active_resolver.get_prefetch_related(model, fields) if select: qs = qs.select_related(*select) if prefetch: qs = qs.prefetch_related(*prefetch) # check sync state desync = [] if progress: with tqdm(total=amount, desc=' Check', unit=' rec', disable=self.silent) as bar: for obj in slice_iterator(qs, qsize): if not check_instance(model, fields, obj): desync.append(obj.pk) bar.update(1) else: for obj in slice_iterator(qs, qsize): if not check_instance(model, fields, obj): desync.append(obj.pk) if not desync: self.eprint(self.style.SUCCESS(f' Desync: 0 records')) else: has_desync = True self.eprint( self.style.WARNING( f' Desync: {len(desync)} records ({percent(len(desync), amount)})' )) if not self.silent and not self.skip_tainted: mode, tainted = try_tainted(qs, desync, amount) if tainted: self.eprint( self.style.NOTICE(f' Tainted dependants:')) for level, submodel, fields, count in tainted: records = '' if mode == 'concrete': records = '~' elif mode == 'approx': records = '>>' records += f'{count} records' if count != -1 else 'records unknown' self.eprint( self.style.NOTICE( ' ' * level + f'└─ {modelname(submodel)}: {", ".join(fields)} ({records})' )) if len(tainted) >= TAINTED_MAXLENGTH: self.eprint( self.style.NOTICE(' (listing shortened...)')) if json_out: json_out.write( dumps({ 'model': modelname(model), 'desync': desync })) return has_desync