def test_low_override_wins(self):
# q1000 wins
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q1000'],
10000), 1000)
# override wins
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q1000'], 10),
10)
def test_lowest_in_updates(self):
# the lowest local cf value always wins
self.assertEqual(active_resolver.get_querysize(Querysize), 1)
self.assertEqual(active_resolver.get_querysize(Querysize, None, 10000),
1)
# q10 limits
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q10']), 10)
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q10'],
10000), 10)
self.assertEqual(
active_resolver.get_querysize(Querysize,
['default', 'q10', 'q100', 'q1000']),
10)
# q100 limits
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q100']), 100)
self.assertEqual(
active_resolver.get_querysize(Querysize,
['default', 'q100', 'q1000'], 10000),
100)
# q1000 limits
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default', 'q1000'],
10000), 1000)
def action_loop(self, models, size, show_progress):
print('Update mode: loop')
print(f'Global querysize: {size}')
print('Models:')
if size != settings.COMPUTEDFIELDS_QUERYSIZE:
# patch django settings in case querysize was explicitly given
# needed here, as we have no other API to announce the changed value
from django.conf import settings as ds
ds.COMPUTEDFIELDS_QUERYSIZE = size
for model in models:
qs = model.objects.all()
amount = qs.count()
fields = list(active_resolver.computed_models[model].keys())
qsize = active_resolver.get_querysize(model, fields, size)
print(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
print(f' Fields: {", ".join(fields)}')
print(f' Records: {amount}')
print(f' Querysize: {qsize}')
if not amount:
continue
# also apply select/prefetch rules
select = active_resolver.get_select_related(model, fields)
prefetch = active_resolver.get_prefetch_related(model, fields)
if select:
qs = qs.select_related(*select)
if prefetch:
qs = qs.prefetch_related(*prefetch)
if show_progress:
with tqdm(total=amount, desc=' Progress', unit=' rec') as bar:
for obj in slice_iterator(qs, qsize):
obj.save()
bar.update(1)
else:
for obj in slice_iterator(qs, qsize):
obj.save()
def test_default(self):
self.assertEqual(active_resolver.get_querysize(Querysize, ['default']),
settings.COMPUTEDFIELDS_QUERYSIZE)
self.assertEqual(active_resolver.get_querysize(EmailUser),
settings.COMPUTEDFIELDS_QUERYSIZE)
def test_chain(self):
# c_10_100 can do 100, but is limited by prev q10
mro = active_resolver.get_local_mro(Querysize, ['q10'])
self.assertEqual(active_resolver.get_querysize(Querysize, mro, 10000),
1)
def test_default_altered(self):
self.assertEqual(settings.COMPUTEDFIELDS_QUERYSIZE, 10000)
self.assertEqual(
active_resolver.get_querysize(Querysize, ['default'], 10000),
settings.COMPUTEDFIELDS_QUERYSIZE)
def action_check(self, models, progress, size, json_out):
has_desync = False
for model in models:
qs = model.objects.all()
amount = qs.count()
fields = set(active_resolver.computed_models[model].keys())
qsize = active_resolver.get_querysize(model, fields, size)
self.eprint(f'- {self.style.MIGRATE_LABEL(modelname(model))}')
self.eprint(f' Fields: {", ".join(fields)}')
self.eprint(f' Records: {amount}')
if not amount:
continue
# apply select/prefetch rules
select = active_resolver.get_select_related(model, fields)
prefetch = active_resolver.get_prefetch_related(model, fields)
if select:
qs = qs.select_related(*select)
if prefetch:
qs = qs.prefetch_related(*prefetch)
# check sync state
desync = []
if progress:
with tqdm(total=amount,
desc=' Check',
unit=' rec',
disable=self.silent) as bar:
for obj in slice_iterator(qs, qsize):
if not check_instance(model, fields, obj):
desync.append(obj.pk)
bar.update(1)
else:
for obj in slice_iterator(qs, qsize):
if not check_instance(model, fields, obj):
desync.append(obj.pk)
if not desync:
self.eprint(self.style.SUCCESS(f' Desync: 0 records'))
else:
has_desync = True
self.eprint(
self.style.WARNING(
f' Desync: {len(desync)} records ({percent(len(desync), amount)})'
))
if not self.silent and not self.skip_tainted:
mode, tainted = try_tainted(qs, desync, amount)
if tainted:
self.eprint(
self.style.NOTICE(f' Tainted dependants:'))
for level, submodel, fields, count in tainted:
records = ''
if mode == 'concrete':
records = '~'
elif mode == 'approx':
records = '>>'
records += f'{count} records' if count != -1 else 'records unknown'
self.eprint(
self.style.NOTICE(
' ' * level +
f'└─ {modelname(submodel)}: {", ".join(fields)} ({records})'
))
if len(tainted) >= TAINTED_MAXLENGTH:
self.eprint(
self.style.NOTICE(' (listing shortened...)'))
if json_out:
json_out.write(
dumps({
'model': modelname(model),
'desync': desync
}))
return has_desync