def refresh_scores(self, users=None, existing=None, existing_tps=None):
suppress_tp_scores = keep_data(signals=(update_scores, ),
suppress=(TranslationProject, ))
existing = existing or self.get_store_scores(self.tp)
with bulk_operations(UserTPScore):
with suppress_tp_scores:
with bulk_operations(UserStoreScore):
for store in self.tp.stores.iterator():
score_updater.get(store.__class__)(store).update(
users=users, existing=existing.get(store.id))
self.update(users=users, existing=existing_tps)
def refresh_scores(self, users=None, existing=None, existing_tps=None):
suppress_tp_scores = keep_data(
signals=(update_scores, ),
suppress=(TranslationProject, ))
existing = existing or self.get_store_scores(self.tp)
with bulk_operations(UserTPScore):
with suppress_tp_scores:
with bulk_operations(UserStoreScore):
for store in self.tp.stores.iterator():
score_updater.get(store.__class__)(store).update(
users=users,
existing=existing.get(store.id))
self.update(users=users, existing=existing_tps)
def _callback_handler(sender, updated, **kwargs):
bulk_tps = bulk_operations(models=(get_user_model(), UserTPScore, TPData,
TPChecksData))
with keep_data(signals=(update_revisions, )):
@receiver(update_revisions)
def update_revisions_handler(**kwargs):
if updated.revisions is None:
updated.revisions = set()
instance = kwargs.get("instance")
if isinstance(instance, Store):
updated.revisions.add(kwargs["instance"].parent.pootle_path)
elif isinstance(instance, Directory):
updated.revisions.add(kwargs["instance"].pootle_path)
with bulk_tps:
_update_stores(sender, updated)
if updated.tp_data:
update_data.send(sender.__class__, instance=sender)
if updated.tp_scores:
update_scores.send(sender.__class__,
instance=sender,
users=updated.score_users)
if updated.revisions:
update_revisions.send(Directory,
paths=updated.revisions,
keys=["stats", "checks"])
def _update_stores(sender, updated):
# call signals for stores
bulk_stores = bulk_operations(models=(UserStoreScore, StoreData,
StoreChecksData))
with keep_data(suppress=(sender.__class__, )):
with bulk_stores:
_handle_update_stores(sender, updated)
def _update_stores(sender, updated):
# call signals for stores
bulk_stores = bulk_operations(
models=(
UserStoreScore,
StoreData,
StoreChecksData))
with keep_data(suppress=(sender.__class__, )):
with bulk_stores:
_handle_update_stores(sender, updated)
def refresh_scores(self, users=None, **kwargs):
suppress_user_scores = keep_data(signals=(update_scores, ),
suppress=(get_user_model(), ))
tp_scores = self.get_tp_scores()
with bulk_operations(get_user_model()):
with suppress_user_scores:
for tp in TranslationProject.objects.all():
score_updater.get(tp.__class__)(tp).refresh_scores(
users=users, existing_tps=tp_scores.get(tp.id))
self.update(users=users)
def refresh_scores(self, users=None, **kwargs):
suppress_user_scores = keep_data(
signals=(update_scores, ),
suppress=(get_user_model(), ))
tp_scores = self.get_tp_scores()
with bulk_operations(get_user_model()):
with suppress_user_scores:
for tp in TranslationProject.objects.all():
score_updater.get(tp.__class__)(tp).refresh_scores(
users=users,
existing_tps=tp_scores.get(tp.id))
self.update(users=users)
def update(self, clear_unknown=False, update_data_after=False):
"""Update/purge all QualityChecks for Units, and expire Store caches.
"""
if clear_unknown:
self.clear_unknown_checks()
with bulk_operations(QualityCheck):
self.update_untranslated()
self.update_translated()
updated = self.updated_stores
if update_data_after:
self.update_data(updated)
if "checks" in self.__dict__:
del self.__dict__["checks"]
self._updated_stores = {}
return updated
def test_contextmanager_bulk_ops_delete(tp0, store0):
unit = store0.units.first()
store1 = tp0.stores.filter(name="store1.po").first()
store2 = tp0.stores.filter(name="store2.po").first()
class Update(object):
store_deleted = None
unit_deleted = None
store_called = 0
unit_called = 0
with keep_data(signals=[delete]):
updated = Update()
@receiver(delete, sender=Unit)
def handle_unit_delete(**kwargs):
assert "instance" not in kwargs
updated.unit_deleted = kwargs["objects"]
updated.unit_called += 1
@receiver(delete, sender=Store)
def handle_store_delete(**kwargs):
updated.store_called += 1
if "objects" in kwargs:
updated.store_deleted = kwargs["objects"]
else:
assert "instance" in kwargs
with bulk_operations(Unit):
delete.send(Unit, instance=unit)
delete.send(Unit, objects=store1.unit_set.all())
delete.send(Unit, objects=store2.unit_set.all())
delete.send(Store, instance=store0)
delete.send(
Store,
objects=store1.translation_project.stores.filter(
id=store1.id))
delete.send(
Store,
objects=store2.translation_project.stores.filter(
id=store2.id))
assert updated.unit_called == 1
assert qs_match(
updated.unit_deleted,
(store0.unit_set.filter(id=unit.id)
| store1.unit_set.all()
| store2.unit_set.all()))
assert updated.store_called == 3
def _handle_update_stores(sender, updated):
@receiver(update_data, sender=sender.__class__)
def update_tp_data_handler(**kwargs):
updated.tp_data = True
update_data.disconnect(
update_tp_data_handler,
sender=sender.__class__)
@receiver(update_scores, sender=sender.__class__)
def update_tp_scores_handler(**kwargs):
updated.tp_scores = True
update_scores.disconnect(
update_tp_scores_handler,
sender=sender.__class__)
if updated.checks:
with keep_data(suppress=(Store, ), signals=(update_data, )):
@receiver(update_data, sender=Store)
def extra_update_data_handler_(**kwargs):
updated.data = updated.data or {}
updated.data[kwargs["instance"].id] = kwargs["instance"]
with bulk_operations(QualityCheck):
for to_check in updated.checks.values():
store = to_check["store"]
units = (
[unit for unit in to_check["units"]]
if to_check["units"]
else None)
update_checks.send(
store.__class__,
instance=store,
units=units)
if updated.data:
stores = updated.data.values()
for store in stores:
update_data.send(
Store,
instance=store)
if updated.score_stores:
for store in updated.score_stores.values():
update_scores.send(
store.__class__,
instance=store,
users=updated.score_users)
def update(self, clear_unknown=False, update_data_after=False):
"""Update/purge all QualityChecks for Units, and expire Store caches.
"""
self.log_debug()
if clear_unknown:
self.clear_unknown_checks()
with bulk_operations(QualityCheck):
self.update_untranslated()
self.update_translated()
updated = self.updated_stores
if update_data_after:
self.update_data(updated)
if "checks" in self.__dict__:
del self.__dict__["checks"]
self._updated_stores = {}
return updated
def setup_submissions(self):
from django.contrib.auth import get_user_model
from django.utils import timezone
from pootle.core.contextmanagers import bulk_operations
from pootle_data.models import TPChecksData, TPData
from pootle_score.models import UserTPScore
from pootle_statistics.models import SubmissionTypes
from pootle_store.constants import UNTRANSLATED
from pootle_store.models import Unit, UnitChange
from pootle_translationproject.contextmanagers import update_tp_after
from pootle_translationproject.models import TranslationProject
year_ago = timezone.now() - relativedelta(years=1)
units = Unit.objects.all()
units.update(creation_time=year_ago)
User = get_user_model()
admin = User.objects.get(username="******")
member = User.objects.get(username="******")
member2 = User.objects.get(username="******")
UnitChange.objects.bulk_create(
UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM)
for unit_id
in units.filter(state__gt=UNTRANSLATED).values_list("id", flat=True))
tps = TranslationProject.objects.exclude(
language__code="templates").select_related(
"language", "project__source_language").all()
bulk_pootle = bulk_operations(
models=(
get_user_model(),
UserTPScore,
TPData,
TPChecksData))
with bulk_pootle:
for tp in tps:
with update_tp_after(tp):
self._add_subs_to_stores(
tp.stores, admin, member, member2)
def _callback_handler(sender, updated, **kwargs):
bulk_pootle = bulk_operations(
models=(
get_user_model(),
UserTPScore,
UserStoreScore,
TPData,
TPChecksData,
StoreData,
StoreChecksData))
with keep_data(signals=(update_revisions, )):
with bulk_pootle:
@receiver(update_revisions)
def handle_update_revisions(**kwargs):
updated.revisions = True
if updated.checks:
update_checks.send(
sender.__class__,
instance=sender,
units=updated.checks,
**kwargs)
if updated.data:
update_data.send(
sender.__class__,
instance=sender,
**kwargs)
if updated.scores:
update_scores.send(
sender.__class__,
instance=sender,
users=updated.scores,
**kwargs)
if updated.revisions:
update_revisions.send(
sender.__class__,
instance=sender,
keys=["stats", "checks"])
def _handle_update_stores(sender, updated):
@receiver(update_data, sender=sender.__class__)
def update_tp_data_handler(**kwargs):
updated.tp_data = True
update_data.disconnect(update_tp_data_handler, sender=sender.__class__)
@receiver(update_scores, sender=sender.__class__)
def update_tp_scores_handler(**kwargs):
updated.tp_scores = True
update_scores.disconnect(update_tp_scores_handler,
sender=sender.__class__)
if updated.checks:
with keep_data(suppress=(Store, ), signals=(update_data, )):
@receiver(update_data, sender=Store)
def extra_update_data_handler_(**kwargs):
updated.data = updated.data or {}
updated.data[kwargs["instance"].id] = kwargs["instance"]
with bulk_operations(QualityCheck):
for to_check in updated.checks.values():
store = to_check["store"]
units = ([unit for unit in to_check["units"]]
if to_check["units"] else None)
update_checks.send(store.__class__,
instance=store,
units=units)
if updated.data:
stores = updated.data.values()
for store in stores:
update_data.send(Store, instance=store)
if updated.score_stores:
for store in updated.score_stores.values():
update_scores.send(store.__class__,
instance=store,
users=updated.score_users)
def _callback_handler(sender, updated, **kwargs):
bulk_tps = bulk_operations(
models=(
get_user_model(),
UserTPScore,
TPData,
TPChecksData))
with keep_data(signals=(update_revisions, )):
@receiver(update_revisions)
def update_revisions_handler(**kwargs):
if updated.revisions is None:
updated.revisions = set()
instance = kwargs.get("instance")
if isinstance(instance, Store):
updated.revisions.add(kwargs["instance"].parent.pootle_path)
elif isinstance(instance, Directory):
updated.revisions.add(kwargs["instance"].pootle_path)
with bulk_tps:
_update_stores(sender, updated)
if updated.tp_data:
update_data.send(
sender.__class__,
instance=sender)
if updated.tp_scores:
update_scores.send(
sender.__class__,
instance=sender,
users=updated.score_users)
if updated.revisions:
update_revisions.send(
Directory,
paths=updated.revisions,
keys=["stats", "checks"])
def setup_submissions(self):
from django.contrib.auth import get_user_model
from django.utils import timezone
from pootle.core.contextmanagers import bulk_operations
from pootle_data.models import TPChecksData, TPData
from pootle_score.models import UserTPScore
from pootle_statistics.models import SubmissionTypes
from pootle_store.constants import UNTRANSLATED
from pootle_store.models import Unit, UnitChange
from pootle_translationproject.contextmanagers import update_tp_after
from pootle_translationproject.models import TranslationProject
year_ago = timezone.now() - relativedelta(years=1)
units = Unit.objects.all()
units.update(creation_time=year_ago)
User = get_user_model()
admin = User.objects.get(username="******")
member = User.objects.get(username="******")
member2 = User.objects.get(username="******")
UnitChange.objects.bulk_create(
UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM)
for unit_id in units.filter(
state__gt=UNTRANSLATED).values_list("id", flat=True))
tps = TranslationProject.objects.exclude(
language__code="templates").select_related(
"language", "project__source_language").all()
bulk_pootle = bulk_operations(models=(get_user_model(), UserTPScore,
TPData, TPChecksData))
with bulk_pootle:
for tp in tps:
with update_tp_after(tp):
self._add_subs_to_stores(tp.stores, admin, member, member2)
def test_contextmanager_bulk_operations(store0):
unit = store0.units.first()
class Update(object):
data_created = None
qc_created = None
data_called = 0
qc_called = 0
with keep_data(signals=[create]):
updated = Update()
@receiver(create, sender=QualityCheck)
def handle_qc_create(**kwargs):
assert "instance" not in kwargs
updated.qc_created = kwargs["objects"]
updated.qc_called += 1
@receiver(create, sender=StoreChecksData)
def handle_data_create(**kwargs):
updated.data_called += 1
if "objects" in kwargs:
updated.data_created = kwargs["objects"]
else:
assert "instance" in kwargs
with bulk_operations(QualityCheck):
qc_creates = _create_qc_events(unit)
data_creates = _create_data_events(unit)
assert updated.qc_created == qc_creates
assert updated.qc_called == 1
assert updated.data_created != data_creates
assert updated.data_called == 5
# nested update
updated = Update()
with bulk_operations(QualityCheck):
with bulk_operations(QualityCheck):
qc_creates = _create_qc_events(unit)
data_creates = _create_data_events(unit)
assert updated.qc_created == qc_creates
assert updated.qc_called == 1
assert updated.data_created != data_creates
assert updated.data_called == 5
# nested update - different models
updated = Update()
with bulk_operations(StoreChecksData):
with bulk_operations(QualityCheck):
qc_creates = _create_qc_events(unit)
data_creates = _create_data_events(unit)
assert updated.qc_created == qc_creates
assert updated.qc_called == 1
assert updated.data_created == data_creates
assert updated.data_called == 1
updated = Update()
with bulk_operations(models=(StoreChecksData, QualityCheck)):
qc_creates = _create_qc_events(unit)
data_creates = _create_data_events(unit)
assert updated.qc_created == qc_creates
assert updated.qc_called == 1
assert updated.data_created == data_creates
assert updated.data_called == 1
def test_contextmanager_bulk_ops_update(tp0, store0):
unit = store0.units.first()
store1 = tp0.stores.filter(name="store1.po").first()
store2 = tp0.stores.filter(name="store2.po").first()
class Update(object):
store_updated = None
unit_updated = None
store_called = 0
unit_called = 0
unit_updates = None
with keep_data(signals=[update]):
updated = Update()
@receiver(update, sender=Unit)
def handle_unit_update(**kwargs):
assert "instance" not in kwargs
updated.unit_updated = kwargs["objects"]
updated.unit_called += 1
updated.unit_update_fields = kwargs.get("update_fields")
updated.unit_updates = kwargs.get("updates")
@receiver(update, sender=Store)
def handle_store_update(**kwargs):
updated.store_called += 1
if "objects" in kwargs:
updated.store_updated = kwargs["objects"]
else:
assert "instance" in kwargs
with bulk_operations(Unit):
update.send(Unit, instance=unit)
update.send(Unit, objects=list(store1.unit_set.all()))
update.send(Unit, objects=list(store2.unit_set.all()))
update.send(Unit, update_fields=set(["foo", "bar"]))
update.send(Unit, update_fields=set(["bar", "baz"]))
update.send(Store, instance=store0)
update.send(
Store,
objects=list(store1.translation_project.stores.filter(
id=store1.id)))
update.send(
Store,
objects=list(store2.translation_project.stores.filter(
id=store2.id)))
assert updated.unit_called == 1
assert isinstance(updated.unit_updated, list)
assert qs_match(
Unit.objects.filter(
id__in=(
un.id for un in updated.unit_updated)),
(store0.unit_set.filter(id=unit.id)
| store1.unit_set.all()
| store2.unit_set.all()))
assert updated.store_called == 3
assert updated.unit_update_fields == set(["bar", "baz", "foo"])
updated = Update()
d1 = {23: dict(foo=True), 45: dict(bar=False)}
d2 = {67: dict(baz=89)}
with bulk_operations(Unit):
update.send(Unit, updates=d1)
update.send(Unit, updates=d2)
d1.update(d2)
assert updated.unit_updates == d1
def init_db(self, create_projects=True):
with keep_data(signals=(update_revisions, )):
with bulk_operations(models=(TPData, TPChecksData)):
self._init_db(create_projects)