def refresh_scores(self, users=None, existing=None, existing_tps=None): suppress_tp_scores = keep_data(signals=(update_scores, ), suppress=(TranslationProject, )) existing = existing or self.get_store_scores(self.tp) with bulk_operations(UserTPScore): with suppress_tp_scores: with bulk_operations(UserStoreScore): for store in self.tp.stores.iterator(): score_updater.get(store.__class__)(store).update( users=users, existing=existing.get(store.id)) self.update(users=users, existing=existing_tps)
def refresh_scores(self, users=None, existing=None, existing_tps=None): suppress_tp_scores = keep_data( signals=(update_scores, ), suppress=(TranslationProject, )) existing = existing or self.get_store_scores(self.tp) with bulk_operations(UserTPScore): with suppress_tp_scores: with bulk_operations(UserStoreScore): for store in self.tp.stores.iterator(): score_updater.get(store.__class__)(store).update( users=users, existing=existing.get(store.id)) self.update(users=users, existing=existing_tps)
def _callback_handler(sender, updated, **kwargs): bulk_tps = bulk_operations(models=(get_user_model(), UserTPScore, TPData, TPChecksData)) with keep_data(signals=(update_revisions, )): @receiver(update_revisions) def update_revisions_handler(**kwargs): if updated.revisions is None: updated.revisions = set() instance = kwargs.get("instance") if isinstance(instance, Store): updated.revisions.add(kwargs["instance"].parent.pootle_path) elif isinstance(instance, Directory): updated.revisions.add(kwargs["instance"].pootle_path) with bulk_tps: _update_stores(sender, updated) if updated.tp_data: update_data.send(sender.__class__, instance=sender) if updated.tp_scores: update_scores.send(sender.__class__, instance=sender, users=updated.score_users) if updated.revisions: update_revisions.send(Directory, paths=updated.revisions, keys=["stats", "checks"])
def _update_stores(sender, updated): # call signals for stores bulk_stores = bulk_operations(models=(UserStoreScore, StoreData, StoreChecksData)) with keep_data(suppress=(sender.__class__, )): with bulk_stores: _handle_update_stores(sender, updated)
def _update_stores(sender, updated): # call signals for stores bulk_stores = bulk_operations( models=( UserStoreScore, StoreData, StoreChecksData)) with keep_data(suppress=(sender.__class__, )): with bulk_stores: _handle_update_stores(sender, updated)
def refresh_scores(self, users=None, **kwargs): suppress_user_scores = keep_data(signals=(update_scores, ), suppress=(get_user_model(), )) tp_scores = self.get_tp_scores() with bulk_operations(get_user_model()): with suppress_user_scores: for tp in TranslationProject.objects.all(): score_updater.get(tp.__class__)(tp).refresh_scores( users=users, existing_tps=tp_scores.get(tp.id)) self.update(users=users)
def refresh_scores(self, users=None, **kwargs): suppress_user_scores = keep_data( signals=(update_scores, ), suppress=(get_user_model(), )) tp_scores = self.get_tp_scores() with bulk_operations(get_user_model()): with suppress_user_scores: for tp in TranslationProject.objects.all(): score_updater.get(tp.__class__)(tp).refresh_scores( users=users, existing_tps=tp_scores.get(tp.id)) self.update(users=users)
def update(self, clear_unknown=False, update_data_after=False): """Update/purge all QualityChecks for Units, and expire Store caches. """ if clear_unknown: self.clear_unknown_checks() with bulk_operations(QualityCheck): self.update_untranslated() self.update_translated() updated = self.updated_stores if update_data_after: self.update_data(updated) if "checks" in self.__dict__: del self.__dict__["checks"] self._updated_stores = {} return updated
def test_contextmanager_bulk_ops_delete(tp0, store0): unit = store0.units.first() store1 = tp0.stores.filter(name="store1.po").first() store2 = tp0.stores.filter(name="store2.po").first() class Update(object): store_deleted = None unit_deleted = None store_called = 0 unit_called = 0 with keep_data(signals=[delete]): updated = Update() @receiver(delete, sender=Unit) def handle_unit_delete(**kwargs): assert "instance" not in kwargs updated.unit_deleted = kwargs["objects"] updated.unit_called += 1 @receiver(delete, sender=Store) def handle_store_delete(**kwargs): updated.store_called += 1 if "objects" in kwargs: updated.store_deleted = kwargs["objects"] else: assert "instance" in kwargs with bulk_operations(Unit): delete.send(Unit, instance=unit) delete.send(Unit, objects=store1.unit_set.all()) delete.send(Unit, objects=store2.unit_set.all()) delete.send(Store, instance=store0) delete.send( Store, objects=store1.translation_project.stores.filter( id=store1.id)) delete.send( Store, objects=store2.translation_project.stores.filter( id=store2.id)) assert updated.unit_called == 1 assert qs_match( updated.unit_deleted, (store0.unit_set.filter(id=unit.id) | store1.unit_set.all() | store2.unit_set.all())) assert updated.store_called == 3
def _handle_update_stores(sender, updated): @receiver(update_data, sender=sender.__class__) def update_tp_data_handler(**kwargs): updated.tp_data = True update_data.disconnect( update_tp_data_handler, sender=sender.__class__) @receiver(update_scores, sender=sender.__class__) def update_tp_scores_handler(**kwargs): updated.tp_scores = True update_scores.disconnect( update_tp_scores_handler, sender=sender.__class__) if updated.checks: with keep_data(suppress=(Store, ), signals=(update_data, )): @receiver(update_data, sender=Store) def extra_update_data_handler_(**kwargs): updated.data = updated.data or {} updated.data[kwargs["instance"].id] = kwargs["instance"] with bulk_operations(QualityCheck): for to_check in updated.checks.values(): store = to_check["store"] units = ( [unit for unit in to_check["units"]] if to_check["units"] else None) update_checks.send( store.__class__, instance=store, units=units) if updated.data: stores = updated.data.values() for store in stores: update_data.send( Store, instance=store) if updated.score_stores: for store in updated.score_stores.values(): update_scores.send( store.__class__, instance=store, users=updated.score_users)
def update(self, clear_unknown=False, update_data_after=False): """Update/purge all QualityChecks for Units, and expire Store caches. """ self.log_debug() if clear_unknown: self.clear_unknown_checks() with bulk_operations(QualityCheck): self.update_untranslated() self.update_translated() updated = self.updated_stores if update_data_after: self.update_data(updated) if "checks" in self.__dict__: del self.__dict__["checks"] self._updated_stores = {} return updated
def setup_submissions(self): from django.contrib.auth import get_user_model from django.utils import timezone from pootle.core.contextmanagers import bulk_operations from pootle_data.models import TPChecksData, TPData from pootle_score.models import UserTPScore from pootle_statistics.models import SubmissionTypes from pootle_store.constants import UNTRANSLATED from pootle_store.models import Unit, UnitChange from pootle_translationproject.contextmanagers import update_tp_after from pootle_translationproject.models import TranslationProject year_ago = timezone.now() - relativedelta(years=1) units = Unit.objects.all() units.update(creation_time=year_ago) User = get_user_model() admin = User.objects.get(username="******") member = User.objects.get(username="******") member2 = User.objects.get(username="******") UnitChange.objects.bulk_create( UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM) for unit_id in units.filter(state__gt=UNTRANSLATED).values_list("id", flat=True)) tps = TranslationProject.objects.exclude( language__code="templates").select_related( "language", "project__source_language").all() bulk_pootle = bulk_operations( models=( get_user_model(), UserTPScore, TPData, TPChecksData)) with bulk_pootle: for tp in tps: with update_tp_after(tp): self._add_subs_to_stores( tp.stores, admin, member, member2)
def _callback_handler(sender, updated, **kwargs): bulk_pootle = bulk_operations( models=( get_user_model(), UserTPScore, UserStoreScore, TPData, TPChecksData, StoreData, StoreChecksData)) with keep_data(signals=(update_revisions, )): with bulk_pootle: @receiver(update_revisions) def handle_update_revisions(**kwargs): updated.revisions = True if updated.checks: update_checks.send( sender.__class__, instance=sender, units=updated.checks, **kwargs) if updated.data: update_data.send( sender.__class__, instance=sender, **kwargs) if updated.scores: update_scores.send( sender.__class__, instance=sender, users=updated.scores, **kwargs) if updated.revisions: update_revisions.send( sender.__class__, instance=sender, keys=["stats", "checks"])
def _handle_update_stores(sender, updated): @receiver(update_data, sender=sender.__class__) def update_tp_data_handler(**kwargs): updated.tp_data = True update_data.disconnect(update_tp_data_handler, sender=sender.__class__) @receiver(update_scores, sender=sender.__class__) def update_tp_scores_handler(**kwargs): updated.tp_scores = True update_scores.disconnect(update_tp_scores_handler, sender=sender.__class__) if updated.checks: with keep_data(suppress=(Store, ), signals=(update_data, )): @receiver(update_data, sender=Store) def extra_update_data_handler_(**kwargs): updated.data = updated.data or {} updated.data[kwargs["instance"].id] = kwargs["instance"] with bulk_operations(QualityCheck): for to_check in updated.checks.values(): store = to_check["store"] units = ([unit for unit in to_check["units"]] if to_check["units"] else None) update_checks.send(store.__class__, instance=store, units=units) if updated.data: stores = updated.data.values() for store in stores: update_data.send(Store, instance=store) if updated.score_stores: for store in updated.score_stores.values(): update_scores.send(store.__class__, instance=store, users=updated.score_users)
def _callback_handler(sender, updated, **kwargs): bulk_tps = bulk_operations( models=( get_user_model(), UserTPScore, TPData, TPChecksData)) with keep_data(signals=(update_revisions, )): @receiver(update_revisions) def update_revisions_handler(**kwargs): if updated.revisions is None: updated.revisions = set() instance = kwargs.get("instance") if isinstance(instance, Store): updated.revisions.add(kwargs["instance"].parent.pootle_path) elif isinstance(instance, Directory): updated.revisions.add(kwargs["instance"].pootle_path) with bulk_tps: _update_stores(sender, updated) if updated.tp_data: update_data.send( sender.__class__, instance=sender) if updated.tp_scores: update_scores.send( sender.__class__, instance=sender, users=updated.score_users) if updated.revisions: update_revisions.send( Directory, paths=updated.revisions, keys=["stats", "checks"])
def setup_submissions(self): from django.contrib.auth import get_user_model from django.utils import timezone from pootle.core.contextmanagers import bulk_operations from pootle_data.models import TPChecksData, TPData from pootle_score.models import UserTPScore from pootle_statistics.models import SubmissionTypes from pootle_store.constants import UNTRANSLATED from pootle_store.models import Unit, UnitChange from pootle_translationproject.contextmanagers import update_tp_after from pootle_translationproject.models import TranslationProject year_ago = timezone.now() - relativedelta(years=1) units = Unit.objects.all() units.update(creation_time=year_ago) User = get_user_model() admin = User.objects.get(username="******") member = User.objects.get(username="******") member2 = User.objects.get(username="******") UnitChange.objects.bulk_create( UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM) for unit_id in units.filter( state__gt=UNTRANSLATED).values_list("id", flat=True)) tps = TranslationProject.objects.exclude( language__code="templates").select_related( "language", "project__source_language").all() bulk_pootle = bulk_operations(models=(get_user_model(), UserTPScore, TPData, TPChecksData)) with bulk_pootle: for tp in tps: with update_tp_after(tp): self._add_subs_to_stores(tp.stores, admin, member, member2)
def test_contextmanager_bulk_operations(store0): unit = store0.units.first() class Update(object): data_created = None qc_created = None data_called = 0 qc_called = 0 with keep_data(signals=[create]): updated = Update() @receiver(create, sender=QualityCheck) def handle_qc_create(**kwargs): assert "instance" not in kwargs updated.qc_created = kwargs["objects"] updated.qc_called += 1 @receiver(create, sender=StoreChecksData) def handle_data_create(**kwargs): updated.data_called += 1 if "objects" in kwargs: updated.data_created = kwargs["objects"] else: assert "instance" in kwargs with bulk_operations(QualityCheck): qc_creates = _create_qc_events(unit) data_creates = _create_data_events(unit) assert updated.qc_created == qc_creates assert updated.qc_called == 1 assert updated.data_created != data_creates assert updated.data_called == 5 # nested update updated = Update() with bulk_operations(QualityCheck): with bulk_operations(QualityCheck): qc_creates = _create_qc_events(unit) data_creates = _create_data_events(unit) assert updated.qc_created == qc_creates assert updated.qc_called == 1 assert updated.data_created != data_creates assert updated.data_called == 5 # nested update - different models updated = Update() with bulk_operations(StoreChecksData): with bulk_operations(QualityCheck): qc_creates = _create_qc_events(unit) data_creates = _create_data_events(unit) assert updated.qc_created == qc_creates assert updated.qc_called == 1 assert updated.data_created == data_creates assert updated.data_called == 1 updated = Update() with bulk_operations(models=(StoreChecksData, QualityCheck)): qc_creates = _create_qc_events(unit) data_creates = _create_data_events(unit) assert updated.qc_created == qc_creates assert updated.qc_called == 1 assert updated.data_created == data_creates assert updated.data_called == 1
def test_contextmanager_bulk_ops_update(tp0, store0): unit = store0.units.first() store1 = tp0.stores.filter(name="store1.po").first() store2 = tp0.stores.filter(name="store2.po").first() class Update(object): store_updated = None unit_updated = None store_called = 0 unit_called = 0 unit_updates = None with keep_data(signals=[update]): updated = Update() @receiver(update, sender=Unit) def handle_unit_update(**kwargs): assert "instance" not in kwargs updated.unit_updated = kwargs["objects"] updated.unit_called += 1 updated.unit_update_fields = kwargs.get("update_fields") updated.unit_updates = kwargs.get("updates") @receiver(update, sender=Store) def handle_store_update(**kwargs): updated.store_called += 1 if "objects" in kwargs: updated.store_updated = kwargs["objects"] else: assert "instance" in kwargs with bulk_operations(Unit): update.send(Unit, instance=unit) update.send(Unit, objects=list(store1.unit_set.all())) update.send(Unit, objects=list(store2.unit_set.all())) update.send(Unit, update_fields=set(["foo", "bar"])) update.send(Unit, update_fields=set(["bar", "baz"])) update.send(Store, instance=store0) update.send( Store, objects=list(store1.translation_project.stores.filter( id=store1.id))) update.send( Store, objects=list(store2.translation_project.stores.filter( id=store2.id))) assert updated.unit_called == 1 assert isinstance(updated.unit_updated, list) assert qs_match( Unit.objects.filter( id__in=( un.id for un in updated.unit_updated)), (store0.unit_set.filter(id=unit.id) | store1.unit_set.all() | store2.unit_set.all())) assert updated.store_called == 3 assert updated.unit_update_fields == set(["bar", "baz", "foo"]) updated = Update() d1 = {23: dict(foo=True), 45: dict(bar=False)} d2 = {67: dict(baz=89)} with bulk_operations(Unit): update.send(Unit, updates=d1) update.send(Unit, updates=d2) d1.update(d2) assert updated.unit_updates == d1
def init_db(self, create_projects=True): with keep_data(signals=(update_revisions, )): with bulk_operations(models=(TPData, TPChecksData)): self._init_db(create_projects)