def setup_submissions(self): from pootle.core.contextmanagers import update_data_after from pootle_statistics.models import SubmissionTypes from pootle_store.constants import UNTRANSLATED from pootle_store.models import Store, Unit, UnitChange from django.utils import timezone year_ago = timezone.now() - relativedelta(years=1) Unit.objects.update(creation_time=year_ago) stores = Store.objects.select_related( "translation_project__project", "translation_project__language") units = Unit.objects.filter(store__in=stores) UnitChange.objects.bulk_create( UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM) for unit_id in units.filter(state__gt=UNTRANSLATED).values_list("id", flat=True)) for store in stores.all(): with update_data_after(store): for unit in store.unit_set.all(): unit.store = store self._add_submissions(unit, year_ago)
def setup_submissions(self): from pootle.core.contextmanagers import update_data_after from pootle_statistics.models import SubmissionTypes from pootle_store.constants import UNTRANSLATED from pootle_store.models import Store, Unit, UnitChange from django.utils import timezone year_ago = timezone.now() - relativedelta(years=1) Unit.objects.update(creation_time=year_ago) stores = Store.objects.select_related("translation_project__project", "translation_project__language") units = Unit.objects.filter(store__in=stores) UnitChange.objects.bulk_create( UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM) for unit_id in units.filter( state__gt=UNTRANSLATED).values_list("id", flat=True)) for store in stores.all(): with update_data_after(store): for unit in store.unit_set.all(): unit.store = store self._add_submissions(unit, year_ago)
def test_data_store_critical_checks(store0): qc_qs = QualityCheck.objects qc_qs = ( qc_qs.filter(unit__store=store0) .filter(unit__state__gt=UNTRANSLATED) .filter(category=Category.CRITICAL) .exclude(false_positive=True)) check_count = qc_qs.count() assert ( store0.data.critical_checks == check_count) unit = store0.units.exclude( qualitycheck__isnull=True, qualitycheck__name__in=["xmltags", "endpunc"]).first() unit.target = "<foo></bar>;" unit.save() unit_critical = unit.qualitycheck_set.filter( category=Category.CRITICAL).count() assert ( store0.data.critical_checks == check_count + unit_critical) # lets make another unit false positive other_qc = unit.qualitycheck_set.exclude( name="xmltags").filter(category=Category.CRITICAL).first() other_qc.false_positive = True other_qc.save() # trigger refresh with update_data_after(unit.store): unit.update_qualitychecks(keep_false_positives=True) assert ( store0.data.critical_checks == check_count + unit_critical - 1)
def save(self, *args, **kwargs): kwargs["commit"] = False unit = super(UnitForm, self).save(*args, **kwargs) with update_data_after(unit.store): current_time = timezone.now() if SubmissionFields.TARGET in (f[0] for f in self.updated_fields): unit.submitted_by = self.user unit.submitted_on = current_time unit.reviewed_by = None unit.reviewed_on = None unit._log_user = self.user unit.save( action=self.cleaned_data.get("save_action"), state_updated=self.cleaned_data.get("state_updated"), target_updated=self.cleaned_data.get("target_updated")) suggestion = self.cleaned_data["suggestion"] translation_project = unit.store.translation_project for field, old_value, new_value in self.updated_fields: if field == SubmissionFields.TARGET and suggestion: old_value = str(suggestion.target_f) sub = Submission(creation_time=current_time, translation_project=translation_project, submitter=self.user, unit=unit, store=unit.store, field=field, type=SubmissionTypes.NORMAL, old_value=old_value, new_value=new_value) sub.save() return unit
def save(self, *args, **kwargs): changed_with = kwargs.pop("changed_with", None) kwargs["commit"] = False unit = super(UnitForm, self).save(*args, **kwargs) with update_data_after(unit.store): current_time = timezone.now() if SubmissionFields.TARGET in (f[0] for f in self.updated_fields): unit.submitted_by = self.user unit.submitted_on = current_time unit.reviewed_by = None unit.reviewed_on = None suggestion = self.cleaned_data["suggestion"] user = (suggestion.user if suggestion else self.user) unit.save(submitted_on=current_time, submitted_by=user, changed_with=changed_with) translation_project = unit.store.translation_project for field, old_value, new_value in self.updated_fields: if field == SubmissionFields.TARGET and suggestion: old_value = str(suggestion.target_f) sub = Submission(creation_time=current_time, translation_project=translation_project, submitter=self.user, unit=unit, store=unit.store, field=field, type=SubmissionTypes.NORMAL, old_value=old_value, new_value=new_value) sub.save() return unit
def test_data_tp_qc_stats(tp0): units = Unit.objects.filter(state__gt=OBSOLETE, store__translation_project=tp0) qc_qs = QualityCheck.objects qc_qs = (qc_qs.filter(unit__store__translation_project=tp0).filter( unit__state__gt=UNTRANSLATED).filter( category=Category.CRITICAL).exclude(false_positive=True)) check_count = qc_qs.count() store_data = tp0.data_tool.updater.get_store_data() assert (store_data["critical_checks"] == tp0.data.critical_checks == check_count) unit = units.exclude(qualitycheck__isnull=True, qualitycheck__name__in=["xmltags", "endpunc"]).first() unit.target = "<foo></bar>;" unit.save() unit_critical = unit.qualitycheck_set.filter( category=Category.CRITICAL).count() store_data = tp0.data_tool.updater.get_store_data() tp0.data.refresh_from_db() assert (store_data["critical_checks"] == tp0.data.critical_checks == check_count + unit_critical) # lets make another unit false positive other_qc = unit.qualitycheck_set.exclude(name="xmltags").filter( category=Category.CRITICAL).first() other_qc.false_positive = True other_qc.save() # trigger refresh with update_data_after(unit.store): unit.update_qualitychecks(keep_false_positives=True) store_data = tp0.data_tool.updater.get_store_data() tp0.data.refresh_from_db() assert (store_data["critical_checks"] == tp0.data.critical_checks == check_count + unit_critical - 1)
def save(self, *args, **kwargs): changed_with = kwargs.pop("changed_with", None) kwargs["commit"] = False unit = super(UnitForm, self).save(*args, **kwargs) with update_data_after(unit.store): current_time = timezone.now() if SubmissionFields.TARGET in (f[0] for f in self.updated_fields): unit.submitted_by = self.user unit.submitted_on = current_time unit.reviewed_by = None unit.reviewed_on = None suggestion = self.cleaned_data["suggestion"] user = ( suggestion.user if suggestion else self.user) unit.save( submitted_on=current_time, submitted_by=user, changed_with=changed_with) translation_project = unit.store.translation_project for field, old_value, new_value in self.updated_fields: if field == SubmissionFields.TARGET and suggestion: old_value = str(suggestion.target_f) sub = Submission( creation_time=current_time, translation_project=translation_project, submitter=self.user, unit=unit, field=field, type=SubmissionTypes.WEB, old_value=old_value, new_value=new_value) sub.save() return unit
def _make_member_updates(store, member): from pootle.core.contextmanagers import update_data_after # Member updates first unit, adding a suggestion, and marking unit as fuzzy with update_data_after(store): _create_submission_and_suggestion(store, member) _create_comment_on_unit(store.units[0], member, "NICE COMMENT") _mark_unit_fuzzy(store.units[0], member)
def save(self, *args, **kwargs): if not self.updated_fields: return changed_with = kwargs.pop("changed_with", None) suggestion = self.cleaned_data["suggestion"] with update_data_after(self.instance.store): user = (suggestion.user if suggestion else self.user) self.instance.save(user=user, changed_with=changed_with) return self.instance
def store_diff_tests(request, tp0, member, member2): from pootle.core.contextmanagers import update_data_after from pootle_store.diff import StoreDiff store = StoreDBFactory(translation_project=tp0, parent=tp0.directory) with update_data_after(store): test = _setup_store_test(store, member, member2, UPDATE_STORE_TESTS[request.param]) test_store = create_store(units=test[1]) return [StoreDiff(test[0], test_store, test[2])] + list(test[:3])
def update_from_disk(self, overwrite=False): """Update DB with units from the disk Store. :param overwrite: make db match file regardless of last_sync_revision. """ changed = False if not self.target_store.file: return changed with update_data_after(self.target_store): self._update_from_disk(overwrite)
def increment_unsynced_unit_revision(self, update_revision): filter_by = { 'revision__gt': self.target_store.last_sync_revision, 'revision__lt': update_revision, 'state__gt': OBSOLETE} units = self.target_store.unit_set.filter(**filter_by) count = units.count() if count: # we update after here to trigger a stats update # for the store after doing Unit.objects.update() with update_data_after(self.target_store): units.update(revision=Revision.incr()) return count
def store_diff_tests(request, tp0, member, member2): from pootle.core.contextmanagers import update_data_after from pootle_store.diff import StoreDiff store = StoreDBFactory( translation_project=tp0, parent=tp0.directory) with update_data_after(store): test = _setup_store_test(store, member, member2, UPDATE_STORE_TESTS[request.param]) test_store = create_store(units=test[1]) return [StoreDiff(test[0], test_store, test[2])] + list(test[:3])
def param_update_store_test(request, tp0, member, member2): from pootle.core.contextmanagers import update_data_after store = StoreDBFactory(translation_project=tp0, parent=tp0.directory) with update_data_after(store): test = _setup_store_test(store, member, member2, UPDATE_STORE_TESTS[request.param]) update_store(test[0], units=test[1], store_revision=test[2], user=member2, resolve_conflict=test[3]) return test
def update_units(self, update): update_count = 0 suggestion_count = 0 if not update.uids: return update_count, suggestion_count with update_data_after(self.target_store): for unit in self.units(update.uids): updated, suggested = self.unit_updater_class( unit, update).update_unit() if updated: update_count += 1 if suggested: suggestion_count += 1 return update_count, suggestion_count
def test_contextmanager_keep_data_kwargs(store0, no_update_data_): result = [] @receiver(update_data, sender=Store) def update_data_handler(**kwargs): result.append(kwargs) with update_data_after(store0): update_data.send(Store, instance=store0) # update_data was not called assert result == [] # update_data called now assert len(result) == 1 assert result[0]["instance"] == store0 # you can pass args to pass to the update receiver result.remove(result[0]) with update_data_after(store0, foo="apples", bar="oranges"): update_data.send(Store, instance=store0) assert result == [] assert len(result) == 1 assert result[0]["instance"] == store0 assert result[0]["foo"] == "apples" assert result[0]["bar"] == "oranges" # you can control the kwargs passed to send inside the context result.remove(result[0]) kwargs = dict(foo="apples", bar="oranges") with update_data_after(store0, kwargs=kwargs): update_data.send(Store, instance=store0) kwargs["foo"] = "plums" assert result == [] assert len(result) == 1 assert result[0]["instance"] == store0 assert result[0]["foo"] == "plums" assert result[0]["bar"] == "oranges"
def setup_submissions(self): from pootle.core.contextmanagers import update_data_after from pootle_store.models import Store, Unit from django.utils import timezone year_ago = timezone.now() - relativedelta(years=1) Unit.objects.update(creation_time=year_ago) stores = Store.objects.select_related("translation_project__project", "translation_project__language") for store in stores.all(): with update_data_after(store): for unit in store.unit_set.all(): unit.store = store self._add_submissions(unit, year_ago)
def handle(self, **options): users = (list(get_user_model().objects.filter( username__in=options["users"]).values_list("pk", flat=True)) if options["users"] else None) if options["reset"]: score_updater.get(get_user_model())(users=users).clear() return for tp in TranslationProject.objects.all(): data_update = update_data_after(tp, signals=(update_scores, ), suppress=(tp.__class__, ), kwargs=dict(users=users)) with data_update: for store in tp.stores.all(): update_scores.send(store.__class__, instance=store, users=users)
def setup_submissions(self): from pootle.core.contextmanagers import update_data_after from pootle_store.models import Store, Unit from django.utils import timezone year_ago = timezone.now() - relativedelta(years=1) Unit.objects.update(creation_time=year_ago) stores = Store.objects.select_related( "translation_project__project", "translation_project__language") for store in stores.all(): with update_data_after(store): for unit in store.unit_set.all(): unit.store = store self._add_submissions(unit, year_ago)
def param_update_store_test(request, tp0, member, member2): from pootle.core.contextmanagers import update_data_after store = StoreDBFactory( translation_project=tp0, parent=tp0.directory) with update_data_after(store): test = _setup_store_test( store, member, member2, UPDATE_STORE_TESTS[request.param]) update_store( test[0], units=test[1], store_revision=test[2], user=member2, resolve_conflict=test[3]) return test
def update_from_diff(self, store, store_revision, to_change, update_revision, user, submission_type, resolve_conflict=POOTLE_WINS, allow_add_and_obsolete=True): changes = {} if allow_add_and_obsolete: # Update indexes for start, delta in to_change["index"]: self.target_store.update_index(start=start, delta=delta) # Add new units with update_data_after(self.target_store): for unit, new_unit_index in to_change["add"]: self.target_store.addunit(unit, new_unit_index, user=user, changed_with=submission_type, update_revision=update_revision) changes["added"] = len(to_change["add"]) # Obsolete units changes["obsoleted"] = self.target_store.mark_units_obsolete( to_change["obsolete"], update_revision) # Update units update_dbids, uid_index_map = to_change['update'] update = StoreUpdate(store, user=user, submission_type=submission_type, resolve_conflict=resolve_conflict, change_indices=allow_add_and_obsolete, uids=update_dbids, indices=uid_index_map, store_revision=store_revision, update_revision=update_revision) changes['updated'], changes['suggested'] = self.update_units(update) return changes
def test_data_tp_qc_stats(tp0): units = Unit.objects.filter( state__gt=OBSOLETE, store__translation_project=tp0) qc_qs = QualityCheck.objects qc_qs = ( qc_qs.filter(unit__store__translation_project=tp0) .filter(unit__state__gt=UNTRANSLATED) .filter(category=Category.CRITICAL) .exclude(false_positive=True)) check_count = qc_qs.count() store_data = tp0.data_tool.updater.get_store_data() assert ( store_data["critical_checks"] == tp0.data.critical_checks == check_count) unit = units.exclude( qualitycheck__isnull=True, qualitycheck__name__in=["xmltags", "endpunc"]).first() unit.target = "<foo></bar>;" unit.save() unit_critical = unit.qualitycheck_set.filter( category=Category.CRITICAL).count() store_data = tp0.data_tool.updater.get_store_data() tp0.data.refresh_from_db() assert ( store_data["critical_checks"] == tp0.data.critical_checks == check_count + unit_critical) # lets make another unit false positive other_qc = unit.qualitycheck_set.exclude( name="xmltags").filter(category=Category.CRITICAL).first() other_qc.false_positive = True other_qc.save() # trigger refresh with update_data_after(unit.store): unit.update_qualitychecks(keep_false_positives=True) store_data = tp0.data_tool.updater.get_store_data() tp0.data.refresh_from_db() assert ( store_data["critical_checks"] == tp0.data.critical_checks == check_count + unit_critical - 1)
def update_store(self, source, target): """Update a target Store from a given source Store""" source_revision = target.data.max_unit_revision + 1 differ = StoreDiff(target, source, source_revision) diff = differ.diff() if diff is None: return system = User.objects.get_system_user() update_revision = Revision.incr() with update_data_after(target): return target.updater.update_from_diff( source, source_revision, diff, update_revision, system, SubmissionTypes.SYSTEM, SOURCE_WINS, True)
def update_from_diff(self, store, store_revision, to_change, update_revision, user, submission_type, resolve_conflict=POOTLE_WINS, allow_add_and_obsolete=True): changes = {} if allow_add_and_obsolete: # Update indexes for start, delta in to_change["index"]: self.target_store.update_index(start=start, delta=delta) # Add new units with update_data_after(self.target_store): for unit, new_unit_index in to_change["add"]: self.target_store.addunit( unit, new_unit_index, user=user, update_revision=update_revision) changes["added"] = len(to_change["add"]) # Obsolete units changes["obsoleted"] = self.target_store.mark_units_obsolete( to_change["obsolete"], update_revision) # Update units update_dbids, uid_index_map = to_change['update'] update = StoreUpdate( store, user=user, submission_type=submission_type, resolve_conflict=resolve_conflict, change_indices=allow_add_and_obsolete, uids=update_dbids, indices=uid_index_map, store_revision=store_revision, update_revision=update_revision) changes['updated'], changes['suggested'] = self.update_units(update) return changes
def update_index(self, start, delta): with update_data_after(self): Unit.objects.filter( store_id=self.id, index__gte=start).update(index=operator.add(F('index'), delta))
def update_index(self, start, delta): with update_data_after(self): Unit.objects.filter(store_id=self.id, index__gte=start).update( index=operator.add(F('index'), delta))
def save(self): with update_data_after(self.unit.store): self.save_unit()
def save(self): with update_data_after(self.unit.store): self.save_unit() if self.cleaned_data['comment']: super(SuggestionSubmitForm, self).save()
def update(self, *args, **kwargs): with update_data_after(self.target_store): return self._update(*args, **kwargs)