def setup_submissions(self):
from pootle.core.contextmanagers import update_data_after
from pootle_statistics.models import SubmissionTypes
from pootle_store.constants import UNTRANSLATED
from pootle_store.models import Store, Unit, UnitChange
from django.utils import timezone
year_ago = timezone.now() - relativedelta(years=1)
Unit.objects.update(creation_time=year_ago)
stores = Store.objects.select_related(
"translation_project__project",
"translation_project__language")
units = Unit.objects.filter(store__in=stores)
UnitChange.objects.bulk_create(
UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM)
for unit_id
in units.filter(state__gt=UNTRANSLATED).values_list("id", flat=True))
for store in stores.all():
with update_data_after(store):
for unit in store.unit_set.all():
unit.store = store
self._add_submissions(unit, year_ago)
def setup_submissions(self):
from pootle.core.contextmanagers import update_data_after
from pootle_statistics.models import SubmissionTypes
from pootle_store.constants import UNTRANSLATED
from pootle_store.models import Store, Unit, UnitChange
from django.utils import timezone
year_ago = timezone.now() - relativedelta(years=1)
Unit.objects.update(creation_time=year_ago)
stores = Store.objects.select_related("translation_project__project",
"translation_project__language")
units = Unit.objects.filter(store__in=stores)
UnitChange.objects.bulk_create(
UnitChange(unit_id=unit_id, changed_with=SubmissionTypes.SYSTEM)
for unit_id in units.filter(
state__gt=UNTRANSLATED).values_list("id", flat=True))
for store in stores.all():
with update_data_after(store):
for unit in store.unit_set.all():
unit.store = store
self._add_submissions(unit, year_ago)
def test_data_store_critical_checks(store0):
qc_qs = QualityCheck.objects
qc_qs = (
qc_qs.filter(unit__store=store0)
.filter(unit__state__gt=UNTRANSLATED)
.filter(category=Category.CRITICAL)
.exclude(false_positive=True))
check_count = qc_qs.count()
assert (
store0.data.critical_checks
== check_count)
unit = store0.units.exclude(
qualitycheck__isnull=True,
qualitycheck__name__in=["xmltags", "endpunc"]).first()
unit.target = "<foo></bar>;"
unit.save()
unit_critical = unit.qualitycheck_set.filter(
category=Category.CRITICAL).count()
assert (
store0.data.critical_checks
== check_count + unit_critical)
# lets make another unit false positive
other_qc = unit.qualitycheck_set.exclude(
name="xmltags").filter(category=Category.CRITICAL).first()
other_qc.false_positive = True
other_qc.save()
# trigger refresh
with update_data_after(unit.store):
unit.update_qualitychecks(keep_false_positives=True)
assert (
store0.data.critical_checks
== check_count + unit_critical - 1)
def save(self, *args, **kwargs):
kwargs["commit"] = False
unit = super(UnitForm, self).save(*args, **kwargs)
with update_data_after(unit.store):
current_time = timezone.now()
if SubmissionFields.TARGET in (f[0]
for f in self.updated_fields):
unit.submitted_by = self.user
unit.submitted_on = current_time
unit.reviewed_by = None
unit.reviewed_on = None
unit._log_user = self.user
unit.save(
action=self.cleaned_data.get("save_action"),
state_updated=self.cleaned_data.get("state_updated"),
target_updated=self.cleaned_data.get("target_updated"))
suggestion = self.cleaned_data["suggestion"]
translation_project = unit.store.translation_project
for field, old_value, new_value in self.updated_fields:
if field == SubmissionFields.TARGET and suggestion:
old_value = str(suggestion.target_f)
sub = Submission(creation_time=current_time,
translation_project=translation_project,
submitter=self.user,
unit=unit,
store=unit.store,
field=field,
type=SubmissionTypes.NORMAL,
old_value=old_value,
new_value=new_value)
sub.save()
return unit
def save(self, *args, **kwargs):
changed_with = kwargs.pop("changed_with", None)
kwargs["commit"] = False
unit = super(UnitForm, self).save(*args, **kwargs)
with update_data_after(unit.store):
current_time = timezone.now()
if SubmissionFields.TARGET in (f[0]
for f in self.updated_fields):
unit.submitted_by = self.user
unit.submitted_on = current_time
unit.reviewed_by = None
unit.reviewed_on = None
suggestion = self.cleaned_data["suggestion"]
user = (suggestion.user if suggestion else self.user)
unit.save(submitted_on=current_time,
submitted_by=user,
changed_with=changed_with)
translation_project = unit.store.translation_project
for field, old_value, new_value in self.updated_fields:
if field == SubmissionFields.TARGET and suggestion:
old_value = str(suggestion.target_f)
sub = Submission(creation_time=current_time,
translation_project=translation_project,
submitter=self.user,
unit=unit,
store=unit.store,
field=field,
type=SubmissionTypes.NORMAL,
old_value=old_value,
new_value=new_value)
sub.save()
return unit
def test_data_store_critical_checks(store0):
qc_qs = QualityCheck.objects
qc_qs = (
qc_qs.filter(unit__store=store0)
.filter(unit__state__gt=UNTRANSLATED)
.filter(category=Category.CRITICAL)
.exclude(false_positive=True))
check_count = qc_qs.count()
assert (
store0.data.critical_checks
== check_count)
unit = store0.units.exclude(
qualitycheck__isnull=True,
qualitycheck__name__in=["xmltags", "endpunc"]).first()
unit.target = "<foo></bar>;"
unit.save()
unit_critical = unit.qualitycheck_set.filter(
category=Category.CRITICAL).count()
assert (
store0.data.critical_checks
== check_count + unit_critical)
# lets make another unit false positive
other_qc = unit.qualitycheck_set.exclude(
name="xmltags").filter(category=Category.CRITICAL).first()
other_qc.false_positive = True
other_qc.save()
# trigger refresh
with update_data_after(unit.store):
unit.update_qualitychecks(keep_false_positives=True)
assert (
store0.data.critical_checks
== check_count + unit_critical - 1)
def test_data_tp_qc_stats(tp0):
units = Unit.objects.filter(state__gt=OBSOLETE,
store__translation_project=tp0)
qc_qs = QualityCheck.objects
qc_qs = (qc_qs.filter(unit__store__translation_project=tp0).filter(
unit__state__gt=UNTRANSLATED).filter(
category=Category.CRITICAL).exclude(false_positive=True))
check_count = qc_qs.count()
store_data = tp0.data_tool.updater.get_store_data()
assert (store_data["critical_checks"] == tp0.data.critical_checks ==
check_count)
unit = units.exclude(qualitycheck__isnull=True,
qualitycheck__name__in=["xmltags",
"endpunc"]).first()
unit.target = "<foo></bar>;"
unit.save()
unit_critical = unit.qualitycheck_set.filter(
category=Category.CRITICAL).count()
store_data = tp0.data_tool.updater.get_store_data()
tp0.data.refresh_from_db()
assert (store_data["critical_checks"] == tp0.data.critical_checks ==
check_count + unit_critical)
# lets make another unit false positive
other_qc = unit.qualitycheck_set.exclude(name="xmltags").filter(
category=Category.CRITICAL).first()
other_qc.false_positive = True
other_qc.save()
# trigger refresh
with update_data_after(unit.store):
unit.update_qualitychecks(keep_false_positives=True)
store_data = tp0.data_tool.updater.get_store_data()
tp0.data.refresh_from_db()
assert (store_data["critical_checks"] == tp0.data.critical_checks ==
check_count + unit_critical - 1)
def save(self, *args, **kwargs):
changed_with = kwargs.pop("changed_with", None)
kwargs["commit"] = False
unit = super(UnitForm, self).save(*args, **kwargs)
with update_data_after(unit.store):
current_time = timezone.now()
if SubmissionFields.TARGET in (f[0] for f in self.updated_fields):
unit.submitted_by = self.user
unit.submitted_on = current_time
unit.reviewed_by = None
unit.reviewed_on = None
suggestion = self.cleaned_data["suggestion"]
user = (
suggestion.user
if suggestion
else self.user)
unit.save(
submitted_on=current_time,
submitted_by=user,
changed_with=changed_with)
translation_project = unit.store.translation_project
for field, old_value, new_value in self.updated_fields:
if field == SubmissionFields.TARGET and suggestion:
old_value = str(suggestion.target_f)
sub = Submission(
creation_time=current_time,
translation_project=translation_project,
submitter=self.user,
unit=unit,
field=field,
type=SubmissionTypes.WEB,
old_value=old_value,
new_value=new_value)
sub.save()
return unit
def _make_member_updates(store, member):
from pootle.core.contextmanagers import update_data_after
# Member updates first unit, adding a suggestion, and marking unit as fuzzy
with update_data_after(store):
_create_submission_and_suggestion(store, member)
_create_comment_on_unit(store.units[0], member, "NICE COMMENT")
_mark_unit_fuzzy(store.units[0], member)
def _make_member_updates(store, member):
from pootle.core.contextmanagers import update_data_after
# Member updates first unit, adding a suggestion, and marking unit as fuzzy
with update_data_after(store):
_create_submission_and_suggestion(store, member)
_create_comment_on_unit(store.units[0], member, "NICE COMMENT")
_mark_unit_fuzzy(store.units[0], member)
def save(self, *args, **kwargs):
if not self.updated_fields:
return
changed_with = kwargs.pop("changed_with", None)
suggestion = self.cleaned_data["suggestion"]
with update_data_after(self.instance.store):
user = (suggestion.user if suggestion else self.user)
self.instance.save(user=user, changed_with=changed_with)
return self.instance
def store_diff_tests(request, tp0, member, member2):
from pootle.core.contextmanagers import update_data_after
from pootle_store.diff import StoreDiff
store = StoreDBFactory(translation_project=tp0, parent=tp0.directory)
with update_data_after(store):
test = _setup_store_test(store, member, member2,
UPDATE_STORE_TESTS[request.param])
test_store = create_store(units=test[1])
return [StoreDiff(test[0], test_store, test[2])] + list(test[:3])
def update_from_disk(self, overwrite=False):
"""Update DB with units from the disk Store.
:param overwrite: make db match file regardless of last_sync_revision.
"""
changed = False
if not self.target_store.file:
return changed
with update_data_after(self.target_store):
self._update_from_disk(overwrite)
def increment_unsynced_unit_revision(self, update_revision):
filter_by = {
'revision__gt': self.target_store.last_sync_revision,
'revision__lt': update_revision,
'state__gt': OBSOLETE}
units = self.target_store.unit_set.filter(**filter_by)
count = units.count()
if count:
# we update after here to trigger a stats update
# for the store after doing Unit.objects.update()
with update_data_after(self.target_store):
units.update(revision=Revision.incr())
return count
def increment_unsynced_unit_revision(self, update_revision):
filter_by = {
'revision__gt': self.target_store.last_sync_revision,
'revision__lt': update_revision,
'state__gt': OBSOLETE}
units = self.target_store.unit_set.filter(**filter_by)
count = units.count()
if count:
# we update after here to trigger a stats update
# for the store after doing Unit.objects.update()
with update_data_after(self.target_store):
units.update(revision=Revision.incr())
return count
def store_diff_tests(request, tp0, member, member2):
from pootle.core.contextmanagers import update_data_after
from pootle_store.diff import StoreDiff
store = StoreDBFactory(
translation_project=tp0,
parent=tp0.directory)
with update_data_after(store):
test = _setup_store_test(store, member, member2,
UPDATE_STORE_TESTS[request.param])
test_store = create_store(units=test[1])
return [StoreDiff(test[0], test_store, test[2])] + list(test[:3])
def param_update_store_test(request, tp0, member, member2):
from pootle.core.contextmanagers import update_data_after
store = StoreDBFactory(translation_project=tp0, parent=tp0.directory)
with update_data_after(store):
test = _setup_store_test(store, member, member2,
UPDATE_STORE_TESTS[request.param])
update_store(test[0],
units=test[1],
store_revision=test[2],
user=member2,
resolve_conflict=test[3])
return test
def update_units(self, update):
update_count = 0
suggestion_count = 0
if not update.uids:
return update_count, suggestion_count
with update_data_after(self.target_store):
for unit in self.units(update.uids):
updated, suggested = self.unit_updater_class(
unit, update).update_unit()
if updated:
update_count += 1
if suggested:
suggestion_count += 1
return update_count, suggestion_count
def update_units(self, update):
update_count = 0
suggestion_count = 0
if not update.uids:
return update_count, suggestion_count
with update_data_after(self.target_store):
for unit in self.units(update.uids):
updated, suggested = self.unit_updater_class(
unit, update).update_unit()
if updated:
update_count += 1
if suggested:
suggestion_count += 1
return update_count, suggestion_count
def test_contextmanager_keep_data_kwargs(store0, no_update_data_):
result = []
@receiver(update_data, sender=Store)
def update_data_handler(**kwargs):
result.append(kwargs)
with update_data_after(store0):
update_data.send(Store, instance=store0)
# update_data was not called
assert result == []
# update_data called now
assert len(result) == 1
assert result[0]["instance"] == store0
# you can pass args to pass to the update receiver
result.remove(result[0])
with update_data_after(store0, foo="apples", bar="oranges"):
update_data.send(Store, instance=store0)
assert result == []
assert len(result) == 1
assert result[0]["instance"] == store0
assert result[0]["foo"] == "apples"
assert result[0]["bar"] == "oranges"
# you can control the kwargs passed to send inside the context
result.remove(result[0])
kwargs = dict(foo="apples", bar="oranges")
with update_data_after(store0, kwargs=kwargs):
update_data.send(Store, instance=store0)
kwargs["foo"] = "plums"
assert result == []
assert len(result) == 1
assert result[0]["instance"] == store0
assert result[0]["foo"] == "plums"
assert result[0]["bar"] == "oranges"
def test_contextmanager_keep_data_kwargs(store0, no_update_data_):
result = []
@receiver(update_data, sender=Store)
def update_data_handler(**kwargs):
result.append(kwargs)
with update_data_after(store0):
update_data.send(Store, instance=store0)
# update_data was not called
assert result == []
# update_data called now
assert len(result) == 1
assert result[0]["instance"] == store0
# you can pass args to pass to the update receiver
result.remove(result[0])
with update_data_after(store0, foo="apples", bar="oranges"):
update_data.send(Store, instance=store0)
assert result == []
assert len(result) == 1
assert result[0]["instance"] == store0
assert result[0]["foo"] == "apples"
assert result[0]["bar"] == "oranges"
# you can control the kwargs passed to send inside the context
result.remove(result[0])
kwargs = dict(foo="apples", bar="oranges")
with update_data_after(store0, kwargs=kwargs):
update_data.send(Store, instance=store0)
kwargs["foo"] = "plums"
assert result == []
assert len(result) == 1
assert result[0]["instance"] == store0
assert result[0]["foo"] == "plums"
assert result[0]["bar"] == "oranges"
def setup_submissions(self):
from pootle.core.contextmanagers import update_data_after
from pootle_store.models import Store, Unit
from django.utils import timezone
year_ago = timezone.now() - relativedelta(years=1)
Unit.objects.update(creation_time=year_ago)
stores = Store.objects.select_related("translation_project__project",
"translation_project__language")
for store in stores.all():
with update_data_after(store):
for unit in store.unit_set.all():
unit.store = store
self._add_submissions(unit, year_ago)
def handle(self, **options):
users = (list(get_user_model().objects.filter(
username__in=options["users"]).values_list("pk", flat=True))
if options["users"] else None)
if options["reset"]:
score_updater.get(get_user_model())(users=users).clear()
return
for tp in TranslationProject.objects.all():
data_update = update_data_after(tp,
signals=(update_scores, ),
suppress=(tp.__class__, ),
kwargs=dict(users=users))
with data_update:
for store in tp.stores.all():
update_scores.send(store.__class__,
instance=store,
users=users)
def setup_submissions(self):
from pootle.core.contextmanagers import update_data_after
from pootle_store.models import Store, Unit
from django.utils import timezone
year_ago = timezone.now() - relativedelta(years=1)
Unit.objects.update(creation_time=year_ago)
stores = Store.objects.select_related(
"translation_project__project",
"translation_project__language")
for store in stores.all():
with update_data_after(store):
for unit in store.unit_set.all():
unit.store = store
self._add_submissions(unit, year_ago)
def param_update_store_test(request, tp0, member, member2):
from pootle.core.contextmanagers import update_data_after
store = StoreDBFactory(
translation_project=tp0,
parent=tp0.directory)
with update_data_after(store):
test = _setup_store_test(
store, member, member2,
UPDATE_STORE_TESTS[request.param])
update_store(
test[0],
units=test[1],
store_revision=test[2],
user=member2,
resolve_conflict=test[3])
return test
def update_from_diff(self,
store,
store_revision,
to_change,
update_revision,
user,
submission_type,
resolve_conflict=POOTLE_WINS,
allow_add_and_obsolete=True):
changes = {}
if allow_add_and_obsolete:
# Update indexes
for start, delta in to_change["index"]:
self.target_store.update_index(start=start, delta=delta)
# Add new units
with update_data_after(self.target_store):
for unit, new_unit_index in to_change["add"]:
self.target_store.addunit(unit,
new_unit_index,
user=user,
changed_with=submission_type,
update_revision=update_revision)
changes["added"] = len(to_change["add"])
# Obsolete units
changes["obsoleted"] = self.target_store.mark_units_obsolete(
to_change["obsolete"], update_revision)
# Update units
update_dbids, uid_index_map = to_change['update']
update = StoreUpdate(store,
user=user,
submission_type=submission_type,
resolve_conflict=resolve_conflict,
change_indices=allow_add_and_obsolete,
uids=update_dbids,
indices=uid_index_map,
store_revision=store_revision,
update_revision=update_revision)
changes['updated'], changes['suggested'] = self.update_units(update)
return changes
def test_data_tp_qc_stats(tp0):
units = Unit.objects.filter(
state__gt=OBSOLETE,
store__translation_project=tp0)
qc_qs = QualityCheck.objects
qc_qs = (
qc_qs.filter(unit__store__translation_project=tp0)
.filter(unit__state__gt=UNTRANSLATED)
.filter(category=Category.CRITICAL)
.exclude(false_positive=True))
check_count = qc_qs.count()
store_data = tp0.data_tool.updater.get_store_data()
assert (
store_data["critical_checks"]
== tp0.data.critical_checks
== check_count)
unit = units.exclude(
qualitycheck__isnull=True,
qualitycheck__name__in=["xmltags", "endpunc"]).first()
unit.target = "<foo></bar>;"
unit.save()
unit_critical = unit.qualitycheck_set.filter(
category=Category.CRITICAL).count()
store_data = tp0.data_tool.updater.get_store_data()
tp0.data.refresh_from_db()
assert (
store_data["critical_checks"]
== tp0.data.critical_checks
== check_count + unit_critical)
# lets make another unit false positive
other_qc = unit.qualitycheck_set.exclude(
name="xmltags").filter(category=Category.CRITICAL).first()
other_qc.false_positive = True
other_qc.save()
# trigger refresh
with update_data_after(unit.store):
unit.update_qualitychecks(keep_false_positives=True)
store_data = tp0.data_tool.updater.get_store_data()
tp0.data.refresh_from_db()
assert (
store_data["critical_checks"]
== tp0.data.critical_checks
== check_count + unit_critical - 1)
def update_store(self, source, target):
"""Update a target Store from a given source Store"""
source_revision = target.data.max_unit_revision + 1
differ = StoreDiff(target, source, source_revision)
diff = differ.diff()
if diff is None:
return
system = User.objects.get_system_user()
update_revision = Revision.incr()
with update_data_after(target):
return target.updater.update_from_diff(
source,
source_revision,
diff,
update_revision,
system,
SubmissionTypes.SYSTEM,
SOURCE_WINS,
True)
def update_store(self, source, target):
"""Update a target Store from a given source Store"""
source_revision = target.data.max_unit_revision + 1
differ = StoreDiff(target, source, source_revision)
diff = differ.diff()
if diff is None:
return
system = User.objects.get_system_user()
update_revision = Revision.incr()
with update_data_after(target):
return target.updater.update_from_diff(
source,
source_revision,
diff,
update_revision,
system,
SubmissionTypes.SYSTEM,
SOURCE_WINS,
True)
def update_from_diff(self, store, store_revision,
to_change, update_revision, user,
submission_type, resolve_conflict=POOTLE_WINS,
allow_add_and_obsolete=True):
changes = {}
if allow_add_and_obsolete:
# Update indexes
for start, delta in to_change["index"]:
self.target_store.update_index(start=start, delta=delta)
# Add new units
with update_data_after(self.target_store):
for unit, new_unit_index in to_change["add"]:
self.target_store.addunit(
unit, new_unit_index, user=user,
update_revision=update_revision)
changes["added"] = len(to_change["add"])
# Obsolete units
changes["obsoleted"] = self.target_store.mark_units_obsolete(
to_change["obsolete"],
update_revision)
# Update units
update_dbids, uid_index_map = to_change['update']
update = StoreUpdate(
store,
user=user,
submission_type=submission_type,
resolve_conflict=resolve_conflict,
change_indices=allow_add_and_obsolete,
uids=update_dbids,
indices=uid_index_map,
store_revision=store_revision,
update_revision=update_revision)
changes['updated'], changes['suggested'] = self.update_units(update)
return changes
def update_index(self, start, delta):
with update_data_after(self):
Unit.objects.filter(
store_id=self.id,
index__gte=start).update(index=operator.add(F('index'), delta))
def update_index(self, start, delta):
with update_data_after(self):
Unit.objects.filter(store_id=self.id, index__gte=start).update(
index=operator.add(F('index'), delta))
def save(self):
with update_data_after(self.unit.store):
self.save_unit()
def save(self):
with update_data_after(self.unit.store):
self.save_unit()
if self.cleaned_data['comment']:
super(SuggestionSubmitForm, self).save()
def save(self):
with update_data_after(self.unit.store):
self.save_unit()
if self.cleaned_data['comment']:
super(SuggestionSubmitForm, self).save()
def update(self, *args, **kwargs):
with update_data_after(self.target_store):
return self._update(*args, **kwargs)
def save(self):
with update_data_after(self.unit.store):
self.save_unit()
