def _update(self, store, user=None, store_revision=None,
submission_type=None, resolve_conflict=POOTLE_WINS,
allow_add_and_obsolete=True):
logging.debug(u"Updating %s", self.target_store.pootle_path)
old_state = self.target_store.state
if user is None:
User = get_user_model()
user = User.objects.get_system_user()
update_revision = None
changes = {}
try:
diff = StoreDiff(self.target_store, store, store_revision).diff()
if diff is not None:
update_revision = Revision.incr()
changes = self.update_from_diff(
store,
store_revision,
diff, update_revision,
user, submission_type,
resolve_conflict,
allow_add_and_obsolete)
finally:
if old_state < PARSED:
self.target_store.state = PARSED
self.target_store.save()
has_changed = any(x > 0 for x in changes.values())
if has_changed:
log(u"[update] %s units in %s [revision: %d]"
% (get_change_str(changes),
self.target_store.pootle_path,
(self.target_store.data.max_unit_revision or 0)))
return update_revision, changes
def log(self):
d = {
"user": self.user,
"action": SCORE_CHANGED,
"score_delta": self.score_delta,
"code": TranslationActionCodes.NAMES_MAP[self.action_code],
"unit": self.submission.unit.id,
"wordcount": self.wordcount,
"similarity": self.similarity,
"total": self.user.score,
}
params = ["%(user)s", "%(action)s", "%(score_delta)s", "%(code)s", "#%(unit)s"]
zero_types = [TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED]
no_similarity_types = [
TranslationActionCodes.SUGG_REVIEWED_REJECTED,
TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
TranslationActionCodes.REVIEW_PENALTY,
TranslationActionCodes.REVIEWED,
]
if self.action_code not in zero_types:
params.append("NS=%(wordcount)s")
if self.action_code not in no_similarity_types:
params.append("S=%(similarity)s")
params.append("(total: %(total)s)")
log("\t".join(params) % d)
def log(self):
d = {
'user': self.user,
'action': SCORE_CHANGED,
'score_delta': self.score_delta,
'code': TranslationActionCodes.NAMES_MAP[self.action_code],
'unit': self.submission.unit.id,
'wordcount': self.wordcount,
'similarity': self.similarity,
'total': self.user.score,
}
params = ['%(user)s', '%(action)s', '%(score_delta)s',
'%(code)s', '#%(unit)s']
zero_types = [
TranslationActionCodes.MARKED_FUZZY,
TranslationActionCodes.DELETED,
]
no_similarity_types = [
TranslationActionCodes.SUGG_REVIEWED_REJECTED,
TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
TranslationActionCodes.REVIEW_PENALTY,
TranslationActionCodes.REVIEWED,
]
if self.action_code not in zero_types:
params.append('NS=%(wordcount)s')
if self.action_code not in no_similarity_types:
params.append('S=%(similarity)s')
params.append('(total: %(total)s)')
log("\t".join(params) % d)
def _statslog(instance, *args, **kwargs):
start = datetime.now()
result = function(instance, *args, **kwargs)
end = datetime.now()
log("%s(%s)\t%s\t%s" % (function.__name__, ', '.join(args), end - start,
instance.get_cachekey()))
return result
def update_file_store(self, changes, last_revision, user):
self.syncer.update_store_header(self.file.store, user=user)
self.file.savestore()
self.file_mtime = self.get_file_mtime()
log(u"[sync] File saved; %s units in %s [revision: %d]" %
(get_change_str(changes),
self.pootle_path,
last_revision))
def add_paid_task(request):
form = PaidTaskForm(request.POST)
if form.is_valid():
form.save()
obj = form.instance
log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj))
return JsonResponse({'result': obj.id})
return JsonResponseBadRequest({'errors': form.errors})
def add_paid_task(request):
form = PaidTaskForm(request.POST)
if form.is_valid():
form.save()
obj = form.instance
log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj))
return HttpResponse(jsonify({'result': obj.id}),
content_type="application/json")
return HttpResponseBadRequest(jsonify({'errors': form.errors}),
content_type="application/json")
def remove_paid_task(request, task_id=None):
if request.method == 'DELETE':
try:
obj = PaidTask.objects.get(id=task_id)
str = '%s\t%s\t%s' % (request.user.username,
PAID_TASK_DELETED, obj)
obj.delete()
log(str)
return JsonResponse({'removed': 1})
except PaidTask.DoesNotExist:
return JsonResponseNotFound({})
return JsonResponseBadRequest({'error': _('Invalid request method')})
def create_store_file(self, last_revision, user):
logging.debug(u"Creating file %s", self.store.pootle_path)
store = self.convert()
if not os.path.exists(os.path.dirname(self.store_file_path)):
os.makedirs(os.path.dirname(self.store_file_path))
self.store.file = self.relative_file_path
store.savefile(self.store_file_path)
log(u"Created file for %s [revision: %d]" %
(self.store.pootle_path, last_revision))
self.update_store_header(user=user)
self.store.file.savestore()
self.store.file_mtime = self.store.get_file_mtime()
self.store.last_sync_revision = last_revision
self.store.save()
def create_store_file(self, last_revision, user):
logging.debug(u"Creating file %s", self.store.pootle_path)
store = self.convert()
if not os.path.exists(os.path.dirname(self.store_file_path)):
os.makedirs(os.path.dirname(self.store_file_path))
self.store.file = self.relative_file_path
store.savefile(self.store_file_path)
log(u"Created file for %s [revision: %d]" %
(self.store.pootle_path, last_revision))
self.update_store_header(user=user)
self.store.file.savestore()
self.store.file_mtime = self.store.get_file_mtime()
self.store.last_sync_revision = last_revision
self.store.save()
def remove_paid_task(request, task_id=None):
if request.method == 'DELETE':
try:
obj = PaidTask.objects.get(id=task_id)
str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED,
obj)
obj.delete()
log(str)
return JsonResponse({'removed': 1})
except PaidTask.DoesNotExist:
return JsonResponseNotFound({})
return JsonResponseBadRequest({'error': _('Invalid request method')})
def _clear_cache(self, keys, parents=True, children=False):
itemkey = self.get_cachekey()
for key in keys:
cachekey = iri_to_uri(itemkey + ":" + key)
cache.delete(cachekey)
if keys:
log("%s deleted from %s cache" % (keys, itemkey))
if parents:
item_parents = self.get_parents()
for p in item_parents:
p._clear_cache(keys, parents=parents, children=False)
if children:
self.initialize_children()
for item in self.children:
item._clear_cache(keys, parents=False, children=True)
def save_store(self, last_revision, user, changes, updated):
# TODO conservative -> not overwrite
if updated:
self.update_store_header(user=user)
self.store.file.savestore()
self.store.file_mtime = self.store.get_file_mtime()
log(u"[sync] File saved; %s units in %s [revision: %d]" %
(get_change_str(changes), self.store.pootle_path,
last_revision))
else:
logging.info(
u"[sync] nothing changed in %s [revision: %d]",
self.store.pootle_path,
last_revision,
)
self.store.last_sync_revision = last_revision
self.store.save()
def remove_paid_task(request, task_id=None):
if request.method == 'DELETE':
try:
obj = PaidTask.objects.get(id=task_id)
str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED,
obj)
obj.delete()
log(str)
return HttpResponse(jsonify({'removed': 1}),
content_type="application/json")
except PaidTask.DoesNotExist:
return HttpResponseNotFound({}, content_type="application/json")
return HttpResponseBadRequest(jsonify(
{'error': _('Invalid request method')}),
content_type="application/json")
def _clear_cache(self, keys, parents=True, children=False):
itemkey = self.get_cachekey()
for key in keys:
cachekey = iri_to_uri(itemkey + ":" + key)
cache.delete(cachekey)
if keys:
log("%s deleted from %s cache" % (keys, itemkey))
if parents:
item_parents = self.get_parents()
for p in item_parents:
p._clear_cache(keys, parents=parents, children=False)
if children:
self.initialize_children()
for item in self.children:
item._clear_cache(keys, parents=False, children=True)
def save_store(self, last_revision, user, changes, updated):
# TODO conservative -> not overwrite
if updated:
self.update_store_header(user=user)
self.store.file.savestore()
self.store.file_mtime = self.store.get_file_mtime()
log(u"[sync] File saved; %s units in %s [revision: %d]" %
(get_change_str(changes),
self.store.pootle_path,
last_revision))
else:
logging.info(
u"[sync] nothing changed in %s [revision: %d]",
self.store.pootle_path,
last_revision)
self.store.last_sync_revision = last_revision
self.store.save()
def remove_paid_task(request, task_id=None):
if request.method == 'DELETE':
try:
obj = PaidTask.objects.get(id=task_id)
str = '%s\t%s\t%s' % (request.user.username,
PAID_TASK_DELETED, obj)
obj.delete()
log(str)
return HttpResponse(jsonify({'removed': 1}),
content_type="application/json")
except PaidTask.DoesNotExist:
return HttpResponseNotFound({}, content_type="application/json")
return HttpResponseBadRequest(
jsonify({'error': _('Invalid request method')}),
content_type="application/json"
)
def update(self,
store,
user=None,
store_revision=None,
submission_type=None):
logging.debug(u"Updating %s", self.target_store.pootle_path)
old_state = self.target_store.state
if user is None:
User = get_user_model()
user = User.objects.get_system_user()
update_revision = None
changes = {}
unsynced_uids = []
try:
diff = StoreDiff(self.target_store, store, store_revision).diff()
if diff is not None:
update_revision = Revision.incr()
changes, unsynced_uids = self.update_from_diff(
store,
store_revision,
diff,
update_revision,
user,
submission_type,
)
finally:
if old_state < PARSED:
self.target_store.state = PARSED
else:
self.target_store.state = old_state
has_changed = any(x > 0 for x in changes.values())
self.target_store.save(update_cache=has_changed)
if has_changed:
log(u"[update] %s units in %s [revision: %d]" % (
get_change_str(changes),
self.target_store.pootle_path,
self.target_store.get_max_unit_revision(),
))
return update_revision, changes, unsynced_uids
def create_file_store(self, last_revision, user):
path_parts = split_pootle_path(self.pootle_path)
file_path = os.path.join(
self.translation_project.abs_real_path,
*path_parts[2:])
path_prefix = [path_parts[1]]
if self.translation_project.project.get_treestyle() != "gnu":
path_prefix.append(path_parts[0])
relative_file_path = os.path.join(*(path_prefix + list(path_parts[2:])))
logging.debug(u"Creating file %s", self.pootle_path)
store = self.syncer.convert()
if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
self.file = relative_file_path
store.savefile(file_path)
log(u"Created file for %s [revision: %d]" %
(self.pootle_path, last_revision))
self.syncer.update_store_header(store, user=user)
self.file.savestore()
self.file_mtime = self.get_file_mtime()
self.last_sync_revision = last_revision
self.save()
def create_file_store(self, last_revision, user):
path_parts = split_pootle_path(self.pootle_path)
file_path = os.path.join(self.translation_project.abs_real_path,
*path_parts[2:])
path_prefix = [path_parts[1]]
if self.translation_project.project.get_treestyle() != "gnu":
path_prefix.append(path_parts[0])
relative_file_path = os.path.join(*(path_prefix +
list(path_parts[2:])))
logging.debug(u"Creating file %s", self.pootle_path)
store = self.syncer.convert()
if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
self.file = relative_file_path
store.savefile(file_path)
log(u"Created file for %s [revision: %d]" %
(self.pootle_path, last_revision))
self.syncer.update_store_header(store, user=user)
self.file.savestore()
self.file_mtime = self.get_file_mtime()
self.last_sync_revision = last_revision
self.save()
def log(self):
d = {
'user': self.user,
'action': SCORE_CHANGED,
'score_delta': self.score_delta,
'code': TranslationActionCodes.NAMES_MAP[self.action_code],
'unit': self.submission.unit.id,
'wordcount': self.wordcount,
'similarity': self.similarity,
'total': self.user.score,
}
params = [
'%(user)s', '%(action)s', '%(score_delta)s', '%(code)s',
'#%(unit)s'
]
zero_types = [
TranslationActionCodes.MARKED_FUZZY,
TranslationActionCodes.DELETED,
]
no_similarity_types = [
TranslationActionCodes.SUGG_REVIEWED_REJECTED,
TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
TranslationActionCodes.REVIEW_PENALTY,
TranslationActionCodes.REVIEWED,
]
if self.action_code not in zero_types:
params.append('NS=%(wordcount)s')
if self.action_code not in no_similarity_types:
params.append('S=%(similarity)s')
params.append('(total: %(total)s)')
log("\t".join(params) % d)
def log(self):
d = {
"user": self.user,
"action": SCORE_CHANGED,
"score_delta": self.score_delta,
"code": TranslationActionCodes.NAMES_MAP[self.action_code],
"unit": self.submission.unit.id,
"wordcount": self.wordcount,
"similarity": self.similarity,
"total": self.user.score,
}
params = [
"%(user)s", "%(action)s", "%(score_delta)s", "%(code)s",
"#%(unit)s"
]
zero_types = [
TranslationActionCodes.MARKED_FUZZY,
TranslationActionCodes.DELETED,
]
no_similarity_types = [
TranslationActionCodes.SUGG_REVIEWED_REJECTED,
TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
TranslationActionCodes.REVIEW_PENALTY,
TranslationActionCodes.REVIEWED,
]
if self.action_code not in zero_types:
params.append("NS=%(wordcount)s")
if self.action_code not in no_similarity_types:
params.append("S=%(similarity)s")
params.append("(total: %(total)s)")
log("\t".join(params) % d)
def _update(self,
store,
user=None,
store_revision=None,
submission_type=None,
resolve_conflict=POOTLE_WINS,
allow_add_and_obsolete=True):
logging.debug(u"Updating %s", self.target_store.pootle_path)
old_state = self.target_store.state
if user is None:
User = get_user_model()
user = User.objects.get_system_user()
update_revision = None
changes = {}
try:
diff = StoreDiff(self.target_store, store, store_revision).diff()
if diff is not None:
update_revision = Revision.incr()
changes = self.update_from_diff(store, store_revision, diff,
update_revision, user,
submission_type,
resolve_conflict,
allow_add_and_obsolete)
finally:
if old_state < PARSED:
self.target_store.state = PARSED
else:
self.target_store.state = old_state
has_changed = any(x > 0 for x in changes.values())
self.target_store.save()
if has_changed:
log(u"[update] %s units in %s [revision: %d]" %
(get_change_str(changes), self.target_store.pootle_path,
(self.target_store.data.max_unit_revision or 0)))
return update_revision, changes
def forwards(self, orm):
stats = {
'ONE_TO_ONE_COPY': 0,
'MANY_TO_MANY_COPY': 0, #!!! impossible ?
'PAS_LESS_THAN_PSS': 0,
'NO_PAS_FOR_PSS': 0,
'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
'SUG_CREATED_FROM_SUB': 0,
'NO_SUB_FOR_PAS': 0,
'NO_PSS_FOR_PENDING_PAS': 0,
'DEL_REJECTED_PAS': 0,
'NO_UNIT_FOR_PAS': 0,
'DUPLICATED_SUG': 0,
'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0,
'SUG_CREATION_ERROR': 0,
}
system = orm['pootle_profile.PootleProfile'].objects.get(
user__username='******')
# units with suggestions
units = orm['pootle_store.Unit'].objects.filter(
suggestion__isnull=False).distinct()
for unit in units:
store_suggestions = unit.suggestion_set.all().order_by('user')
for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
ss = list(ss)
app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
.filter(unit=unit.id, suggester__id=user_id, state='pending')
len_ss = len(ss)
len_app_pending_suggestions = len(app_pending_suggestions)
if len_ss == len_app_pending_suggestions:
if len_ss == 1:
pas = app_pending_suggestions[0]
sugg = ss[0]
sugg.state = pas.state # state == pending
sugg.creation_time = pas.creation_time
sugg.save()
log("pas %d copied and will be deleted" % pas.id)
stats['ONE_TO_ONE_COPY'] += 1
pas.delete()
else:
# there should be no such data in the database
log("%d pending suggestions by %d for %d" %
(len_ss, user_id, unit.id))
stats['MANY_TO_MANY_COPY'] += 1
elif len_ss > len_app_pending_suggestions:
log("%d pss > %d pas for unit %d" %
(len_ss, len_app_pending_suggestions, unit.id))
if len_app_pending_suggestions > 0:
stats['PAS_LESS_THAN_PSS'] += 1
else:
stats['NO_PAS_FOR_PSS'] += 1
app_suggestion_ids = map(lambda x: x.id,
app_pending_suggestions)
orm['pootle_statistics.Submission'].objects \
.filter(from_suggestion__id__in=app_suggestion_ids) \
.update(from_suggestion=None)
app_pending_suggestions.delete()
else:
# there should be no such data in the database
log("%d pss < %d pas for %d" %
(len_ss, len_app_pending_suggestions, unit.id))
stats['PSS_LESS_THAN_PAS'] += 1
# all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted
for pas in orm['pootle_app.Suggestion'].objects.all():
try:
# unit isn't deleted
unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)
if pas.state == 'accepted':
try:
# to restore suggestion by submission
sub = orm['pootle_statistics.Submission'].objects.get(
unit=unit, from_suggestion=pas)
target = u"%s" % sub.new_value
sugg = {
'target_f': target,
'target_hash':
md5(target.encode("utf-8")).hexdigest(),
'unit': unit,
'user': pas.suggester,
'reviewer': pas.reviewer,
'review_time': sub.creation_time,
'state': pas.state, #accepted
'creation_time': None,
}
try:
sugg = orm[
'pootle_store.Suggestion'].objects.create(
**sugg)
sub.suggestion = sugg
sub.from_suggestion = None
sub.save()
log("suggestion created from pas %d and sub %d" %
(pas.id, sub.id))
stats['SUG_CREATED_FROM_SUB'] += 1
except IntegrityError:
log("failed to create duplicated suggestion from pas %d and sub %d"
% (pas.id, sub.id))
sub.delete()
stats['DUPLICATED_SUG'] += 1
except orm['pootle_statistics.Submission'].DoesNotExist:
log('No submission found for pas %d' % pas.id)
stats['NO_SUB_FOR_PAS'] += 1
elif pas.state == 'pending':
log('No suggestion found for pending pas %d' % pas.id)
stats['NO_PSS_FOR_PENDING_PAS'] += 1
else:
log('Delete rejected pas %d' % pas.id)
stats['DEL_REJECTED_PAS'] += 1
except orm['pootle_store.Unit'].DoesNotExist:
log('no unit for %d' % pas.id)
stats['NO_UNIT_FOR_PAS'] += 1
pas.delete()
for sub in orm['pootle_statistics.Submission'].objects \
.filter(type=SubmissionTypes.SUGG_ACCEPT,
suggestion=None,
field=SubmissionFields.TARGET):
sugg = {
'target_f': sub.new_value,
'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(),
'unit': sub.unit,
'user': sub.submitter,
'reviewer': system,
'review_time': sub.creation_time,
'state': 'accepted',
'creation_time': None,
}
try:
sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
sub.suggestion = sugg
sub.save()
stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1
except:
stats['SUG_CREATION_ERROR'] += 1
log("%s" % stats)
def form_valid(self, form):
response = super(PaidTaskFormView, self).form_valid(form)
# ignore redirect response
log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED,
self.object))
return self.render_to_json_response({'result': self.object.id})
def forwards(self, orm):
stats = {
'ONE_TO_ONE_COPY': 0,
'MANY_TO_MANY_COPY': 0, #!!! impossible ?
'PAS_LESS_THAN_PSS': 0,
'NO_PAS_FOR_PSS': 0,
'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
'SUG_CREATED_FROM_SUB': 0,
'NO_SUB_FOR_PAS': 0,
'NO_PSS_FOR_PENDING_PAS': 0,
'DEL_REJECTED_PAS': 0,
'NO_UNIT_FOR_PAS': 0,
'DUPLICATED_SUG': 0,
}
# units with suggestions
units = orm['pootle_store.Unit'].objects.filter(
suggestion__isnull=False).distinct()
for unit in units:
store_suggestions = unit.suggestion_set.filter(
user__isnull=False).order_by('user')
for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
ss = list(ss)
app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
.filter(unit=unit.id, suggester__id=user_id, state='pending')
len_ss = len(ss)
len_app_pending_suggestions = len(app_pending_suggestions)
if len_ss == len_app_pending_suggestions:
if len_ss == 1:
pas = app_pending_suggestions[0]
sugg = ss[0]
sugg.state = pas.state # state == pending
sugg.creation_time = pas.creation_time
sugg.translation_project = pas.translation_project
sugg.save()
log("pas %d copied and will be deleted" % pas.id)
stats['ONE_TO_ONE_COPY'] += 1
pas.delete()
else:
# there should be no such data in the database
log("%d pending suggestions by %d for %d" %
(len_ss, user_id, unit.id))
stats['MANY_TO_MANY_COPY'] += 1
elif len_ss > len_app_pending_suggestions:
log("%d pss > %d pas for unit %d" %
(len_ss, len_app_pending_suggestions, unit.id))
if len_app_pending_suggestions > 0:
stats['PAS_LESS_THAN_PSS'] += 1
else:
stats['NO_PAS_FOR_PSS'] += 1
app_pending_suggestions.delete()
else:
# there should be no such data in the database
log("%d pss < %d pas for %d" %
(len_ss, len_app_pending_suggestions, unit.id))
stats['PSS_LESS_THAN_PAS'] += 1
# all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted
for pas in orm['pootle_app.Suggestion'].objects.all():
try:
# unit isn't deleted
unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)
if pas.state == 'accepted':
try:
# to restore suggestion by submission
sub = orm['pootle_statistics.Submission'].objects.get(
unit=unit, from_suggestion=pas)
target = u"%s" % sub.new_value
sugg = {
"target_hash":
md5(target.encode("utf-8")).hexdigest(),
"target_f": target,
"creation_time": pas.creation_time,
"review_time": sub.creation_time,
"user_id": pas.suggester_id,
"reviewer_id": pas.reviewer_id,
"unit_id": unit.pk,
"state": pas.state, #accepted
"translator_comment_f": None,
"translation_project_id":
sub.translation_project_id,
}
try:
keys = [
"target_hash", "target_f", "creation_time",
"review_time", "user_id", "reviewer_id",
"unit_id", "state", "translator_comment_f",
"translation_project_id"
]
values = [sugg[k] for k in keys]
db.execute(
"""
INSERT INTO pootle_store_suggestion
(target_hash, target_f, creation_time, review_time,
user_id, reviewer_id, unit_id, state,
translator_comment_f, translation_project_id)
VALUES
(%s, %s, %s, %s,
%s, %s, %s, %s,
%s, %s)""", values)
sub.suggestion_id = connection.cursor().lastrowid
sub.save()
log("suggestion created from pas %d and sub %d" %
(pas.id, sub.id))
stats['SUG_CREATED_FROM_SUB'] += 1
except IntegrityError:
log("failed to create duplicated suggestion from pas %d and sub %d"
% (pas.id, sub.id))
sub.delete()
stats['DUPLICATED_SUG'] += 1
except orm['pootle_statistics.Submission'].DoesNotExist:
log('No submission found for pas %d' % pas.id)
stats['NO_SUB_FOR_PAS'] += 1
elif pas.state == 'pending':
log('No suggestion found for pending pas %d' % pas.id)
stats['NO_PSS_FOR_PENDING_PAS'] += 1
else:
log('Delete rejected pas %d' % pas.id)
stats['DEL_REJECTED_PAS'] += 1
except orm['pootle_store.Unit'].DoesNotExist:
log('no unit for %d' % pas.id)
stats['NO_UNIT_FOR_PAS'] += 1
pas.delete()
log("%s" % stats)
def form_valid(self, form):
super().form_valid(form)
# ignore redirect response
log("%s\t%s\t%s" %
(self.object.user.username, PAID_TASK_ADDED, self.object))
return JsonResponse({"result": self.object.id})
def forwards(self, orm):
stats = {
'ONE_TO_ONE_COPY': 0,
'MANY_TO_MANY_COPY': 0, #!!! impossible ?
'PAS_LESS_THAN_PSS': 0,
'NO_PAS_FOR_PSS': 0,
'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
'SUG_CREATED_FROM_SUB': 0,
'NO_SUB_FOR_PAS': 0,
'NO_PSS_FOR_PENDING_PAS': 0,
'DEL_REJECTED_PAS': 0,
'NO_UNIT_FOR_PAS': 0,
'DUPLICATED_SUG': 0,
}
# units with suggestions
units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct()
for unit in units:
store_suggestions = unit.suggestion_set.filter(user__isnull=False).order_by('user')
for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
ss = list(ss)
app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
.filter(unit=unit.id, suggester__id=user_id, state='pending')
len_ss = len(ss)
len_app_pending_suggestions = len(app_pending_suggestions)
if len_ss == len_app_pending_suggestions:
if len_ss == 1:
pas = app_pending_suggestions[0]
sugg = ss[0]
sugg.state = pas.state # state == pending
sugg.creation_time = pas.creation_time
sugg.translation_project = pas.translation_project
sugg.save()
log("pas %d copied and will be deleted" % pas.id)
stats['ONE_TO_ONE_COPY'] += 1
pas.delete()
else:
# there should be no such data in the database
log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id))
stats['MANY_TO_MANY_COPY'] += 1
elif len_ss > len_app_pending_suggestions:
log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id))
if len_app_pending_suggestions > 0:
stats['PAS_LESS_THAN_PSS'] += 1
else:
stats['NO_PAS_FOR_PSS'] += 1
app_pending_suggestions.delete()
else:
# there should be no such data in the database
log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id))
stats['PSS_LESS_THAN_PAS'] += 1
# all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted
for pas in orm['pootle_app.Suggestion'].objects.all():
try:
# unit isn't deleted
unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)
if pas.state == 'accepted':
try:
# to restore suggestion by submission
sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas)
target = u"%s" % sub.new_value
sugg = {
"target_hash": md5(target.encode("utf-8")).hexdigest(),
"target_f": target,
"creation_time": pas.creation_time,
"review_time": sub.creation_time,
"user_id": pas.suggester_id,
"reviewer_id": pas.reviewer_id,
"unit_id": unit.pk,
"state": pas.state, #accepted
"translator_comment_f": None,
"translation_project_id": sub.translation_project_id,
}
try:
keys = [
"target_hash", "target_f", "creation_time", "review_time",
"user_id", "reviewer_id", "unit_id", "state",
"translator_comment_f", "translation_project_id"
]
values = [sugg[k] for k in keys]
db.execute("""
INSERT INTO pootle_store_suggestion
(target_hash, target_f, creation_time, review_time,
user_id, reviewer_id, unit_id, state,
translator_comment_f, translation_project_id)
VALUES
(%s, %s, %s, %s,
%s, %s, %s, %s,
%s, %s)""", values)
sub.suggestion_id = connection.cursor().lastrowid
sub.save()
log("suggestion created from pas %d and sub %d" % (pas.id, sub.id))
stats['SUG_CREATED_FROM_SUB'] += 1
except IntegrityError:
log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id))
sub.delete()
stats['DUPLICATED_SUG'] += 1
except orm['pootle_statistics.Submission'].DoesNotExist:
log('No submission found for pas %d' % pas.id)
stats['NO_SUB_FOR_PAS'] += 1
elif pas.state == 'pending':
log('No suggestion found for pending pas %d' % pas.id)
stats['NO_PSS_FOR_PENDING_PAS'] += 1
else:
log('Delete rejected pas %d' % pas.id)
stats['DEL_REJECTED_PAS'] += 1
except orm['pootle_store.Unit'].DoesNotExist:
log('no unit for %d' % pas.id)
stats['NO_UNIT_FOR_PAS'] += 1
pas.delete()
log("%s" % stats)
def update_file_store(self, changes, last_revision, user):
self.syncer.update_store_header(self.file.store, user=user)
self.file.savestore()
self.file_mtime = self.get_file_mtime()
log(u"[sync] File saved; %s units in %s [revision: %d]" %
(get_change_str(changes), self.pootle_path, last_revision))
def form_valid(self, form):
super(AddUserPaidTaskView, self).form_valid(form)
# ignore redirect response
log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED,
self.object))
return JsonResponse({'result': self.object.id})
def form_valid(self, form):
response = super(PaidTaskFormView, self).form_valid(form)
# ignore redirect response
log('%s\t%s\t%s' %
(self.object.user.username, PAID_TASK_ADDED, self.object))
return self.render_to_json_response({'result': self.object.id})
def forwards(self, orm):
stats = {
'ONE_TO_ONE_COPY': 0,
'MANY_TO_MANY_COPY': 0, #!!! impossible ?
'PAS_LESS_THAN_PSS': 0,
'NO_PAS_FOR_PSS': 0,
'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
'SUG_CREATED_FROM_SUB': 0,
'NO_SUB_FOR_PAS': 0,
'NO_PSS_FOR_PENDING_PAS': 0,
'DEL_REJECTED_PAS': 0,
'NO_UNIT_FOR_PAS': 0,
'DUPLICATED_SUG': 0,
'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0,
'SUG_CREATION_ERROR': 0,
}
system = orm['pootle_profile.PootleProfile'].objects.get(user__username='******')
# units with suggestions
units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct()
for unit in units:
store_suggestions = unit.suggestion_set.all().order_by('user')
for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
ss = list(ss)
app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
.filter(unit=unit.id, suggester__id=user_id, state='pending')
len_ss = len(ss)
len_app_pending_suggestions = len(app_pending_suggestions)
if len_ss == len_app_pending_suggestions:
if len_ss == 1:
pas = app_pending_suggestions[0]
sugg = ss[0]
sugg.state = pas.state # state == pending
sugg.creation_time = pas.creation_time
sugg.save()
log("pas %d copied and will be deleted" % pas.id)
stats['ONE_TO_ONE_COPY'] += 1
pas.delete()
else:
# there should be no such data in the database
log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id))
stats['MANY_TO_MANY_COPY'] += 1
elif len_ss > len_app_pending_suggestions:
log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id))
if len_app_pending_suggestions > 0:
stats['PAS_LESS_THAN_PSS'] += 1
else:
stats['NO_PAS_FOR_PSS'] += 1
app_suggestion_ids = map(lambda x: x.id, app_pending_suggestions)
orm['pootle_statistics.Submission'].objects \
.filter(from_suggestion__id__in=app_suggestion_ids) \
.update(from_suggestion=None)
app_pending_suggestions.delete()
else:
# there should be no such data in the database
log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id))
stats['PSS_LESS_THAN_PAS'] += 1
# all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted
for pas in orm['pootle_app.Suggestion'].objects.all():
try:
# unit isn't deleted
unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)
if pas.state == 'accepted':
try:
# to restore suggestion by submission
sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas)
target = u"%s" % sub.new_value
sugg = {
'target_f': target,
'target_hash': md5(target.encode("utf-8")).hexdigest(),
'unit': unit,
'user': pas.suggester,
'reviewer': pas.reviewer,
'review_time': sub.creation_time,
'state': pas.state, #accepted
'creation_time': None,
}
try:
sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
sub.suggestion = sugg
sub.from_suggestion = None
sub.save()
log("suggestion created from pas %d and sub %d" % (pas.id, sub.id))
stats['SUG_CREATED_FROM_SUB'] += 1
except IntegrityError:
log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id))
sub.delete()
stats['DUPLICATED_SUG'] += 1
except orm['pootle_statistics.Submission'].DoesNotExist:
log('No submission found for pas %d' % pas.id)
stats['NO_SUB_FOR_PAS'] += 1
elif pas.state == 'pending':
log('No suggestion found for pending pas %d' % pas.id)
stats['NO_PSS_FOR_PENDING_PAS'] += 1
else:
log('Delete rejected pas %d' % pas.id)
stats['DEL_REJECTED_PAS'] += 1
except orm['pootle_store.Unit'].DoesNotExist:
log('no unit for %d' % pas.id)
stats['NO_UNIT_FOR_PAS'] += 1
pas.delete()
for sub in orm['pootle_statistics.Submission'].objects \
.filter(type=SubmissionTypes.SUGG_ACCEPT,
suggestion=None,
field=SubmissionFields.TARGET):
sugg = {
'target_f': sub.new_value,
'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(),
'unit': sub.unit,
'user': sub.submitter,
'reviewer': system,
'review_time': sub.creation_time,
'state': 'accepted',
'creation_time': None,
}
try:
sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
sub.suggestion = sugg
sub.save()
stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1
except:
stats['SUG_CREATION_ERROR'] += 1
log("%s" % stats)