def _update(self, store, user=None, store_revision=None, submission_type=None, resolve_conflict=POOTLE_WINS, allow_add_and_obsolete=True): logging.debug(u"Updating %s", self.target_store.pootle_path) old_state = self.target_store.state if user is None: User = get_user_model() user = User.objects.get_system_user() update_revision = None changes = {} try: diff = StoreDiff(self.target_store, store, store_revision).diff() if diff is not None: update_revision = Revision.incr() changes = self.update_from_diff( store, store_revision, diff, update_revision, user, submission_type, resolve_conflict, allow_add_and_obsolete) finally: if old_state < PARSED: self.target_store.state = PARSED self.target_store.save() has_changed = any(x > 0 for x in changes.values()) if has_changed: log(u"[update] %s units in %s [revision: %d]" % (get_change_str(changes), self.target_store.pootle_path, (self.target_store.data.max_unit_revision or 0))) return update_revision, changes
def log(self): d = { "user": self.user, "action": SCORE_CHANGED, "score_delta": self.score_delta, "code": TranslationActionCodes.NAMES_MAP[self.action_code], "unit": self.submission.unit.id, "wordcount": self.wordcount, "similarity": self.similarity, "total": self.user.score, } params = ["%(user)s", "%(action)s", "%(score_delta)s", "%(code)s", "#%(unit)s"] zero_types = [TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED] no_similarity_types = [ TranslationActionCodes.SUGG_REVIEWED_REJECTED, TranslationActionCodes.SUGG_REVIEWED_ACCEPTED, TranslationActionCodes.REVIEW_PENALTY, TranslationActionCodes.REVIEWED, ] if self.action_code not in zero_types: params.append("NS=%(wordcount)s") if self.action_code not in no_similarity_types: params.append("S=%(similarity)s") params.append("(total: %(total)s)") log("\t".join(params) % d)
def log(self): d = { 'user': self.user, 'action': SCORE_CHANGED, 'score_delta': self.score_delta, 'code': TranslationActionCodes.NAMES_MAP[self.action_code], 'unit': self.submission.unit.id, 'wordcount': self.wordcount, 'similarity': self.similarity, 'total': self.user.score, } params = ['%(user)s', '%(action)s', '%(score_delta)s', '%(code)s', '#%(unit)s'] zero_types = [ TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED, ] no_similarity_types = [ TranslationActionCodes.SUGG_REVIEWED_REJECTED, TranslationActionCodes.SUGG_REVIEWED_ACCEPTED, TranslationActionCodes.REVIEW_PENALTY, TranslationActionCodes.REVIEWED, ] if self.action_code not in zero_types: params.append('NS=%(wordcount)s') if self.action_code not in no_similarity_types: params.append('S=%(similarity)s') params.append('(total: %(total)s)') log("\t".join(params) % d)
def _statslog(instance, *args, **kwargs): start = datetime.now() result = function(instance, *args, **kwargs) end = datetime.now() log("%s(%s)\t%s\t%s" % (function.__name__, ', '.join(args), end - start, instance.get_cachekey())) return result
def update_file_store(self, changes, last_revision, user): self.syncer.update_store_header(self.file.store, user=user) self.file.savestore() self.file_mtime = self.get_file_mtime() log(u"[sync] File saved; %s units in %s [revision: %d]" % (get_change_str(changes), self.pootle_path, last_revision))
def add_paid_task(request): form = PaidTaskForm(request.POST) if form.is_valid(): form.save() obj = form.instance log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj)) return JsonResponse({'result': obj.id}) return JsonResponseBadRequest({'errors': form.errors})
def add_paid_task(request): form = PaidTaskForm(request.POST) if form.is_valid(): form.save() obj = form.instance log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj)) return HttpResponse(jsonify({'result': obj.id}), content_type="application/json") return HttpResponseBadRequest(jsonify({'errors': form.errors}), content_type="application/json")
def remove_paid_task(request, task_id=None): if request.method == 'DELETE': try: obj = PaidTask.objects.get(id=task_id) str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED, obj) obj.delete() log(str) return JsonResponse({'removed': 1}) except PaidTask.DoesNotExist: return JsonResponseNotFound({}) return JsonResponseBadRequest({'error': _('Invalid request method')})
def create_store_file(self, last_revision, user): logging.debug(u"Creating file %s", self.store.pootle_path) store = self.convert() if not os.path.exists(os.path.dirname(self.store_file_path)): os.makedirs(os.path.dirname(self.store_file_path)) self.store.file = self.relative_file_path store.savefile(self.store_file_path) log(u"Created file for %s [revision: %d]" % (self.store.pootle_path, last_revision)) self.update_store_header(user=user) self.store.file.savestore() self.store.file_mtime = self.store.get_file_mtime() self.store.last_sync_revision = last_revision self.store.save()
def _clear_cache(self, keys, parents=True, children=False): itemkey = self.get_cachekey() for key in keys: cachekey = iri_to_uri(itemkey + ":" + key) cache.delete(cachekey) if keys: log("%s deleted from %s cache" % (keys, itemkey)) if parents: item_parents = self.get_parents() for p in item_parents: p._clear_cache(keys, parents=parents, children=False) if children: self.initialize_children() for item in self.children: item._clear_cache(keys, parents=False, children=True)
def save_store(self, last_revision, user, changes, updated): # TODO conservative -> not overwrite if updated: self.update_store_header(user=user) self.store.file.savestore() self.store.file_mtime = self.store.get_file_mtime() log(u"[sync] File saved; %s units in %s [revision: %d]" % (get_change_str(changes), self.store.pootle_path, last_revision)) else: logging.info( u"[sync] nothing changed in %s [revision: %d]", self.store.pootle_path, last_revision, ) self.store.last_sync_revision = last_revision self.store.save()
def remove_paid_task(request, task_id=None): if request.method == 'DELETE': try: obj = PaidTask.objects.get(id=task_id) str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED, obj) obj.delete() log(str) return HttpResponse(jsonify({'removed': 1}), content_type="application/json") except PaidTask.DoesNotExist: return HttpResponseNotFound({}, content_type="application/json") return HttpResponseBadRequest(jsonify( {'error': _('Invalid request method')}), content_type="application/json")
def save_store(self, last_revision, user, changes, updated): # TODO conservative -> not overwrite if updated: self.update_store_header(user=user) self.store.file.savestore() self.store.file_mtime = self.store.get_file_mtime() log(u"[sync] File saved; %s units in %s [revision: %d]" % (get_change_str(changes), self.store.pootle_path, last_revision)) else: logging.info( u"[sync] nothing changed in %s [revision: %d]", self.store.pootle_path, last_revision) self.store.last_sync_revision = last_revision self.store.save()
def remove_paid_task(request, task_id=None): if request.method == 'DELETE': try: obj = PaidTask.objects.get(id=task_id) str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED, obj) obj.delete() log(str) return HttpResponse(jsonify({'removed': 1}), content_type="application/json") except PaidTask.DoesNotExist: return HttpResponseNotFound({}, content_type="application/json") return HttpResponseBadRequest( jsonify({'error': _('Invalid request method')}), content_type="application/json" )
def update(self, store, user=None, store_revision=None, submission_type=None): logging.debug(u"Updating %s", self.target_store.pootle_path) old_state = self.target_store.state if user is None: User = get_user_model() user = User.objects.get_system_user() update_revision = None changes = {} unsynced_uids = [] try: diff = StoreDiff(self.target_store, store, store_revision).diff() if diff is not None: update_revision = Revision.incr() changes, unsynced_uids = self.update_from_diff( store, store_revision, diff, update_revision, user, submission_type, ) finally: if old_state < PARSED: self.target_store.state = PARSED else: self.target_store.state = old_state has_changed = any(x > 0 for x in changes.values()) self.target_store.save(update_cache=has_changed) if has_changed: log(u"[update] %s units in %s [revision: %d]" % ( get_change_str(changes), self.target_store.pootle_path, self.target_store.get_max_unit_revision(), )) return update_revision, changes, unsynced_uids
def create_file_store(self, last_revision, user): path_parts = split_pootle_path(self.pootle_path) file_path = os.path.join( self.translation_project.abs_real_path, *path_parts[2:]) path_prefix = [path_parts[1]] if self.translation_project.project.get_treestyle() != "gnu": path_prefix.append(path_parts[0]) relative_file_path = os.path.join(*(path_prefix + list(path_parts[2:]))) logging.debug(u"Creating file %s", self.pootle_path) store = self.syncer.convert() if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) self.file = relative_file_path store.savefile(file_path) log(u"Created file for %s [revision: %d]" % (self.pootle_path, last_revision)) self.syncer.update_store_header(store, user=user) self.file.savestore() self.file_mtime = self.get_file_mtime() self.last_sync_revision = last_revision self.save()
def create_file_store(self, last_revision, user): path_parts = split_pootle_path(self.pootle_path) file_path = os.path.join(self.translation_project.abs_real_path, *path_parts[2:]) path_prefix = [path_parts[1]] if self.translation_project.project.get_treestyle() != "gnu": path_prefix.append(path_parts[0]) relative_file_path = os.path.join(*(path_prefix + list(path_parts[2:]))) logging.debug(u"Creating file %s", self.pootle_path) store = self.syncer.convert() if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) self.file = relative_file_path store.savefile(file_path) log(u"Created file for %s [revision: %d]" % (self.pootle_path, last_revision)) self.syncer.update_store_header(store, user=user) self.file.savestore() self.file_mtime = self.get_file_mtime() self.last_sync_revision = last_revision self.save()
def log(self): d = { 'user': self.user, 'action': SCORE_CHANGED, 'score_delta': self.score_delta, 'code': TranslationActionCodes.NAMES_MAP[self.action_code], 'unit': self.submission.unit.id, 'wordcount': self.wordcount, 'similarity': self.similarity, 'total': self.user.score, } params = [ '%(user)s', '%(action)s', '%(score_delta)s', '%(code)s', '#%(unit)s' ] zero_types = [ TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED, ] no_similarity_types = [ TranslationActionCodes.SUGG_REVIEWED_REJECTED, TranslationActionCodes.SUGG_REVIEWED_ACCEPTED, TranslationActionCodes.REVIEW_PENALTY, TranslationActionCodes.REVIEWED, ] if self.action_code not in zero_types: params.append('NS=%(wordcount)s') if self.action_code not in no_similarity_types: params.append('S=%(similarity)s') params.append('(total: %(total)s)') log("\t".join(params) % d)
def log(self): d = { "user": self.user, "action": SCORE_CHANGED, "score_delta": self.score_delta, "code": TranslationActionCodes.NAMES_MAP[self.action_code], "unit": self.submission.unit.id, "wordcount": self.wordcount, "similarity": self.similarity, "total": self.user.score, } params = [ "%(user)s", "%(action)s", "%(score_delta)s", "%(code)s", "#%(unit)s" ] zero_types = [ TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED, ] no_similarity_types = [ TranslationActionCodes.SUGG_REVIEWED_REJECTED, TranslationActionCodes.SUGG_REVIEWED_ACCEPTED, TranslationActionCodes.REVIEW_PENALTY, TranslationActionCodes.REVIEWED, ] if self.action_code not in zero_types: params.append("NS=%(wordcount)s") if self.action_code not in no_similarity_types: params.append("S=%(similarity)s") params.append("(total: %(total)s)") log("\t".join(params) % d)
def _update(self, store, user=None, store_revision=None, submission_type=None, resolve_conflict=POOTLE_WINS, allow_add_and_obsolete=True): logging.debug(u"Updating %s", self.target_store.pootle_path) old_state = self.target_store.state if user is None: User = get_user_model() user = User.objects.get_system_user() update_revision = None changes = {} try: diff = StoreDiff(self.target_store, store, store_revision).diff() if diff is not None: update_revision = Revision.incr() changes = self.update_from_diff(store, store_revision, diff, update_revision, user, submission_type, resolve_conflict, allow_add_and_obsolete) finally: if old_state < PARSED: self.target_store.state = PARSED else: self.target_store.state = old_state has_changed = any(x > 0 for x in changes.values()) self.target_store.save() if has_changed: log(u"[update] %s units in %s [revision: %d]" % (get_change_str(changes), self.target_store.pootle_path, (self.target_store.data.max_unit_revision or 0))) return update_revision, changes
def forwards(self, orm): stats = { 'ONE_TO_ONE_COPY': 0, 'MANY_TO_MANY_COPY': 0, #!!! impossible ? 'PAS_LESS_THAN_PSS': 0, 'NO_PAS_FOR_PSS': 0, 'PSS_LESS_THAN_PAS': 0, #!!! impossible ? 'SUG_CREATED_FROM_SUB': 0, 'NO_SUB_FOR_PAS': 0, 'NO_PSS_FOR_PENDING_PAS': 0, 'DEL_REJECTED_PAS': 0, 'NO_UNIT_FOR_PAS': 0, 'DUPLICATED_SUG': 0, 'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0, 'SUG_CREATION_ERROR': 0, } system = orm['pootle_profile.PootleProfile'].objects.get( user__username='******') # units with suggestions units = orm['pootle_store.Unit'].objects.filter( suggestion__isnull=False).distinct() for unit in units: store_suggestions = unit.suggestion_set.all().order_by('user') for user_id, ss in groupby(store_suggestions, lambda x: x.user.id): ss = list(ss) app_pending_suggestions = orm['pootle_app.Suggestion'].objects \ .filter(unit=unit.id, suggester__id=user_id, state='pending') len_ss = len(ss) len_app_pending_suggestions = len(app_pending_suggestions) if len_ss == len_app_pending_suggestions: if len_ss == 1: pas = app_pending_suggestions[0] sugg = ss[0] sugg.state = pas.state # state == pending sugg.creation_time = pas.creation_time sugg.save() log("pas %d copied and will be deleted" % pas.id) stats['ONE_TO_ONE_COPY'] += 1 pas.delete() else: # there should be no such data in the database log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id)) stats['MANY_TO_MANY_COPY'] += 1 elif len_ss > len_app_pending_suggestions: log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id)) if len_app_pending_suggestions > 0: stats['PAS_LESS_THAN_PSS'] += 1 else: stats['NO_PAS_FOR_PSS'] += 1 app_suggestion_ids = map(lambda x: x.id, app_pending_suggestions) orm['pootle_statistics.Submission'].objects \ .filter(from_suggestion__id__in=app_suggestion_ids) \ .update(from_suggestion=None) app_pending_suggestions.delete() else: # there should be no such data in the database log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id)) stats['PSS_LESS_THAN_PAS'] += 1 # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted for pas in orm['pootle_app.Suggestion'].objects.all(): try: # unit isn't deleted unit = orm['pootle_store.Unit'].objects.get(id=pas.unit) if pas.state == 'accepted': try: # to restore suggestion by submission sub = orm['pootle_statistics.Submission'].objects.get( unit=unit, from_suggestion=pas) target = u"%s" % sub.new_value sugg = { 'target_f': target, 'target_hash': md5(target.encode("utf-8")).hexdigest(), 'unit': unit, 'user': pas.suggester, 'reviewer': pas.reviewer, 'review_time': sub.creation_time, 'state': pas.state, #accepted 'creation_time': None, } try: sugg = orm[ 'pootle_store.Suggestion'].objects.create( **sugg) sub.suggestion = sugg sub.from_suggestion = None sub.save() log("suggestion created from pas %d and sub %d" % (pas.id, sub.id)) stats['SUG_CREATED_FROM_SUB'] += 1 except IntegrityError: log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id)) sub.delete() stats['DUPLICATED_SUG'] += 1 except orm['pootle_statistics.Submission'].DoesNotExist: log('No submission found for pas %d' % pas.id) stats['NO_SUB_FOR_PAS'] += 1 elif pas.state == 'pending': log('No suggestion found for pending pas %d' % pas.id) stats['NO_PSS_FOR_PENDING_PAS'] += 1 else: log('Delete rejected pas %d' % pas.id) stats['DEL_REJECTED_PAS'] += 1 except orm['pootle_store.Unit'].DoesNotExist: log('no unit for %d' % pas.id) stats['NO_UNIT_FOR_PAS'] += 1 pas.delete() for sub in orm['pootle_statistics.Submission'].objects \ .filter(type=SubmissionTypes.SUGG_ACCEPT, suggestion=None, field=SubmissionFields.TARGET): sugg = { 'target_f': sub.new_value, 'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(), 'unit': sub.unit, 'user': sub.submitter, 'reviewer': system, 'review_time': sub.creation_time, 'state': 'accepted', 'creation_time': None, } try: sugg = orm['pootle_store.Suggestion'].objects.create(**sugg) sub.suggestion = sugg sub.save() stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1 except: stats['SUG_CREATION_ERROR'] += 1 log("%s" % stats)
def form_valid(self, form): response = super(PaidTaskFormView, self).form_valid(form) # ignore redirect response log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED, self.object)) return self.render_to_json_response({'result': self.object.id})
def forwards(self, orm): stats = { 'ONE_TO_ONE_COPY': 0, 'MANY_TO_MANY_COPY': 0, #!!! impossible ? 'PAS_LESS_THAN_PSS': 0, 'NO_PAS_FOR_PSS': 0, 'PSS_LESS_THAN_PAS': 0, #!!! impossible ? 'SUG_CREATED_FROM_SUB': 0, 'NO_SUB_FOR_PAS': 0, 'NO_PSS_FOR_PENDING_PAS': 0, 'DEL_REJECTED_PAS': 0, 'NO_UNIT_FOR_PAS': 0, 'DUPLICATED_SUG': 0, } # units with suggestions units = orm['pootle_store.Unit'].objects.filter( suggestion__isnull=False).distinct() for unit in units: store_suggestions = unit.suggestion_set.filter( user__isnull=False).order_by('user') for user_id, ss in groupby(store_suggestions, lambda x: x.user.id): ss = list(ss) app_pending_suggestions = orm['pootle_app.Suggestion'].objects \ .filter(unit=unit.id, suggester__id=user_id, state='pending') len_ss = len(ss) len_app_pending_suggestions = len(app_pending_suggestions) if len_ss == len_app_pending_suggestions: if len_ss == 1: pas = app_pending_suggestions[0] sugg = ss[0] sugg.state = pas.state # state == pending sugg.creation_time = pas.creation_time sugg.translation_project = pas.translation_project sugg.save() log("pas %d copied and will be deleted" % pas.id) stats['ONE_TO_ONE_COPY'] += 1 pas.delete() else: # there should be no such data in the database log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id)) stats['MANY_TO_MANY_COPY'] += 1 elif len_ss > len_app_pending_suggestions: log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id)) if len_app_pending_suggestions > 0: stats['PAS_LESS_THAN_PSS'] += 1 else: stats['NO_PAS_FOR_PSS'] += 1 app_pending_suggestions.delete() else: # there should be no such data in the database log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id)) stats['PSS_LESS_THAN_PAS'] += 1 # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted for pas in orm['pootle_app.Suggestion'].objects.all(): try: # unit isn't deleted unit = orm['pootle_store.Unit'].objects.get(id=pas.unit) if pas.state == 'accepted': try: # to restore suggestion by submission sub = orm['pootle_statistics.Submission'].objects.get( unit=unit, from_suggestion=pas) target = u"%s" % sub.new_value sugg = { "target_hash": md5(target.encode("utf-8")).hexdigest(), "target_f": target, "creation_time": pas.creation_time, "review_time": sub.creation_time, "user_id": pas.suggester_id, "reviewer_id": pas.reviewer_id, "unit_id": unit.pk, "state": pas.state, #accepted "translator_comment_f": None, "translation_project_id": sub.translation_project_id, } try: keys = [ "target_hash", "target_f", "creation_time", "review_time", "user_id", "reviewer_id", "unit_id", "state", "translator_comment_f", "translation_project_id" ] values = [sugg[k] for k in keys] db.execute( """ INSERT INTO pootle_store_suggestion (target_hash, target_f, creation_time, review_time, user_id, reviewer_id, unit_id, state, translator_comment_f, translation_project_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", values) sub.suggestion_id = connection.cursor().lastrowid sub.save() log("suggestion created from pas %d and sub %d" % (pas.id, sub.id)) stats['SUG_CREATED_FROM_SUB'] += 1 except IntegrityError: log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id)) sub.delete() stats['DUPLICATED_SUG'] += 1 except orm['pootle_statistics.Submission'].DoesNotExist: log('No submission found for pas %d' % pas.id) stats['NO_SUB_FOR_PAS'] += 1 elif pas.state == 'pending': log('No suggestion found for pending pas %d' % pas.id) stats['NO_PSS_FOR_PENDING_PAS'] += 1 else: log('Delete rejected pas %d' % pas.id) stats['DEL_REJECTED_PAS'] += 1 except orm['pootle_store.Unit'].DoesNotExist: log('no unit for %d' % pas.id) stats['NO_UNIT_FOR_PAS'] += 1 pas.delete() log("%s" % stats)
def form_valid(self, form): super().form_valid(form) # ignore redirect response log("%s\t%s\t%s" % (self.object.user.username, PAID_TASK_ADDED, self.object)) return JsonResponse({"result": self.object.id})
def forwards(self, orm): stats = { 'ONE_TO_ONE_COPY': 0, 'MANY_TO_MANY_COPY': 0, #!!! impossible ? 'PAS_LESS_THAN_PSS': 0, 'NO_PAS_FOR_PSS': 0, 'PSS_LESS_THAN_PAS': 0, #!!! impossible ? 'SUG_CREATED_FROM_SUB': 0, 'NO_SUB_FOR_PAS': 0, 'NO_PSS_FOR_PENDING_PAS': 0, 'DEL_REJECTED_PAS': 0, 'NO_UNIT_FOR_PAS': 0, 'DUPLICATED_SUG': 0, } # units with suggestions units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct() for unit in units: store_suggestions = unit.suggestion_set.filter(user__isnull=False).order_by('user') for user_id, ss in groupby(store_suggestions, lambda x: x.user.id): ss = list(ss) app_pending_suggestions = orm['pootle_app.Suggestion'].objects \ .filter(unit=unit.id, suggester__id=user_id, state='pending') len_ss = len(ss) len_app_pending_suggestions = len(app_pending_suggestions) if len_ss == len_app_pending_suggestions: if len_ss == 1: pas = app_pending_suggestions[0] sugg = ss[0] sugg.state = pas.state # state == pending sugg.creation_time = pas.creation_time sugg.translation_project = pas.translation_project sugg.save() log("pas %d copied and will be deleted" % pas.id) stats['ONE_TO_ONE_COPY'] += 1 pas.delete() else: # there should be no such data in the database log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id)) stats['MANY_TO_MANY_COPY'] += 1 elif len_ss > len_app_pending_suggestions: log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id)) if len_app_pending_suggestions > 0: stats['PAS_LESS_THAN_PSS'] += 1 else: stats['NO_PAS_FOR_PSS'] += 1 app_pending_suggestions.delete() else: # there should be no such data in the database log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id)) stats['PSS_LESS_THAN_PAS'] += 1 # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted for pas in orm['pootle_app.Suggestion'].objects.all(): try: # unit isn't deleted unit = orm['pootle_store.Unit'].objects.get(id=pas.unit) if pas.state == 'accepted': try: # to restore suggestion by submission sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas) target = u"%s" % sub.new_value sugg = { "target_hash": md5(target.encode("utf-8")).hexdigest(), "target_f": target, "creation_time": pas.creation_time, "review_time": sub.creation_time, "user_id": pas.suggester_id, "reviewer_id": pas.reviewer_id, "unit_id": unit.pk, "state": pas.state, #accepted "translator_comment_f": None, "translation_project_id": sub.translation_project_id, } try: keys = [ "target_hash", "target_f", "creation_time", "review_time", "user_id", "reviewer_id", "unit_id", "state", "translator_comment_f", "translation_project_id" ] values = [sugg[k] for k in keys] db.execute(""" INSERT INTO pootle_store_suggestion (target_hash, target_f, creation_time, review_time, user_id, reviewer_id, unit_id, state, translator_comment_f, translation_project_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""", values) sub.suggestion_id = connection.cursor().lastrowid sub.save() log("suggestion created from pas %d and sub %d" % (pas.id, sub.id)) stats['SUG_CREATED_FROM_SUB'] += 1 except IntegrityError: log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id)) sub.delete() stats['DUPLICATED_SUG'] += 1 except orm['pootle_statistics.Submission'].DoesNotExist: log('No submission found for pas %d' % pas.id) stats['NO_SUB_FOR_PAS'] += 1 elif pas.state == 'pending': log('No suggestion found for pending pas %d' % pas.id) stats['NO_PSS_FOR_PENDING_PAS'] += 1 else: log('Delete rejected pas %d' % pas.id) stats['DEL_REJECTED_PAS'] += 1 except orm['pootle_store.Unit'].DoesNotExist: log('no unit for %d' % pas.id) stats['NO_UNIT_FOR_PAS'] += 1 pas.delete() log("%s" % stats)
def form_valid(self, form): super(AddUserPaidTaskView, self).form_valid(form) # ignore redirect response log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED, self.object)) return JsonResponse({'result': self.object.id})
def forwards(self, orm): stats = { 'ONE_TO_ONE_COPY': 0, 'MANY_TO_MANY_COPY': 0, #!!! impossible ? 'PAS_LESS_THAN_PSS': 0, 'NO_PAS_FOR_PSS': 0, 'PSS_LESS_THAN_PAS': 0, #!!! impossible ? 'SUG_CREATED_FROM_SUB': 0, 'NO_SUB_FOR_PAS': 0, 'NO_PSS_FOR_PENDING_PAS': 0, 'DEL_REJECTED_PAS': 0, 'NO_UNIT_FOR_PAS': 0, 'DUPLICATED_SUG': 0, 'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0, 'SUG_CREATION_ERROR': 0, } system = orm['pootle_profile.PootleProfile'].objects.get(user__username='******') # units with suggestions units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct() for unit in units: store_suggestions = unit.suggestion_set.all().order_by('user') for user_id, ss in groupby(store_suggestions, lambda x: x.user.id): ss = list(ss) app_pending_suggestions = orm['pootle_app.Suggestion'].objects \ .filter(unit=unit.id, suggester__id=user_id, state='pending') len_ss = len(ss) len_app_pending_suggestions = len(app_pending_suggestions) if len_ss == len_app_pending_suggestions: if len_ss == 1: pas = app_pending_suggestions[0] sugg = ss[0] sugg.state = pas.state # state == pending sugg.creation_time = pas.creation_time sugg.save() log("pas %d copied and will be deleted" % pas.id) stats['ONE_TO_ONE_COPY'] += 1 pas.delete() else: # there should be no such data in the database log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id)) stats['MANY_TO_MANY_COPY'] += 1 elif len_ss > len_app_pending_suggestions: log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id)) if len_app_pending_suggestions > 0: stats['PAS_LESS_THAN_PSS'] += 1 else: stats['NO_PAS_FOR_PSS'] += 1 app_suggestion_ids = map(lambda x: x.id, app_pending_suggestions) orm['pootle_statistics.Submission'].objects \ .filter(from_suggestion__id__in=app_suggestion_ids) \ .update(from_suggestion=None) app_pending_suggestions.delete() else: # there should be no such data in the database log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id)) stats['PSS_LESS_THAN_PAS'] += 1 # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted for pas in orm['pootle_app.Suggestion'].objects.all(): try: # unit isn't deleted unit = orm['pootle_store.Unit'].objects.get(id=pas.unit) if pas.state == 'accepted': try: # to restore suggestion by submission sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas) target = u"%s" % sub.new_value sugg = { 'target_f': target, 'target_hash': md5(target.encode("utf-8")).hexdigest(), 'unit': unit, 'user': pas.suggester, 'reviewer': pas.reviewer, 'review_time': sub.creation_time, 'state': pas.state, #accepted 'creation_time': None, } try: sugg = orm['pootle_store.Suggestion'].objects.create(**sugg) sub.suggestion = sugg sub.from_suggestion = None sub.save() log("suggestion created from pas %d and sub %d" % (pas.id, sub.id)) stats['SUG_CREATED_FROM_SUB'] += 1 except IntegrityError: log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id)) sub.delete() stats['DUPLICATED_SUG'] += 1 except orm['pootle_statistics.Submission'].DoesNotExist: log('No submission found for pas %d' % pas.id) stats['NO_SUB_FOR_PAS'] += 1 elif pas.state == 'pending': log('No suggestion found for pending pas %d' % pas.id) stats['NO_PSS_FOR_PENDING_PAS'] += 1 else: log('Delete rejected pas %d' % pas.id) stats['DEL_REJECTED_PAS'] += 1 except orm['pootle_store.Unit'].DoesNotExist: log('no unit for %d' % pas.id) stats['NO_UNIT_FOR_PAS'] += 1 pas.delete() for sub in orm['pootle_statistics.Submission'].objects \ .filter(type=SubmissionTypes.SUGG_ACCEPT, suggestion=None, field=SubmissionFields.TARGET): sugg = { 'target_f': sub.new_value, 'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(), 'unit': sub.unit, 'user': sub.submitter, 'reviewer': system, 'review_time': sub.creation_time, 'state': 'accepted', 'creation_time': None, } try: sugg = orm['pootle_store.Suggestion'].objects.create(**sugg) sub.suggestion = sugg sub.save() stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1 except: stats['SUG_CREATION_ERROR'] += 1 log("%s" % stats)