Esempio n. 1
0
    def _update(self, store, user=None, store_revision=None,
                submission_type=None, resolve_conflict=POOTLE_WINS,
                allow_add_and_obsolete=True):
        logging.debug(u"Updating %s", self.target_store.pootle_path)
        old_state = self.target_store.state

        if user is None:
            User = get_user_model()
            user = User.objects.get_system_user()

        update_revision = None
        changes = {}
        try:
            diff = StoreDiff(self.target_store, store, store_revision).diff()
            if diff is not None:
                update_revision = Revision.incr()
                changes = self.update_from_diff(
                    store,
                    store_revision,
                    diff, update_revision,
                    user, submission_type,
                    resolve_conflict,
                    allow_add_and_obsolete)
        finally:
            if old_state < PARSED:
                self.target_store.state = PARSED
                self.target_store.save()
            has_changed = any(x > 0 for x in changes.values())
            if has_changed:
                log(u"[update] %s units in %s [revision: %d]"
                    % (get_change_str(changes),
                       self.target_store.pootle_path,
                       (self.target_store.data.max_unit_revision or 0)))
        return update_revision, changes
Esempio n. 2
0
    def log(self):
        d = {
            "user": self.user,
            "action": SCORE_CHANGED,
            "score_delta": self.score_delta,
            "code": TranslationActionCodes.NAMES_MAP[self.action_code],
            "unit": self.submission.unit.id,
            "wordcount": self.wordcount,
            "similarity": self.similarity,
            "total": self.user.score,
        }

        params = ["%(user)s", "%(action)s", "%(score_delta)s", "%(code)s", "#%(unit)s"]

        zero_types = [TranslationActionCodes.MARKED_FUZZY, TranslationActionCodes.DELETED]
        no_similarity_types = [
            TranslationActionCodes.SUGG_REVIEWED_REJECTED,
            TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
            TranslationActionCodes.REVIEW_PENALTY,
            TranslationActionCodes.REVIEWED,
        ]

        if self.action_code not in zero_types:
            params.append("NS=%(wordcount)s")

            if self.action_code not in no_similarity_types:
                params.append("S=%(similarity)s")

        params.append("(total: %(total)s)")

        log("\t".join(params) % d)
Esempio n. 3
0
    def log(self):
        d = {
            'user': self.user,
            'action': SCORE_CHANGED,
            'score_delta': self.score_delta,
            'code': TranslationActionCodes.NAMES_MAP[self.action_code],
            'unit': self.submission.unit.id,
            'wordcount': self.wordcount,
            'similarity': self.similarity,
            'total': self.user.score,
        }

        params = ['%(user)s', '%(action)s', '%(score_delta)s',
                  '%(code)s', '#%(unit)s']

        zero_types = [
            TranslationActionCodes.MARKED_FUZZY,
            TranslationActionCodes.DELETED,
        ]
        no_similarity_types = [
            TranslationActionCodes.SUGG_REVIEWED_REJECTED,
            TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
            TranslationActionCodes.REVIEW_PENALTY,
            TranslationActionCodes.REVIEWED,
        ]

        if self.action_code not in zero_types:
            params.append('NS=%(wordcount)s')

            if self.action_code not in no_similarity_types:
                params.append('S=%(similarity)s')

        params.append('(total: %(total)s)')

        log("\t".join(params) % d)
Esempio n. 4
0
 def _statslog(instance, *args, **kwargs):
     start = datetime.now()
     result = function(instance, *args, **kwargs)
     end = datetime.now()
     log("%s(%s)\t%s\t%s" % (function.__name__, ', '.join(args), end - start,
                             instance.get_cachekey()))
     return result
Esempio n. 5
0
 def update_file_store(self, changes, last_revision, user):
     self.syncer.update_store_header(self.file.store, user=user)
     self.file.savestore()
     self.file_mtime = self.get_file_mtime()
     log(u"[sync] File saved; %s units in %s [revision: %d]" %
         (get_change_str(changes),
          self.pootle_path,
          last_revision))
Esempio n. 6
0
def add_paid_task(request):
    form = PaidTaskForm(request.POST)
    if form.is_valid():
        form.save()
        obj = form.instance
        log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj))
        return JsonResponse({'result': obj.id})

    return JsonResponseBadRequest({'errors': form.errors})
Esempio n. 7
0
def add_paid_task(request):
    form = PaidTaskForm(request.POST)
    if form.is_valid():
        form.save()
        obj = form.instance
        log('%s\t%s\t%s' % (request.user.username, PAID_TASK_ADDED, obj))
        return HttpResponse(jsonify({'result': obj.id}),
                            content_type="application/json")

    return HttpResponseBadRequest(jsonify({'errors': form.errors}),
                                  content_type="application/json")
Esempio n. 8
0
def remove_paid_task(request, task_id=None):
    if request.method == 'DELETE':
        try:
            obj = PaidTask.objects.get(id=task_id)
            str = '%s\t%s\t%s' % (request.user.username,
                                  PAID_TASK_DELETED, obj)
            obj.delete()
            log(str)
            return JsonResponse({'removed': 1})

        except PaidTask.DoesNotExist:
            return JsonResponseNotFound({})

    return JsonResponseBadRequest({'error': _('Invalid request method')})
Esempio n. 9
0
 def create_store_file(self, last_revision, user):
     logging.debug(u"Creating file %s", self.store.pootle_path)
     store = self.convert()
     if not os.path.exists(os.path.dirname(self.store_file_path)):
         os.makedirs(os.path.dirname(self.store_file_path))
     self.store.file = self.relative_file_path
     store.savefile(self.store_file_path)
     log(u"Created file for %s [revision: %d]" %
         (self.store.pootle_path, last_revision))
     self.update_store_header(user=user)
     self.store.file.savestore()
     self.store.file_mtime = self.store.get_file_mtime()
     self.store.last_sync_revision = last_revision
     self.store.save()
Esempio n. 10
0
 def create_store_file(self, last_revision, user):
     logging.debug(u"Creating file %s", self.store.pootle_path)
     store = self.convert()
     if not os.path.exists(os.path.dirname(self.store_file_path)):
         os.makedirs(os.path.dirname(self.store_file_path))
     self.store.file = self.relative_file_path
     store.savefile(self.store_file_path)
     log(u"Created file for %s [revision: %d]" %
         (self.store.pootle_path, last_revision))
     self.update_store_header(user=user)
     self.store.file.savestore()
     self.store.file_mtime = self.store.get_file_mtime()
     self.store.last_sync_revision = last_revision
     self.store.save()
Esempio n. 11
0
File: views.py Progetto: Yelp/pootle
def remove_paid_task(request, task_id=None):
    if request.method == 'DELETE':
        try:
            obj = PaidTask.objects.get(id=task_id)
            str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED,
                                  obj)
            obj.delete()
            log(str)
            return JsonResponse({'removed': 1})

        except PaidTask.DoesNotExist:
            return JsonResponseNotFound({})

    return JsonResponseBadRequest({'error': _('Invalid request method')})
Esempio n. 12
0
    def _clear_cache(self, keys, parents=True, children=False):
        itemkey = self.get_cachekey()
        for key in keys:
            cachekey = iri_to_uri(itemkey + ":" + key)
            cache.delete(cachekey)
        if keys:
            log("%s deleted from %s cache" % (keys, itemkey))

        if parents:
            item_parents = self.get_parents()
            for p in item_parents:
                p._clear_cache(keys, parents=parents, children=False)

        if children:
            self.initialize_children()
            for item in self.children:
                item._clear_cache(keys, parents=False, children=True)
Esempio n. 13
0
 def save_store(self, last_revision, user, changes, updated):
     # TODO conservative -> not overwrite
     if updated:
         self.update_store_header(user=user)
         self.store.file.savestore()
         self.store.file_mtime = self.store.get_file_mtime()
         log(u"[sync] File saved; %s units in %s [revision: %d]" %
             (get_change_str(changes), self.store.pootle_path,
              last_revision))
     else:
         logging.info(
             u"[sync] nothing changed in %s [revision: %d]",
             self.store.pootle_path,
             last_revision,
         )
     self.store.last_sync_revision = last_revision
     self.store.save()
Esempio n. 14
0
def remove_paid_task(request, task_id=None):
    if request.method == 'DELETE':
        try:
            obj = PaidTask.objects.get(id=task_id)
            str = '%s\t%s\t%s' % (request.user.username, PAID_TASK_DELETED,
                                  obj)
            obj.delete()
            log(str)
            return HttpResponse(jsonify({'removed': 1}),
                                content_type="application/json")

        except PaidTask.DoesNotExist:
            return HttpResponseNotFound({}, content_type="application/json")

    return HttpResponseBadRequest(jsonify(
        {'error': _('Invalid request method')}),
                                  content_type="application/json")
Esempio n. 15
0
    def _clear_cache(self, keys, parents=True, children=False):
        itemkey = self.get_cachekey()
        for key in keys:
            cachekey = iri_to_uri(itemkey + ":" + key)
            cache.delete(cachekey)
        if keys:
            log("%s deleted from %s cache" % (keys, itemkey))

        if parents:
            item_parents = self.get_parents()
            for p in item_parents:
                p._clear_cache(keys, parents=parents, children=False)

        if children:
            self.initialize_children()
            for item in self.children:
                item._clear_cache(keys, parents=False, children=True)
Esempio n. 16
0
 def save_store(self, last_revision, user, changes, updated):
     # TODO conservative -> not overwrite
     if updated:
         self.update_store_header(user=user)
         self.store.file.savestore()
         self.store.file_mtime = self.store.get_file_mtime()
         log(u"[sync] File saved; %s units in %s [revision: %d]" %
             (get_change_str(changes),
              self.store.pootle_path,
              last_revision))
     else:
         logging.info(
             u"[sync] nothing changed in %s [revision: %d]",
             self.store.pootle_path,
             last_revision)
     self.store.last_sync_revision = last_revision
     self.store.save()
Esempio n. 17
0
def remove_paid_task(request, task_id=None):
    if request.method == 'DELETE':
        try:
            obj = PaidTask.objects.get(id=task_id)
            str = '%s\t%s\t%s' % (request.user.username,
                                  PAID_TASK_DELETED, obj)
            obj.delete()
            log(str)
            return HttpResponse(jsonify({'removed': 1}),
                                content_type="application/json")

        except PaidTask.DoesNotExist:
            return HttpResponseNotFound({}, content_type="application/json")

    return HttpResponseBadRequest(
        jsonify({'error': _('Invalid request method')}),
        content_type="application/json"
    )
Esempio n. 18
0
    def update(self,
               store,
               user=None,
               store_revision=None,
               submission_type=None):
        logging.debug(u"Updating %s", self.target_store.pootle_path)
        old_state = self.target_store.state

        if user is None:
            User = get_user_model()
            user = User.objects.get_system_user()

        update_revision = None
        changes = {}
        unsynced_uids = []
        try:
            diff = StoreDiff(self.target_store, store, store_revision).diff()
            if diff is not None:
                update_revision = Revision.incr()
                changes, unsynced_uids = self.update_from_diff(
                    store,
                    store_revision,
                    diff,
                    update_revision,
                    user,
                    submission_type,
                )
        finally:
            if old_state < PARSED:
                self.target_store.state = PARSED
            else:
                self.target_store.state = old_state
            has_changed = any(x > 0 for x in changes.values())
            self.target_store.save(update_cache=has_changed)
            if has_changed:
                log(u"[update] %s units in %s [revision: %d]" % (
                    get_change_str(changes),
                    self.target_store.pootle_path,
                    self.target_store.get_max_unit_revision(),
                ))

        return update_revision, changes, unsynced_uids
Esempio n. 19
0
 def create_file_store(self, last_revision, user):
     path_parts = split_pootle_path(self.pootle_path)
     file_path = os.path.join(
         self.translation_project.abs_real_path,
         *path_parts[2:])
     path_prefix = [path_parts[1]]
     if self.translation_project.project.get_treestyle() != "gnu":
         path_prefix.append(path_parts[0])
     relative_file_path = os.path.join(*(path_prefix + list(path_parts[2:])))
     logging.debug(u"Creating file %s", self.pootle_path)
     store = self.syncer.convert()
     if not os.path.exists(os.path.dirname(file_path)):
         os.makedirs(os.path.dirname(file_path))
     self.file = relative_file_path
     store.savefile(file_path)
     log(u"Created file for %s [revision: %d]" %
         (self.pootle_path, last_revision))
     self.syncer.update_store_header(store, user=user)
     self.file.savestore()
     self.file_mtime = self.get_file_mtime()
     self.last_sync_revision = last_revision
     self.save()
Esempio n. 20
0
 def create_file_store(self, last_revision, user):
     path_parts = split_pootle_path(self.pootle_path)
     file_path = os.path.join(self.translation_project.abs_real_path,
                              *path_parts[2:])
     path_prefix = [path_parts[1]]
     if self.translation_project.project.get_treestyle() != "gnu":
         path_prefix.append(path_parts[0])
     relative_file_path = os.path.join(*(path_prefix +
                                         list(path_parts[2:])))
     logging.debug(u"Creating file %s", self.pootle_path)
     store = self.syncer.convert()
     if not os.path.exists(os.path.dirname(file_path)):
         os.makedirs(os.path.dirname(file_path))
     self.file = relative_file_path
     store.savefile(file_path)
     log(u"Created file for %s [revision: %d]" %
         (self.pootle_path, last_revision))
     self.syncer.update_store_header(store, user=user)
     self.file.savestore()
     self.file_mtime = self.get_file_mtime()
     self.last_sync_revision = last_revision
     self.save()
Esempio n. 21
0
    def log(self):
        d = {
            'user': self.user,
            'action': SCORE_CHANGED,
            'score_delta': self.score_delta,
            'code': TranslationActionCodes.NAMES_MAP[self.action_code],
            'unit': self.submission.unit.id,
            'wordcount': self.wordcount,
            'similarity': self.similarity,
            'total': self.user.score,
        }

        params = [
            '%(user)s', '%(action)s', '%(score_delta)s', '%(code)s',
            '#%(unit)s'
        ]

        zero_types = [
            TranslationActionCodes.MARKED_FUZZY,
            TranslationActionCodes.DELETED,
        ]
        no_similarity_types = [
            TranslationActionCodes.SUGG_REVIEWED_REJECTED,
            TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
            TranslationActionCodes.REVIEW_PENALTY,
            TranslationActionCodes.REVIEWED,
        ]

        if self.action_code not in zero_types:
            params.append('NS=%(wordcount)s')

            if self.action_code not in no_similarity_types:
                params.append('S=%(similarity)s')

        params.append('(total: %(total)s)')

        log("\t".join(params) % d)
Esempio n. 22
0
    def log(self):
        d = {
            "user": self.user,
            "action": SCORE_CHANGED,
            "score_delta": self.score_delta,
            "code": TranslationActionCodes.NAMES_MAP[self.action_code],
            "unit": self.submission.unit.id,
            "wordcount": self.wordcount,
            "similarity": self.similarity,
            "total": self.user.score,
        }

        params = [
            "%(user)s", "%(action)s", "%(score_delta)s", "%(code)s",
            "#%(unit)s"
        ]

        zero_types = [
            TranslationActionCodes.MARKED_FUZZY,
            TranslationActionCodes.DELETED,
        ]
        no_similarity_types = [
            TranslationActionCodes.SUGG_REVIEWED_REJECTED,
            TranslationActionCodes.SUGG_REVIEWED_ACCEPTED,
            TranslationActionCodes.REVIEW_PENALTY,
            TranslationActionCodes.REVIEWED,
        ]

        if self.action_code not in zero_types:
            params.append("NS=%(wordcount)s")

            if self.action_code not in no_similarity_types:
                params.append("S=%(similarity)s")

        params.append("(total: %(total)s)")

        log("\t".join(params) % d)
Esempio n. 23
0
    def _update(self,
                store,
                user=None,
                store_revision=None,
                submission_type=None,
                resolve_conflict=POOTLE_WINS,
                allow_add_and_obsolete=True):
        logging.debug(u"Updating %s", self.target_store.pootle_path)
        old_state = self.target_store.state

        if user is None:
            User = get_user_model()
            user = User.objects.get_system_user()

        update_revision = None
        changes = {}
        try:
            diff = StoreDiff(self.target_store, store, store_revision).diff()
            if diff is not None:
                update_revision = Revision.incr()
                changes = self.update_from_diff(store, store_revision, diff,
                                                update_revision, user,
                                                submission_type,
                                                resolve_conflict,
                                                allow_add_and_obsolete)
        finally:
            if old_state < PARSED:
                self.target_store.state = PARSED
            else:
                self.target_store.state = old_state
            has_changed = any(x > 0 for x in changes.values())
            self.target_store.save()
            if has_changed:
                log(u"[update] %s units in %s [revision: %d]" %
                    (get_change_str(changes), self.target_store.pootle_path,
                     (self.target_store.data.max_unit_revision or 0)))
        return update_revision, changes
    def forwards(self, orm):

        stats = {
            'ONE_TO_ONE_COPY': 0,
            'MANY_TO_MANY_COPY': 0,  #!!! impossible ?
            'PAS_LESS_THAN_PSS': 0,
            'NO_PAS_FOR_PSS': 0,
            'PSS_LESS_THAN_PAS': 0,  #!!! impossible ?
            'SUG_CREATED_FROM_SUB': 0,
            'NO_SUB_FOR_PAS': 0,
            'NO_PSS_FOR_PENDING_PAS': 0,
            'DEL_REJECTED_PAS': 0,
            'NO_UNIT_FOR_PAS': 0,
            'DUPLICATED_SUG': 0,
            'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0,
            'SUG_CREATION_ERROR': 0,
        }

        system = orm['pootle_profile.PootleProfile'].objects.get(
            user__username='******')

        # units with suggestions
        units = orm['pootle_store.Unit'].objects.filter(
            suggestion__isnull=False).distinct()
        for unit in units:
            store_suggestions = unit.suggestion_set.all().order_by('user')
            for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
                ss = list(ss)
                app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
                    .filter(unit=unit.id, suggester__id=user_id, state='pending')

                len_ss = len(ss)
                len_app_pending_suggestions = len(app_pending_suggestions)

                if len_ss == len_app_pending_suggestions:
                    if len_ss == 1:
                        pas = app_pending_suggestions[0]
                        sugg = ss[0]
                        sugg.state = pas.state  # state == pending
                        sugg.creation_time = pas.creation_time
                        sugg.save()

                        log("pas %d copied and will be deleted" % pas.id)
                        stats['ONE_TO_ONE_COPY'] += 1
                        pas.delete()

                    else:
                        # there should be no such data in the database
                        log("%d pending suggestions by %d for %d" %
                            (len_ss, user_id, unit.id))
                        stats['MANY_TO_MANY_COPY'] += 1

                elif len_ss > len_app_pending_suggestions:
                    log("%d pss > %d pas for unit %d" %
                        (len_ss, len_app_pending_suggestions, unit.id))
                    if len_app_pending_suggestions > 0:
                        stats['PAS_LESS_THAN_PSS'] += 1
                    else:
                        stats['NO_PAS_FOR_PSS'] += 1

                    app_suggestion_ids = map(lambda x: x.id,
                                             app_pending_suggestions)
                    orm['pootle_statistics.Submission'].objects \
                        .filter(from_suggestion__id__in=app_suggestion_ids) \
                        .update(from_suggestion=None)
                    app_pending_suggestions.delete()

                else:
                    # there should be no such data in the database
                    log("%d pss < %d pas for %d" %
                        (len_ss, len_app_pending_suggestions, unit.id))
                    stats['PSS_LESS_THAN_PAS'] += 1

        # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted

        for pas in orm['pootle_app.Suggestion'].objects.all():
            try:
                # unit isn't deleted
                unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)

                if pas.state == 'accepted':
                    try:
                        # to restore suggestion by submission
                        sub = orm['pootle_statistics.Submission'].objects.get(
                            unit=unit, from_suggestion=pas)
                        target = u"%s" % sub.new_value
                        sugg = {
                            'target_f': target,
                            'target_hash':
                            md5(target.encode("utf-8")).hexdigest(),
                            'unit': unit,
                            'user': pas.suggester,
                            'reviewer': pas.reviewer,
                            'review_time': sub.creation_time,
                            'state': pas.state,  #accepted
                            'creation_time': None,
                        }

                        try:
                            sugg = orm[
                                'pootle_store.Suggestion'].objects.create(
                                    **sugg)
                            sub.suggestion = sugg
                            sub.from_suggestion = None
                            sub.save()

                            log("suggestion created from pas %d and sub %d" %
                                (pas.id, sub.id))
                            stats['SUG_CREATED_FROM_SUB'] += 1

                        except IntegrityError:
                            log("failed to create duplicated suggestion from pas %d and sub %d"
                                % (pas.id, sub.id))

                            sub.delete()
                            stats['DUPLICATED_SUG'] += 1

                    except orm['pootle_statistics.Submission'].DoesNotExist:
                        log('No submission found for pas %d' % pas.id)
                        stats['NO_SUB_FOR_PAS'] += 1

                elif pas.state == 'pending':
                    log('No suggestion found for pending pas %d' % pas.id)
                    stats['NO_PSS_FOR_PENDING_PAS'] += 1
                else:
                    log('Delete rejected pas %d' % pas.id)
                    stats['DEL_REJECTED_PAS'] += 1

            except orm['pootle_store.Unit'].DoesNotExist:
                log('no unit for %d' % pas.id)
                stats['NO_UNIT_FOR_PAS'] += 1

            pas.delete()

        for sub in orm['pootle_statistics.Submission'].objects \
                .filter(type=SubmissionTypes.SUGG_ACCEPT,
                        suggestion=None,
                        field=SubmissionFields.TARGET):

            sugg = {
                'target_f': sub.new_value,
                'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(),
                'unit': sub.unit,
                'user': sub.submitter,
                'reviewer': system,
                'review_time': sub.creation_time,
                'state': 'accepted',
                'creation_time': None,
            }

            try:
                sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
                sub.suggestion = sugg
                sub.save()
                stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1

            except:
                stats['SUG_CREATION_ERROR'] += 1

        log("%s" % stats)
Esempio n. 25
0
 def form_valid(self, form):
     response = super(PaidTaskFormView, self).form_valid(form)
     # ignore redirect response
     log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED,
                         self.object))
     return self.render_to_json_response({'result': self.object.id})
Esempio n. 26
0
    def forwards(self, orm):

        stats = {
            'ONE_TO_ONE_COPY': 0,
            'MANY_TO_MANY_COPY': 0,  #!!! impossible ?
            'PAS_LESS_THAN_PSS': 0,
            'NO_PAS_FOR_PSS': 0,
            'PSS_LESS_THAN_PAS': 0,  #!!! impossible ?
            'SUG_CREATED_FROM_SUB': 0,
            'NO_SUB_FOR_PAS': 0,
            'NO_PSS_FOR_PENDING_PAS': 0,
            'DEL_REJECTED_PAS': 0,
            'NO_UNIT_FOR_PAS': 0,
            'DUPLICATED_SUG': 0,
        }

        # units with suggestions
        units = orm['pootle_store.Unit'].objects.filter(
            suggestion__isnull=False).distinct()
        for unit in units:
            store_suggestions = unit.suggestion_set.filter(
                user__isnull=False).order_by('user')
            for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
                ss = list(ss)
                app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
                    .filter(unit=unit.id, suggester__id=user_id, state='pending')

                len_ss = len(ss)
                len_app_pending_suggestions = len(app_pending_suggestions)

                if len_ss == len_app_pending_suggestions:
                    if len_ss == 1:
                        pas = app_pending_suggestions[0]
                        sugg = ss[0]
                        sugg.state = pas.state  # state == pending
                        sugg.creation_time = pas.creation_time
                        sugg.translation_project = pas.translation_project
                        sugg.save()

                        log("pas %d copied and will be deleted" % pas.id)
                        stats['ONE_TO_ONE_COPY'] += 1
                        pas.delete()

                    else:
                        # there should be no such data in the database
                        log("%d pending suggestions by %d for %d" %
                            (len_ss, user_id, unit.id))
                        stats['MANY_TO_MANY_COPY'] += 1

                elif len_ss > len_app_pending_suggestions:
                    log("%d pss > %d pas for unit %d" %
                        (len_ss, len_app_pending_suggestions, unit.id))
                    if len_app_pending_suggestions > 0:
                        stats['PAS_LESS_THAN_PSS'] += 1
                    else:
                        stats['NO_PAS_FOR_PSS'] += 1

                    app_pending_suggestions.delete()

                else:
                    # there should be no such data in the database
                    log("%d pss < %d pas for %d" %
                        (len_ss, len_app_pending_suggestions, unit.id))
                    stats['PSS_LESS_THAN_PAS'] += 1

        # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted

        for pas in orm['pootle_app.Suggestion'].objects.all():
            try:
                # unit isn't deleted
                unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)

                if pas.state == 'accepted':
                    try:
                        # to restore suggestion by submission
                        sub = orm['pootle_statistics.Submission'].objects.get(
                            unit=unit, from_suggestion=pas)
                        target = u"%s" % sub.new_value
                        sugg = {
                            "target_hash":
                            md5(target.encode("utf-8")).hexdigest(),
                            "target_f": target,
                            "creation_time": pas.creation_time,
                            "review_time": sub.creation_time,
                            "user_id": pas.suggester_id,
                            "reviewer_id": pas.reviewer_id,
                            "unit_id": unit.pk,
                            "state": pas.state,  #accepted
                            "translator_comment_f": None,
                            "translation_project_id":
                            sub.translation_project_id,
                        }

                        try:
                            keys = [
                                "target_hash", "target_f", "creation_time",
                                "review_time", "user_id", "reviewer_id",
                                "unit_id", "state", "translator_comment_f",
                                "translation_project_id"
                            ]
                            values = [sugg[k] for k in keys]
                            db.execute(
                                """
                                INSERT INTO pootle_store_suggestion
                                (target_hash, target_f, creation_time, review_time,
                                 user_id, reviewer_id, unit_id, state,
                                 translator_comment_f, translation_project_id)
                                VALUES
                                (%s, %s, %s, %s,
                                 %s, %s, %s, %s,
                                 %s, %s)""", values)
                            sub.suggestion_id = connection.cursor().lastrowid
                            sub.save()

                            log("suggestion created from pas %d and sub %d" %
                                (pas.id, sub.id))
                            stats['SUG_CREATED_FROM_SUB'] += 1

                        except IntegrityError:
                            log("failed to create duplicated suggestion from pas %d and sub %d"
                                % (pas.id, sub.id))

                            sub.delete()
                            stats['DUPLICATED_SUG'] += 1

                    except orm['pootle_statistics.Submission'].DoesNotExist:
                        log('No submission found for pas %d' % pas.id)
                        stats['NO_SUB_FOR_PAS'] += 1

                elif pas.state == 'pending':
                    log('No suggestion found for pending pas %d' % pas.id)
                    stats['NO_PSS_FOR_PENDING_PAS'] += 1
                else:
                    log('Delete rejected pas %d' % pas.id)
                    stats['DEL_REJECTED_PAS'] += 1

            except orm['pootle_store.Unit'].DoesNotExist:
                log('no unit for %d' % pas.id)
                stats['NO_UNIT_FOR_PAS'] += 1

            pas.delete()

        log("%s" % stats)
Esempio n. 27
0
 def form_valid(self, form):
     super().form_valid(form)
     # ignore redirect response
     log("%s\t%s\t%s" %
         (self.object.user.username, PAID_TASK_ADDED, self.object))
     return JsonResponse({"result": self.object.id})
    def forwards(self, orm):

        stats = {
            'ONE_TO_ONE_COPY': 0,
            'MANY_TO_MANY_COPY': 0, #!!! impossible ?
            'PAS_LESS_THAN_PSS': 0,
            'NO_PAS_FOR_PSS': 0,
            'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
            'SUG_CREATED_FROM_SUB': 0,
            'NO_SUB_FOR_PAS': 0,
            'NO_PSS_FOR_PENDING_PAS': 0,
            'DEL_REJECTED_PAS': 0,
            'NO_UNIT_FOR_PAS': 0,
            'DUPLICATED_SUG': 0,
        }

        # units with suggestions
        units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct()
        for unit in units:
            store_suggestions = unit.suggestion_set.filter(user__isnull=False).order_by('user')
            for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
                ss = list(ss)
                app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
                    .filter(unit=unit.id, suggester__id=user_id, state='pending')

                len_ss = len(ss)
                len_app_pending_suggestions = len(app_pending_suggestions)

                if len_ss == len_app_pending_suggestions:
                    if len_ss == 1:
                        pas = app_pending_suggestions[0]
                        sugg = ss[0]
                        sugg.state = pas.state # state == pending
                        sugg.creation_time = pas.creation_time
                        sugg.translation_project = pas.translation_project
                        sugg.save()

                        log("pas %d copied and will be deleted" % pas.id)
                        stats['ONE_TO_ONE_COPY'] += 1
                        pas.delete()

                    else:
                        # there should be no such data in the database
                        log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id))
                        stats['MANY_TO_MANY_COPY'] += 1

                elif len_ss > len_app_pending_suggestions:
                    log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id))
                    if len_app_pending_suggestions > 0:
                        stats['PAS_LESS_THAN_PSS'] += 1
                    else:
                        stats['NO_PAS_FOR_PSS'] += 1

                    app_pending_suggestions.delete()

                else:
                    # there should be no such data in the database
                    log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id))
                    stats['PSS_LESS_THAN_PAS'] += 1

        # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted

        for pas in orm['pootle_app.Suggestion'].objects.all():
            try:
                # unit isn't deleted
                unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)

                if pas.state == 'accepted':
                    try:
                        # to restore suggestion by submission
                        sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas)
                        target = u"%s" % sub.new_value
                        sugg = {
                            "target_hash": md5(target.encode("utf-8")).hexdigest(),
                            "target_f": target,
                            "creation_time": pas.creation_time,
                            "review_time": sub.creation_time,
                            "user_id": pas.suggester_id,
                            "reviewer_id": pas.reviewer_id,
                            "unit_id": unit.pk,
                            "state": pas.state, #accepted
                            "translator_comment_f": None,
                            "translation_project_id": sub.translation_project_id,
                        }

                        try:
                            keys = [
                                "target_hash", "target_f", "creation_time", "review_time",
                                "user_id", "reviewer_id", "unit_id", "state",
                                "translator_comment_f", "translation_project_id"
                            ]
                            values = [sugg[k] for k in keys]
                            db.execute("""
                                INSERT INTO pootle_store_suggestion
                                (target_hash, target_f, creation_time, review_time,
                                 user_id, reviewer_id, unit_id, state,
                                 translator_comment_f, translation_project_id)
                                VALUES
                                (%s, %s, %s, %s,
                                 %s, %s, %s, %s,
                                 %s, %s)""", values)
                            sub.suggestion_id = connection.cursor().lastrowid
                            sub.save()

                            log("suggestion created from pas %d and sub %d" % (pas.id, sub.id))
                            stats['SUG_CREATED_FROM_SUB'] += 1

                        except IntegrityError:
                            log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id))

                            sub.delete()
                            stats['DUPLICATED_SUG'] += 1

                    except orm['pootle_statistics.Submission'].DoesNotExist:
                        log('No submission found for pas %d' % pas.id)
                        stats['NO_SUB_FOR_PAS'] += 1

                elif pas.state == 'pending':
                    log('No suggestion found for pending pas %d' % pas.id)
                    stats['NO_PSS_FOR_PENDING_PAS'] += 1
                else:
                    log('Delete rejected pas %d' % pas.id)
                    stats['DEL_REJECTED_PAS'] += 1

            except orm['pootle_store.Unit'].DoesNotExist:
                log('no unit for %d' % pas.id)
                stats['NO_UNIT_FOR_PAS'] += 1

            pas.delete()

        log("%s" % stats)
Esempio n. 29
0
 def update_file_store(self, changes, last_revision, user):
     self.syncer.update_store_header(self.file.store, user=user)
     self.file.savestore()
     self.file_mtime = self.get_file_mtime()
     log(u"[sync] File saved; %s units in %s [revision: %d]" %
         (get_change_str(changes), self.pootle_path, last_revision))
Esempio n. 30
0
 def form_valid(self, form):
     super(AddUserPaidTaskView, self).form_valid(form)
     # ignore redirect response
     log('%s\t%s\t%s' % (self.object.user.username, PAID_TASK_ADDED,
                         self.object))
     return JsonResponse({'result': self.object.id})
Esempio n. 31
0
 def form_valid(self, form):
     response = super(PaidTaskFormView, self).form_valid(form)
     # ignore redirect response
     log('%s\t%s\t%s' %
         (self.object.user.username, PAID_TASK_ADDED, self.object))
     return self.render_to_json_response({'result': self.object.id})
    def forwards(self, orm):

        stats = {
            'ONE_TO_ONE_COPY': 0,
            'MANY_TO_MANY_COPY': 0, #!!! impossible ?
            'PAS_LESS_THAN_PSS': 0,
            'NO_PAS_FOR_PSS': 0,
            'PSS_LESS_THAN_PAS': 0, #!!! impossible ?
            'SUG_CREATED_FROM_SUB': 0,
            'NO_SUB_FOR_PAS': 0,
            'NO_PSS_FOR_PENDING_PAS': 0,
            'DEL_REJECTED_PAS': 0,
            'NO_UNIT_FOR_PAS': 0,
            'DUPLICATED_SUG': 0,
            'SUG_CREATED_FROM_SUB_WITHOUT_PAS': 0,
            'SUG_CREATION_ERROR': 0,
        }

        system = orm['pootle_profile.PootleProfile'].objects.get(user__username='******')

        # units with suggestions
        units = orm['pootle_store.Unit'].objects.filter(suggestion__isnull=False).distinct()
        for unit in units:
            store_suggestions = unit.suggestion_set.all().order_by('user')
            for user_id, ss in groupby(store_suggestions, lambda x: x.user.id):
                ss = list(ss)
                app_pending_suggestions = orm['pootle_app.Suggestion'].objects \
                    .filter(unit=unit.id, suggester__id=user_id, state='pending')

                len_ss = len(ss)
                len_app_pending_suggestions = len(app_pending_suggestions)

                if len_ss == len_app_pending_suggestions:
                    if len_ss == 1:
                        pas = app_pending_suggestions[0]
                        sugg = ss[0]
                        sugg.state = pas.state # state == pending
                        sugg.creation_time = pas.creation_time
                        sugg.save()

                        log("pas %d copied and will be deleted" % pas.id)
                        stats['ONE_TO_ONE_COPY'] += 1
                        pas.delete()

                    else:
                        # there should be no such data in the database
                        log("%d pending suggestions by %d for %d" % (len_ss, user_id, unit.id))
                        stats['MANY_TO_MANY_COPY'] += 1

                elif len_ss > len_app_pending_suggestions:
                    log("%d pss > %d pas for unit %d" % (len_ss, len_app_pending_suggestions, unit.id))
                    if len_app_pending_suggestions > 0:
                        stats['PAS_LESS_THAN_PSS'] += 1
                    else:
                        stats['NO_PAS_FOR_PSS'] += 1

                    app_suggestion_ids = map(lambda x: x.id, app_pending_suggestions)
                    orm['pootle_statistics.Submission'].objects \
                        .filter(from_suggestion__id__in=app_suggestion_ids) \
                        .update(from_suggestion=None)
                    app_pending_suggestions.delete()

                else:
                    # there should be no such data in the database
                    log("%d pss < %d pas for %d" % (len_ss, len_app_pending_suggestions, unit.id))
                    stats['PSS_LESS_THAN_PAS'] += 1

        # all pootle_app_suggestions corresponding to pootle_store_suggestions have been already deleted

        for pas in orm['pootle_app.Suggestion'].objects.all():
            try:
                # unit isn't deleted
                unit = orm['pootle_store.Unit'].objects.get(id=pas.unit)

                if pas.state == 'accepted':
                    try:
                        # to restore suggestion by submission
                        sub = orm['pootle_statistics.Submission'].objects.get(unit=unit, from_suggestion=pas)
                        target = u"%s" % sub.new_value
                        sugg = {
                            'target_f': target,
                            'target_hash': md5(target.encode("utf-8")).hexdigest(),
                            'unit': unit,
                            'user': pas.suggester,
                            'reviewer': pas.reviewer,
                            'review_time': sub.creation_time,
                            'state': pas.state, #accepted
                            'creation_time': None,
                        }
                            
                        try:
                            sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
                            sub.suggestion = sugg
                            sub.from_suggestion = None
                            sub.save()

                            log("suggestion created from pas %d and sub %d" % (pas.id, sub.id))
                            stats['SUG_CREATED_FROM_SUB'] += 1

                        except IntegrityError:
                            log("failed to create duplicated suggestion from pas %d and sub %d" % (pas.id, sub.id))

                            sub.delete()
                            stats['DUPLICATED_SUG'] += 1

                    except orm['pootle_statistics.Submission'].DoesNotExist:
                        log('No submission found for pas %d' % pas.id)
                        stats['NO_SUB_FOR_PAS'] += 1

                elif pas.state == 'pending':
                    log('No suggestion found for pending pas %d' % pas.id)
                    stats['NO_PSS_FOR_PENDING_PAS'] += 1
                else:
                    log('Delete rejected pas %d' % pas.id)
                    stats['DEL_REJECTED_PAS'] += 1

            except orm['pootle_store.Unit'].DoesNotExist:
                log('no unit for %d' % pas.id)
                stats['NO_UNIT_FOR_PAS'] += 1

            pas.delete()

        for sub in orm['pootle_statistics.Submission'].objects \
                .filter(type=SubmissionTypes.SUGG_ACCEPT,
                        suggestion=None,
                        field=SubmissionFields.TARGET):

            sugg = {
                'target_f': sub.new_value,
                'target_hash': md5(sub.new_value.encode("utf-8")).hexdigest(),
                'unit': sub.unit,
                'user': sub.submitter,
                'reviewer': system,
                'review_time': sub.creation_time,
                'state': 'accepted',
                'creation_time': None,
            }

            try:
                sugg = orm['pootle_store.Suggestion'].objects.create(**sugg)
                sub.suggestion = sugg
                sub.save()
                stats['SUG_CREATED_FROM_SUB_WITHOUT_PAS'] += 1

            except:
                stats['SUG_CREATION_ERROR'] += 1

        log("%s" % stats)