def get_qualitychecks():
available_checks = {}
if settings.POOTLE_QUALITY_CHECKER:
checkers = [import_func(settings.POOTLE_QUALITY_CHECKER)()]
else:
checkers = [checker() for checker in checks.projectcheckers.values()]
for checker in checkers:
for filt in checker.defaultfilters:
if filt not in excluded_filters:
# don't use an empty string because of
# http://bugs.python.org/issue18190
try:
getattr(checker, filt)(u'_', u'_')
except Exception as e:
# FIXME there must be a better way to get a list of
# available checks. Some error because we're not actually
# using them on real units.
logging.error("Problem with check filter '%s': %s", filt, e)
continue
available_checks.update(checker.categories)
return available_checks
def get_unit_wordcount(**kwargs_):
global wordcounter
if not wordcounter:
wordcounter = UnitWordcount(
import_func(settings.POOTLE_WORDCOUNT_FUNC))
return wordcounter
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.POOTLE_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.POOTLE_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardUnitChecker)
]
checker = checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
# patch checker configuration with Pootle-provided language information
def _update_lang(ch):
ch.config.lang.fullname = self.language.fullname
ch.config.lang.nplurals = self.language.nplurals
ch.config.lang.pluralequation = self.language.pluralequation
if hasattr(ch, 'checkers'):
for child_ch in ch.checkers:
_update_lang(child_ch)
_update_lang(checker)
return checker
def get_unit_wordcount(**kwargs_):
global wordcounter
if not wordcounter:
wordcounter = UnitWordcount(import_func(
settings.POOTLE_WORDCOUNT_FUNC))
return wordcounter
def get_checker(self, tp_pk):
"""Return the site QualityChecker or the QualityCheck associated with
the a Unit's TP otherwise.
"""
if settings.POOTLE_QUALITY_CHECKER:
return import_func(settings.POOTLE_QUALITY_CHECKER)()
try:
return TranslationProject.objects.get(id=tp_pk).checker
except TranslationProject.DoesNotExist:
# There seems to be a risk of dangling Stores with no TP
return None
def get_checker(self, tp_pk):
"""Return the site QualityChecker or the QualityCheck associated with
the a Unit's TP otherwise.
"""
if settings.POOTLE_QUALITY_CHECKER:
return import_func(settings.POOTLE_QUALITY_CHECKER)()
try:
return TranslationProject.objects.get(id=tp_pk).checker
except TranslationProject.DoesNotExist:
# There seems to be a risk of dangling Stores with no TP
return None
def test_tp_checker(po_directory, tp_checker_tests, settings):
language = Language.objects.get(code="language0")
checker_name_, project = tp_checker_tests
tp = TranslationProject.objects.create(project=project, language=language)
if settings.ZING_QUALITY_CHECKER:
checkerclasses = [import_func(settings.ZING_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(tp.project.checkstyle,
checks.StandardChecker)
]
assert [x.__class__ for x in tp.checker.checkers] == checkerclasses
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.POOTLE_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.POOTLE_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardUnitChecker)
]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
def checker(self):
from translate.filters import checks
# We do not use default Translate Toolkit checkers; instead use
# our own one
if settings.POOTLE_QUALITY_CHECKER:
from pootle_misc.util import import_func
checkerclasses = [import_func(settings.POOTLE_QUALITY_CHECKER)]
else:
checkerclasses = [
checks.projectcheckers.get(self.project.checkstyle,
checks.StandardUnitChecker)
]
return checks.TeeChecker(checkerclasses=checkerclasses,
excludefilters=excluded_filters,
errorhandler=self.filtererrorhandler,
languagecode=self.language.code)
checks = options.get('checks', [])
if options['unit'] is not None:
unit_id = options.get('unit', '')
try:
unit = Unit.objects.get(id=unit_id)
source = unit.source
target = unit.target
except Unit.DoesNotExist, e:
raise CommandError(e.message)
else:
source = options.get('source', '').decode('utf-8')
target = options.get('target', '').decode('utf-8')
if settings.POOTLE_QUALITY_CHECKER:
checkers = [import_func(settings.POOTLE_QUALITY_CHECKER)()]
else:
checkers = [checker() for checker in projectcheckers.values()]
if not checks:
checks = get_qualitychecks().keys()
error_checks = []
for checker in checkers:
for check in checks:
filtermessage = ''
if check in error_checks:
continue
try:
if hasattr(checker, check):
test = getattr(checker, check)
def get_daily_activity(user, scores, start, end):
result_translated = {
'label': PaidTask.get_task_type_title(
PaidTaskTypes.TRANSLATION),
'data': [],
}
result_reviewed = {
'label': PaidTask.get_task_type_title(
PaidTaskTypes.REVIEW),
'data': [],
}
result = {
'data': [result_translated, result_reviewed],
'max_day_score': 10,
'min_ts': "%d" % (calendar.timegm(start.timetuple()) * 1000),
'max_ts': "%d" % (calendar.timegm(end.timetuple()) * 1000),
'nonempty': False,
}
if settings.POOTLE_REPORTS_MARK_FUNC:
try:
get_mark_data = import_func(settings.POOTLE_REPORTS_MARK_FUNC)
result['data'].append({
'data': [],
'marks': {'show': True},
'markdata': get_mark_data(user, start, end)
})
except ImproperlyConfigured:
pass
saved_date = None
current_day_score = 0
translated_group = {}
reviewed_group = {}
for score in scores:
score_time = make_naive(score.creation_time)
date = score_time.date()
translated, reviewed = score.get_paid_words()
if translated or reviewed:
translated = 0 if translated is None else translated
reviewed = 0 if reviewed is None else reviewed
if saved_date != date:
saved_date = date
reviewed_group[date] = 0
translated_group[date] = 0
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
current_day_score = 0
current_day_score += int(reviewed + translated)
result['nonempty'] |= current_day_score > 0
translated_group[date] += translated
reviewed_group[date] += reviewed
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
for date, item in sorted(translated_group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
result_translated['data'].append((ts, item))
for date, item in sorted(reviewed_group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
result_reviewed['data'].append((ts, item))
return result
def get_checker(unit):
checker_class = getattr(settings, 'QUALITY_CHECKER', '')
if checker_class:
return import_func(checker_class)()
else:
return unit.store.translation_project.checker
def get_daily_activity(user, scores, start, end):
result_translated = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.TRANSLATION],
'data': [],
}
result_reviewed = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.REVIEW],
'data': [],
}
result_suggested = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.SUGGESTION],
'data': [],
}
result = {
'data': [result_translated, result_reviewed, result_suggested],
'max_day_score': 10,
'min_ts': "%d" % (calendar.timegm(start.timetuple()) * 1000),
'max_ts': "%d" % (calendar.timegm(end.timetuple()) * 1000),
'nonempty': False,
}
if settings.POOTLE_REPORTS_MARK_FUNC:
try:
get_mark_data = import_func(settings.POOTLE_REPORTS_MARK_FUNC)
result['data'].append({
'data': [],
'marks': {'show': True},
'markdata': get_mark_data(user, start, end)
})
except ImproperlyConfigured:
pass
saved_date = None
current_day_score = 0
translated_group = {}
reviewed_group = {}
suggested_group = {}
for score in scores:
score_time = make_naive(score.creation_time)
date = score_time.date()
translated, reviewed = score.get_paid_wordcounts()
suggested = score.get_suggested_wordcount()
if any(map(lambda x: x is not None, [translated, reviewed,
suggested])):
translated = 0 if translated is None else translated
reviewed = 0 if reviewed is None else reviewed
suggested = 0 if suggested is None else suggested
if saved_date != date:
saved_date = date
reviewed_group[date] = 0
translated_group[date] = 0
suggested_group[date] = 0
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
current_day_score = 0
current_day_score += int(reviewed + translated + suggested)
result['nonempty'] |= current_day_score > 0
translated_group[date] += translated
reviewed_group[date] += reviewed
suggested_group[date] += suggested
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
for group, res in [(translated_group, result_translated),
(reviewed_group, result_reviewed),
(suggested_group, result_suggested)]:
for date, item in sorted(group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
res['data'].append((ts, item))
return result
def get_daily_activity(user, scores, start, end):
result_translated = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.TRANSLATION],
'data': [],
}
result_reviewed = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.REVIEW],
'data': [],
}
result_suggested = {
'label': ReportActionTypes.NAMES_MAP[ReportActionTypes.SUGGESTION],
'data': [],
}
result = {
'data': [result_translated, result_reviewed, result_suggested],
'max_day_score': 10,
'min_ts': "%d" % (calendar.timegm(start.timetuple()) * 1000),
'max_ts': "%d" % (calendar.timegm(end.timetuple()) * 1000),
'nonempty': False,
}
if settings.POOTLE_REPORTS_MARK_FUNC:
try:
get_mark_data = import_func(settings.POOTLE_REPORTS_MARK_FUNC)
result['data'].append({
'data': [],
'marks': {'show': True},
'markdata': get_mark_data(user, start, end)
})
except ImproperlyConfigured:
pass
saved_date = None
current_day_score = 0
translated_group = {}
reviewed_group = {}
suggested_group = {}
for score in scores:
score_time = make_naive(score.creation_time)
date = score_time.date()
translated, reviewed = score.get_paid_wordcounts()
suggested = score.get_suggested_wordcount()
if any(map(lambda x: x is not None, [translated, reviewed,
suggested])):
translated = 0 if translated is None else translated
reviewed = 0 if reviewed is None else reviewed
suggested = 0 if suggested is None else suggested
if saved_date != date:
saved_date = date
reviewed_group[date] = 0
translated_group[date] = 0
suggested_group[date] = 0
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
current_day_score = 0
current_day_score += int(reviewed + translated + suggested)
result['nonempty'] |= current_day_score > 0
translated_group[date] += translated
reviewed_group[date] += reviewed
suggested_group[date] += suggested
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
for group, res in [(translated_group, result_translated),
(reviewed_group, result_reviewed),
(suggested_group, result_suggested)]:
for date, item in sorted(group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
res['data'].append((ts, item))
return result
def __unicode__(self):
return unicode(self.target)
def _set_hash(self):
string = self.translator_comment_f
if string:
string = self.target_f + SEPARATOR + string
else:
string = self.target_f
self.target_hash = md5(string.encode("utf-8")).hexdigest()
# # # # # # # # Unit # # # # # # # # # #
wordcount_f = import_func(settings.POOTLE_WORDCOUNT_FUNC)
def count_words(strings):
wordcount = 0
for string in strings:
wordcount += wordcount_f(string)
return wordcount
def stringcount(string):
try:
return len(string.strings)
except AttributeError:
def __unicode__(self):
return unicode(self.target)
def _set_hash(self):
string = self.translator_comment_f
if string:
string = self.target_f + SEPARATOR + string
else:
string = self.target_f
self.target_hash = md5(string.encode("utf-8")).hexdigest()
# # # # # # # # Unit # # # # # # # # # #
wordcount_f = import_func(settings.POOTLE_WORDCOUNT_FUNC)
def count_words(strings):
wordcount = 0
for string in strings:
wordcount += wordcount_f(string)
return wordcount
def stringcount(string):
try:
return len(string.strings)
except AttributeError:
def handle(self, **options):
# adjust debug level to the verbosity option
debug_levels = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG
}
debug_level = debug_levels.get(options['verbosity'], logging.DEBUG)
logging.getLogger().setLevel(debug_level)
self.name = self.__class__.__module__.split('.')[-1]
if ((options['unit'] is not None and
(options['source'] or options['target']))
or (options['unit'] is None and not options['source']
and not options['target'])):
raise CommandError("Either --unit or a pair of --source "
"and --target must be provided.")
if bool(options['source']) != bool(options['target']):
raise CommandError("Use a pair of --source and --target.")
checks = options.get('checks', [])
if options['unit'] is not None:
try:
unit = Unit.objects.get(id=options['unit'])
source = unit.source
target = unit.target
except Unit.DoesNotExist as e:
raise CommandError(e)
else:
source = options['source'].decode('utf-8')
target = options['target'].decode('utf-8')
if settings.POOTLE_QUALITY_CHECKER:
checkers = [import_func(settings.POOTLE_QUALITY_CHECKER)()]
else:
checkers = [checker() for checker in projectcheckers.values()]
if not checks:
checks = get_qualitychecks().keys()
error_checks = []
for checker in checkers:
for check in checks:
filtermessage = ''
filterresult = True
if check in error_checks:
continue
try:
if hasattr(checker, check):
test = getattr(checker, check)
try:
filterresult = test(source, target)
except AttributeError:
continue
except FilterFailure as e:
filterresult = False
filtermessage = unicode(e)
message = "%s - %s" % (filterresult, check)
if filtermessage:
message += ": %s" % filtermessage
logging.info(message)
if not filterresult:
error_checks.append(check)
if error_checks:
self.stdout.write('Failing checks: %s' % ', '.join(error_checks))
else:
self.stdout.write('No errors found.')
def handle(self, **options):
# adjust debug level to the verbosity option
debug_levels = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
3: logging.DEBUG
}
debug_level = debug_levels.get(options['verbosity'], logging.DEBUG)
logging.getLogger().setLevel(debug_level)
self.name = self.__class__.__module__.split('.')[-1]
if ((options['unit'] is not None
and (options['source'] or options['target']))
or (options['unit'] is None
and not options['source']
and not options['target'])):
raise CommandError("Either --unit or a pair of --source "
"and --target must be provided.")
if bool(options['source']) != bool(options['target']):
raise CommandError("Use a pair of --source and --target.")
checks = options.get('checks', [])
if options['unit'] is not None:
try:
unit = Unit.objects.get(id=options['unit'])
source = unit.source
target = unit.target
except Unit.DoesNotExist as e:
raise CommandError(e)
else:
source = options['source'].decode('utf-8')
target = options['target'].decode('utf-8')
if settings.POOTLE_QUALITY_CHECKER:
checkers = [import_func(settings.POOTLE_QUALITY_CHECKER)()]
else:
checkers = [checker() for checker in projectcheckers.values()]
if not checks:
checks = get_qualitychecks().keys()
error_checks = []
for checker in checkers:
for check in checks:
filtermessage = ''
filterresult = True
if check in error_checks:
continue
try:
if hasattr(checker, check):
test = getattr(checker, check)
try:
filterresult = test(source, target)
except AttributeError:
continue
except FilterFailure as e:
filterresult = False
filtermessage = unicode(e)
message = "%s - %s" % (filterresult, check)
if filtermessage:
message += ": %s" % filtermessage
logging.info(message)
if not filterresult:
error_checks.append(check)
if error_checks:
self.stdout.write('Failing checks: %s' % ', '.join(error_checks))
else:
self.stdout.write('No errors found.')
def get_checker(unit):
if settings.POOTLE_QUALITY_CHECKER:
return import_func(settings.POOTLE_QUALITY_CHECKER)()
return unit.store.translation_project.checker
checks = options.get('checks', [])
if options['unit'] is not None:
unit_id = options.get('unit', '')
try:
unit = Unit.objects.get(id=unit_id)
source = unit.source
target = unit.target
except Unit.DoesNotExist, e:
raise CommandError(e.message)
else:
source = options.get('source', '').decode('utf-8')
target = options.get('target', '').decode('utf-8')
if settings.POOTLE_QUALITY_CHECKER:
checkers = [import_func(settings.POOTLE_QUALITY_CHECKER)()]
else:
checkers = [checker() for checker in projectcheckers.values()]
if not checks:
checks = get_qualitychecks().keys()
error_checks = []
for checker in checkers:
for check in checks:
filtermessage = ''
if check in error_checks:
continue
try:
if hasattr(checker, check):
test = getattr(checker, check)
def get_daily_activity(user, scores, start, end):
result_translated = {
'label': PaidTask.get_task_type_title(PaidTaskTypes.TRANSLATION),
'data': [],
}
result_reviewed = {
'label': PaidTask.get_task_type_title(PaidTaskTypes.REVIEW),
'data': [],
}
result = {
'data': [result_translated, result_reviewed],
'max_day_score': 10,
'min_ts': "%d" % (calendar.timegm(start.timetuple()) * 1000),
'max_ts': "%d" % (calendar.timegm(end.timetuple()) * 1000),
'nonempty': False,
}
if settings.POOTLE_REPORTS_MARK_FUNC:
try:
get_mark_data = import_func(settings.POOTLE_REPORTS_MARK_FUNC)
result['data'].append({
'data': [],
'marks': {
'show': True
},
'markdata': get_mark_data(user, start, end)
})
except ImproperlyConfigured:
pass
saved_date = None
current_day_score = 0
translated_group = {}
reviewed_group = {}
for score in scores:
score_time = make_naive(score.creation_time)
date = score_time.date()
translated, reviewed = score.get_paid_words()
if translated or reviewed:
translated = 0 if translated is None else translated
reviewed = 0 if reviewed is None else reviewed
if saved_date != date:
saved_date = date
reviewed_group[date] = 0
translated_group[date] = 0
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
current_day_score = 0
current_day_score += int(reviewed + translated)
result['nonempty'] |= current_day_score > 0
translated_group[date] += translated
reviewed_group[date] += reviewed
if result['max_day_score'] < current_day_score:
result['max_day_score'] = current_day_score
for date, item in sorted(translated_group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
result_translated['data'].append((ts, item))
for date, item in sorted(reviewed_group.items(), key=lambda x: x[0]):
ts = int(calendar.timegm(date.timetuple()) * 1000)
result_reviewed['data'].append((ts, item))
return result