def _create_qc_events(unit):
create_instances = [
QualityCheck(
name=("Foo%s" % i), category=2, unit=unit)
for i in range(0, 3)]
for instance in create_instances:
create.send(
QualityCheck,
instance=instance)
create_objects = [
QualityCheck(
name=("Bar%s" % i),
category=2,
unit=unit)
for i in range(0, 3)]
create_more_objects = [
QualityCheck(
name=("Baz%s" % i),
category=2,
unit=unit)
for i in range(0, 3)]
create.send(
QualityCheck,
objects=create_objects)
create.send(
QualityCheck,
objects=create_more_objects)
return create_instances + create_objects + create_more_objects
def calculate_checks(self, check_names, unit_fk_filter, store_fk_filter):
logger.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
checks = QualityCheck.objects.filter(**unit_fk_filter)
if check_names:
checks = checks.filter(name__in=check_names)
checks = checks.values('id', 'name', 'unit_id',
'category', 'false_positive')
all_units_checks = {}
for check in checks:
all_units_checks.setdefault(check['unit_id'], {})[check['name']] = check
unit_count = 0
units = Unit.simple_objects.select_related('store')
units.query.clear_ordering(True)
for unit in units.filter(**store_fk_filter).iterator():
unit_count += 1
unit_checks = {}
if unit.id in all_units_checks:
unit_checks = all_units_checks[unit.id]
if unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names,
existing=unit_checks):
# update unit.mtime
# TODO: add new action type `quality checks were updated`?
Unit.simple_objects.filter(id=unit.id).update(mtime=timezone.now())
if unit_count % 10000 == 0:
logger.info("%d units processed" % unit_count)
def calculate_checks(self, check_names, unit_fk_filter, store_fk_filter):
logger.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
checks = QualityCheck.objects.filter(**unit_fk_filter)
if check_names:
checks = checks.filter(name__in=check_names)
checks = checks.values('id', 'name', 'unit_id',
'category', 'false_positive')
all_units_checks = {}
for check in checks:
all_units_checks.setdefault(check['unit_id'], {})[check['name']] = check
unit_count = 0
units = Unit.simple_objects.select_related('store')
units.query.clear_ordering(True)
for unit in units.filter(**store_fk_filter).iterator():
unit_count += 1
unit_checks = {}
if unit.id in all_units_checks:
unit_checks = all_units_checks[unit.id]
if unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names,
existing=unit_checks):
# update unit.mtime
# TODO: add new action type `quality checks were updated`?
Unit.simple_objects.filter(id=unit.id).update(mtime=timezone.now())
if unit_count % 10000 == 0:
logger.info("%d units processed" % unit_count)
def test_unit_lifecycle_unmute_qc(store0, member):
unit = store0.UnitClass()
unit.store = store0
unit.source_f = multistring("Foo")
unit.save()
unit_lifecycle = lifecycle.get(Unit)(unit)
qc = QualityCheck(
unit=unit,
name="foo-check",
message="Check foo!",
category="Footile")
with pytest.raises(KeyError):
unit_lifecycle.sub_unmute_qc()
with pytest.raises(KeyError):
unit_lifecycle.sub_unmute_qc(submitter=member)
with pytest.raises(KeyError):
unit_lifecycle.sub_unmute_qc(quality_check=qc)
sub_unmute_qc = unit_lifecycle.sub_unmute_qc(
quality_check=qc, submitter=member)
assert sub_unmute_qc.unit == unit
assert sub_unmute_qc.store == store0
assert sub_unmute_qc.translation_project == store0.translation_project
assert sub_unmute_qc.revision == unit.revision
assert sub_unmute_qc.type == SubmissionTypes.UNMUTE_CHECK
assert sub_unmute_qc.field == SubmissionFields.NONE
assert sub_unmute_qc.new_value == ""
assert sub_unmute_qc.old_value == ""
assert not sub_unmute_qc.pk
def clone_checks(self, source_store, target_store):
"""Clone checks from source store to target store."""
fields = ('unit__unitid_hash', 'category', 'name',
'false_positive', 'message')
checks = QualityCheck.objects.filter(
unit__store=source_store,
unit__state__gt=OBSOLETE,
).values(*fields)
unitid_hashes = [x['unit__unitid_hash'] for x in checks]
units = target_store.units.filter(unitid_hash__in=unitid_hashes)
unit_map = {
x['unitid_hash']: x['id']
for x in units.values('id', 'unitid_hash')}
cloned_checks = []
for check in checks:
cloned_checks.append(QualityCheck(
unit_id=unit_map[check['unit__unitid_hash']],
category=check['category'],
name=check['name'],
false_positive=check['false_positive'],
message=check['message']))
create.send(QualityCheck, objects=cloned_checks)
def clear_checks(self):
QualityCheck.delete_unknown_checks()
def calculate_checks(check_names=None, translation_project=None):
store_fk_filter = {}
unit_fk_filter = {}
if translation_project is not None:
store_fk_filter = {
'store__translation_project': translation_project,
}
unit_fk_filter = {
'unit__store__translation_project': translation_project,
}
logging.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
checks = QualityCheck.objects.filter(**unit_fk_filter)
if check_names is not None:
checks = checks.filter(name__in=check_names)
checks = checks.values('id', 'name', 'unit_id',
'category', 'false_positive')
all_units_checks = {}
for check in checks:
all_units_checks.setdefault(check['unit_id'], {})[check['name']] = check
unit_filter = {
'state__gt': OBSOLETE
}
unit_filter.update(store_fk_filter)
# unit's query is faster without `select_related('store')`
units = Unit.simple_objects.filter(**unit_filter) \
.order_by('store__id')
store = None
# units are ordered by store, we update dirty cache after we switch
# to another store
for unit_count, unit in enumerate(units.iterator(), start=1):
if store is None or unit.store_id != store.id:
if store is not None:
store.update_dirty_cache()
# we get unit.store only if the store differs from previous
store = Store.simple_objects.get(id=unit.store_id)
# HACKISH: set unit.store to avoid extra querying in
# `unit.update_quality_checks()` method
unit.store = store
unit_checks = {}
if unit.id in all_units_checks:
unit_checks = all_units_checks[unit.id]
if unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names,
existing=unit_checks):
# update unit.mtime but avoid to use unit.save()
# because it can trigger unnecessary things:
# logging, stats cache updating
# TODO: add new action type `quality checks were updated`?
Unit.simple_objects.filter(id=unit.id).update(mtime=timezone.now())
if unit_count % 10000 == 0:
logging.info("%d units processed" % unit_count)
if store is not None:
store.update_dirty_cache()
def clear_checks(self):
QualityCheck.delete_unknown_checks()
def process(self, **options):
calculate_checks = options.get('calculate_checks', False)
calculate_wordcount = options.get('calculate_wordcount', False)
check_names = options.get('check_names', [])
store_filter = options.get('store_filter', {})
unit_fk_filter = options.get('unit_fk_filter', {})
store_fk_filter = options.get('store_fk_filter', {})
logging.info('Initializing stores...')
stores = Store.objects.all()
if store_filter:
stores = stores.filter(**store_filter)
self._init_stores(stores)
# if check_names is non-empty then stats for only these checks
# will be updated
if not check_names:
self._init_stats()
self._init_checks()
if calculate_checks:
logging.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
unit_count = 0
for i, store in enumerate(stores.iterator(), start=1):
logging.info("update_qualitychecks for %s" % store.pootle_path)
for unit in store.units.iterator():
unit_count += 1
unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names)
if i % 20 == 0:
logging.info("%d units processed" % unit_count)
if calculate_wordcount:
logging.info('Calculating wordcount for all units...')
unit_count = 0
for i, store in enumerate(stores.iterator(), start=1):
logging.info("calculate wordcount for %s" % store.pootle_path)
for unit in store.unit_set.iterator():
unit_count += 1
unit.update_wordcount()
unit.save()
if i % 20 == 0:
logging.info("%d units processed" % unit_count)
logging.info('Setting quality check stats values for all stores...')
self._set_qualitycheck_stats(unit_fk_filter)
if not check_names:
logging.info('Setting last action values for all stores...')
self._set_last_action_stats(store_fk_filter)
logging.info('Setting last updated values for all stores...')
self._set_last_updated_stats(store_fk_filter)
logging.info('Setting mtime values for all stores...')
self._set_mtime_stats(store_fk_filter)
logging.info('Setting wordcount stats values for all stores...')
self._set_wordcount_stats(store_fk_filter)
logging.info('Setting suggestion count values for all stores...')
self._set_suggestion_stats(unit_fk_filter)
logging.info('Setting empty values for other cache entries...')
self._set_empty_values()
def calculate_checks(check_names=None, translation_project=None):
store_fk_filter = {}
unit_fk_filter = {}
if translation_project is not None:
store_fk_filter = {
'store__translation_project': translation_project,
}
unit_fk_filter = {
'unit__store__translation_project': translation_project,
}
logging.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
checks = QualityCheck.objects.filter(**unit_fk_filter)
if check_names is not None:
checks = checks.filter(name__in=check_names)
checks = checks.values('id', 'name', 'unit_id', 'category',
'false_positive')
all_units_checks = {}
for check in checks:
all_units_checks.setdefault(check['unit_id'],
{})[check['name']] = check
unit_filter = {'state__gt': OBSOLETE}
unit_filter.update(store_fk_filter)
# unit's query is faster without `select_related('store')`
units = Unit.simple_objects.filter(**unit_filter) \
.order_by('store__id')
store = None
# units are ordered by store, we update dirty cache after we switch
# to another store
for unit_count, unit in enumerate(units.iterator(), start=1):
if store is None or unit.store_id != store.id:
if store is not None:
store.update_dirty_cache()
# we get unit.store only if the store differs from previous
store = Store.simple_objects.get(id=unit.store_id)
# HACKISH: set unit.store to avoid extra querying in
# `unit.update_quality_checks()` method
unit.store = store
unit_checks = {}
if unit.id in all_units_checks:
unit_checks = all_units_checks[unit.id]
if unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names,
existing=unit_checks):
# update unit.mtime but avoid to use unit.save()
# because it can trigger unnecessary things:
# logging, stats cache updating
# TODO: add new action type `quality checks were updated`?
Unit.simple_objects.filter(id=unit.id).update(mtime=timezone.now())
if unit_count % 10000 == 0:
logging.info("%d units processed" % unit_count)
if store is not None:
store.update_dirty_cache()
def process(self, **options):
calculate_checks = options.get('calculate_checks', False)
calculate_wordcount = options.get('calculate_wordcount', False)
check_names = options.get('check_names', [])
store_filter = options.get('store_filter', {})
unit_fk_filter = options.get('unit_fk_filter', {})
store_fk_filter = options.get('store_fk_filter', {})
logging.info('Initializing stores...')
stores = Store.objects.all()
if store_filter:
stores = stores.filter(**store_filter)
self._init_stores(stores)
# if check_names is non-empty then stats for only these checks
# will be updated
if not check_names:
self._init_stats()
self._init_checks()
if calculate_checks:
logging.info('Calculating quality checks for all units...')
QualityCheck.delete_unknown_checks()
unit_count = 0
for i, store in enumerate(stores.iterator(), start=1):
logging.info("update_qualitychecks for %s" % store.pootle_path)
for unit in store.units.iterator():
unit_count += 1
unit.update_qualitychecks(keep_false_positives=True,
check_names=check_names)
if i % 20 == 0:
logging.info("%d units processed" % unit_count)
if calculate_wordcount:
logging.info('Calculating wordcount for all units...')
unit_count = 0
for i, store in enumerate(stores.iterator(), start=1):
logging.info("calculate wordcount for %s" % store.pootle_path)
for unit in store.unit_set.iterator():
unit_count += 1
unit.update_wordcount()
unit.save()
if i % 20 == 0:
logging.info("%d units processed" % unit_count)
logging.info('Setting quality check stats values for all stores...')
self._set_qualitycheck_stats(unit_fk_filter)
if not check_names:
logging.info('Setting last action values for all stores...')
self._set_last_action_stats(store_fk_filter)
logging.info('Setting last updated values for all stores...')
self._set_last_updated_stats(store_fk_filter)
logging.info('Setting mtime values for all stores...')
self._set_mtime_stats(store_fk_filter)
logging.info('Setting wordcount stats values for all stores...')
self._set_wordcount_stats(store_fk_filter)
logging.info('Setting suggestion count values for all stores...')
self._set_suggestion_stats(unit_fk_filter)
logging.info('Setting empty values for other cache entries...')
self._set_empty_values()
