def set_unit_created_by(apps, schema_editor): subs = apps.get_model("pootle_statistics.Submission").objects.all() UnitSource = apps.get_model("pootle_store.UnitSource") sources = UnitSource.objects.all() units = apps.get_model("pootle_store.Unit").objects.all() total = units.count() offset = 0 step = 10000 start = time.time() # type 10 is the now deleted UNIT_CREATE creators = dict( subs.filter(type=10).exclude(submitter__username="******").values_list( "unit_id", "submitter")) sysuser = get_system_user_id() while True: UnitSource.objects.bulk_create([ UnitSource(unit_id=pk, created_by_id=creators.get(pk, sysuser)) for pk in units[offset:offset + step].values_list("id", flat=True) ]) logger.debug("added %s/%s in %s seconds" % (offset + step, total, (time.time() - start))) if offset > total: break offset = offset + step
def _create_method(unit, timestamp, user): return dict( unit_id=unit, changed_with=(SubmissionTypes.SYSTEM if user == get_system_user_id() else SubmissionTypes.WEB), submitted_by_id=user, submitted_on=timestamp)
def set_unit_created_by(apps, schema_editor): subs = apps.get_model("pootle_statistics.Submission").objects.all() UnitSource = apps.get_model("pootle_store.UnitSource") sources = UnitSource.objects.all() units = apps.get_model("pootle_store.Unit").objects.all() total = units.count() offset = 0 step = 10000 start = time.time() # type 10 is the now deleted UNIT_CREATE creators = dict( subs.filter(type=10) .exclude(submitter__username="******") .values_list("unit_id", "submitter")) sysuser = get_system_user_id() while True: UnitSource.objects.bulk_create( [UnitSource( unit_id=pk, created_by_id=creators.get(pk, sysuser)) for pk in units[offset: offset + step].values_list("id", flat=True)]) logger.debug( "added %s/%s in %s seconds" % (offset + step, total, (time.time() - start))) if offset > total: break offset = offset + step
def _create_method(unit, timestamp, user): return dict( unit_id=unit, changed_with=( SubmissionTypes.SYSTEM if user == get_system_user_id() else SubmissionTypes.WEB), reviewed_by_id=user, reviewed_on=timestamp)
def add_unit_sources(apps, schema_editor): subs = apps.get_model("pootle_statistics.Submission").objects.all() sysuser = get_system_user_id() creators = dict( subs.filter(type=UNIT_CREATE_TYPE).exclude( submitter_id=sysuser).values_list("unit_id", "submitter")) if schema_editor.connection.vendor == "mysql" and settings.POOTLE_SQL_MIGRATIONS: create_sources_with_sql(schema_editor) update_sources_with_orm(apps, creators) else: create_sources_with_orm(apps, creators)
def add_unit_sources(apps, schema_editor): subs = apps.get_model("pootle_statistics.Submission").objects.all() sysuser = get_system_user_id() creators = dict( subs.filter(type=UNIT_CREATE_TYPE) .exclude(submitter_id=sysuser) .values_list("unit_id", "submitter")) if schema_editor.connection.vendor == "mysql" and settings.POOTLE_SQL_MIGRATIONS: create_sources_with_sql(schema_editor) update_sources_with_orm(apps, creators) else: create_sources_with_orm(apps, creators)
def create_sources_with_orm(apps, creators): sysuser = get_system_user_id() UnitSource = apps.get_model("pootle_store.UnitSource") units = apps.get_model("pootle_store.Unit").objects.all() _units = list( units.values_list("id", "source_hash", "source_length", "source_wordcount").iterator()) def _unit_source_create(pk, source_hash, source_length, source_wordcount): return dict(unit_id=pk, source_hash=source_hash, source_wordcount=source_wordcount, source_length=source_length, created_with=SubmissionTypes.WEB, created_by_id=creators.get(pk, sysuser)) Batch(UnitSource.objects, batch_size=500).create(_units, _unit_source_create, reduces=False)
def create_sources_with_orm(apps, creators): sysuser = get_system_user_id() UnitSource = apps.get_model("pootle_store.UnitSource") units = apps.get_model("pootle_store.Unit").objects.all() _units = list( units.values_list( "id", "source_hash", "source_length", "source_wordcount").iterator()) def _unit_source_create(pk, source_hash, source_length, source_wordcount): return dict( unit_id=pk, source_hash=source_hash, source_wordcount=source_wordcount, source_length=source_length, created_with=SubmissionTypes.WEB, created_by_id=creators.get(pk, sysuser)) Batch(UnitSource.objects, batch_size=500).create( _units, _unit_source_create, reduces=False)
def remove_system_scorelogs(apps, schema_editor): scorelogs = apps.get_model("pootle_statistics.ScoreLog").objects.all() scorelogs.filter(user_id=get_system_user_id()).delete()
def test_user_get_system_user(system): assert get_system_user() == system assert get_system_user_id() == system.id
def create_sources_with_sql(schema_editor): sysuser = get_system_user_id() cursor = schema_editor.connection.cursor() cursor.execute(UNIT_SOURCE_SQL % (SubmissionTypes.SYSTEM, sysuser))
def create_sources_with_sql(schema_editor): sysuser = get_system_user_id() cursor = schema_editor.connection.cursor() cursor.execute(UNIT_SOURCE_SQL % (SubmissionTypes.SYSTEM, sysuser))