def set_unit_created_by(apps, schema_editor):
subs = apps.get_model("pootle_statistics.Submission").objects.all()
UnitSource = apps.get_model("pootle_store.UnitSource")
sources = UnitSource.objects.all()
units = apps.get_model("pootle_store.Unit").objects.all()
total = units.count()
offset = 0
step = 10000
start = time.time()
# type 10 is the now deleted UNIT_CREATE
creators = dict(
subs.filter(type=10).exclude(submitter__username="******").values_list(
"unit_id", "submitter"))
sysuser = get_system_user_id()
while True:
UnitSource.objects.bulk_create([
UnitSource(unit_id=pk, created_by_id=creators.get(pk, sysuser))
for pk in units[offset:offset + step].values_list("id", flat=True)
])
logger.debug("added %s/%s in %s seconds" % (offset + step, total,
(time.time() - start)))
if offset > total:
break
offset = offset + step
def _create_method(unit, timestamp, user):
return dict(
unit_id=unit,
changed_with=(SubmissionTypes.SYSTEM if user
== get_system_user_id() else SubmissionTypes.WEB),
submitted_by_id=user,
submitted_on=timestamp)
def set_unit_created_by(apps, schema_editor):
subs = apps.get_model("pootle_statistics.Submission").objects.all()
UnitSource = apps.get_model("pootle_store.UnitSource")
sources = UnitSource.objects.all()
units = apps.get_model("pootle_store.Unit").objects.all()
total = units.count()
offset = 0
step = 10000
start = time.time()
# type 10 is the now deleted UNIT_CREATE
creators = dict(
subs.filter(type=10)
.exclude(submitter__username="******")
.values_list("unit_id", "submitter"))
sysuser = get_system_user_id()
while True:
UnitSource.objects.bulk_create(
[UnitSource(
unit_id=pk,
created_by_id=creators.get(pk, sysuser))
for pk
in units[offset: offset + step].values_list("id", flat=True)])
logger.debug(
"added %s/%s in %s seconds"
% (offset + step, total, (time.time() - start)))
if offset > total:
break
offset = offset + step
def _create_method(unit, timestamp, user):
return dict(
unit_id=unit,
changed_with=(
SubmissionTypes.SYSTEM
if user == get_system_user_id()
else SubmissionTypes.WEB),
reviewed_by_id=user,
reviewed_on=timestamp)
def add_unit_sources(apps, schema_editor):
subs = apps.get_model("pootle_statistics.Submission").objects.all()
sysuser = get_system_user_id()
creators = dict(
subs.filter(type=UNIT_CREATE_TYPE).exclude(
submitter_id=sysuser).values_list("unit_id", "submitter"))
if schema_editor.connection.vendor == "mysql" and settings.POOTLE_SQL_MIGRATIONS:
create_sources_with_sql(schema_editor)
update_sources_with_orm(apps, creators)
else:
create_sources_with_orm(apps, creators)
def add_unit_sources(apps, schema_editor):
subs = apps.get_model("pootle_statistics.Submission").objects.all()
sysuser = get_system_user_id()
creators = dict(
subs.filter(type=UNIT_CREATE_TYPE)
.exclude(submitter_id=sysuser)
.values_list("unit_id", "submitter"))
if schema_editor.connection.vendor == "mysql" and settings.POOTLE_SQL_MIGRATIONS:
create_sources_with_sql(schema_editor)
update_sources_with_orm(apps, creators)
else:
create_sources_with_orm(apps, creators)
def create_sources_with_orm(apps, creators):
sysuser = get_system_user_id()
UnitSource = apps.get_model("pootle_store.UnitSource")
units = apps.get_model("pootle_store.Unit").objects.all()
_units = list(
units.values_list("id", "source_hash", "source_length",
"source_wordcount").iterator())
def _unit_source_create(pk, source_hash, source_length, source_wordcount):
return dict(unit_id=pk,
source_hash=source_hash,
source_wordcount=source_wordcount,
source_length=source_length,
created_with=SubmissionTypes.WEB,
created_by_id=creators.get(pk, sysuser))
Batch(UnitSource.objects, batch_size=500).create(_units,
_unit_source_create,
reduces=False)
def create_sources_with_orm(apps, creators):
sysuser = get_system_user_id()
UnitSource = apps.get_model("pootle_store.UnitSource")
units = apps.get_model("pootle_store.Unit").objects.all()
_units = list(
units.values_list(
"id",
"source_hash",
"source_length",
"source_wordcount").iterator())
def _unit_source_create(pk, source_hash, source_length, source_wordcount):
return dict(
unit_id=pk,
source_hash=source_hash,
source_wordcount=source_wordcount,
source_length=source_length,
created_with=SubmissionTypes.WEB,
created_by_id=creators.get(pk, sysuser))
Batch(UnitSource.objects, batch_size=500).create(
_units,
_unit_source_create,
reduces=False)
def remove_system_scorelogs(apps, schema_editor):
scorelogs = apps.get_model("pootle_statistics.ScoreLog").objects.all()
scorelogs.filter(user_id=get_system_user_id()).delete()
def test_user_get_system_user(system):
assert get_system_user() == system
assert get_system_user_id() == system.id
def create_sources_with_sql(schema_editor):
sysuser = get_system_user_id()
cursor = schema_editor.connection.cursor()
cursor.execute(UNIT_SOURCE_SQL % (SubmissionTypes.SYSTEM, sysuser))
def create_sources_with_sql(schema_editor):
sysuser = get_system_user_id()
cursor = schema_editor.connection.cursor()
cursor.execute(UNIT_SOURCE_SQL % (SubmissionTypes.SYSTEM, sysuser))