def process_reporting_metadata_staging():
from corehq.apps.users.models import (
CouchUser, UserReportingMetadataStaging
)
start = datetime.utcnow()
with transaction.atomic():
records = (
UserReportingMetadataStaging.objects.select_for_update(skip_locked=True).order_by('pk')
)[:100]
for record in records:
user = CouchUser.get_by_user_id(record.user_id, record.domain)
try:
record.process_record(user)
except ResourceConflict:
# https://sentry.io/organizations/dimagi/issues/1479516073/
user = CouchUser.get_by_user_id(record.user_id, record.domain)
record.process_record(user)
record.delete()
duration = datetime.utcnow() - start
run_again = run_periodic_task_again(process_reporting_metadata_staging_schedule, start, duration)
if run_again and UserReportingMetadataStaging.objects.exists():
process_reporting_metadata_staging.delay()
def process_reporting_metadata_staging():
from corehq.apps.users.models import (CouchUser,
UserReportingMetadataStaging)
lock_key = "PROCESS_REPORTING_METADATA_STAGING_TASK"
process_reporting_metadata_lock = get_redis_lock(
lock_key,
timeout=60 * 60, # one hour
name=lock_key,
)
if not process_reporting_metadata_lock.acquire(blocking=False):
metrics_counter("commcare.process_reporting_metadata.locked_out")
return
try:
start = datetime.utcnow()
for i in range(20):
with transaction.atomic():
records = (
UserReportingMetadataStaging.objects.select_for_update(
skip_locked=True).order_by('pk'))[:5]
for record in records:
user = CouchUser.get_by_user_id(record.user_id,
record.domain)
try:
record.process_record(user)
except ResourceConflict:
# https://sentry.io/organizations/dimagi/issues/1479516073/
user = CouchUser.get_by_user_id(
record.user_id, record.domain)
record.process_record(user)
record.delete()
finally:
process_reporting_metadata_lock.release()
duration = datetime.utcnow() - start
run_again = run_periodic_task_again(
process_reporting_metadata_staging_schedule, start, duration)
if run_again and UserReportingMetadataStaging.objects.exists():
process_reporting_metadata_staging.delay()
def _test(run_every, last_run, duration, expected, now):
with freeze_time(now):
run_again = run_periodic_task_again(run_every, last_run, duration)
eq(run_again, expected)
