repeat_record.doc_type += DELETED_SUFFIX repeat_record.save() elif repeater.paused: # postpone repeat record by MAX_RETRY_WAIT so that these don't get picked in each cycle and # thus clogging the queue with repeat records with paused repeater repeat_record.postpone_by(MAX_RETRY_WAIT) elif repeat_record.state == RECORD_PENDING_STATE or repeat_record.state == RECORD_FAILURE_STATE: repeat_record.fire() except Exception: logging.exception('Failed to process repeat record: {}'.format( repeat_record._id)) metrics_gauge_task( 'commcare.repeaters.overdue', get_overdue_repeat_record_count, run_every=crontab(), # every minute multiprocess_mode=MPM_MAX) @task(queue=settings.CELERY_REPEAT_RECORD_QUEUE) def process_repeater(repeater_id: int): """ Worker task to send SQLRepeatRecords in chronological order. This function assumes that ``repeater`` checks have already been performed. Call via ``models.attempt_forward_now()``. """ repeater = SQLRepeater.objects.get(id=repeater_id) with CriticalSection( [f'process-repeater-{repeater.repeater_id}'],
# Delete entries that should not exist for phone_number in phone_entries.keys(): if phone_number not in numbers_that_should_exist: phone_entries[phone_number].delete() # Create entries that should exist but do not exist for phone_number in numbers_that_should_exist: if phone_number not in phone_entries: try: couch_user.create_phone_entry(phone_number) except InvalidFormatException: pass @no_result_task(queue='background_queue', acks_late=True, default_retry_delay=5 * 60, max_retries=10, bind=True) def publish_sms_change(self, sms_id): sms = SMS.objects.get(sms_id) try: publish_sms_saved(sms) except Exception as e: self.retry(exc=e) def queued_sms(): return QueuedSMS.objects.count() metrics_gauge_task('commcare.sms.queued', queued_sms, run_every=crontab(), multiprocess_mode=MPM_MAX)
numbers_that_should_exist = [apply_leniency(phone_number) for phone_number in couch_user.phone_numbers] # Delete entries that should not exist for phone_number in phone_entries.keys(): if phone_number not in numbers_that_should_exist: phone_entries[phone_number].delete() # Create entries that should exist but do not exist for phone_number in numbers_that_should_exist: if phone_number not in phone_entries: try: couch_user.create_phone_entry(phone_number) except InvalidFormatException: pass @no_result_task(serializer='pickle', queue='background_queue', acks_late=True, default_retry_delay=5 * 60, max_retries=10, bind=True) def publish_sms_change(self, sms): try: publish_sms_saved(sms) except Exception as e: self.retry(exc=e) def queued_sms(): return QueuedSMS.objects.count() metrics_gauge_task('commcare.sms.queued', queued_sms, run_every=crontab())
@periodic_task(run_every=crontab(minute=0, hour=3), queue='background_queue') def clean_expired_transient_emails(): try: TransientBounceEmail.delete_expired_bounces() except Exception as e: notify_exception(None, message="Encountered error while deleting expired " "transient bounce emails", details={ 'error': e, }) def get_maintenance_alert_active(): from corehq.apps.hqwebapp.models import MaintenanceAlert return 1 if MaintenanceAlert.get_latest_alert() else 0 metrics_gauge_task('commcare.maintenance_alerts.active', get_maintenance_alert_active, run_every=crontab(minute=1), multiprocess_mode=MPM_MAX) @periodic_task(run_every=crontab(minute=0, hour=4)) def clear_expired_oauth_tokens(): # https://django-oauth-toolkit.readthedocs.io/en/latest/management_commands.html#cleartokens call_command('cleartokens')
return exit_celery_with_error_message(bulk_import_async, get_importer_error_message(e)) finally: store_task_result.delay(excel_id) @task(serializer='pickle', queue='case_import_queue') def store_task_result(upload_id): case_upload = CaseUpload.get(upload_id) case_upload.store_task_result() def _alert_on_result(result, domain): """ Check import result and send internal alerts based on result :param result: dict that should include key "created_count" pointing to an int """ if result['created_count'] > 10000: message = "A case import just uploaded {num} new cases to HQ. {domain} might be scaling operations".format( num=result['created_count'], domain=domain) alert = AbnormalUsageAlert(source="case importer", domain=domain, message=message) send_abnormal_usage_alert.delay(alert) total_bytes = metrics_gauge_task('commcare.case_importer.files.total_bytes', get_case_upload_files_total_bytes, run_every=crontab(minute=0))
return repeater = repeat_record.repeater if not repeater: repeat_record.cancel() repeat_record.save() return try: if repeater.paused: # postpone repeat record by 1 day so that these don't get picked in each cycle and # thus clogging the queue with repeat records with paused repeater repeat_record.postpone_by(timedelta(days=1)) return if repeater.doc_type.endswith(DELETED_SUFFIX): if not repeat_record.doc_type.endswith(DELETED_SUFFIX): repeat_record.doc_type += DELETED_SUFFIX repeat_record.save() elif repeat_record.state == RECORD_PENDING_STATE or repeat_record.state == RECORD_FAILURE_STATE: repeat_record.fire() except Exception: logging.exception('Failed to process repeat record: {}'.format( repeat_record._id)) repeaters_overdue = metrics_gauge_task( 'commcare.repeaters.overdue', get_overdue_repeat_record_count, run_every=crontab() # every minute )
) self.retry(exc=e) @periodic_task(run_every=crontab(minute=0, hour=0), queue='background_queue') def process_bounced_emails(): if settings.RETURN_PATH_EMAIL and settings.RETURN_PATH_EMAIL_PASSWORD: try: with BouncedEmailManager( delete_processed_messages=True ) as bounced_manager, metrics_track_errors('process_bounced_emails_task'): bounced_manager.process_aws_notifications() bounced_manager.process_daemon_messages() except Exception as e: notify_exception( None, message="Encountered error while processing bounced emails", details={ 'error': e, } ) def get_maintenance_alert_active(): from corehq.apps.hqwebapp.models import MaintenanceAlert return 1 if MaintenanceAlert.get_latest_alert() else 0 metrics_gauge_task('commcare.maintenance_alerts.active', get_maintenance_alert_active, run_every=crontab(minute=1))
finally: if not result_stored: store_failed_task_result.delay(excel_id) @task(queue='case_import_queue') def store_failed_task_result(upload_id): case_upload = CaseUpload.get(upload_id) case_upload.store_failed_task_result() def _alert_on_result(result, domain): """ Check import result and send internal alerts based on result :param result: dict that should include key "created_count" pointing to an int """ if result['created_count'] > 10000: message = "A case import just uploaded {num} new cases to HQ. {domain} might be scaling operations".format( num=result['created_count'], domain=domain) alert = AbnormalUsageAlert(source="case importer", domain=domain, message=message) send_abnormal_usage_alert.delay(alert) metrics_gauge_task('commcare.case_importer.files.total_bytes', get_case_upload_files_total_bytes, run_every=crontab(minute=0), multiprocess_mode=MPM_MAX)