def create_activity_upcoming_notifications():
# Oh oh, this is a bit complex. As notification.context is a JSONField, the participants_already_notified subquery
# would return a jsonb object by default (which can't be compared to integer).
# We can work around this by transforming the property value to text ("->>" lookup) and then casting to integer
with timer() as t:
participants_already_notified = Notification.objects.\
order_by().\
filter(type=NotificationType.ACTIVITY_UPCOMING.value).\
exclude(context__activity_participant=None).\
values_list(Cast(KeyTextTransform('activity_participant', 'context'), IntegerField()), flat=True)
activities_due_soon = Activity.objects.order_by().due_soon()
participants = ActivityParticipant.objects.\
filter(activity__in=activities_due_soon).\
exclude(id__in=participants_already_notified).\
distinct()
for participant in participants:
Notification.objects.create(
type=NotificationType.ACTIVITY_UPCOMING.value,
user=participant.user,
expires_at=participant.activity.date.start,
context={
'group': participant.activity.place.group.id,
'place': participant.activity.place.id,
'activity': participant.activity.id,
'activity_participant': participant.id,
},
)
stats_utils.periodic_task(
'notifications__create_activity_upcoming_notifications',
seconds=t.elapsed_seconds)
def process_inactive_users():
now = timezone.now()
count_users_flagged_inactive = 0
count_users_flagged_for_removal = 0
count_users_removed = 0
# first, we mark them as inactive
inactive_threshold_date = now - timedelta(
days=settings.NUMBER_OF_DAYS_UNTIL_INACTIVE_IN_GROUP)
for membership in GroupMembership.objects.filter(
lastseen_at__lte=inactive_threshold_date,
inactive_at=None,
):
# only send emails if group itself is marked as active
if membership.group.status == GroupStatus.ACTIVE.value:
prepare_user_inactive_in_group_email(membership.user,
membership.group).send()
membership.inactive_at = now
membership.save()
count_users_flagged_inactive += 1
# then, if they have been inactive for some time, we warn them we will remove them
removal_notification_date = now - relativedelta(
months=settings.
NUMBER_OF_INACTIVE_MONTHS_UNTIL_REMOVAL_FROM_GROUP_NOTIFICATION)
for membership in GroupMembership.objects.filter(
removal_notification_at=None,
inactive_at__lte=removal_notification_date):
if membership.group.status == GroupStatus.ACTIVE.value:
prepare_user_removal_from_group_email(membership.user,
membership.group).send()
membership.removal_notification_at = now
membership.save()
count_users_flagged_for_removal += 1
# and finally, actually remove them
removal_date = now - timedelta(
days=settings.
NUMBER_OF_DAYS_AFTER_REMOVAL_NOTIFICATION_WE_ACTUALLY_REMOVE_THEM)
for membership in GroupMembership.objects.filter(
removal_notification_at__lte=removal_date):
membership.delete()
History.objects.create(
typus=HistoryTypus.GROUP_LEAVE_INACTIVE,
group=membership.group,
users=[membership.user],
payload={'display_name': membership.user.display_name})
count_users_removed += 1
stats_utils.periodic_task(
'group__process_inactive_users', {
'count_users_flagged_inactive': count_users_flagged_inactive,
'count_users_flagged_for_removal': count_users_flagged_for_removal,
'count_users_removed': count_users_removed,
})
def delete_old_email_events():
with timer() as t:
# delete email events after some months
EmailEvent.objects.filter(created_at__lt=timezone.now() -
timedelta(days=3 * 30))
stats_utils.periodic_task('webhooks__delete_old_email_events',
seconds=t.elapsed_seconds)
def mark_inactive_groups():
with timer() as t:
for group in Group.objects.filter(status=GroupStatus.ACTIVE.value):
if not group.has_recent_activity():
group.status = GroupStatus.INACTIVE.value
group.save()
stats_utils.periodic_task('group__mark_inactive_groups',
seconds=t.elapsed_seconds)
def record_user_stats():
stats_utils.periodic_task('users__record_user_stats')
fields = get_users_stats()
write_points([{
'measurement': 'karrot.users',
'fields': fields,
}])
def record_group_stats():
stats_utils.periodic_task('group__record_group_stats')
points = []
for group in Group.objects.all():
points.extend(get_group_members_stats(group))
points.extend(get_group_places_stats(group))
points.extend(get_application_stats(group))
points.extend(get_issue_stats(group))
write_points(points)
def record_group_stats():
with timer() as t:
points = []
for group in Group.objects.all():
points.extend(get_group_members_stats(group))
points.extend(get_group_places_stats(group))
points.extend(get_application_stats(group))
points.extend(get_issue_stats(group))
write_points(points)
stats_utils.periodic_task('group__record_group_stats',
seconds=t.elapsed_seconds)
def process_expired_votings():
with timer() as t:
for voting in Voting.objects.filter(
expires_at__lte=timezone.now(),
accepted_option__isnull=True,
issue__status=IssueStatus.ONGOING.value,
):
# if nobody participated in the voting, cancel it!
# otherwise it would result in a tie and continue forever
if voting.participant_count() == 0:
voting.issue.cancel()
continue
voting.calculate_results()
stats_utils.periodic_task('issues__process_expired_votings', seconds=t.elapsed_seconds)
def mark_conversations_as_closed():
with timer() as t:
close_threshold = timezone.now() - relativedelta(
days=settings.CONVERSATION_CLOSED_DAYS)
for conversation in Conversation.objects.filter(
is_closed=False,
target_id__isnull=False,
).exclude(latest_message__created_at__gte=close_threshold):
ended_at = conversation.target.ended_at
if ended_at is not None and ended_at < close_threshold:
conversation.is_closed = True
conversation.save()
stats_utils.periodic_task('conversations__mark_conversations_as_closed',
seconds=t.elapsed_seconds)
def daily_pickup_notifications():
stats_utils.periodic_task('pickups__daily_pickup_notifications')
for group in Group.objects.all():
with timezone.override(group.timezone):
if timezone.localtime().hour != 20: # only at 8pm local time
continue
for data in fetch_pickup_notification_data_for_group(group):
prepare_pickup_notification_email(**data).send()
stats.pickup_notification_email(group=data['group'],
**{
k: v.count()
for k, v in data.items()
if isinstance(v, QuerySet)
})
def daily_activity_notifications():
with timer() as t:
for group in Group.objects.all():
with timezone.override(group.timezone):
if timezone.localtime().hour != 20: # only at 8pm local time
continue
for data in fetch_activity_notification_data_for_group(group):
prepare_activity_notification_email(**data).send()
stats.activity_notification_email(
group=data['group'], **{k: v.count()
for k, v in data.items() if isinstance(v, QuerySet)}
)
stats_utils.periodic_task('activities__daily_activity_notifications', seconds=t.elapsed_seconds)
def record_user_stats():
with timer() as t:
fields = get_users_stats()
language_fields = get_user_language_stats()
write_points([{
'measurement': 'karrot.users',
'fields': fields,
}])
write_points([{
'measurement': 'karrot.users.language',
'fields': language_fields,
}])
stats_utils.periodic_task('users__record_user_stats',
seconds=t.elapsed_seconds)
def create_voting_ends_soon_notifications():
with timer() as t:
existing_notifications = Notification.objects.order_by().filter(
type=NotificationType.VOTING_ENDS_SOON.value).values_list(
'user_id', 'context__issue')
for voting in Voting.objects.order_by().due_soon().filter(
issue__status=IssueStatus.ONGOING.value):
# only notify users that haven't voted already
for user in voting.issue.group.members.exclude(
votes_given__option__voting=voting):
if (user.id, voting.issue_id) not in existing_notifications:
Notification.objects.create(
type=NotificationType.VOTING_ENDS_SOON.value,
user=user,
expires_at=voting.expires_at,
context={
'group': voting.issue.group_id,
'issue': voting.issue_id,
},
)
stats_utils.periodic_task(
'notifications__create_voting_ends_soon_notification',
seconds=t.elapsed_seconds)
def process_finished_pickup_dates():
with timer() as t:
PickupDate.objects.process_finished_pickup_dates()
stats_utils.periodic_task('pickups__process_finished_pickup_dates', seconds=t.elapsed_seconds)
def delete_expired_invitations():
with timer() as t:
Invitation.objects.delete_expired_invitations()
stats_utils.periodic_task('invitations__deleted_expired_invitations',
seconds=t.elapsed_seconds)
def delete_expired_notifications():
with timer() as t:
Notification.objects.expired().delete()
stats_utils.periodic_task('notifications__delete_expired_notifications',
seconds=t.elapsed_seconds)
def update_activities():
with timer() as t:
ActivitySeries.objects.update_activities()
stats_utils.periodic_task('activities__update_activities', seconds=t.elapsed_seconds)
def process_finished_activities():
with timer() as t:
Activity.objects.process_finished_activities()
stats_utils.periodic_task('activities__process_finished_activities', seconds=t.elapsed_seconds)
def update_pickups():
with timer() as t:
PickupDateSeries.objects.update_pickups()
stats_utils.periodic_task('pickups__update_pickups', seconds=t.elapsed_seconds)
def send_summary_emails():
"""
We want to send them on Sunday at 8am in the local time of the group
So we run this each hour to check if any group needs their summary email sending.
We limit it to just the weekend (Saturday, Sunday) as this will cover any possible timezone offset (-11/+12)
In cron Sunday is day 0 and Saturday is day 6
"""
with timer() as t:
email_count = 0
recipient_count = 0
def is_8am_localtime(group):
with timezone.override(group.timezone):
localtime = timezone.localtime()
# Sunday is day 6 here, confusing!
return localtime.hour == 8 and localtime.weekday() == 6
groups = Group.objects.annotate(member_count=Count('members')).filter(
member_count__gt=0)
for group in groups:
if not is_8am_localtime(group):
continue
from_date, to_date = calculate_group_summary_dates(group)
if not group.sent_summary_up_to or group.sent_summary_up_to < to_date:
email_recipient_count = 0
context = prepare_group_summary_data(group, from_date, to_date)
if context['has_activity']:
for email in prepare_group_summary_emails(group, context):
try:
email.send()
email_count += 1
email_recipient_count += len(email.to)
except AnymailAPIError:
sentry_client.captureException()
# we save this even if some of the email sending fails, no retries right now basically...
# we also save if no emails were sent due to missing activity, to not try again over and over.
group.sent_summary_up_to = to_date
group.save()
group_summary_email(
group,
email_recipient_count=email_recipient_count,
feedback_count=context['feedbacks'].count(),
message_count=context['messages'].count(),
new_user_count=context['new_users'].count(),
activities_done_count=context['activities_done_count'],
activities_missed_count=context['activities_missed_count'],
has_activity=context['has_activity'],
)
recipient_count += email_recipient_count
stats_utils.periodic_task('group__send_summary_emails',
seconds=t.elapsed_seconds,
extra_fields={
'recipient_count': recipient_count,
'email_count': email_count,
})
