def handle(self, *args, **options):
"""Run Populate ocw courses"""
if options["delete"]:
self.stdout.write("Deleting all existing OCW courses")
for course in Course.objects.filter(platform="ocw"):
course.delete()
delete_course(course)
else:
task = get_ocw_data.delay(
force_overwrite=options["force_overwrite"],
upload_to_s3=options["upload_to_s3"],
ignore_flag=True,
)
self.stdout.write(
"Started task {task} to get ocw course data w/force_overwrite={overwrite}, upload_to_s3={s3}"
.format(
task=task,
overwrite=options["force_overwrite"],
s3=options["upload_to_s3"],
))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of ocw data finished, took {} seconds".format(
total_seconds))
def test_send_notification(mocker, user):
"""Tests send_notification"""
ns = NotificationSettingsFactory.create(via_email=True, weekly=True)
notifier = frontpage.FrontpageDigestNotifier(ns)
post = PostFactory.create()
send_messages_mock = mocker.patch("mail.api.send_messages")
serializer_mock = mocker.patch("channels.serializers.posts.PostSerializer")
serializer_mock.return_value.data = {
"id": post.post_id,
"title": "post's title",
"slug": "a_slug",
"channel_name": "micromasters",
"channel_title": "MicroMasters",
"created": now_in_utc().isoformat(),
"author_id": user.username,
}
submission = mocker.Mock(id=post.post_id,
created=int(now_in_utc().timestamp()),
stickied=False)
api_mock = mocker.patch("channels.api.Api")
api_mock.return_value.front_page.return_value = [submission]
note = EmailNotificationFactory.create(
user=ns.user, notification_type=ns.notification_type, sending=True)
notifier.send_notification(note)
serializer_mock.assert_called_once_with(
PostProxy(submission, post), context={"current_user": note.user})
send_messages_mock.assert_called_once_with([any_instance_of(EmailMessage)])
def handle(self, *args, **options):
task = tasks.update_memberships_for_managed_channels.delay(
channel_names=options["channel_names"])
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Updated user channel memberships, took {} seconds".format(
total_seconds))
def handle(self, *args, **options):
task = tasks.subscribe_all_users_to_channels.delay(
channel_names=options["channel_names"]
)
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Subscribed all users to channels, took {} seconds".format(total_seconds)
)
def handle(self, *args, **options):
"""Run Populate youtube videos"""
command = options["command"]
if command == "delete":
videos = Video.objects.filter(platform=PlatformType.youtube.value)
self.stdout.write(
f"Deleting {videos.count()} existing YouTube videos from database and ElasticSearch"
)
for video in videos:
video.delete()
delete_video(video)
self.stdout.write("Complete")
elif command == "fetch":
channel_ids = options["channel_ids"]
task = get_youtube_data.delay(channel_ids=channel_ids)
self.stdout.write(f"Started task {task} to get YouTube video data")
self.stdout.write("Waiting on task...")
start = now_in_utc()
result = task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
f"Fetched {result} YouTube channel in {total_seconds} seconds")
elif command == "transcripts":
created_after = options["created_after"]
created_minutes = options["created_minutes"]
overwrite = options["overwrite"]
if created_after:
try:
created_after = datetime.strptime(
created_after, ISOFORMAT).replace(tzinfo=pytz.UTC)
except ValueError:
self.stdout.write("Invalid date format")
return
if created_minutes:
try:
created_minutes = int(created_minutes)
except ValueError:
self.stdout.write("created_minutes must be an integer")
return
task = get_youtube_transcripts.delay(
created_after=created_after,
created_minutes=created_minutes,
overwrite=overwrite,
)
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(f"Completed in {total_seconds} seconds")
def handle(self, *args, **options):
"""Run celery task to create/update channel subscriptions and moderator/contributor roles"""
task = populate_subscriptions_and_roles.delay()
self.stdout.write(
"Started celery task {task} to populate channel user roles and subscriptions"
.format(task=task))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of channel user roles and subscriptions finished, took {} seconds"
.format(total_seconds))
def handle(self, *args, **options):
"""Run celery task to backpopulate channel fields from reddit"""
task = populate_channel_fields.delay()
self.stdout.write(
"Started celery task {task} to populate channel fieldss".format(
task=task))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of channel fields finished, took {} seconds".format(
total_seconds))
def handle(self, *args, **options):
"""Backpopulates post and comment fields"""
task = populate_post_and_comment_fields.delay()
self.stdout.write(
"Started celery task {task} to backpopulate post and comment fields"
.format(task=task))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Backpopulate of post and comment fields finished, took {} seconds"
.format(total_seconds))
def handle(self, *args, **options):
"""Run Upload OCW master json"""
task = upload_ocw_master_json.delay()
self.stdout.write(
"Started celery task {task} to upload ocw master json files to s3".
format(task=task))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Finished uploading ocw master json files to s3, took {} seconds".
format(total_seconds))
def handle(self, *args, **options):
"""Run Populate ocw course run files"""
chunk_size = options["chunk_size"]
task = import_all_ocw_files.delay(chunk_size=chunk_size)
self.stdout.write(
f"Started task {task} to get ocw course run file data w/chunk size {chunk_size}"
)
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of ocw file data finished, took {} seconds".format(
total_seconds))
def handle(self, *args, **options):
"""Index the comments and posts for the channels the user is subscribed to"""
task = start_recreate_index.delay()
self.stdout.write(
"Started celery task {task} to index content".format(task=task))
self.stdout.write("Waiting on task...")
start = now_in_utc()
error = task.get()
if error:
raise CommandError(f"Recreate index errored: {error}")
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Recreate index finished, took {} seconds".format(total_seconds))
def _track_activity(self, request):
"""
Updates activity date on the user record
Args:
request (django.http.request.Request): the request to inspect
Returns:
bool: True if the activity was tracked
"""
try:
payload = self.jwt_decode_handler(
request.COOKIES.get(settings.OPEN_DISCUSSIONS_COOKIE_NAME,
None))
except jwt.InvalidTokenError:
return None
if not payload:
return None
if not payload.get("tracked", False) and "username" in payload:
Profile.objects.filter(user__username=payload["username"]).update(
last_active_on=now_in_utc())
payload["tracked"] = True
return payload
return None
def get_youtube_videos_for_transcripts_job(*,
created_after=None,
created_minutes=None,
overwrite=False):
"""
course_catalog.Video object filtered to tasks.get_youtube_transcripts job params
Args:
created_after (date or None):
if a date inclued only videos with a created_on after that date
created_minutes (int or None):
if an int include only videos with created_on in the last created_minutes minutes
overwrite (bool):
if true include videos that already have transcripts
Returns
Django filtered course_catalog.videos object
"""
videos = Video.objects.filter(published=True)
if not overwrite:
videos = videos.filter(transcript="")
if created_after:
videos = videos.filter(created_on__gte=created_after)
elif created_minutes:
date = now_in_utc() - timedelta(minutes=created_minutes)
videos = videos.filter(created_on__gte=date)
return videos
def test_can_notify_invalid_frequency(notifier, mocker):
"""Tests that this raises an error if an unsupported trigger_frequency is used"""
notifier.notification_settings = NotificationSettingsFactory.create(
trigger_frequency="bananas"
)
notification = mocker.Mock()
notification.created_on = now_in_utc()
with pytest.raises(InvalidTriggerFrequencyError):
notifier.can_notify(notification)
def update(self, **kwargs):
"""
Automatically update updated_on timestamp when .update(). This is because .update()
does not go through .save(), thus will not auto_now, because it happens on the
database level without loading objects into memory.
"""
if "updated_on" not in kwargs:
kwargs["updated_on"] = now_in_utc()
return super().update(**kwargs)
def handle(self, *args, **options):
"""Run Populate edx courses"""
if options["delete"]:
self.stdout.write(
"Deleting all existing MITx courses from database and ElasticSearch"
)
for course in Course.objects.filter(platform=PlatformType.mitx.value):
course.delete()
delete_course(course)
else:
task = get_mitx_data.delay()
self.stdout.write(f"Started task {task} to get edx course data")
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of edx data finished, took {} seconds".format(total_seconds)
)
def handle(self, *args, **options):
"""Run Populate micromasters courses"""
if options["delete"]:
self.stdout.write(
"Deleting all existing xPro programs from database and ElasticSearch"
)
# NOTE: we only delete programs, because courses are owned by the MITx integration
for program in Program.objects.filter(
platform=PlatformType.micromasters.value):
delete_program(program)
else:
task = get_micromasters_data.delay()
self.stdout.write(
f"Started task {task} to get micromasters course data")
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of micromasters data finished, took {} seconds".
format(total_seconds))
def handle(self, *args, **options):
"""Run Populate bootcamp courses"""
if options["delete"]:
self.stdout.write(
"Deleting all existing bootcamps from database and ElasticSearch"
)
for bootcamp in Bootcamp.objects.iterator():
bootcamp.delete()
delete_bootcamp(bootcamp)
else:
task = get_bootcamp_data.delay(
force_overwrite=options["force_overwrite"])
self.stdout.write(
"Started task {task} to get bootcamp data w/force_overwrite={overwrite}"
.format(task=task, overwrite=options["force_overwrite"]))
self.stdout.write("Waiting on task...")
start = now_in_utc()
task.get()
total_seconds = (now_in_utc() - start).total_seconds()
self.stdout.write(
"Population of bootcamp data finished, took {} seconds".format(
total_seconds))
def test_can_notify(
settings,
mocker,
is_enabled,
can_notify,
has_posts,
trigger_frequency,
has_last_notification,
has_posts_after,
): # pylint: disable=too-many-arguments, too-many-locals
"""Test can_notify"""
notification_settings = NotificationSettingsFactory.create(
trigger_frequency=trigger_frequency)
notification = (EmailNotificationFactory.create(
user=notification_settings.user, frontpage_type=True, sent=True)
if has_last_notification else None)
settings.FEATURES[features.FRONTPAGE_EMAIL_DIGESTS] = is_enabled
can_notify_mock = mocker.patch(
"notifications.notifiers.email.EmailNotifier.can_notify",
return_value=can_notify,
)
created_on = notification.created_on if notification is not None else now_in_utc(
)
post = PostFactory.create()
api_mock = mocker.patch("channels.api.Api")
api_mock.return_value.front_page.return_value = ([
mocker.Mock(
id=post.post_id,
created=int(
(created_on +
timedelta(days=10 if has_posts_after else -10)).timestamp()),
stickied=False,
)
] if has_posts else [])
notifier = frontpage.FrontpageDigestNotifier(notification_settings)
expected = (is_enabled and can_notify and has_posts
and (not has_last_notification or has_posts_after)
and trigger_frequency in [FREQUENCY_DAILY, FREQUENCY_WEEKLY])
assert notifier.can_notify(notification) is expected
if is_enabled:
can_notify_mock.assert_called_once_with(notification)
def can_notify(self, last_notification):
"""
Returns true if we can notify this user based on their settings and when the last notification occurred
Args:
last_notification (NotificationBase): last notification that was triggered for this NotificationSettings
Raises:
InvalidTriggerFrequencyError: if the frequency is invalid
Returns:
bool: True if we're due to send another notification
"""
if (not self.notification_settings.user.is_active
or self.notification_settings.is_triggered_never):
return False
# special case if we've never sent a notification or the setting is for an immediate send
if (last_notification is None
or self.notification_settings.is_triggered_immediate):
return True
# normalize now and created_on to the start of their respective days
normalized_now = normalize_to_start_of_day(now_in_utc())
normalized_created_on = normalize_to_start_of_day(
last_notification.created_on)
if self.notification_settings.is_triggered_daily:
trigger_offset = DELTA_ONE_DAY
elif self.notification_settings.is_triggered_weekly:
trigger_offset = DELTA_ONE_WEEK
else:
# practically, we'd only see this if our code called a notifier invalidly for a NEVER trigger_frequency
raise InvalidTriggerFrequencyError(
"Unsupported trigger_frequency: {}".format(
self.notification_settings.trigger_frequency))
# return true if the normalized value is at least trigger_offset in the past
return (normalized_created_on + trigger_offset) <= normalized_now
def mark_as_sent_or_canceled(notification):
"""
Marks the message as sent if it hasn't been yet or canceled if a cancel exception is raised
Yeilds:
bool: True if the email is being sent
Args:
notification (NotificationBase): notification to mark as sent
"""
with transaction.atomic():
notification = EmailNotification.objects.select_for_update().get(
id=notification.id
)
if notification.state != EmailNotification.STATE_SENDING:
# prevent sending an email that is not ready to be sent or already has been
log.debug("EmailNotification not in sending state: %s", notification.id)
yield False
return
try:
yield True
log.debug("EmailNotification sent: %s", notification.id)
notification.state = EmailNotification.STATE_SENT
notification.sent_at = now_in_utc()
notification.save()
except CancelNotificationError:
log.debug("EmailNotification canceled: %s", notification.id)
notification.state = EmailNotification.STATE_CANCELED
notification.save()
except: # pylint: disable=bare-except
# if any other error happens, roll back to pending so that the crontask picks it up again
log.exception(
"EmailNotification rolled back to pending: %s", notification.id
)
notification.state = EmailNotification.STATE_PENDING
notification.save()
def channel_name(channel, index): # pylint: disable=unused-argument
"""Generate a channel name from the current time and a random word"""
now = now_in_utc().timestamp()
return "{}_{}_{}".format(
int(now), index, FAKE.word())[:21] # maximum of 21-char channel names
def test_now_in_utc():
"""now_in_utc() should return the current time set to the UTC time zone"""
now = now_in_utc()
assert is_near_now(now)
assert now.tzinfo == pytz.UTC
