def handle(self, *args, **options): """Run Populate ocw courses""" if options["delete"]: self.stdout.write("Deleting all existing OCW courses") for course in Course.objects.filter(platform="ocw"): course.delete() delete_course(course) else: task = get_ocw_data.delay( force_overwrite=options["force_overwrite"], upload_to_s3=options["upload_to_s3"], ignore_flag=True, ) self.stdout.write( "Started task {task} to get ocw course data w/force_overwrite={overwrite}, upload_to_s3={s3}" .format( task=task, overwrite=options["force_overwrite"], s3=options["upload_to_s3"], )) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of ocw data finished, took {} seconds".format( total_seconds))
def test_send_notification(mocker, user): """Tests send_notification""" ns = NotificationSettingsFactory.create(via_email=True, weekly=True) notifier = frontpage.FrontpageDigestNotifier(ns) post = PostFactory.create() send_messages_mock = mocker.patch("mail.api.send_messages") serializer_mock = mocker.patch("channels.serializers.posts.PostSerializer") serializer_mock.return_value.data = { "id": post.post_id, "title": "post's title", "slug": "a_slug", "channel_name": "micromasters", "channel_title": "MicroMasters", "created": now_in_utc().isoformat(), "author_id": user.username, } submission = mocker.Mock(id=post.post_id, created=int(now_in_utc().timestamp()), stickied=False) api_mock = mocker.patch("channels.api.Api") api_mock.return_value.front_page.return_value = [submission] note = EmailNotificationFactory.create( user=ns.user, notification_type=ns.notification_type, sending=True) notifier.send_notification(note) serializer_mock.assert_called_once_with( PostProxy(submission, post), context={"current_user": note.user}) send_messages_mock.assert_called_once_with([any_instance_of(EmailMessage)])
def handle(self, *args, **options): task = tasks.update_memberships_for_managed_channels.delay( channel_names=options["channel_names"]) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Updated user channel memberships, took {} seconds".format( total_seconds))
def handle(self, *args, **options): task = tasks.subscribe_all_users_to_channels.delay( channel_names=options["channel_names"] ) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Subscribed all users to channels, took {} seconds".format(total_seconds) )
def handle(self, *args, **options): """Run Populate youtube videos""" command = options["command"] if command == "delete": videos = Video.objects.filter(platform=PlatformType.youtube.value) self.stdout.write( f"Deleting {videos.count()} existing YouTube videos from database and ElasticSearch" ) for video in videos: video.delete() delete_video(video) self.stdout.write("Complete") elif command == "fetch": channel_ids = options["channel_ids"] task = get_youtube_data.delay(channel_ids=channel_ids) self.stdout.write(f"Started task {task} to get YouTube video data") self.stdout.write("Waiting on task...") start = now_in_utc() result = task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( f"Fetched {result} YouTube channel in {total_seconds} seconds") elif command == "transcripts": created_after = options["created_after"] created_minutes = options["created_minutes"] overwrite = options["overwrite"] if created_after: try: created_after = datetime.strptime( created_after, ISOFORMAT).replace(tzinfo=pytz.UTC) except ValueError: self.stdout.write("Invalid date format") return if created_minutes: try: created_minutes = int(created_minutes) except ValueError: self.stdout.write("created_minutes must be an integer") return task = get_youtube_transcripts.delay( created_after=created_after, created_minutes=created_minutes, overwrite=overwrite, ) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write(f"Completed in {total_seconds} seconds")
def handle(self, *args, **options): """Run celery task to create/update channel subscriptions and moderator/contributor roles""" task = populate_subscriptions_and_roles.delay() self.stdout.write( "Started celery task {task} to populate channel user roles and subscriptions" .format(task=task)) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of channel user roles and subscriptions finished, took {} seconds" .format(total_seconds))
def handle(self, *args, **options): """Run celery task to backpopulate channel fields from reddit""" task = populate_channel_fields.delay() self.stdout.write( "Started celery task {task} to populate channel fieldss".format( task=task)) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of channel fields finished, took {} seconds".format( total_seconds))
def handle(self, *args, **options): """Backpopulates post and comment fields""" task = populate_post_and_comment_fields.delay() self.stdout.write( "Started celery task {task} to backpopulate post and comment fields" .format(task=task)) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Backpopulate of post and comment fields finished, took {} seconds" .format(total_seconds))
def handle(self, *args, **options): """Run Upload OCW master json""" task = upload_ocw_master_json.delay() self.stdout.write( "Started celery task {task} to upload ocw master json files to s3". format(task=task)) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Finished uploading ocw master json files to s3, took {} seconds". format(total_seconds))
def handle(self, *args, **options): """Run Populate ocw course run files""" chunk_size = options["chunk_size"] task = import_all_ocw_files.delay(chunk_size=chunk_size) self.stdout.write( f"Started task {task} to get ocw course run file data w/chunk size {chunk_size}" ) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of ocw file data finished, took {} seconds".format( total_seconds))
def handle(self, *args, **options): """Index the comments and posts for the channels the user is subscribed to""" task = start_recreate_index.delay() self.stdout.write( "Started celery task {task} to index content".format(task=task)) self.stdout.write("Waiting on task...") start = now_in_utc() error = task.get() if error: raise CommandError(f"Recreate index errored: {error}") total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Recreate index finished, took {} seconds".format(total_seconds))
def _track_activity(self, request): """ Updates activity date on the user record Args: request (django.http.request.Request): the request to inspect Returns: bool: True if the activity was tracked """ try: payload = self.jwt_decode_handler( request.COOKIES.get(settings.OPEN_DISCUSSIONS_COOKIE_NAME, None)) except jwt.InvalidTokenError: return None if not payload: return None if not payload.get("tracked", False) and "username" in payload: Profile.objects.filter(user__username=payload["username"]).update( last_active_on=now_in_utc()) payload["tracked"] = True return payload return None
def get_youtube_videos_for_transcripts_job(*, created_after=None, created_minutes=None, overwrite=False): """ course_catalog.Video object filtered to tasks.get_youtube_transcripts job params Args: created_after (date or None): if a date inclued only videos with a created_on after that date created_minutes (int or None): if an int include only videos with created_on in the last created_minutes minutes overwrite (bool): if true include videos that already have transcripts Returns Django filtered course_catalog.videos object """ videos = Video.objects.filter(published=True) if not overwrite: videos = videos.filter(transcript="") if created_after: videos = videos.filter(created_on__gte=created_after) elif created_minutes: date = now_in_utc() - timedelta(minutes=created_minutes) videos = videos.filter(created_on__gte=date) return videos
def test_can_notify_invalid_frequency(notifier, mocker): """Tests that this raises an error if an unsupported trigger_frequency is used""" notifier.notification_settings = NotificationSettingsFactory.create( trigger_frequency="bananas" ) notification = mocker.Mock() notification.created_on = now_in_utc() with pytest.raises(InvalidTriggerFrequencyError): notifier.can_notify(notification)
def update(self, **kwargs): """ Automatically update updated_on timestamp when .update(). This is because .update() does not go through .save(), thus will not auto_now, because it happens on the database level without loading objects into memory. """ if "updated_on" not in kwargs: kwargs["updated_on"] = now_in_utc() return super().update(**kwargs)
def handle(self, *args, **options): """Run Populate edx courses""" if options["delete"]: self.stdout.write( "Deleting all existing MITx courses from database and ElasticSearch" ) for course in Course.objects.filter(platform=PlatformType.mitx.value): course.delete() delete_course(course) else: task = get_mitx_data.delay() self.stdout.write(f"Started task {task} to get edx course data") self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of edx data finished, took {} seconds".format(total_seconds) )
def handle(self, *args, **options): """Run Populate micromasters courses""" if options["delete"]: self.stdout.write( "Deleting all existing xPro programs from database and ElasticSearch" ) # NOTE: we only delete programs, because courses are owned by the MITx integration for program in Program.objects.filter( platform=PlatformType.micromasters.value): delete_program(program) else: task = get_micromasters_data.delay() self.stdout.write( f"Started task {task} to get micromasters course data") self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of micromasters data finished, took {} seconds". format(total_seconds))
def handle(self, *args, **options): """Run Populate bootcamp courses""" if options["delete"]: self.stdout.write( "Deleting all existing bootcamps from database and ElasticSearch" ) for bootcamp in Bootcamp.objects.iterator(): bootcamp.delete() delete_bootcamp(bootcamp) else: task = get_bootcamp_data.delay( force_overwrite=options["force_overwrite"]) self.stdout.write( "Started task {task} to get bootcamp data w/force_overwrite={overwrite}" .format(task=task, overwrite=options["force_overwrite"])) self.stdout.write("Waiting on task...") start = now_in_utc() task.get() total_seconds = (now_in_utc() - start).total_seconds() self.stdout.write( "Population of bootcamp data finished, took {} seconds".format( total_seconds))
def test_can_notify( settings, mocker, is_enabled, can_notify, has_posts, trigger_frequency, has_last_notification, has_posts_after, ): # pylint: disable=too-many-arguments, too-many-locals """Test can_notify""" notification_settings = NotificationSettingsFactory.create( trigger_frequency=trigger_frequency) notification = (EmailNotificationFactory.create( user=notification_settings.user, frontpage_type=True, sent=True) if has_last_notification else None) settings.FEATURES[features.FRONTPAGE_EMAIL_DIGESTS] = is_enabled can_notify_mock = mocker.patch( "notifications.notifiers.email.EmailNotifier.can_notify", return_value=can_notify, ) created_on = notification.created_on if notification is not None else now_in_utc( ) post = PostFactory.create() api_mock = mocker.patch("channels.api.Api") api_mock.return_value.front_page.return_value = ([ mocker.Mock( id=post.post_id, created=int( (created_on + timedelta(days=10 if has_posts_after else -10)).timestamp()), stickied=False, ) ] if has_posts else []) notifier = frontpage.FrontpageDigestNotifier(notification_settings) expected = (is_enabled and can_notify and has_posts and (not has_last_notification or has_posts_after) and trigger_frequency in [FREQUENCY_DAILY, FREQUENCY_WEEKLY]) assert notifier.can_notify(notification) is expected if is_enabled: can_notify_mock.assert_called_once_with(notification)
def can_notify(self, last_notification): """ Returns true if we can notify this user based on their settings and when the last notification occurred Args: last_notification (NotificationBase): last notification that was triggered for this NotificationSettings Raises: InvalidTriggerFrequencyError: if the frequency is invalid Returns: bool: True if we're due to send another notification """ if (not self.notification_settings.user.is_active or self.notification_settings.is_triggered_never): return False # special case if we've never sent a notification or the setting is for an immediate send if (last_notification is None or self.notification_settings.is_triggered_immediate): return True # normalize now and created_on to the start of their respective days normalized_now = normalize_to_start_of_day(now_in_utc()) normalized_created_on = normalize_to_start_of_day( last_notification.created_on) if self.notification_settings.is_triggered_daily: trigger_offset = DELTA_ONE_DAY elif self.notification_settings.is_triggered_weekly: trigger_offset = DELTA_ONE_WEEK else: # practically, we'd only see this if our code called a notifier invalidly for a NEVER trigger_frequency raise InvalidTriggerFrequencyError( "Unsupported trigger_frequency: {}".format( self.notification_settings.trigger_frequency)) # return true if the normalized value is at least trigger_offset in the past return (normalized_created_on + trigger_offset) <= normalized_now
def mark_as_sent_or_canceled(notification): """ Marks the message as sent if it hasn't been yet or canceled if a cancel exception is raised Yeilds: bool: True if the email is being sent Args: notification (NotificationBase): notification to mark as sent """ with transaction.atomic(): notification = EmailNotification.objects.select_for_update().get( id=notification.id ) if notification.state != EmailNotification.STATE_SENDING: # prevent sending an email that is not ready to be sent or already has been log.debug("EmailNotification not in sending state: %s", notification.id) yield False return try: yield True log.debug("EmailNotification sent: %s", notification.id) notification.state = EmailNotification.STATE_SENT notification.sent_at = now_in_utc() notification.save() except CancelNotificationError: log.debug("EmailNotification canceled: %s", notification.id) notification.state = EmailNotification.STATE_CANCELED notification.save() except: # pylint: disable=bare-except # if any other error happens, roll back to pending so that the crontask picks it up again log.exception( "EmailNotification rolled back to pending: %s", notification.id ) notification.state = EmailNotification.STATE_PENDING notification.save()
def channel_name(channel, index): # pylint: disable=unused-argument """Generate a channel name from the current time and a random word""" now = now_in_utc().timestamp() return "{}_{}_{}".format( int(now), index, FAKE.word())[:21] # maximum of 21-char channel names
def test_now_in_utc(): """now_in_utc() should return the current time set to the UTC time zone""" now = now_in_utc() assert is_near_now(now) assert now.tzinfo == pytz.UTC