def update_youtube_statuses(self):
"""
Update the status of recently uploaded YouTube videos and upload captions if complete
"""
youtube = YouTubeApi()
videos_processing = YouTubeVideo.objects.filter(status=YouTubeStatus.UPLOADED)
for yt_video in videos_processing:
try:
yt_video.status = youtube.video_status(yt_video.id)
yt_video.save()
if yt_video.status == YouTubeStatus.PROCESSED:
for subtitle in yt_video.video.videosubtitle_set.all():
youtube.upload_caption(subtitle, yt_video.id)
except IndexError:
# Video might be a dupe or deleted, mark it as failed and continue to next one.
yt_video.status = YouTubeStatus.FAILED
yt_video.save()
log.exception(
"Status of YoutubeVideo not found.",
youtubevideo_id=yt_video.id,
youtubevideo_video_id=yt_video.video_id,
)
except HttpError as error:
if API_QUOTA_ERROR_MSG in error.content.decode("utf-8"):
# Don't raise the error, task will try on next run until daily quota is reset
break
raise
def remove_youtube_caption(self, video_id, language):
"""
Remove Youtube captions not matching a video's subtitle language)
"""
video = Video.objects.get(id=video_id)
captions = YouTubeApi().list_captions(video.youtube_id)
if language in captions.keys():
YouTubeApi().delete_caption(captions[language])
def upload_youtube_caption(self, caption_id):
"""
Upload a video caption file to YouTube
"""
caption = VideoSubtitle.objects.get(id=caption_id)
yt_video = YouTubeVideo.objects.get(video=caption.video)
youtube = YouTubeApi()
youtube.upload_caption(caption, yt_video.id)
def test_upload_video(mocker):
"""
Test that the upload_video task calls the YouTube API execute method
"""
videofile = VideoFileFactory()
youtube_id = "M6LymW_8qVk"
video_upload_response = {
"id": youtube_id,
"kind": "youtube#video",
"snippet": {
"description": "Testing description",
"title": "Testing123"
},
"status": {
"uploadStatus": "uploaded"
},
}
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.side_effect = [
(None, None),
(None, video_upload_response),
]
response = YouTubeApi().upload_video(videofile.video)
assert response == video_upload_response
def test_video_status(mocker):
"""
Test that the 'video_status' method returns the correct value from the API response
"""
expected_status = "processed"
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).videos.return_value.list.return_value.execute.return_value = {
"etag":
'"ld9biNPKjAjgjV7EZ4EKeEGrhao/Lf7oS5V-Gjw0XHBBKFJRpn60z3w"',
"items": [{
"etag":
'"ld9biNPKjAjgjV7EZ4EKeEGrhao/-UL82wRXbq3YJiMZuZpqCWKoq6Q"',
"id": "wAjoqsZng_M",
"kind": "youtube#video",
"status": {
"embeddable": True,
"license": "youtube",
"privacyStatus": "unlisted",
"publicStatsViewable": True,
"uploadStatus": expected_status,
},
}],
"kind":
"youtube#videoListResponse",
"pageInfo": {
"resultsPerPage": 1,
"totalResults": 1
},
}
assert YouTubeApi().video_status("foo") == expected_status
youtube_mocker().videos.return_value.list.assert_called_once_with(
id="foo", part="status")
def test_list_captions(mocker):
"""
Test that the 'list_captions' method executes a YouTube API request and returns a dict with correct values
"""
key = "en"
value = "Srnr982VEC79QzEBGcBOL_UFmu9U2e-JgOw-EWIxJXEB5Bjltl3Yvg="
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).captions.return_value.list.return_value.execute.return_value = {
"etag":
"foo",
"items": [{
"id": value,
"kind": "youtube#caption",
"snippet": {
"audioTrackType": "unknown",
"language": key,
"lastUpdated": "2017-11-15T14:53:21.839Z",
"name": "English",
"videoId": "3h-0mkTVbRg",
},
}],
"kind":
"youtube#captionListResponse",
}
assert YouTubeApi().list_captions("foo") == {key: value}
youtube_mocker().captions.return_value.list.assert_called_once_with(
videoId="foo", part="snippet")
def test_delete_video(mocker):
"""
Test that the 'delete_video' method executes a YouTube API deletion request and returns the status code
"""
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).videos.return_value.delete.return_value.execute.return_value = 204
assert YouTubeApi().delete_video("foo") == 204
youtube_mocker().videos.return_value.delete.assert_called_with(id="foo")
def remove_youtube_video(self, video_id):
"""
Delete a video from Youtube
"""
try:
YouTubeApi().delete_video(video_id)
except HttpError as error:
if error.resp.status == 404:
log.info("Not found on Youtube, already deleted?", video_id=video_id)
else:
raise
def test_upload_errors_retryable(mocker, error, retryable):
"""
Test that uploads are retried 10x for retryable exceptions
"""
youtube_mocker = mocker.patch("cloudsync.youtube.build")
mocker.patch("cloudsync.youtube.time")
videofile = VideoFileFactory()
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.side_effect = (error)
with pytest.raises(Exception) as exc:
YouTubeApi().upload_video(videofile.video)
assert str(exc.value).startswith("Retried YouTube upload 10x") == retryable
def test_upload_video_no_id(mocker):
"""
Test that the upload_video task fails if the response contains no id
"""
videofile = VideoFileFactory()
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.return_value = (
None,
{},
)
with pytest.raises(YouTubeUploadException):
YouTubeApi().upload_video(videofile.video)
def upload_youtube_videos():
"""
Upload public videos one at a time to YouTube (if not already there) until the daily maximum is reached.
"""
yt_queue = (
Video.objects.filter(is_public=True)
.filter(status=VideoStatus.COMPLETE)
.filter(youtubevideo__id__isnull=True)
.exclude(collection__stream_source=StreamSource.CLOUDFRONT)
.order_by("-created_at")[: settings.YT_UPLOAD_LIMIT]
)
for video in yt_queue.all():
youtube_video = YouTubeVideo.objects.create(video=video)
try:
youtube = YouTubeApi()
response = youtube.upload_video(video)
youtube_video.id = response["id"]
youtube_video.status = response["status"]["uploadStatus"]
youtube_video.save()
except HttpError as error:
log.exception(
"HttpError uploading video to Youtube",
video_hexkey=video.hexkey,
status=youtube_video.status,
)
if API_QUOTA_ERROR_MSG in error.content.decode("utf-8"):
break
except: # pylint: disable=bare-except
log.exception(
"Error uploading video to Youtube",
video_hexkey=video.hexkey,
status=youtube_video.status,
)
finally:
# If anything went wrong with the upload, delete the YouTubeVideo object.
# Another upload attempt will be made the next time the task is run.
if youtube_video.id is None:
youtube_video.delete()
def test_upload_caption_calls_insert(mocker):
"""
Test that the upload_caption task calls insert_caption for a YouTube video if no caption for that language exists
"""
subtitle = VideoSubtitleFactory()
caption_id = "foo"
caption_response = {"id": caption_id}
mocker.patch("cloudsync.youtube.YouTubeApi.list_captions", return_value={})
youtube_mocker = mocker.patch("cloudsync.youtube.build")
youtube_mocker(
).captions.return_value.insert.return_value.next_chunk.return_value = (
None,
caption_response,
)
response = YouTubeApi().upload_caption(subtitle, caption_id)
assert response == caption_response
def test_upload_video_long_fields(mocker):
"""
Test that the upload_youtube_video task truncates title and description if too long
"""
title = "".join(random.choice(string.ascii_lowercase) for c in range(105))
desc = "".join(random.choice(string.ascii_lowercase) for c in range(5005))
video = VideoFactory.create(title=title,
description=desc,
is_public=True,
status=VideoStatus.COMPLETE)
VideoFileFactory(video=video)
mocker.patch("cloudsync.youtube.resumable_upload")
youtube_mocker = mocker.patch("cloudsync.youtube.build")
mock_upload = youtube_mocker().videos.return_value.insert
YouTubeApi().upload_video(video)
called_args, called_kwargs = mock_upload.call_args
assert called_kwargs["body"]["snippet"]["title"] == title[:100]
assert called_kwargs["body"]["snippet"]["description"] == desc[:5000]
def test_youtube_settings(mocker, settings):
"""
Test that Youtube object creation uses YT_* settings for credentials
"""
settings.YT_ACCESS_TOKEN = "yt_access_token"
settings.YT_CLIENT_ID = "yt_client_id"
settings.YT_CLIENT_SECRET = "yt_secret"
settings.YT_REFRESH_TOKEN = "yt_refresh"
mock_oauth = mocker.patch(
"cloudsync.youtube.oauth2client.client.GoogleCredentials")
YouTubeApi()
mock_oauth.assert_called_with(
settings.YT_ACCESS_TOKEN,
settings.YT_CLIENT_ID,
settings.YT_CLIENT_SECRET,
settings.YT_REFRESH_TOKEN,
None,
"https://accounts.google.com/o/oauth2/token",
None,
)