def update_youtube_statuses():
"""
Update the status of recently uploaded YouTube videos if complete
"""
if not is_youtube_enabled():
return
videos_processing = VideoFile.objects.filter(
Q(status=VideoFileStatus.UPLOADED)
& Q(destination=DESTINATION_YOUTUBE))
if videos_processing.count() == 0:
return
youtube = YouTubeApi()
for video_file in videos_processing:
try:
with transaction.atomic():
video_file.destination_status = youtube.video_status(
video_file.destination_id)
if video_file.destination_status == YouTubeStatus.PROCESSED:
video_file.status = VideoFileStatus.COMPLETE
video_file.save()
drive_file = DriveFile.objects.filter(
video=video_file.video).first()
if drive_file and drive_file.resource:
resource = drive_file.resource
set_dict_field(
resource.metadata,
settings.YT_FIELD_ID,
video_file.destination_id,
)
set_dict_field(
resource.metadata,
settings.YT_FIELD_THUMBNAIL,
YT_THUMBNAIL_IMG.format(
video_id=video_file.destination_id),
)
resource.save()
mail_youtube_upload_success(video_file)
except IndexError:
# Video might be a dupe or deleted, mark it as failed and continue to next one.
video_file.status = VideoFileStatus.FAILED
video_file.save()
log.exception(
"Status of YouTube video not found: youtube_id %s",
video_file.destination_id,
)
mail_youtube_upload_failure(video_file)
except HttpError as error:
if API_QUOTA_ERROR_MSG in error.content.decode("utf-8"):
# Don't raise the error, task will try on next run until daily quota is reset
break
log.exception(
"Error for youtube_id %s: %s",
video_file.destination_id,
error.content.decode("utf-8"),
)
mail_youtube_upload_failure(video_file)
def test_video_status(youtube_mocker):
"""
Test that the 'video_status' method returns the correct value from the API response
"""
expected_status = "processed"
youtube_mocker(
).videos.return_value.list.return_value.execute.return_value = {
"etag":
'"ld9biNPKjAjgjV7EZ4EKeEGrhao/Lf7oS5V-Gjw0XHBBKFJRpn60z3w"',
"items": [{
"etag":
'"ld9biNPKjAjgjV7EZ4EKeEGrhao/-UL82wRXbq3YJiMZuZpqCWKoq6Q"',
"id": "wAjoqsZng_M",
"kind": "youtube#video",
"status": {
"embeddable": True,
"license": "youtube",
"privacyStatus": "unlisted",
"publicStatsViewable": True,
"uploadStatus": expected_status,
},
}],
"kind":
"youtube#videoListResponse",
"pageInfo": {
"resultsPerPage": 1,
"totalResults": 1
},
}
assert YouTubeApi().video_status("foo") == expected_status
youtube_mocker().videos.return_value.list.assert_called_once_with(
id="foo", part="status")
def test_delete_video(youtube_mocker):
"""
Test that the 'delete_video' method executes a YouTube API deletion request and returns the status code
"""
youtube_mocker(
).videos.return_value.delete.return_value.execute.return_value = 204
assert YouTubeApi().delete_video("foo") == 204
youtube_mocker().videos.return_value.delete.assert_called_with(id="foo")
def test_update_video(settings, mocker, youtube_mocker, privacy):
"""update_video should send the correct data in a request to update youtube metadata"""
speakers = "speaker1, speaker2"
tags = "tag1, tag2"
youtube_id = "test video description"
title = "TitleLngt>"
description = "DescLngth>"
content = WebsiteContentFactory.create(
title=" ".join([title for i in range(11)]),
metadata={
"resourcetype": RESOURCE_TYPE_VIDEO,
"description": " ".join([description for _ in range(501)]),
"video_metadata": {
"youtube_id": youtube_id,
"video_tags": tags,
"video_speakers": speakers,
},
},
)
expected_title = f'{" ".join([title.replace(">", "") for _ in range(9)])}...'
expected_desc = f'{" ".join([description.replace(">", "") for _ in range(499)])}...'
assert len(content.title) > YT_MAX_LENGTH_TITLE
assert len(content.metadata["description"]) > YT_MAX_LENGTH_DESCRIPTION
assert len(expected_title) <= YT_MAX_LENGTH_TITLE
assert len(expected_desc) <= YT_MAX_LENGTH_DESCRIPTION
mock_update_caption = mocker.patch(
"videos.youtube.YouTubeApi.update_captions")
YouTubeApi().update_video(content, privacy=privacy)
youtube_mocker().videos.return_value.update.assert_any_call(
part="snippet",
body={
"id": youtube_id,
"snippet": {
"title": expected_title,
"description": expected_desc,
"tags": tags,
"categoryId": settings.YT_CATEGORY_ID,
},
},
)
if privacy is not None:
youtube_mocker().videos.return_value.update.assert_any_call(
part="status",
body={
"id": youtube_id,
"status": {
"privacyStatus": privacy,
"embeddable": True
},
},
)
mock_update_caption.assert_called_once_with(content, youtube_id)
def test_upload_errors_retryable(mocker, youtube_mocker, error, retryable):
"""
Test that uploads are retried 10x for retryable exceptions
"""
mocker.patch("videos.youtube.time")
videofile = VideoFileFactory()
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.side_effect = (error)
with pytest.raises(Exception) as exc:
YouTubeApi().upload_video(videofile)
assert str(exc.value).startswith("Retried YouTube upload 10x") == retryable
def upload_youtube_videos(self):
"""
Upload public videos one at a time to YouTube (if not already there) until the daily maximum is reached.
"""
if not is_youtube_enabled():
return
yt_queue = VideoFile.objects.filter(
Q(destination=DESTINATION_YOUTUBE)
& Q(destination_id__isnull=True)
& Q(status=STATUS_CREATED)).order_by(
"-created_on")[:settings.YT_UPLOAD_LIMIT]
if yt_queue.count() == 0:
return
youtube = YouTubeApi()
group_tasks = []
for video_file in yt_queue:
error_msg = None
try:
response = youtube.upload_video(video_file)
video_file.destination_id = response["id"]
video_file.destination_status = response["status"]["uploadStatus"]
video_file.status = VideoFileStatus.UPLOADED
group_tasks.append(start_transcript_job.s(video_file.video.id))
except HttpError as error:
error_msg = error.content.decode("utf-8")
if API_QUOTA_ERROR_MSG in error_msg:
break
log.exception("HttpError uploading video to Youtube: %s",
video_file.s3_key)
video_file.status = VideoFileStatus.FAILED
except: # pylint: disable=bare-except
log.exception("Error uploading video to Youtube: %s",
video_file.s3_key)
video_file.status = VideoFileStatus.FAILED
video_file.save()
if error_msg:
mail_youtube_upload_failure(video_file)
if group_tasks:
raise self.replace(celery.group(group_tasks))
def test_upload_video_long_fields(mocker, youtube_mocker):
"""
Test that the upload_youtube_video task truncates title and description if too long
"""
name = "".join(random.choice(string.ascii_lowercase) for c in range(105))
video_file = VideoFileFactory.create()
video_file.video.source_key = video_file.s3_key.replace("file_", name)
mocker.patch("videos.youtube.resumable_upload")
mock_upload = youtube_mocker().videos.return_value.insert
YouTubeApi().upload_video(video_file)
called_args, called_kwargs = mock_upload.call_args
assert called_kwargs["body"]["snippet"]["title"] == f"{name[:97]}..."
def test_upload_video_no_id(youtube_mocker):
"""
Test that the upload_video task fails if the response contains no id
"""
videofile = VideoFileFactory()
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.return_value = (
None,
{},
)
with pytest.raises(YouTubeUploadException):
YouTubeApi().upload_video(videofile)
def remove_youtube_video(video_id):
"""
Delete a video from Youtube
"""
if not is_youtube_enabled():
return
try:
YouTubeApi().delete_video(video_id)
except HttpError as error:
if error.resp.status == 404:
log.info("Not found on Youtube, already deleted?",
video_id=video_id)
else:
raise
def test_youtube_settings(mocker, settings):
"""
Test that Youtube object creation uses YT_* settings for credentials
"""
settings.YT_ACCESS_TOKEN = "yt_access_token"
settings.YT_CLIENT_ID = "yt_client_id"
settings.YT_CLIENT_SECRET = "yt_secret"
settings.YT_REFRESH_TOKEN = "yt_refresh"
mock_oauth = mocker.patch(
"videos.youtube.oauth2client.client.GoogleCredentials")
YouTubeApi()
mock_oauth.assert_called_with(
settings.YT_ACCESS_TOKEN,
settings.YT_CLIENT_ID,
settings.YT_CLIENT_SECRET,
settings.YT_REFRESH_TOKEN,
None,
"https://accounts.google.com/o/oauth2/token",
None,
)
def test_upload_video(youtube_mocker):
"""
Test that the upload_video task calls the YouTube API execute method
"""
videofile = VideoFileFactory()
youtube_id = "M6LymW_8qVk"
video_upload_response = {
"id": youtube_id,
"kind": "youtube#video",
"snippet": {
"description": "Testing description",
"title": "Testing123"
},
"status": {
"uploadStatus": "uploaded"
},
}
youtube_mocker(
).videos.return_value.insert.return_value.next_chunk.side_effect = [
(None, None),
(None, video_upload_response),
]
response = YouTubeApi().upload_video(videofile)
assert response == video_upload_response
def test_update_captions(settings, mocker, youtube_mocker, existing_captions):
"""
Test update_captions
"""
youtube_id = "abc123"
captions = b"these are the file contents!"
videofile = VideoFileFactory.create(destination=DESTINATION_YOUTUBE,
destination_id=youtube_id)
video = videofile.video
video.webvtt_transcript_file = SimpleUploadedFile("file.txt", captions)
video.save()
content = WebsiteContentFactory.create(
metadata={
"resourcetype": RESOURCE_TYPE_VIDEO,
"video_metadata": {
"youtube_id": youtube_id,
},
},
website=video.website,
)
if existing_captions:
existing_captions_response = {
"items": [{
"id": "youtube_caption_id",
"snippet": {
"name": CAPTION_UPLOAD_NAME
}
}]
}
else:
existing_captions_response = {"items": []}
mock_media_upload = mocker.patch("videos.youtube.MediaIoBaseUpload")
mock_bytes_io = mocker.patch("videos.youtube.BytesIO")
youtube_mocker(
).captions.return_value.list.return_value.execute.return_value = (
existing_captions_response)
YouTubeApi().update_captions(content, youtube_id)
youtube_mocker().captions.return_value.list.assert_any_call(
part="snippet", videoId=youtube_id)
mock_bytes_io.assert_called_once_with(captions)
mock_media_upload.assert_called_once_with(mock_bytes_io.return_value,
mimetype="text/vtt",
chunksize=-1,
resumable=True)
if existing_captions:
youtube_mocker().captions.return_value.update.assert_any_call(
part="snippet",
body={"id": "youtube_caption_id"},
media_body=mock_media_upload.return_value,
)
else:
youtube_mocker().captions.return_value.insert.assert_any_call(
part="snippet",
sync=False,
body={
"snippet": {
"language": "en",
"name": CAPTION_UPLOAD_NAME,
"videoId": youtube_id,
}
},
media_body=mock_media_upload.return_value,
)