def upload_transcripts(request):
"""
Upload transcripts for current module.
returns: response dict::
status: 'Success' and HTTP 200 or 'Error' and HTTP 400.
subs: Value of uploaded and saved html5 sub field in video item.
"""
error, validated_data = validate_transcript_upload_data(request)
if error:
response = JsonResponse({'status': error}, status=400)
else:
video = validated_data['video']
edx_video_id = validated_data['edx_video_id']
transcript_file = validated_data['transcript_file']
# check if we need to create an external VAL video to associate the transcript
# and save its ID on the video component.
if not edx_video_id:
edx_video_id = create_external_video(display_name=u'external video')
video.edx_video_id = edx_video_id
video.save_with_metadata(request.user)
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
try:
# Convert 'srt' transcript into the 'sjson' and upload it to
# configured transcript storage. For example, S3.
sjson_subs = Transcript.convert(
content=transcript_file.read(),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
)
transcript_created = create_or_update_video_transcript(
video_id=edx_video_id,
language_code=u'en',
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': u'en'
},
file_data=ContentFile(sjson_subs),
)
if transcript_created is None:
response = JsonResponse({'status': 'Invalid Video ID'}, status=400)
except (TranscriptsGenerationException, UnicodeDecodeError):
response = JsonResponse({
'status': _(u'There is a problem with this transcript file. Try to upload a different file.')
}, status=400)
return response
def upload_transcripts(request):
"""
Upload transcripts for current module.
returns: response dict::
status: 'Success' and HTTP 200 or 'Error' and HTTP 400.
subs: Value of uploaded and saved html5 sub field in video item.
"""
error, validated_data = validate_transcript_upload_data(request)
if error:
response = JsonResponse({'status': error}, status=400)
else:
video = validated_data['video']
edx_video_id = validated_data['edx_video_id']
transcript_file = validated_data['transcript_file']
# check if we need to create an external VAL video to associate the transcript
# and save its ID on the video component.
if not edx_video_id:
edx_video_id = create_external_video(display_name=u'external video')
video.edx_video_id = edx_video_id
video.save_with_metadata(request.user)
response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200)
try:
# Convert 'srt' transcript into the 'sjson' and upload it to
# configured transcript storage. For example, S3.
sjson_subs = Transcript.convert(
content=transcript_file.read(),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
)
transcript_created = create_or_update_video_transcript(
video_id=edx_video_id,
language_code=u'en',
metadata={
'provider': TranscriptProvider.CUSTOM,
'file_format': Transcript.SJSON,
'language_code': u'en'
},
file_data=ContentFile(sjson_subs),
)
if transcript_created is None:
response = JsonResponse({'status': 'Invalid Video ID'}, status=400)
except (TranscriptsGenerationException, UnicodeDecodeError):
response = JsonResponse({
'status': _(u'There is a problem with this transcript file. Try to upload a different file.')
}, status=400)
return response
def link_video_to_component(video_component, user):
"""
Links a VAL video to the video component.
Arguments:
video_component: video descriptor item.
user: A requesting user.
Returns:
A cleaned Video ID.
"""
edx_video_id = clean_video_id(video_component.edx_video_id)
if not edx_video_id:
edx_video_id = create_external_video(display_name=u'external video')
video_component.edx_video_id = edx_video_id
video_component.save_with_metadata(user)
return edx_video_id
def link_video_to_component(video_component, user):
"""
Links a VAL video to the video component.
Arguments:
video_component: video descriptor item.
user: A requesting user.
Returns:
A cleaned Video ID.
"""
edx_video_id = clean_video_id(video_component.edx_video_id)
if not edx_video_id:
edx_video_id = create_external_video(display_name=u'external video')
video_component.edx_video_id = edx_video_id
video_component.save_with_metadata(user)
return edx_video_id
def async_migrate_transcript_subtask(self, *args, **kwargs):
#pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
video, language_code, force_update = args
commit = kwargs['commit']
result = None
if commit is not True:
return 'Language {0} transcript of video {1} will be migrated'.format(
language_code,
video.edx_video_id
)
LOGGER.info("[Transcript migration] process for %s transcript started", language_code)
try:
transcript_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video, language_code, Transcript.SJSON, transcript_info)
edx_video_id = clean_video_id(video.edx_video_id)
if not edx_video_id:
video.edx_video_id = create_external_video('external-video')
video.save_with_metadata(user=User.objects.get(username='******'))
if edx_video_id:
result = save_transcript_to_storage(
edx_video_id,
language_code,
transcript_content,
Transcript.SJSON,
force_update
)
except (NotFoundError, TranscriptsGenerationException, ValCannotCreateError) as exc:
LOGGER.exception('[Transcript migration] Exception: %r', text_type(exc))
return 'Failed: language {language} of video {video} with exception {exception}'.format(
language=language_code,
video=video.edx_video_id,
exception=text_type(exc)
)
LOGGER.info("[Transcript migration] process for %s transcript ended", language_code)
if result is not None:
return 'Success: language {0} of video {1}'.format(language_code, video.edx_video_id)
else:
return 'Failed: language {0} of video {1}'.format(language_code, video.edx_video_id)
def async_migrate_transcript_subtask(self, *args, **kwargs):
#pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
video, language_code, force_update = args
commit = kwargs['commit']
result = None
if commit is not True:
return 'Language {0} transcript of video {1} will be migrated'.format(
language_code, video.edx_video_id)
LOGGER.info("[Transcript migration] process for %s transcript started",
language_code)
try:
transcript_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video, language_code, Transcript.SJSON, transcript_info)
edx_video_id = clean_video_id(video.edx_video_id)
if not edx_video_id:
video.edx_video_id = create_external_video('external-video')
video.save_with_metadata(user=User.objects.get(username='******'))
if edx_video_id:
result = save_transcript_to_storage(edx_video_id, language_code,
transcript_content,
Transcript.SJSON, force_update)
except (NotFoundError, TranscriptsGenerationException,
ValCannotCreateError) as exc:
LOGGER.exception('[Transcript migration] Exception: %r',
text_type(exc))
return 'Failed: language {language} of video {video} with exception {exception}'.format(
language=language_code,
video=video.edx_video_id,
exception=text_type(exc))
LOGGER.info("[Transcript migration] process for %s transcript ended",
language_code)
if result is not None:
return 'Success: language {0} of video {1}'.format(
language_code, video.edx_video_id)
else:
return 'Failed: language {0} of video {1}'.format(
language_code, video.edx_video_id)
def studio_transcript(self, request, dispatch):
"""
Entry point for Studio transcript handlers.
Dispatches:
/translation/[language_id] - language_id sould be in url.
`translation` dispatch support following HTTP methods:
`POST`:
Upload srt file. Check possibility of generation of proper sjson files.
For now, it works only for self.transcripts, not for `en`.
Do not update self.transcripts, as fields are updated on save in Studio.
`GET:
Return filename from storage. SRT format is sent back on success. Filename should be in GET dict.
We raise all exceptions right in Studio:
NotFoundError:
Video or asset was deleted from module/contentstore, but request came later.
Seems impossible to be raised. module_render.py catches NotFoundErrors from here.
/translation POST:
TypeError:
Unjsonable filename or content.
TranscriptsGenerationException, TranscriptException:
no SRT extension or not parse-able by PySRT
UnicodeDecodeError: non-UTF8 uploaded file content encoding.
"""
_ = self.runtime.service(self, "i18n").ugettext
if dispatch.startswith('translation'):
if request.method == 'POST':
error = self.validate_transcript_upload_data(data=request.POST)
if error:
response = Response(json={'error': error}, status=400)
else:
edx_video_id = clean_video_id(request.POST['edx_video_id'])
language_code = request.POST['language_code']
new_language_code = request.POST['new_language_code']
transcript_file = request.POST['file'].file
if not edx_video_id:
# Back-populate the video ID for an external video.
# pylint: disable=attribute-defined-outside-init
self.edx_video_id = edx_video_id = create_external_video(display_name=u'external video')
try:
# Convert SRT transcript into an SJSON format
# and upload it to S3.
sjson_subs = Transcript.convert(
content=transcript_file.read(),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
)
create_or_update_video_transcript(
video_id=edx_video_id,
language_code=language_code,
metadata={
'file_format': Transcript.SJSON,
'language_code': new_language_code
},
file_data=ContentFile(sjson_subs),
)
payload = {
'edx_video_id': edx_video_id,
'language_code': new_language_code
}
response = Response(json.dumps(payload), status=201)
except (TranscriptsGenerationException, UnicodeDecodeError):
response = Response(
json={
'error': _(
u'There is a problem with this transcript file. Try to upload a different file.'
)
},
status=400
)
elif request.method == 'DELETE':
request_data = request.json
if 'lang' not in request_data or 'edx_video_id' not in request_data:
return Response(status=400)
language = request_data['lang']
edx_video_id = clean_video_id(request_data['edx_video_id'])
if edx_video_id:
delete_video_transcript(video_id=edx_video_id, language_code=language)
if language == u'en':
# remove any transcript file from content store for the video ids
possible_sub_ids = [
self.sub, # pylint: disable=access-member-before-definition
self.youtube_id_1_0
] + get_html5_ids(self.html5_sources)
for sub_id in possible_sub_ids:
remove_subs_from_store(sub_id, self, language)
# update metadata as `en` can also be present in `transcripts` field
remove_subs_from_store(self.transcripts.pop(language, None), self, language)
# also empty `sub` field
self.sub = '' # pylint: disable=attribute-defined-outside-init
else:
remove_subs_from_store(self.transcripts.pop(language, None), self, language)
return Response(status=200)
elif request.method == 'GET':
language = request.GET.get('language_code')
if not language:
return Response(json={'error': _(u'Language is required.')}, status=400)
try:
transcript_content, transcript_name, mime_type = get_transcript(
video=self, lang=language, output_format=Transcript.SRT
)
response = Response(transcript_content, headerlist=[
(
'Content-Disposition',
'attachment; filename="{}"'.format(
transcript_name.encode('utf8') if six.PY2 else transcript_name
)
),
('Content-Language', language),
('Content-Type', mime_type)
])
except (UnicodeDecodeError, TranscriptsGenerationException, NotFoundError):
response = Response(status=404)
else:
# Any other HTTP method is not allowed.
response = Response(status=404)
else: # unknown dispatch
log.debug("Dispatch is not allowed")
response = Response(status=404)
return response
def async_migrate_transcript_subtask(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
success, failure = 'Success', 'Failure'
video_location, revision, language_code, force_update = args
command_run = kwargs['command_run']
store = modulestore()
video = store.get_item(
usage_key=BlockUsageLocator.from_string(video_location),
revision=revision)
edx_video_id = clean_video_id(video.edx_video_id)
if not kwargs['commit']:
LOGGER.info(
('[%s] [run=%s] [video-transcript-will-be-migrated] '
'[revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location,
edx_video_id, language_code)
return success
LOGGER.info((
'[%s] [run=%s] [transcripts-migration-process-started-for-video-transcript] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location,
edx_video_id, language_code)
try:
transcripts_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video=video,
language=language_code,
output_format=Transcript.SJSON,
transcripts_info=transcripts_info,
)
is_video_valid = edx_video_id and is_video_available(edx_video_id)
if not is_video_valid:
edx_video_id = create_external_video('external-video')
video.edx_video_id = edx_video_id
# determine branch published/draft
branch_setting = (ModuleStoreEnum.Branch.published_only if revision
== ModuleStoreEnum.RevisionOption.published_only
else ModuleStoreEnum.Branch.draft_preferred)
with store.branch_setting(branch_setting):
store.update_item(video, ModuleStoreEnum.UserID.mgmt_command)
LOGGER.info(
'[%s] [run=%s] [generated-edx-video-id] [revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]',
MIGRATION_LOGS_PREFIX, command_run, revision, video_location,
edx_video_id, language_code)
save_transcript_to_storage(
command_run=command_run,
edx_video_id=edx_video_id,
language_code=language_code,
transcript_content=transcript_content,
file_format=Transcript.SJSON,
force_update=force_update,
)
except (NotFoundError, TranscriptsGenerationException,
ValCannotCreateError):
LOGGER.exception((
'[%s] [run=%s] [video-transcript-migration-failed-with-known-exc] [revision=%s] [video=%s] '
'[edx_video_id=%s] [language_code=%s]'), MIGRATION_LOGS_PREFIX,
command_run, revision, video_location, edx_video_id,
language_code)
return failure
except Exception:
LOGGER.exception((
'[%s] [run=%s] [video-transcript-migration-failed-with-unknown-exc] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision,
video_location, edx_video_id, language_code)
raise
LOGGER.info((
'[%s] [run=%s] [video-transcript-migration-succeeded-for-a-video] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location,
edx_video_id, language_code)
return success
def async_migrate_transcript_subtask(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
success, failure = 'Success', 'Failure'
video_location, revision, language_code, force_update = args
command_run = kwargs['command_run']
store = modulestore()
video = store.get_item(usage_key=BlockUsageLocator.from_string(video_location), revision=revision)
edx_video_id = clean_video_id(video.edx_video_id)
if not kwargs['commit']:
LOGGER.info(
('[%s] [run=%s] [video-transcript-will-be-migrated] '
'[revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return success
LOGGER.info(
('[%s] [run=%s] [transcripts-migration-process-started-for-video-transcript] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
try:
transcripts_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video=video,
language=language_code,
output_format=Transcript.SJSON,
transcripts_info=transcripts_info,
)
is_video_valid = edx_video_id and is_video_available(edx_video_id)
if not is_video_valid:
edx_video_id = create_external_video('external-video')
video.edx_video_id = edx_video_id
# determine branch published/draft
branch_setting = (
ModuleStoreEnum.Branch.published_only
if revision == ModuleStoreEnum.RevisionOption.published_only else
ModuleStoreEnum.Branch.draft_preferred
)
with store.branch_setting(branch_setting):
store.update_item(video, ModuleStoreEnum.UserID.mgmt_command)
LOGGER.info(
'[%s] [run=%s] [generated-edx-video-id] [revision=%s] [video=%s] [edx_video_id=%s] [language_code=%s]',
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
save_transcript_to_storage(
command_run=command_run,
edx_video_id=edx_video_id,
language_code=language_code,
transcript_content=transcript_content,
file_format=Transcript.SJSON,
force_update=force_update,
)
except (NotFoundError, TranscriptsGenerationException, ValCannotCreateError):
LOGGER.exception(
('[%s] [run=%s] [video-transcript-migration-failed-with-known-exc] [revision=%s] [video=%s] '
'[edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return failure
except Exception:
LOGGER.exception(
('[%s] [run=%s] [video-transcript-migration-failed-with-unknown-exc] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
raise
LOGGER.info(
('[%s] [run=%s] [video-transcript-migration-succeeded-for-a-video] [revision=%s] '
'[video=%s] [edx_video_id=%s] [language_code=%s]'),
MIGRATION_LOGS_PREFIX, command_run, revision, video_location, edx_video_id, language_code
)
return success
def studio_transcript(self, request, dispatch):
"""
Entry point for Studio transcript handlers.
Dispatches:
/translation/[language_id] - language_id sould be in url.
`translation` dispatch support following HTTP methods:
`POST`:
Upload srt file. Check possibility of generation of proper sjson files.
For now, it works only for self.transcripts, not for `en`.
Do not update self.transcripts, as fields are updated on save in Studio.
`GET:
Return filename from storage. SRT format is sent back on success. Filename should be in GET dict.
We raise all exceptions right in Studio:
NotFoundError:
Video or asset was deleted from module/contentstore, but request came later.
Seems impossible to be raised. module_render.py catches NotFoundErrors from here.
/translation POST:
TypeError:
Unjsonable filename or content.
TranscriptsGenerationException, TranscriptException:
no SRT extension or not parse-able by PySRT
UnicodeDecodeError: non-UTF8 uploaded file content encoding.
"""
_ = self.runtime.service(self, "i18n").ugettext
if dispatch.startswith('translation'):
if request.method == 'POST':
error = self.validate_transcript_upload_data(data=request.POST)
if error:
response = Response(json={'error': error}, status=400)
else:
edx_video_id = clean_video_id(request.POST['edx_video_id'])
language_code = request.POST['language_code']
new_language_code = request.POST['new_language_code']
transcript_file = request.POST['file'].file
if not edx_video_id:
# Back-populate the video ID for an external video.
# pylint: disable=attribute-defined-outside-init
self.edx_video_id = edx_video_id = create_external_video(display_name=u'external video')
try:
# Convert SRT transcript into an SJSON format
# and upload it to S3.
sjson_subs = Transcript.convert(
content=transcript_file.read(),
input_format=Transcript.SRT,
output_format=Transcript.SJSON
)
create_or_update_video_transcript(
video_id=edx_video_id,
language_code=language_code,
metadata={
'file_format': Transcript.SJSON,
'language_code': new_language_code
},
file_data=ContentFile(sjson_subs),
)
payload = {
'edx_video_id': edx_video_id,
'language_code': new_language_code
}
response = Response(json.dumps(payload), status=201)
except (TranscriptsGenerationException, UnicodeDecodeError):
response = Response(
json={
'error': _(
u'There is a problem with this transcript file. Try to upload a different file.'
)
},
status=400
)
elif request.method == 'DELETE':
request_data = request.json
if 'lang' not in request_data or 'edx_video_id' not in request_data:
return Response(status=400)
language = request_data['lang']
edx_video_id = clean_video_id(request_data['edx_video_id'])
if edx_video_id:
delete_video_transcript(video_id=edx_video_id, language_code=language)
if language == u'en':
# remove any transcript file from content store for the video ids
possible_sub_ids = [
self.sub, # pylint: disable=access-member-before-definition
self.youtube_id_1_0
] + get_html5_ids(self.html5_sources)
for sub_id in possible_sub_ids:
remove_subs_from_store(sub_id, self, language)
# update metadata as `en` can also be present in `transcripts` field
remove_subs_from_store(self.transcripts.pop(language, None), self, language)
# also empty `sub` field
self.sub = '' # pylint: disable=attribute-defined-outside-init
else:
remove_subs_from_store(self.transcripts.pop(language, None), self, language)
return Response(status=200)
elif request.method == 'GET':
language = request.GET.get('language_code')
if not language:
return Response(json={'error': _(u'Language is required.')}, status=400)
try:
transcript_content, transcript_name, mime_type = get_transcript(
video=self, lang=language, output_format=Transcript.SRT
)
response = Response(transcript_content, headerlist=[
('Content-Disposition', 'attachment; filename="{}"'.format(transcript_name.encode('utf8'))),
('Content-Language', language),
('Content-Type', mime_type)
])
except (UnicodeDecodeError, TranscriptsGenerationException, NotFoundError):
response = Response(status=404)
else:
# Any other HTTP method is not allowed.
response = Response(status=404)
else: # unknown dispatch
log.debug("Dispatch is not allowed")
response = Response(status=404)
return response
def async_migrate_transcript_subtask(self, *args, **kwargs): # pylint: disable=unused-argument
"""
Migrates a transcript of a given video in a course as a new celery task.
"""
video_location, language_code, force_update = args
store = modulestore()
video = store.get_item(usage_key=BlockUsageLocator.from_string(video_location))
commit = kwargs['commit']
if not commit:
return 'Language {language_code} transcript of video {edx_video_id} will be migrated'.format(
language_code=language_code,
edx_video_id=video.edx_video_id
)
# Start transcript's migration
edx_video_id = clean_video_id(video.edx_video_id)
LOGGER.info(
"[Transcript migration] migration process is started for video [%s] language [%s].",
edx_video_id, language_code
)
try:
transcripts_info = video.get_transcripts_info()
transcript_content, _, _ = get_transcript_from_contentstore(
video=video,
language=language_code,
output_format=Transcript.SJSON,
transcripts_info=transcripts_info,
)
if not edx_video_id:
edx_video_id = create_external_video('external-video')
video.edx_video_id = edx_video_id
store.update_item(video, ModuleStoreEnum.UserID.mgmt_command)
save_transcript_to_storage(
edx_video_id=edx_video_id,
language_code=language_code,
transcript_content=transcript_content,
file_format=Transcript.SJSON,
force_update=force_update,
)
except (NotFoundError, TranscriptsGenerationException, ValCannotCreateError) as exc:
LOGGER.exception(
'[Transcript migration] transcript migration failed for video [%s] and language [%s].',
edx_video_id, language_code
)
message = 'Failed: language {language} of video {video} with exception {exception}'.format(
language=language_code,
video=video.edx_video_id,
exception=text_type(exc)
)
except Exception:
LOGGER.exception(
'[Transcript migration] transcript migration failed for video [%s] and language [%s].',
edx_video_id, language_code
)
raise
else:
message = (
'Success: transcript (language: {language_code}, edx_video_id: {edx_video_id}) has been migrated '
'for video [{location}].'
).format(edx_video_id=edx_video_id, language_code=language_code, location=unicode(video.location))
return message