def three_play_transcription_callback(sender, **kwargs):
"""
This is a receiver for 3Play Media callback signal.
Arguments:
sender: sender of the signal
kwargs(dict): video transcription metadata
Process:
* download transcript(SRT) from 3PlayMedia
* convert SRT to SJSON
* upload SJSON to AWS S3
* order translations for all the preferred languages
* update transcript status in VAL
"""
log_prefix = u'3PlayMedia Callback'
# Extract all the must have attributes
org = kwargs['org']
edx_video_id = kwargs['edx_video_id']
lang_code = kwargs['lang_code']
file_id = kwargs['file_id']
state = kwargs['status']
try:
process = TranscriptProcessMetadata.objects.filter(
provider=TranscriptProvider.THREE_PLAY,
process_id=file_id,
lang_code=lang_code,
).latest()
except TranscriptProcessMetadata.DoesNotExist:
LOGGER.exception(
u'[3PlayMedia Callback] Unable to get transcript process for org=%s, edx_video_id=%s, file_id=%s.',
org,
edx_video_id,
file_id,
)
return
# On completion of a transcript
# Indicates that the default video speech transcription has been done successfully.
if state == COMPLETE:
log_args = (edx_video_id, lang_code, file_id)
# 1 - Retrieve transcript credentials
transcript_secrets = get_transcript_credentials(
provider=TranscriptProvider.THREE_PLAY,
org=org,
edx_video_id=edx_video_id,
file_id=file_id,
log_prefix=log_prefix,
)
if not transcript_secrets:
process.update(status=TranscriptStatus.FAILED)
return
# 2 - Fetch the transcript from 3Play Media.
try:
srt_transcript = fetch_srt_data(
THREE_PLAY_TRANSCRIPT_URL.format(file_id=file_id),
apikey=transcript_secrets.api_key,
)
except TranscriptFetchError:
LOGGER.exception(
u'[3PlayMedia Callback] Fetch request failed for video=%s -- lang_code=%s -- process_id=%s',
*log_args)
process.update(status=TranscriptStatus.FAILED)
return
# 3 - Validate transcript content received from 3Play Media and mark the transcription process.
is_valid_transcript = validate_transcript_response(
edx_video_id=edx_video_id,
file_id=file_id,
transcript=srt_transcript,
lang_code=lang_code,
log_prefix=log_prefix,
)
if is_valid_transcript:
process.update(status=TranscriptStatus.READY)
else:
process.update(status=TranscriptStatus.FAILED)
# 4 - Convert SRT transcript to SJson format and upload it to S3.
try:
sjson_transcript = convert_srt_to_sjson(srt_transcript)
sjson_file = upload_sjson_to_s3(CONFIG, sjson_transcript)
except Exception:
# in case of any exception, log and raise.
LOGGER.exception(
u'[3PlayMedia Callback] Request failed for video=%s -- lang_code=%s -- process_id=%s',
*log_args)
raise
# 5 - Update edx-val with completed transcript information.
val_api = VALAPICall(video_proto=None, val_status=None)
val_api.update_val_transcript(
video_id=process.video.studio_id,
lang_code=lang_code,
name=sjson_file,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.THREE_PLAY,
)
# 6 - Translation Phase
# That's the phase for kicking off translation processes for all the
# preferred languages except the video's speech language.
target_languages = list(process.video.preferred_languages)
target_languages.remove(lang_code)
# Create the translation tracking processes for all the target languages.
for target_language in target_languages:
TranscriptProcessMetadata.objects.create(
video=process.video,
provider=TranscriptProvider.THREE_PLAY,
process_id=file_id,
lang_code=target_language,
status=TranscriptStatus.PENDING,
)
# Order translations for target languages
try:
order_translations(file_id,
transcript_secrets.api_key,
transcript_secrets.api_secret,
source_language=lang_code,
target_languages=target_languages)
except TranscriptTranslationError:
LOGGER.exception(
u'[3PlayMedia Callback] Translation could not be performed - video=%s, lang_code=%s, file_id=%s.',
*log_args)
except Exception:
LOGGER.exception(
u'[3PlayMedia Callback] Error while translating the transcripts - video=%s, lang_code=%s, file_id=%s',
*log_args)
raise
# 7 - Update transcript status.
# It will be for edx-val as well as edx-video-pipeline and this will be the case when
# there is only one transcript language for a video(that is, already been processed).
if not target_languages:
utils.update_video_status(val_api_client=val_api,
video=process.video,
status=TranscriptStatus.READY)
# On success, a happy farewell log.
LOGGER.info((
u'[3PlayMedia Callback] Video speech transcription was successful for'
u' video=%s -- lang_code=%s -- process_id=%s'), *log_args)
elif state == ERROR:
# Fail the process
process.status = TranscriptStatus.FAILED
process.save()
# Log the error information
LOGGER.error(
u'[3PlayMedia Callback] Error while transcription - error=%s, org=%s, edx_video_id=%s, file_id=%s.',
kwargs['error_description'],
org,
edx_video_id,
file_id,
)
else:
# Status must be either 'complete' or 'error'
# more details on http://support.3playmedia.com/hc/en-us/articles/227729828-Files-API-Methods
LOGGER.error(
u'[3PlayMedia Callback] Got invalid status - status=%s, org=%s, edx_video_id=%s, file_id=%s.',
state,
org,
edx_video_id,
file_id,
)
def handle_video_translations(video, translations, file_id, api_key,
log_prefix):
"""
It is a sub-module of `retrieve_three_play_translations` to handle
all the completed translations for a single video.
Arguments:
video: Video data object whose translations need to be handled here.
translations: A list containing translations metadata information received from 3play Media.
file_id: It is file identifier that is assigned to a Video by 3Play Media.
api_key: An api key to communicate to the 3Play Media.
log_prefix: A logging prefix used by the main process.
Steps include:
- Fetch translated transcript content from 3Play Media.
- Validate the content of received translated transcript.
- Convert translated SRT transcript to SJson format and upload it to S3.
- Update edx-val for a completed transcript.
- update transcript status for video in edx-val as well as edx-video-pipeline.
"""
video_translation_processes = get_in_progress_translation_processes(video)
for translation_metadata in translations:
translation_id = translation_metadata['id']
translation_state = translation_metadata['state']
target_language = translation_metadata[
'target_language_iso_639_1_code']
LOGGER.info(
'[3PlayMedia Task] Translation metadata retrieved -- video=%s, translation_id=%s, language=%s, status=%s.',
video.studio_id, translation_id, target_language,
translation_state)
if translation_state == COMPLETE:
# Fetch the corresponding tracking process.
translation_process = get_in_progress_translation_process(
video_translation_processes,
file_id=file_id,
translation_id=translation_id,
target_language=target_language)
if translation_process is None:
continue
# 1 - Fetch translated transcript content from 3Play Media.
srt_transcript = get_transcript_content_from_3play_media(
api_key=api_key,
edx_video_id=video.studio_id,
file_id=file_id,
translation_id=translation_id,
target_language=target_language,
)
if srt_transcript is None:
continue
# 2 - Validate the content of received translated transcript.
is_transcript_valid = validate_transcript_response(
edx_video_id=video.studio_id,
file_id=file_id,
transcript=srt_transcript,
lang_code=target_language,
log_prefix=log_prefix)
if is_transcript_valid:
translation_process.update(status=TranscriptStatus.READY)
else:
translation_process.update(status=TranscriptStatus.FAILED)
continue
# 3 - Convert SRT translation to SJson format and upload it to S3.
sjson_file = convert_to_sjson_and_upload_to_s3(
srt_transcript=srt_transcript,
target_language=target_language,
edx_video_id=video.studio_id,
file_id=file_id,
)
# 4 Update edx-val with completed transcript information
val_api = VALAPICall(video_proto=None, val_status=None)
val_api.update_val_transcript(
video_id=video.studio_id,
lang_code=target_language,
name=sjson_file,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.THREE_PLAY,
)
LOGGER.info(
'[3PlayMedia Task] Translation retrieval was successful -- video=%s, translation_id=%s, language=%s.',
video.studio_id, translation_id, target_language)
# 5 - if all the processes for this video are complete, update transcript status
# for video in edx-val as well as edx-video-pipeline.
video_jobs = TranscriptProcessMetadata.objects.filter(video=video)
if all(video_job.status == TranscriptStatus.READY
for video_job in video_jobs):
utils.update_video_status(val_api_client=val_api,
video=video,
status=TranscriptStatus.READY)
def cielo24_transcript_callback(sender, **kwargs):
"""
* download transcript(SRT) from Cielo24
* convert SRT to SJSON
* upload SJSON to AWS S3
* update transcript status in VAL
"""
process_metadata = None
transcript_prefs = None
org = kwargs['org']
job_id = kwargs['job_id']
video_id = kwargs['video_id']
iwp_name = kwargs['iwp_name']
lang_code = kwargs['lang_code']
LOGGER.info(
'[CIELO24 TRANSCRIPTS] Transcript complete request received for '
'video=%s -- org=%s -- lang=%s -- job_id=%s -- iwp_name=%s', video_id,
org, lang_code, job_id, iwp_name)
# get transcript credentials for an organization
try:
transcript_prefs = TranscriptCredentials.objects.get(
org=org,
provider=TranscriptProvider.CIELO24,
)
except TranscriptCredentials.DoesNotExist:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Unable to get transcript credentials for job_id=%s',
job_id)
# mark the transcript for a particular language as ready
try:
process_metadata = TranscriptProcessMetadata.objects.filter(
provider=TranscriptProvider.CIELO24,
process_id=job_id,
lang_code=lang_code).latest()
except TranscriptProcessMetadata.DoesNotExist:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Unable to get transcript process metadata for job_id=%s',
job_id)
# if transcript credentials are missing then we can do nothing
if not transcript_prefs and process_metadata:
process_metadata.status = TranscriptStatus.FAILED
process_metadata.save()
if transcript_prefs and process_metadata:
api_key = transcript_prefs.api_key
try:
srt_data = fetch_srt_data(CIELO24_GET_CAPTION_URL,
v=CIELO24_API_VERSION,
job_id=job_id,
api_token=api_key,
caption_format='SRT')
except TranscriptFetchError:
process_metadata.status = TranscriptStatus.FAILED
process_metadata.save()
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Fetch request failed for video=%s -- lang=%s -- job_id=%s',
video_id, lang_code, job_id)
return
process_metadata.status = TranscriptStatus.READY
process_metadata.save()
try:
sjson = convert_srt_to_sjson(srt_data)
sjson_file_name = upload_sjson_to_s3(CONFIG, sjson)
except Exception:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s.',
video_id, lang_code, job_id)
raise
# update edx-val with completed transcript information
val_api = VALAPICall(process_metadata.video, val_status=None)
val_api.update_val_transcript(
video_id=process_metadata.video.studio_id,
lang_code=lang_code,
name=sjson_file_name,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.CIELO24)
# update transcript status for video in edx-val only if all langauge transcripts are ready
video_jobs = TranscriptProcessMetadata.objects.filter(
video__studio_id=video_id)
if all(video_job.status == TranscriptStatus.READY
for video_job in video_jobs):
utils.update_video_status(val_api_client=val_api,
video=process_metadata.video,
status=TranscriptStatus.READY)