def setUp(self):
self.VP = VideoProto(client_title='Test Title', veda_id='TESTID')
with patch.object(VALAPICall, '_AUTH',
PropertyMock(return_value=lambda: CONFIG_DATA)):
self.VAC = VALAPICall(video_proto=self.VP, val_status='complete')
self.auth_yaml = CONFIG_DATA
def urlpatch(upload_data):
"""
# :param upload_data: dict
# :return:
"""
if upload_data['status'] == 'Failure':
return None
try:
test_id = Video.objects.filter(edx_id=upload_data['edx_id']).latest()
except:
upload_data['status'] = 'Failure'
return
if upload_data['status'] == 'Successful':
LOGGER.info('[YOUTUBE CALLBACK] : Urlpatch : Upload status is successful : {upload_data}'.format(
upload_data=upload_data
))
url_query = URL.objects.filter(
encode_url=upload_data['youtube_id']
)
if len(url_query) == 0:
u1 = URL(
videoID=Video.objects.filter(
edx_id=test_id.edx_id
).latest()
)
u1.encode_profile = Encode.objects.get(
encode_suffix=upload_data['file_suffix']
)
u1.encode_url = upload_data['youtube_id']
u1.url_date = upload_data['datetime']
u1.encode_duration = test_id.video_orig_duration
u1.encode_bitdepth = 0
u1.encode_size = 0
u1.save()
"""
Report to Email
"""
if 'EDXABVID' in upload_data['edx_id']:
v1 = Video.objects.filter(edx_id=upload_data['edx_id']).latest()
if v1.abvid_serial is not None:
report_status(
status="Complete",
abvid_serial=v1.abvid_serial,
youtube_id=upload_data['youtube_id']
)
video_check = Video.objects.filter(edx_id=test_id.edx_id).latest()
if video_check.video_trans_status == 'Youtube Duplicate':
Video.objects.filter(
edx_id=video_check.edx_id
).update(
video_trans_status='Progress'
)
"""
Update Status & VAL
"""
video_proto = VideoProto(
veda_id=test_id.edx_id,
val_id=test_id.studio_id,
client_title=test_id.client_title,
duration=test_id.video_orig_duration,
bitrate='0',
s3_filename=test_id.studio_id
)
VF = Metadata(
video_object=test_id
)
encode_list = VF._FAULT(
video_object=test_id
)
"""
Review can stop here
"""
if upload_data['file_suffix'] == 'RVW':
return None
if len(encode_list) == 0:
Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Complete')
val_status = 'file_complete'
else:
val_status = 'transcode_active'
ApiConn = VALAPICall(
video_proto=video_proto,
val_status=val_status,
endpoint_url=upload_data['youtube_id'],
encode_profile='youtube'
)
ApiConn.call()
elif upload_data['status'] == 'Duplicate' and \
upload_data['file_suffix'] == '100':
LOGGER.info('[YOUTUBE CALLBACK] : Urlpatch : Upload status is duplicate : {upload_data}'.format(
upload_data=upload_data
))
url_query = URL.objects.filter(
videoID=Video.objects.filter(
edx_id=upload_data['edx_id']
).latest(),
encode_profile=Encode.objects.get(
encode_suffix=upload_data['file_suffix']
)
)
if len(url_query) == 0:
if 'EDXABVID' in upload_data['edx_id']:
report_status(
status="Youtube Duplicate",
abvid_serial=test_id.abvid_serial,
youtube_id=''
)
Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Youtube Duplicate')
video_proto = VideoProto(
veda_id=test_id.edx_id,
val_id=test_id.studio_id,
client_title=test_id.client_title,
duration=test_id.video_orig_duration,
bitrate='0',
s3_filename=test_id.studio_id
)
ApiConn = VALAPICall(
video_proto=video_proto,
val_status="duplicate",
endpoint_url="DUPLICATE",
encode_profile='youtube'
)
ApiConn.call()
class TestVALAPI(TestCase):
def setUp(self):
self.VP = VideoProto(client_title='Test Title', veda_id='TESTID')
with patch.object(VALAPICall, '_AUTH',
PropertyMock(return_value=lambda: CONFIG_DATA)):
self.VAC = VALAPICall(video_proto=self.VP, val_status='complete')
self.auth_yaml = CONFIG_DATA
def test_val_setup(self):
# register val url to send api response
responses.add(responses.POST,
CONFIG_DATA['val_token_url'],
'{"access_token": "1234567890"}',
status=200)
salient_variables = [
'val_api_url',
'val_client_id',
'val_password',
'val_secret_key',
'val_username',
'val_token_url',
]
for salient_variable in salient_variables:
self.assertTrue(len(self.VAC.auth_dict[salient_variable]) > 0)
@responses.activate
def test_val_connection(self):
# register val url to send api response
responses.add(responses.POST,
CONFIG_DATA['val_token_url'],
'{"access_token": "1234567890"}',
status=200)
responses.add(responses.GET, CONFIG_DATA['val_api_url'], status=200)
self.VAC.val_tokengen()
self.assertFalse(self.VAC.val_token is None)
response = requests.get(self.VAC.auth_dict['val_api_url'],
headers=self.VAC.headers,
timeout=20)
self.assertFalse(response.status_code == 404)
self.assertFalse(response.status_code > 299)
@data(
{
'encode_list': [],
'val_status': 'file_complete',
'expected_response': False
},
{
'encode_list': [],
'val_status': ValTranscriptStatus.TRANSCRIPT_READY,
'expected_response': False
},
{
'encode_list': [],
'val_status': ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS,
'expected_response': False
},
{
'encode_list': ['abc.mp4'],
'val_status': 'file_complete',
'expected_response': True
},
{
'encode_list': ['abc.mp4'],
'val_status': ValTranscriptStatus.TRANSCRIPT_READY,
'expected_response': True
},
{
'encode_list': ['abc.mp4'],
'val_status': ValTranscriptStatus.TRANSCRIPTION_IN_PROGRESS,
'expected_response': True
},
)
@unpack
def test_val_should_update_status(self, encode_list, val_status,
expected_response):
"""
Verify that `should_update_status` works as expected.
"""
response = self.VAC.should_update_status(encode_list, val_status)
self.assertEqual(response, expected_response)
def handle_video_translations(video, translations, file_id, api_key,
log_prefix):
"""
It is a sub-module of `retrieve_three_play_translations` to handle
all the completed translations for a single video.
Arguments:
video: Video data object whose translations need to be handled here.
translations: A list containing translations metadata information received from 3play Media.
file_id: It is file identifier that is assigned to a Video by 3Play Media.
api_key: An api key to communicate to the 3Play Media.
log_prefix: A logging prefix used by the main process.
Steps include:
- Fetch translated transcript content from 3Play Media.
- Validate the content of received translated transcript.
- Convert translated SRT transcript to SJson format and upload it to S3.
- Update edx-val for a completed transcript.
- update transcript status for video in edx-val as well as edx-video-pipeline.
"""
video_translation_processes = get_in_progress_translation_processes(video)
for translation_metadata in translations:
translation_id = translation_metadata['id']
translation_state = translation_metadata['state']
target_language = translation_metadata[
'target_language_iso_639_1_code']
LOGGER.info(
'[3PlayMedia Task] Translation metadata retrieved -- video=%s, translation_id=%s, language=%s, status=%s.',
video.studio_id, translation_id, target_language,
translation_state)
if translation_state == COMPLETE:
# Fetch the corresponding tracking process.
translation_process = get_in_progress_translation_process(
video_translation_processes,
file_id=file_id,
translation_id=translation_id,
target_language=target_language)
if translation_process is None:
continue
# 1 - Fetch translated transcript content from 3Play Media.
srt_transcript = get_transcript_content_from_3play_media(
api_key=api_key,
edx_video_id=video.studio_id,
file_id=file_id,
translation_id=translation_id,
target_language=target_language,
)
if srt_transcript is None:
continue
# 2 - Validate the content of received translated transcript.
is_transcript_valid = validate_transcript_response(
edx_video_id=video.studio_id,
file_id=file_id,
transcript=srt_transcript,
lang_code=target_language,
log_prefix=log_prefix)
if is_transcript_valid:
translation_process.update(status=TranscriptStatus.READY)
else:
translation_process.update(status=TranscriptStatus.FAILED)
continue
# 3 - Convert SRT translation to SJson format and upload it to S3.
sjson_file = convert_to_sjson_and_upload_to_s3(
srt_transcript=srt_transcript,
target_language=target_language,
edx_video_id=video.studio_id,
file_id=file_id,
)
# 4 Update edx-val with completed transcript information
val_api = VALAPICall(video_proto=None, val_status=None)
val_api.update_val_transcript(
video_id=video.studio_id,
lang_code=target_language,
name=sjson_file,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.THREE_PLAY,
)
LOGGER.info(
'[3PlayMedia Task] Translation retrieval was successful -- video=%s, translation_id=%s, language=%s.',
video.studio_id, translation_id, target_language)
# 5 - if all the processes for this video are complete, update transcript status
# for video in edx-val as well as edx-video-pipeline.
video_jobs = TranscriptProcessMetadata.objects.filter(video=video)
if all(video_job.status == TranscriptStatus.READY
for video_job in video_jobs):
utils.update_video_status(val_api_client=val_api,
video=video,
status=TranscriptStatus.READY)
def three_play_transcription_callback(sender, **kwargs):
"""
This is a receiver for 3Play Media callback signal.
Arguments:
sender: sender of the signal
kwargs(dict): video transcription metadata
Process:
* download transcript(SRT) from 3PlayMedia
* convert SRT to SJSON
* upload SJSON to AWS S3
* order translations for all the preferred languages
* update transcript status in VAL
"""
log_prefix = u'3PlayMedia Callback'
# Extract all the must have attributes
org = kwargs['org']
edx_video_id = kwargs['edx_video_id']
lang_code = kwargs['lang_code']
file_id = kwargs['file_id']
state = kwargs['status']
try:
process = TranscriptProcessMetadata.objects.filter(
provider=TranscriptProvider.THREE_PLAY,
process_id=file_id,
lang_code=lang_code,
).latest()
except TranscriptProcessMetadata.DoesNotExist:
LOGGER.exception(
u'[3PlayMedia Callback] Unable to get transcript process for org=%s, edx_video_id=%s, file_id=%s.',
org,
edx_video_id,
file_id,
)
return
# On completion of a transcript
# Indicates that the default video speech transcription has been done successfully.
if state == COMPLETE:
log_args = (edx_video_id, lang_code, file_id)
# 1 - Retrieve transcript credentials
transcript_secrets = get_transcript_credentials(
provider=TranscriptProvider.THREE_PLAY,
org=org,
edx_video_id=edx_video_id,
file_id=file_id,
log_prefix=log_prefix,
)
if not transcript_secrets:
process.update(status=TranscriptStatus.FAILED)
return
# 2 - Fetch the transcript from 3Play Media.
try:
srt_transcript = fetch_srt_data(
THREE_PLAY_TRANSCRIPT_URL.format(file_id=file_id),
apikey=transcript_secrets.api_key,
)
except TranscriptFetchError:
LOGGER.exception(
u'[3PlayMedia Callback] Fetch request failed for video=%s -- lang_code=%s -- process_id=%s',
*log_args)
process.update(status=TranscriptStatus.FAILED)
return
# 3 - Validate transcript content received from 3Play Media and mark the transcription process.
is_valid_transcript = validate_transcript_response(
edx_video_id=edx_video_id,
file_id=file_id,
transcript=srt_transcript,
lang_code=lang_code,
log_prefix=log_prefix,
)
if is_valid_transcript:
process.update(status=TranscriptStatus.READY)
else:
process.update(status=TranscriptStatus.FAILED)
# 4 - Convert SRT transcript to SJson format and upload it to S3.
try:
sjson_transcript = convert_srt_to_sjson(srt_transcript)
sjson_file = upload_sjson_to_s3(CONFIG, sjson_transcript)
except Exception:
# in case of any exception, log and raise.
LOGGER.exception(
u'[3PlayMedia Callback] Request failed for video=%s -- lang_code=%s -- process_id=%s',
*log_args)
raise
# 5 - Update edx-val with completed transcript information.
val_api = VALAPICall(video_proto=None, val_status=None)
val_api.update_val_transcript(
video_id=process.video.studio_id,
lang_code=lang_code,
name=sjson_file,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.THREE_PLAY,
)
# 6 - Translation Phase
# That's the phase for kicking off translation processes for all the
# preferred languages except the video's speech language.
target_languages = list(process.video.preferred_languages)
target_languages.remove(lang_code)
# Create the translation tracking processes for all the target languages.
for target_language in target_languages:
TranscriptProcessMetadata.objects.create(
video=process.video,
provider=TranscriptProvider.THREE_PLAY,
process_id=file_id,
lang_code=target_language,
status=TranscriptStatus.PENDING,
)
# Order translations for target languages
try:
order_translations(file_id,
transcript_secrets.api_key,
transcript_secrets.api_secret,
source_language=lang_code,
target_languages=target_languages)
except TranscriptTranslationError:
LOGGER.exception(
u'[3PlayMedia Callback] Translation could not be performed - video=%s, lang_code=%s, file_id=%s.',
*log_args)
except Exception:
LOGGER.exception(
u'[3PlayMedia Callback] Error while translating the transcripts - video=%s, lang_code=%s, file_id=%s',
*log_args)
raise
# 7 - Update transcript status.
# It will be for edx-val as well as edx-video-pipeline and this will be the case when
# there is only one transcript language for a video(that is, already been processed).
if not target_languages:
utils.update_video_status(val_api_client=val_api,
video=process.video,
status=TranscriptStatus.READY)
# On success, a happy farewell log.
LOGGER.info((
u'[3PlayMedia Callback] Video speech transcription was successful for'
u' video=%s -- lang_code=%s -- process_id=%s'), *log_args)
elif state == ERROR:
# Fail the process
process.status = TranscriptStatus.FAILED
process.save()
# Log the error information
LOGGER.error(
u'[3PlayMedia Callback] Error while transcription - error=%s, org=%s, edx_video_id=%s, file_id=%s.',
kwargs['error_description'],
org,
edx_video_id,
file_id,
)
else:
# Status must be either 'complete' or 'error'
# more details on http://support.3playmedia.com/hc/en-us/articles/227729828-Files-API-Methods
LOGGER.error(
u'[3PlayMedia Callback] Got invalid status - status=%s, org=%s, edx_video_id=%s, file_id=%s.',
state,
org,
edx_video_id,
file_id,
)
def cielo24_transcript_callback(sender, **kwargs):
"""
* download transcript(SRT) from Cielo24
* convert SRT to SJSON
* upload SJSON to AWS S3
* update transcript status in VAL
"""
process_metadata = None
transcript_prefs = None
org = kwargs['org']
job_id = kwargs['job_id']
video_id = kwargs['video_id']
iwp_name = kwargs['iwp_name']
lang_code = kwargs['lang_code']
LOGGER.info(
'[CIELO24 TRANSCRIPTS] Transcript complete request received for '
'video=%s -- org=%s -- lang=%s -- job_id=%s -- iwp_name=%s', video_id,
org, lang_code, job_id, iwp_name)
# get transcript credentials for an organization
try:
transcript_prefs = TranscriptCredentials.objects.get(
org=org,
provider=TranscriptProvider.CIELO24,
)
except TranscriptCredentials.DoesNotExist:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Unable to get transcript credentials for job_id=%s',
job_id)
# mark the transcript for a particular language as ready
try:
process_metadata = TranscriptProcessMetadata.objects.filter(
provider=TranscriptProvider.CIELO24,
process_id=job_id,
lang_code=lang_code).latest()
except TranscriptProcessMetadata.DoesNotExist:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Unable to get transcript process metadata for job_id=%s',
job_id)
# if transcript credentials are missing then we can do nothing
if not transcript_prefs and process_metadata:
process_metadata.status = TranscriptStatus.FAILED
process_metadata.save()
if transcript_prefs and process_metadata:
api_key = transcript_prefs.api_key
try:
srt_data = fetch_srt_data(CIELO24_GET_CAPTION_URL,
v=CIELO24_API_VERSION,
job_id=job_id,
api_token=api_key,
caption_format='SRT')
except TranscriptFetchError:
process_metadata.status = TranscriptStatus.FAILED
process_metadata.save()
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Fetch request failed for video=%s -- lang=%s -- job_id=%s',
video_id, lang_code, job_id)
return
process_metadata.status = TranscriptStatus.READY
process_metadata.save()
try:
sjson = convert_srt_to_sjson(srt_data)
sjson_file_name = upload_sjson_to_s3(CONFIG, sjson)
except Exception:
LOGGER.exception(
'[CIELO24 TRANSCRIPTS] Request failed for video=%s -- lang=%s -- job_id=%s.',
video_id, lang_code, job_id)
raise
# update edx-val with completed transcript information
val_api = VALAPICall(process_metadata.video, val_status=None)
val_api.update_val_transcript(
video_id=process_metadata.video.studio_id,
lang_code=lang_code,
name=sjson_file_name,
transcript_format=TRANSCRIPT_SJSON,
provider=TranscriptProvider.CIELO24)
# update transcript status for video in edx-val only if all langauge transcripts are ready
video_jobs = TranscriptProcessMetadata.objects.filter(
video__studio_id=video_id)
if all(video_job.status == TranscriptStatus.READY
for video_job in video_jobs):
utils.update_video_status(val_api_client=val_api,
video=process_metadata.video,
status=TranscriptStatus.READY)