def storage_service_bucket(course_key=None):
"""
Returns an S3 bucket for video upload. The S3 bucket returned depends on
which pipeline, VEDA or VEM, is enabled.
"""
if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled():
credentials = AssumeRole.get_instance().credentials
params = {
'aws_access_key_id': credentials['access_key'],
'aws_secret_access_key': credentials['secret_key'],
'security_token': credentials['session_token']
}
else:
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
}
conn = s3.connection.S3Connection(**params)
vem_pipeline = VEMPipelineIntegration.current()
course_hash_value = get_course_hash_value(course_key)
vem_override = course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled(course_key)
allow_course_to_use_vem = vem_pipeline.enabled and course_hash_value < vem_pipeline.vem_enabled_courses_percentage
# We don't need to validate our bucket, it requires a very permissive IAM permission
# set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
# meaning it would need ListObjects on the whole bucket, not just the path used in each
# environment (since we share a single bucket for multiple deployments in some configurations)
if vem_override or allow_course_to_use_vem:
LOGGER.info('Uploading course: {} to VEM bucket.'.format(course_key))
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False)
else:
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['BUCKET'], validate=False)
def storage_service_bucket(course_key=None):
"""
Returns an S3 bucket for video upload. The S3 bucket returned depends on
which pipeline, VEDA or VEM, is enabled.
"""
if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled():
credentials = AssumeRole.get_instance().credentials
params = {
'aws_access_key_id': credentials['access_key'],
'aws_secret_access_key': credentials['secret_key'],
'security_token': credentials['session_token']
}
else:
params = {
'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
}
conn = s3.connection.S3Connection(**params)
# We don't need to validate our bucket, it requires a very permissive IAM permission
# set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
# meaning it would need ListObjects on the whole bucket, not just the path used in each
# environment (since we share a single bucket for multiple deployments in some configurations)
return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'],
validate=False)
def transcript_credentials_handler(request, course_key_string):
"""
JSON view handler to update the transcript organization credentials.
Arguments:
request: WSGI request object
course_key_string: A course identifier to extract the org.
Returns:
- A 200 response if credentials are valid and successfully updated in edx-video-pipeline.
- A 404 response if transcript feature is not enabled for this course.
- A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline.
"""
course_key = CourseKey.from_string(course_key_string)
if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
return HttpResponseNotFound()
provider = request.json.pop('provider')
error_message, validated_credentials = validate_transcript_credentials(
provider=provider, **request.json)
if error_message:
response = JsonResponse({'error': error_message}, status=400)
else:
# Send the validated credentials to edx-video-pipeline.
credentials_payload = dict(validated_credentials,
org=course_key.org,
provider=provider)
if waffle_flags()[SAVE_CREDENTIALS_IN_VAL].is_enabled(course_key):
from edxval.api import create_or_update_transcript_credentials
response = create_or_update_transcript_credentials(
**credentials_payload)
error_response, is_updated = response, not response.get(
'error_type')
else:
error_response, is_updated = update_3rd_party_transcription_service_credentials(
**credentials_payload)
# Send appropriate response based on whether credentials were updated or not.
if is_updated:
# Cache credentials state in edx-val.
update_transcript_credentials_state_for_org(org=course_key.org,
provider=provider,
exists=is_updated)
response = JsonResponse(status=200)
else:
# Error response would contain error types and the following
# error type is received from edx-video-pipeline whenever we've
# got invalid credentials for a provider. Its kept this way because
# edx-video-pipeline doesn't support i18n translations yet.
error_type = error_response.get('error_type')
if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS:
error_message = _('The information you entered is incorrect.')
response = JsonResponse({'error': error_message}, status=400)
return response
def youtube_deprecated(self):
"""
Return True if youtube is deprecated and hls as primary playback is enabled else False
"""
# Return False if `hls` playback feature is disabled.
if not HLSPlaybackEnabledFlag.feature_enabled(self.location.course_key):
return False
# check if youtube has been deprecated and hls as primary playback
# is enabled for this course
return waffle_flags()[DEPRECATE_YOUTUBE].is_enabled(self.location.course_key)
def update_3rd_party_transcription_service_credentials(**credentials_payload):
"""
Updates the 3rd party transcription service's credentials.
Arguments:
credentials_payload(dict): A payload containing org, provider and its credentials.
Returns:
A Boolean specifying whether the credentials were updated or not
and an error response received from pipeline.
"""
error_response, is_updated = {}, False
course_key = credentials_payload.pop('course_key', None)
if course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled(
course_key):
pipeline_integration = VEMPipelineIntegration.current()
else:
pipeline_integration = VideoPipelineIntegration.current()
if pipeline_integration.enabled:
try:
oauth_client = Application.objects.get(
name=pipeline_integration.client_name)
except ObjectDoesNotExist:
return error_response, is_updated
client = create_video_pipeline_api_client(oauth_client.client_id,
oauth_client.client_secret)
error_message = "Unable to update transcript credentials -- org={}, provider={}, response={}"
try:
response = client.request("POST",
pipeline_integration.api_url,
json=credentials_payload)
if response.ok:
is_updated = True
else:
is_updated = False
error_response = json.loads(response.text)
log.error(
error_message.format(credentials_payload.get('org'),
credentials_payload.get('provider'),
response.text))
except HttpClientError as ex:
is_updated = False
log.exception(
error_message.format(credentials_payload.get('org'),
credentials_payload.get('provider'),
ex.content))
error_response = json.loads(ex.content)
return error_response, is_updated
def test_update_transcription_service_credentials_for_vem(self, mock_client, mock_logger):
"""
Test that if waffle flag `ENABLE_VEM_PIPELINE` is on for course, then credentials
are successfully posted to VEM.
"""
self.add_vem_client()
course_key = CourseLocator("test_org", "test_course_num", "test_run")
credentials_payload = {
'username': '******',
'api_key': '12345678',
'course_key': course_key
}
mock_client.request.return_value.ok = True
# Try updating the transcription service credentials
with override_waffle_flag(waffle_flags()[ENABLE_VEM_PIPELINE], active=True):
error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)
# Making sure log.exception is not called.
self.assertDictEqual(error_response, {})
self.assertFalse(mock_logger.exception.called)
self.assertTrue(is_updated)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
"""
error = None
data = request.json
if 'files' not in data:
error = "Request object is not JSON or does not contain 'files'"
elif any(
'file_name' not in file or 'content_type' not in file
for file in data['files']
):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
elif any(
file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
for file in data['files']
):
error = "Request 'files' entry contain unsupported content_type"
if error:
return JsonResponse({'error': error}, status=400)
bucket = storage_service_bucket()
req_files = data['files']
resp_files = []
for req_file in req_files:
file_name = req_file['file_name']
try:
file_name.encode('ascii')
except UnicodeEncodeError:
error_msg = u'The file name for %s must contain only ASCII characters.' % file_name
return JsonResponse({'error': error_msg}, status=400)
edx_video_id = unicode(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
metadata_list = [
('client_video_id', file_name),
('course_key', unicode(course.id)),
]
deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE]
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
# Only include `course_video_upload_token` if youtube has not been deprecated
# for this course.
if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token:
metadata_list.append(('course_video_upload_token', course_video_upload_token))
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
if is_video_transcript_enabled:
transcript_preferences = get_transcript_preferences(unicode(course.id))
if transcript_preferences is not None:
metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))
for metadata_name, value in metadata_list:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
'PUT',
headers={'Content-Type': req_file['content_type']}
)
# persist edx_video_id in VAL
create_video({
'edx_video_id': edx_video_id,
'status': 'upload',
'client_video_id': file_name,
'duration': 0,
'encoded_videos': [],
'courses': [unicode(course.id)]
})
resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})
return JsonResponse({'files': resp_files}, status=200)
class TranscriptCredentialsTest(CourseTestCase):
"""
Tests for transcript credentials handler.
"""
VIEW_NAME = 'transcript_credentials_handler'
def get_url_for_course_key(self, course_id):
return reverse_course_url(self.VIEW_NAME, course_id)
def test_302_with_anonymous_user(self):
"""
Verify that redirection happens in case of unauthorized request.
"""
self.client.logout()
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.post(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 302)
def test_405_with_not_allowed_request_method(self):
"""
Verify that 405 is returned in case of not-allowed request methods.
Allowed request methods include POST.
"""
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.get(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 405)
def test_404_with_feature_disabled(self):
"""
Verify that 404 is returned if the corresponding feature is disabled.
"""
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
with patch('openedx.core.djangoapps.video_config.models.VideoTranscriptEnabledFlag.feature_enabled') as feature:
feature.return_value = False
response = self.client.post(transcript_credentials_url, content_type='application/json')
self.assertEqual(response.status_code, 404)
@ddt.data(
(
{
'provider': 'abc_provider',
'api_key': '1234'
},
({}, None),
400,
'{\n "error": "Invalid Provider abc_provider."\n}'
),
(
{
'provider': '3PlayMedia',
'api_key': '11111',
'api_secret_key': '44444'
},
({'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS}, False),
400,
'{\n "error": "The information you entered is incorrect."\n}'
),
(
{
'provider': 'Cielo24',
'api_key': '12345',
'username': '******'
},
({}, True),
200,
''
)
)
@ddt.unpack
@patch('contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials')
def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code,
expected_response, mock_update_credentials):
"""
Tests that transcript credentials handler works as expected.
"""
mock_update_credentials.return_value = update_credentials_response
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.post(
transcript_credentials_url,
data=json.dumps(request_payload),
content_type='application/json'
)
self.assertEqual(response.status_code, expected_status_code)
self.assertEqual(response.content.decode('utf-8'), expected_response)
@override_waffle_flag(waffle_flags()[SAVE_CREDENTIALS_IN_VAL], True)
@ddt.data(
(
{
'provider': '3PlayMedia',
'api_key': '11111',
'api_secret_key': '44444'
},
{'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS},
400,
'{\n "error": "The information you entered is incorrect."\n}'
),
(
{
'provider': 'Cielo24',
'api_key': '12345',
'username': '******'
},
{'error_type': None},
200,
''
),
(
{
'provider': '3PlayMedia',
'api_key': '12345',
'api_secret_key': '44444'
},
{'error_type': None},
200,
''
)
)
@ddt.unpack
@patch('edxval.api.create_or_update_transcript_credentials')
def test_val_transcript_credentials_handler(self, request_payload, update_credentials_response,
expected_status_code, expected_response, api_patch):
"""
Test that credentials handler works fine with VAL api endpoint.
"""
api_patch.return_value = update_credentials_response
transcript_credentials_url = self.get_url_for_course_key(self.course.id)
response = self.client.post(
transcript_credentials_url,
data=json.dumps(request_payload),
content_type='application/json'
)
self.assertEqual(response.status_code, expected_status_code)
self.assertEqual(response.content.decode('utf-8'), expected_response)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
"""
error = None
data = request.json
if 'files' not in data:
error = "Request object is not JSON or does not contain 'files'"
elif any(
'file_name' not in file or 'content_type' not in file
for file in data['files']
):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
elif any(
file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
for file in data['files']
):
error = "Request 'files' entry contain unsupported content_type"
if error:
return JsonResponse({'error': error}, status=400)
bucket = storage_service_bucket()
req_files = data['files']
resp_files = []
for req_file in req_files:
file_name = req_file['file_name']
try:
file_name.encode('ascii')
except UnicodeEncodeError:
error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
return JsonResponse({'error': error_msg}, status=400)
edx_video_id = unicode(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
metadata_list = [
('client_video_id', file_name),
('course_key', unicode(course.id)),
]
deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE]
course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')
# Only include `course_video_upload_token` if youtube has not been deprecated
# for this course.
if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token:
metadata_list.append(('course_video_upload_token', course_video_upload_token))
is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
if is_video_transcript_enabled:
transcript_preferences = get_transcript_preferences(unicode(course.id))
if transcript_preferences is not None:
metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))
for metadata_name, value in metadata_list:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
'PUT',
headers={'Content-Type': req_file['content_type']}
)
# persist edx_video_id in VAL
create_video({
'edx_video_id': edx_video_id,
'status': 'upload',
'client_video_id': file_name,
'duration': 0,
'encoded_videos': [],
'courses': [unicode(course.id)]
})
resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})
return JsonResponse({'files': resp_files}, status=200)
