def storage_service_bucket(course_key=None): """ Returns an S3 bucket for video upload. The S3 bucket returned depends on which pipeline, VEDA or VEM, is enabled. """ if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled(): credentials = AssumeRole.get_instance().credentials params = { 'aws_access_key_id': credentials['access_key'], 'aws_secret_access_key': credentials['secret_key'], 'security_token': credentials['session_token'] } else: params = { 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY } conn = s3.connection.S3Connection(**params) vem_pipeline = VEMPipelineIntegration.current() course_hash_value = get_course_hash_value(course_key) vem_override = course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled(course_key) allow_course_to_use_vem = vem_pipeline.enabled and course_hash_value < vem_pipeline.vem_enabled_courses_percentage # We don't need to validate our bucket, it requires a very permissive IAM permission # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() # meaning it would need ListObjects on the whole bucket, not just the path used in each # environment (since we share a single bucket for multiple deployments in some configurations) if vem_override or allow_course_to_use_vem: LOGGER.info('Uploading course: {} to VEM bucket.'.format(course_key)) return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False) else: return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['BUCKET'], validate=False)
def storage_service_bucket(course_key=None): """ Returns an S3 bucket for video upload. The S3 bucket returned depends on which pipeline, VEDA or VEM, is enabled. """ if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled(): credentials = AssumeRole.get_instance().credentials params = { 'aws_access_key_id': credentials['access_key'], 'aws_secret_access_key': credentials['secret_key'], 'security_token': credentials['session_token'] } else: params = { 'aws_access_key_id': settings.AWS_ACCESS_KEY_ID, 'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY } conn = s3.connection.S3Connection(**params) # We don't need to validate our bucket, it requires a very permissive IAM permission # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys() # meaning it would need ListObjects on the whole bucket, not just the path used in each # environment (since we share a single bucket for multiple deployments in some configurations) return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False)
def transcript_credentials_handler(request, course_key_string): """ JSON view handler to update the transcript organization credentials. Arguments: request: WSGI request object course_key_string: A course identifier to extract the org. Returns: - A 200 response if credentials are valid and successfully updated in edx-video-pipeline. - A 404 response if transcript feature is not enabled for this course. - A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline. """ course_key = CourseKey.from_string(course_key_string) if not VideoTranscriptEnabledFlag.feature_enabled(course_key): return HttpResponseNotFound() provider = request.json.pop('provider') error_message, validated_credentials = validate_transcript_credentials( provider=provider, **request.json) if error_message: response = JsonResponse({'error': error_message}, status=400) else: # Send the validated credentials to edx-video-pipeline. credentials_payload = dict(validated_credentials, org=course_key.org, provider=provider) if waffle_flags()[SAVE_CREDENTIALS_IN_VAL].is_enabled(course_key): from edxval.api import create_or_update_transcript_credentials response = create_or_update_transcript_credentials( **credentials_payload) error_response, is_updated = response, not response.get( 'error_type') else: error_response, is_updated = update_3rd_party_transcription_service_credentials( **credentials_payload) # Send appropriate response based on whether credentials were updated or not. if is_updated: # Cache credentials state in edx-val. update_transcript_credentials_state_for_org(org=course_key.org, provider=provider, exists=is_updated) response = JsonResponse(status=200) else: # Error response would contain error types and the following # error type is received from edx-video-pipeline whenever we've # got invalid credentials for a provider. Its kept this way because # edx-video-pipeline doesn't support i18n translations yet. error_type = error_response.get('error_type') if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS: error_message = _('The information you entered is incorrect.') response = JsonResponse({'error': error_message}, status=400) return response
def youtube_deprecated(self): """ Return True if youtube is deprecated and hls as primary playback is enabled else False """ # Return False if `hls` playback feature is disabled. if not HLSPlaybackEnabledFlag.feature_enabled(self.location.course_key): return False # check if youtube has been deprecated and hls as primary playback # is enabled for this course return waffle_flags()[DEPRECATE_YOUTUBE].is_enabled(self.location.course_key)
def update_3rd_party_transcription_service_credentials(**credentials_payload): """ Updates the 3rd party transcription service's credentials. Arguments: credentials_payload(dict): A payload containing org, provider and its credentials. Returns: A Boolean specifying whether the credentials were updated or not and an error response received from pipeline. """ error_response, is_updated = {}, False course_key = credentials_payload.pop('course_key', None) if course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled( course_key): pipeline_integration = VEMPipelineIntegration.current() else: pipeline_integration = VideoPipelineIntegration.current() if pipeline_integration.enabled: try: oauth_client = Application.objects.get( name=pipeline_integration.client_name) except ObjectDoesNotExist: return error_response, is_updated client = create_video_pipeline_api_client(oauth_client.client_id, oauth_client.client_secret) error_message = "Unable to update transcript credentials -- org={}, provider={}, response={}" try: response = client.request("POST", pipeline_integration.api_url, json=credentials_payload) if response.ok: is_updated = True else: is_updated = False error_response = json.loads(response.text) log.error( error_message.format(credentials_payload.get('org'), credentials_payload.get('provider'), response.text)) except HttpClientError as ex: is_updated = False log.exception( error_message.format(credentials_payload.get('org'), credentials_payload.get('provider'), ex.content)) error_response = json.loads(ex.content) return error_response, is_updated
def test_update_transcription_service_credentials_for_vem(self, mock_client, mock_logger): """ Test that if waffle flag `ENABLE_VEM_PIPELINE` is on for course, then credentials are successfully posted to VEM. """ self.add_vem_client() course_key = CourseLocator("test_org", "test_course_num", "test_run") credentials_payload = { 'username': '******', 'api_key': '12345678', 'course_key': course_key } mock_client.request.return_value.ok = True # Try updating the transcription service credentials with override_waffle_flag(waffle_flags()[ENABLE_VEM_PIPELINE], active=True): error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload) # Making sure log.exception is not called. self.assertDictEqual(error_response, {}) self.assertFalse(mock_logger.exception.called) self.assertTrue(is_updated)
def videos_post(course, request): """ Input (JSON): { "files": [{ "file_name": "video.mp4", "content_type": "video/mp4" }] } Returns (JSON): { "files": [{ "file_name": "video.mp4", "upload_url": "http://example.com/put_video" }] } The returned array corresponds exactly to the input array. """ error = None data = request.json if 'files' not in data: error = "Request object is not JSON or does not contain 'files'" elif any( 'file_name' not in file or 'content_type' not in file for file in data['files'] ): error = "Request 'files' entry does not contain 'file_name' and 'content_type'" elif any( file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values() for file in data['files'] ): error = "Request 'files' entry contain unsupported content_type" if error: return JsonResponse({'error': error}, status=400) bucket = storage_service_bucket() req_files = data['files'] resp_files = [] for req_file in req_files: file_name = req_file['file_name'] try: file_name.encode('ascii') except UnicodeEncodeError: error_msg = u'The file name for %s must contain only ASCII characters.' % file_name return JsonResponse({'error': error_msg}, status=400) edx_video_id = unicode(uuid4()) key = storage_service_key(bucket, file_name=edx_video_id) metadata_list = [ ('client_video_id', file_name), ('course_key', unicode(course.id)), ] deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE] course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') # Only include `course_video_upload_token` if youtube has not been deprecated # for this course. if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token: metadata_list.append(('course_video_upload_token', course_video_upload_token)) is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) if is_video_transcript_enabled: transcript_preferences = get_transcript_preferences(unicode(course.id)) if transcript_preferences is not None: metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences))) for metadata_name, value in metadata_list: key.set_metadata(metadata_name, value) upload_url = key.generate_url( KEY_EXPIRATION_IN_SECONDS, 'PUT', headers={'Content-Type': req_file['content_type']} ) # persist edx_video_id in VAL create_video({ 'edx_video_id': edx_video_id, 'status': 'upload', 'client_video_id': file_name, 'duration': 0, 'encoded_videos': [], 'courses': [unicode(course.id)] }) resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id}) return JsonResponse({'files': resp_files}, status=200)
class TranscriptCredentialsTest(CourseTestCase): """ Tests for transcript credentials handler. """ VIEW_NAME = 'transcript_credentials_handler' def get_url_for_course_key(self, course_id): return reverse_course_url(self.VIEW_NAME, course_id) def test_302_with_anonymous_user(self): """ Verify that redirection happens in case of unauthorized request. """ self.client.logout() transcript_credentials_url = self.get_url_for_course_key(self.course.id) response = self.client.post(transcript_credentials_url, content_type='application/json') self.assertEqual(response.status_code, 302) def test_405_with_not_allowed_request_method(self): """ Verify that 405 is returned in case of not-allowed request methods. Allowed request methods include POST. """ transcript_credentials_url = self.get_url_for_course_key(self.course.id) response = self.client.get(transcript_credentials_url, content_type='application/json') self.assertEqual(response.status_code, 405) def test_404_with_feature_disabled(self): """ Verify that 404 is returned if the corresponding feature is disabled. """ transcript_credentials_url = self.get_url_for_course_key(self.course.id) with patch('openedx.core.djangoapps.video_config.models.VideoTranscriptEnabledFlag.feature_enabled') as feature: feature.return_value = False response = self.client.post(transcript_credentials_url, content_type='application/json') self.assertEqual(response.status_code, 404) @ddt.data( ( { 'provider': 'abc_provider', 'api_key': '1234' }, ({}, None), 400, '{\n "error": "Invalid Provider abc_provider."\n}' ), ( { 'provider': '3PlayMedia', 'api_key': '11111', 'api_secret_key': '44444' }, ({'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS}, False), 400, '{\n "error": "The information you entered is incorrect."\n}' ), ( { 'provider': 'Cielo24', 'api_key': '12345', 'username': '******' }, ({}, True), 200, '' ) ) @ddt.unpack @patch('contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials') def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code, expected_response, mock_update_credentials): """ Tests that transcript credentials handler works as expected. """ mock_update_credentials.return_value = update_credentials_response transcript_credentials_url = self.get_url_for_course_key(self.course.id) response = self.client.post( transcript_credentials_url, data=json.dumps(request_payload), content_type='application/json' ) self.assertEqual(response.status_code, expected_status_code) self.assertEqual(response.content.decode('utf-8'), expected_response) @override_waffle_flag(waffle_flags()[SAVE_CREDENTIALS_IN_VAL], True) @ddt.data( ( { 'provider': '3PlayMedia', 'api_key': '11111', 'api_secret_key': '44444' }, {'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS}, 400, '{\n "error": "The information you entered is incorrect."\n}' ), ( { 'provider': 'Cielo24', 'api_key': '12345', 'username': '******' }, {'error_type': None}, 200, '' ), ( { 'provider': '3PlayMedia', 'api_key': '12345', 'api_secret_key': '44444' }, {'error_type': None}, 200, '' ) ) @ddt.unpack @patch('edxval.api.create_or_update_transcript_credentials') def test_val_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code, expected_response, api_patch): """ Test that credentials handler works fine with VAL api endpoint. """ api_patch.return_value = update_credentials_response transcript_credentials_url = self.get_url_for_course_key(self.course.id) response = self.client.post( transcript_credentials_url, data=json.dumps(request_payload), content_type='application/json' ) self.assertEqual(response.status_code, expected_status_code) self.assertEqual(response.content.decode('utf-8'), expected_response)
def videos_post(course, request): """ Input (JSON): { "files": [{ "file_name": "video.mp4", "content_type": "video/mp4" }] } Returns (JSON): { "files": [{ "file_name": "video.mp4", "upload_url": "http://example.com/put_video" }] } The returned array corresponds exactly to the input array. """ error = None data = request.json if 'files' not in data: error = "Request object is not JSON or does not contain 'files'" elif any( 'file_name' not in file or 'content_type' not in file for file in data['files'] ): error = "Request 'files' entry does not contain 'file_name' and 'content_type'" elif any( file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values() for file in data['files'] ): error = "Request 'files' entry contain unsupported content_type" if error: return JsonResponse({'error': error}, status=400) bucket = storage_service_bucket() req_files = data['files'] resp_files = [] for req_file in req_files: file_name = req_file['file_name'] try: file_name.encode('ascii') except UnicodeEncodeError: error_msg = 'The file name for %s must contain only ASCII characters.' % file_name return JsonResponse({'error': error_msg}, status=400) edx_video_id = unicode(uuid4()) key = storage_service_key(bucket, file_name=edx_video_id) metadata_list = [ ('client_video_id', file_name), ('course_key', unicode(course.id)), ] deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE] course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token') # Only include `course_video_upload_token` if youtube has not been deprecated # for this course. if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token: metadata_list.append(('course_video_upload_token', course_video_upload_token)) is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id) if is_video_transcript_enabled: transcript_preferences = get_transcript_preferences(unicode(course.id)) if transcript_preferences is not None: metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences))) for metadata_name, value in metadata_list: key.set_metadata(metadata_name, value) upload_url = key.generate_url( KEY_EXPIRATION_IN_SECONDS, 'PUT', headers={'Content-Type': req_file['content_type']} ) # persist edx_video_id in VAL create_video({ 'edx_video_id': edx_video_id, 'status': 'upload', 'client_video_id': file_name, 'duration': 0, 'encoded_videos': [], 'courses': [unicode(course.id)] }) resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id}) return JsonResponse({'files': resp_files}, status=200)