Exemple #1
0
def storage_service_bucket(course_key=None):
    """
    Returns an S3 bucket for video upload. The S3 bucket returned depends on
    which pipeline, VEDA or VEM, is enabled.
    """
    if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled():
        credentials = AssumeRole.get_instance().credentials
        params = {
            'aws_access_key_id': credentials['access_key'],
            'aws_secret_access_key': credentials['secret_key'],
            'security_token': credentials['session_token']
        }
    else:
        params = {
            'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
            'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
        }

    conn = s3.connection.S3Connection(**params)
    vem_pipeline = VEMPipelineIntegration.current()
    course_hash_value = get_course_hash_value(course_key)

    vem_override = course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled(course_key)
    allow_course_to_use_vem = vem_pipeline.enabled and course_hash_value < vem_pipeline.vem_enabled_courses_percentage

    # We don't need to validate our bucket, it requires a very permissive IAM permission
    # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
    # meaning it would need ListObjects on the whole bucket, not just the path used in each
    # environment (since we share a single bucket for multiple deployments in some configurations)
    if vem_override or allow_course_to_use_vem:
        LOGGER.info('Uploading course: {} to VEM bucket.'.format(course_key))
        return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'], validate=False)
    else:
        return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['BUCKET'], validate=False)
Exemple #2
0
def storage_service_bucket(course_key=None):
    """
    Returns an S3 bucket for video upload. The S3 bucket returned depends on
    which pipeline, VEDA or VEM, is enabled.
    """
    if waffle_flags()[ENABLE_DEVSTACK_VIDEO_UPLOADS].is_enabled():
        credentials = AssumeRole.get_instance().credentials
        params = {
            'aws_access_key_id': credentials['access_key'],
            'aws_secret_access_key': credentials['secret_key'],
            'security_token': credentials['session_token']
        }
    else:
        params = {
            'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
            'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY
        }

    conn = s3.connection.S3Connection(**params)

    # We don't need to validate our bucket, it requires a very permissive IAM permission
    # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
    # meaning it would need ListObjects on the whole bucket, not just the path used in each
    # environment (since we share a single bucket for multiple deployments in some configurations)
    return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE['VEM_S3_BUCKET'],
                           validate=False)
Exemple #3
0
def transcript_credentials_handler(request, course_key_string):
    """
    JSON view handler to update the transcript organization credentials.

    Arguments:
        request: WSGI request object
        course_key_string: A course identifier to extract the org.

    Returns:
        - A 200 response if credentials are valid and successfully updated in edx-video-pipeline.
        - A 404 response if transcript feature is not enabled for this course.
        - A 400 if credentials do not pass validations, hence not updated in edx-video-pipeline.
    """
    course_key = CourseKey.from_string(course_key_string)
    if not VideoTranscriptEnabledFlag.feature_enabled(course_key):
        return HttpResponseNotFound()

    provider = request.json.pop('provider')
    error_message, validated_credentials = validate_transcript_credentials(
        provider=provider, **request.json)
    if error_message:
        response = JsonResponse({'error': error_message}, status=400)
    else:
        # Send the validated credentials to edx-video-pipeline.
        credentials_payload = dict(validated_credentials,
                                   org=course_key.org,
                                   provider=provider)
        if waffle_flags()[SAVE_CREDENTIALS_IN_VAL].is_enabled(course_key):
            from edxval.api import create_or_update_transcript_credentials
            response = create_or_update_transcript_credentials(
                **credentials_payload)
            error_response, is_updated = response, not response.get(
                'error_type')
        else:
            error_response, is_updated = update_3rd_party_transcription_service_credentials(
                **credentials_payload)
        # Send appropriate response based on whether credentials were updated or not.
        if is_updated:
            # Cache credentials state in edx-val.
            update_transcript_credentials_state_for_org(org=course_key.org,
                                                        provider=provider,
                                                        exists=is_updated)
            response = JsonResponse(status=200)
        else:
            # Error response would contain error types and the following
            # error type is received from edx-video-pipeline whenever we've
            # got invalid credentials for a provider. Its kept this way because
            # edx-video-pipeline doesn't support i18n translations yet.
            error_type = error_response.get('error_type')
            if error_type == TranscriptionProviderErrorType.INVALID_CREDENTIALS:
                error_message = _('The information you entered is incorrect.')

            response = JsonResponse({'error': error_message}, status=400)

    return response
Exemple #4
0
    def youtube_deprecated(self):
        """
        Return True if youtube is deprecated and hls as primary playback is enabled else False
        """
        # Return False if `hls` playback feature is disabled.
        if not HLSPlaybackEnabledFlag.feature_enabled(self.location.course_key):
            return False

        # check if youtube has been deprecated and hls as primary playback
        # is enabled for this course
        return waffle_flags()[DEPRECATE_YOUTUBE].is_enabled(self.location.course_key)
Exemple #5
0
def update_3rd_party_transcription_service_credentials(**credentials_payload):
    """
    Updates the 3rd party transcription service's credentials.

    Arguments:
        credentials_payload(dict): A payload containing org, provider and its credentials.

    Returns:
        A Boolean specifying whether the credentials were updated or not
        and an error response received from pipeline.
    """
    error_response, is_updated = {}, False
    course_key = credentials_payload.pop('course_key', None)

    if course_key and waffle_flags()[ENABLE_VEM_PIPELINE].is_enabled(
            course_key):
        pipeline_integration = VEMPipelineIntegration.current()
    else:
        pipeline_integration = VideoPipelineIntegration.current()

    if pipeline_integration.enabled:
        try:
            oauth_client = Application.objects.get(
                name=pipeline_integration.client_name)
        except ObjectDoesNotExist:
            return error_response, is_updated

        client = create_video_pipeline_api_client(oauth_client.client_id,
                                                  oauth_client.client_secret)
        error_message = "Unable to update transcript credentials -- org={}, provider={}, response={}"
        try:
            response = client.request("POST",
                                      pipeline_integration.api_url,
                                      json=credentials_payload)
            if response.ok:
                is_updated = True
            else:
                is_updated = False
                error_response = json.loads(response.text)
                log.error(
                    error_message.format(credentials_payload.get('org'),
                                         credentials_payload.get('provider'),
                                         response.text))
        except HttpClientError as ex:
            is_updated = False
            log.exception(
                error_message.format(credentials_payload.get('org'),
                                     credentials_payload.get('provider'),
                                     ex.content))
            error_response = json.loads(ex.content)

    return error_response, is_updated
Exemple #6
0
    def test_update_transcription_service_credentials_for_vem(self, mock_client, mock_logger):
        """
        Test that if waffle flag `ENABLE_VEM_PIPELINE` is on for course, then credentials
        are successfully posted to VEM.
        """
        self.add_vem_client()
        course_key = CourseLocator("test_org", "test_course_num", "test_run")
        credentials_payload = {
            'username': '******',
            'api_key': '12345678',
            'course_key': course_key
        }
        mock_client.request.return_value.ok = True

        # Try updating the transcription service credentials
        with override_waffle_flag(waffle_flags()[ENABLE_VEM_PIPELINE], active=True):
            error_response, is_updated = update_3rd_party_transcription_service_credentials(**credentials_payload)

        # Making sure log.exception is not called.
        self.assertDictEqual(error_response, {})
        self.assertFalse(mock_logger.exception.called)
        self.assertTrue(is_updated)
Exemple #7
0
def videos_post(course, request):
    """
    Input (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "content_type": "video/mp4"
        }]
    }

    Returns (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "upload_url": "http://example.com/put_video"
        }]
    }

    The returned array corresponds exactly to the input array.
    """
    error = None
    data = request.json
    if 'files' not in data:
        error = "Request object is not JSON or does not contain 'files'"
    elif any(
            'file_name' not in file or 'content_type' not in file
            for file in data['files']
    ):
        error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
    elif any(
            file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
            for file in data['files']
    ):
        error = "Request 'files' entry contain unsupported content_type"

    if error:
        return JsonResponse({'error': error}, status=400)

    bucket = storage_service_bucket()
    req_files = data['files']
    resp_files = []

    for req_file in req_files:
        file_name = req_file['file_name']

        try:
            file_name.encode('ascii')
        except UnicodeEncodeError:
            error_msg = u'The file name for %s must contain only ASCII characters.' % file_name
            return JsonResponse({'error': error_msg}, status=400)

        edx_video_id = unicode(uuid4())
        key = storage_service_key(bucket, file_name=edx_video_id)

        metadata_list = [
            ('client_video_id', file_name),
            ('course_key', unicode(course.id)),
        ]

        deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE]
        course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')

        # Only include `course_video_upload_token` if youtube has not been deprecated
        # for this course.
        if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token:
            metadata_list.append(('course_video_upload_token', course_video_upload_token))

        is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
        if is_video_transcript_enabled:
            transcript_preferences = get_transcript_preferences(unicode(course.id))
            if transcript_preferences is not None:
                metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))

        for metadata_name, value in metadata_list:
            key.set_metadata(metadata_name, value)
        upload_url = key.generate_url(
            KEY_EXPIRATION_IN_SECONDS,
            'PUT',
            headers={'Content-Type': req_file['content_type']}
        )

        # persist edx_video_id in VAL
        create_video({
            'edx_video_id': edx_video_id,
            'status': 'upload',
            'client_video_id': file_name,
            'duration': 0,
            'encoded_videos': [],
            'courses': [unicode(course.id)]
        })

        resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})

    return JsonResponse({'files': resp_files}, status=200)
class TranscriptCredentialsTest(CourseTestCase):
    """
    Tests for transcript credentials handler.
    """
    VIEW_NAME = 'transcript_credentials_handler'

    def get_url_for_course_key(self, course_id):
        return reverse_course_url(self.VIEW_NAME, course_id)

    def test_302_with_anonymous_user(self):
        """
        Verify that redirection happens in case of unauthorized request.
        """
        self.client.logout()
        transcript_credentials_url = self.get_url_for_course_key(self.course.id)
        response = self.client.post(transcript_credentials_url, content_type='application/json')
        self.assertEqual(response.status_code, 302)

    def test_405_with_not_allowed_request_method(self):
        """
        Verify that 405 is returned in case of not-allowed request methods.
        Allowed request methods include POST.
        """
        transcript_credentials_url = self.get_url_for_course_key(self.course.id)
        response = self.client.get(transcript_credentials_url, content_type='application/json')
        self.assertEqual(response.status_code, 405)

    def test_404_with_feature_disabled(self):
        """
        Verify that 404 is returned if the corresponding feature is disabled.
        """
        transcript_credentials_url = self.get_url_for_course_key(self.course.id)
        with patch('openedx.core.djangoapps.video_config.models.VideoTranscriptEnabledFlag.feature_enabled') as feature:
            feature.return_value = False
            response = self.client.post(transcript_credentials_url, content_type='application/json')
            self.assertEqual(response.status_code, 404)

    @ddt.data(
        (
            {
                'provider': 'abc_provider',
                'api_key': '1234'
            },
            ({}, None),
            400,
            '{\n  "error": "Invalid Provider abc_provider."\n}'
        ),
        (
            {
                'provider': '3PlayMedia',
                'api_key': '11111',
                'api_secret_key': '44444'
            },
            ({'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS}, False),
            400,
            '{\n  "error": "The information you entered is incorrect."\n}'
        ),
        (
            {
                'provider': 'Cielo24',
                'api_key': '12345',
                'username': '******'
            },
            ({}, True),
            200,
            ''
        )
    )
    @ddt.unpack
    @patch('contentstore.views.transcript_settings.update_3rd_party_transcription_service_credentials')
    def test_transcript_credentials_handler(self, request_payload, update_credentials_response, expected_status_code,
                                            expected_response, mock_update_credentials):
        """
        Tests that transcript credentials handler works as expected.
        """
        mock_update_credentials.return_value = update_credentials_response
        transcript_credentials_url = self.get_url_for_course_key(self.course.id)
        response = self.client.post(
            transcript_credentials_url,
            data=json.dumps(request_payload),
            content_type='application/json'
        )
        self.assertEqual(response.status_code, expected_status_code)
        self.assertEqual(response.content.decode('utf-8'), expected_response)

    @override_waffle_flag(waffle_flags()[SAVE_CREDENTIALS_IN_VAL], True)
    @ddt.data(
        (
            {
                'provider': '3PlayMedia',
                'api_key': '11111',
                'api_secret_key': '44444'
            },
            {'error_type': TranscriptionProviderErrorType.INVALID_CREDENTIALS},
            400,
            '{\n  "error": "The information you entered is incorrect."\n}'
        ),
        (
            {
                'provider': 'Cielo24',
                'api_key': '12345',
                'username': '******'
            },
            {'error_type': None},
            200,
            ''
        ),
        (
            {
                'provider': '3PlayMedia',
                'api_key': '12345',
                'api_secret_key': '44444'
            },
            {'error_type': None},
            200,
            ''
        )
    )
    @ddt.unpack
    @patch('edxval.api.create_or_update_transcript_credentials')
    def test_val_transcript_credentials_handler(self, request_payload, update_credentials_response,
                                                expected_status_code, expected_response, api_patch):
        """
        Test that credentials handler works fine with VAL api endpoint.
        """
        api_patch.return_value = update_credentials_response
        transcript_credentials_url = self.get_url_for_course_key(self.course.id)
        response = self.client.post(
            transcript_credentials_url,
            data=json.dumps(request_payload),
            content_type='application/json'
        )
        self.assertEqual(response.status_code, expected_status_code)
        self.assertEqual(response.content.decode('utf-8'), expected_response)
Exemple #9
0
def videos_post(course, request):
    """
    Input (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "content_type": "video/mp4"
        }]
    }

    Returns (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "upload_url": "http://example.com/put_video"
        }]
    }

    The returned array corresponds exactly to the input array.
    """
    error = None
    data = request.json
    if 'files' not in data:
        error = "Request object is not JSON or does not contain 'files'"
    elif any(
            'file_name' not in file or 'content_type' not in file
            for file in data['files']
    ):
        error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
    elif any(
            file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
            for file in data['files']
    ):
        error = "Request 'files' entry contain unsupported content_type"

    if error:
        return JsonResponse({'error': error}, status=400)

    bucket = storage_service_bucket()
    req_files = data['files']
    resp_files = []

    for req_file in req_files:
        file_name = req_file['file_name']

        try:
            file_name.encode('ascii')
        except UnicodeEncodeError:
            error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
            return JsonResponse({'error': error_msg}, status=400)

        edx_video_id = unicode(uuid4())
        key = storage_service_key(bucket, file_name=edx_video_id)

        metadata_list = [
            ('client_video_id', file_name),
            ('course_key', unicode(course.id)),
        ]

        deprecate_youtube = waffle_flags()[DEPRECATE_YOUTUBE]
        course_video_upload_token = course.video_upload_pipeline.get('course_video_upload_token')

        # Only include `course_video_upload_token` if youtube has not been deprecated
        # for this course.
        if not deprecate_youtube.is_enabled(course.id) and course_video_upload_token:
            metadata_list.append(('course_video_upload_token', course_video_upload_token))

        is_video_transcript_enabled = VideoTranscriptEnabledFlag.feature_enabled(course.id)
        if is_video_transcript_enabled:
            transcript_preferences = get_transcript_preferences(unicode(course.id))
            if transcript_preferences is not None:
                metadata_list.append(('transcript_preferences', json.dumps(transcript_preferences)))

        for metadata_name, value in metadata_list:
            key.set_metadata(metadata_name, value)
        upload_url = key.generate_url(
            KEY_EXPIRATION_IN_SECONDS,
            'PUT',
            headers={'Content-Type': req_file['content_type']}
        )

        # persist edx_video_id in VAL
        create_video({
            'edx_video_id': edx_video_id,
            'status': 'upload',
            'client_video_id': file_name,
            'duration': 0,
            'encoded_videos': [],
            'courses': [unicode(course.id)]
        })

        resp_files.append({'file_name': file_name, 'upload_url': upload_url, 'edx_video_id': edx_video_id})

    return JsonResponse({'files': resp_files}, status=200)