def test_automatic_token_refresh(self): """ Test that the JWT token is automatically refreshed """ tokens = ['cred2', 'cred1'] def auth_callback(request): resp = {'expires_in': 60} if 'grant_type=client_credentials' in request.body: resp['access_token'] = tokens.pop() return (200, {}, json.dumps(resp)) responses.add_callback( responses.POST, self.base_url + '/oauth2/access_token', callback=auth_callback, content_type='application/json', ) client_session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) self._mock_auth_api(self.base_url + '/endpoint', 200, {'status': 'ok'}) response = client_session.post(self.base_url + '/endpoint', data={'test': 'ok'}) first_call_datetime = datetime.datetime.utcnow() self.assertEqual(client_session.auth.token, 'cred1') self.assertEqual(response.json()['status'], 'ok') # after only 30 seconds should still use the cached token with freeze_time(first_call_datetime + datetime.timedelta(seconds=30)): response = client_session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertEqual(client_session.auth.token, 'cred1') # after just under a minute, should request a new token # - expires early due to ACCESS_TOKEN_EXPIRED_THRESHOLD_SECONDS with freeze_time(first_call_datetime + datetime.timedelta(seconds=56)): response = client_session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertEqual(client_session.auth.token, 'cred2')
def test_automatic_token_refresh(self): """ Test that the JWT token is automatically refreshed """ tokens = ['cred2', 'cred1'] def auth_callback(request): resp = {'expires_in': 60} if 'grant_type=client_credentials' in request.body: resp['access_token'] = tokens.pop() return (200, {}, json.dumps(resp)) responses.add_callback( responses.POST, self.base_url + '/oauth2/access_token', callback=auth_callback, content_type='application/json', ) session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) self._mock_auth_api(self.base_url + '/endpoint', 200, {'status': 'ok'}) response = session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertEqual(session.auth.token, 'cred1') self.assertEqual(response.json()['status'], 'ok') with freeze_time(datetime.datetime.utcnow() + datetime.timedelta(seconds=3600)): response = session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertEqual(session.auth.token, 'cred2')
def load_course_blocks_from_LMS(course_code): """Uses an OAuthAPIClient to recover the course structure from the LMS backend Arguments: course_code string as block-v1:course_name+type@course+block@course Returns: JSON text response with course enumerated blocks Boolean update to modify Course Verticals in the db """ client = OAuthAPIClient(settings.BACKEND_LMS_BASE_URL, settings.BACKEND_SERVICE_EDX_OAUTH2_KEY, settings.BACKEND_SERVICE_EDX_OAUTH2_SECRET) url = '{}/api/courses/v1/blocks/{}?depth=all&all_blocks=true&requested_fields=all,children'.format( settings.BACKEND_LMS_BASE_URL, course_code) blocks = cache.get(url, 'has_expired') update = False if blocks == 'has_expired': update = True response = client.get(url) if response.status_code != 200: raise Exception("Request to LMS failed for {}".format(course_code), str(response.text)) else: cache.set(url, response.text, settings.CACHE_TTL) blocks = response.text return (blocks, update)
def push_ecommerce_entitlement(self, partner, course, entitlement, partial=False): """ Creates or updates the stockrecord information on the ecommerce side. """ api_client = OAuthAPIClient(partner.lms_url, partner.oidc_key, partner.oidc_secret) if partial: method = 'PUT' url = '{0}stockrecords/{1}/'.format(partner.ecommerce_api_url, entitlement.sku) data = { 'price_excl_tax': entitlement.price, } else: method = 'POST' url = '{0}products/'.format(partner.ecommerce_api_url) data = { 'product_class': 'Course Entitlement', 'title': course.title, 'price': entitlement.price, 'certificate_type': entitlement.mode.slug, 'uuid': str(course.uuid), } response = api_client.request(method, url, data=data) if not response.ok: raise EcommerceAPIClientException(response.text) return response
def test_access_token_request_timeout_wiring2(self, mock_access_token_post): mock_access_token_post.return_value.json.return_value = {'access_token': 'token', 'expires_in': 1000} timeout_override = (6.1, 2) client = OAuthAPIClient(self.base_url, self.client_id, self.client_secret, timeout=timeout_override) client._ensure_authentication() # pylint: disable=protected-access assert mock_access_token_post.call_args.kwargs['timeout'] == timeout_override
def test_access_token_bad_response_code(self): responses.add(responses.POST, self.base_url + '/oauth2/access_token', status=500, json={}) client = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) with self.assertRaises(requests.HTTPError): client._ensure_authentication() # pylint: disable=protected-access
def _refresh_user_course_permissions(user): """ Refresh user course permissions from the auth server. Arguments user (User) -- User whose permissions should be refreshed """ response_data = None try: client = OAuthAPIClient( settings.BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, settings.BACKEND_SERVICE_EDX_OAUTH2_KEY, settings.BACKEND_SERVICE_EDX_OAUTH2_SECRET, ) course_ids = [] page = 1 while page: logger.debug('Retrieving page %d of course_ids...', page) response = client.get( settings.COURSE_API_URL + 'course_ids/', params={ 'username': user.username, 'role': ROLE_FOR_ALLOWED_COURSES, 'page': page, 'page_size': 1000, }, timeout=( 3.05, 55 ), # the course_ids API can be slow, so use a higher READ timeout ) response_data = response.json() response.raise_for_status( ) # response_data could be an error response course_ids += response_data['results'] # ensure the next param is a string to avoid infinite loops for mock objects if response_data['pagination']['next'] and isinstance( response_data['pagination']['next'], str): page += 1 else: page = None logger.debug( 'Completed retrieval of course_ids. Retrieved info for %d courses.', len(course_ids)) allowed_courses = list(set(course_ids)) except Exception as e: logger.exception( "Unable to retrieve course permissions for username=%s and response=%s", user.username, response_data) raise PermissionsRetrievalFailedError(e) set_user_course_permissions(user, allowed_courses) return allowed_courses
def test_access_token_invalid_json_response(self): responses.add(responses.POST, self.base_url + '/oauth2/access_token', status=200, body="Not JSON") client = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) with self.assertRaises(requests.RequestException): client._ensure_authentication() # pylint: disable=protected-access
def test_get_jwt_access_token(self): token = 'abcd' self._mock_auth_api(self.base_url + '/oauth2/access_token', 200, { 'access_token': token, 'expires_in': 60 }) client = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) access_token = client.get_jwt_access_token() self.assertEqual(access_token, token)
def test_automatic_auth(self): """ Test that the JWT token is automatically set """ session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) self._mock_auth_api(self.base_url + '/oauth2/access_token', 200, {'access_token': 'abcd', 'expires_in': 60}) self._mock_auth_api(self.base_url + '/endpoint', 200, {'status': 'ok'}) response = session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertIn('client_id=%s' % self.client_id, responses.calls[0].request.body) self.assertEqual(session.auth.token, 'abcd') self.assertEqual(response.json()['status'], 'ok')
def __init__(self, client_id=None, client_secret=None, **kwargs): """ Initialize REST backend. client_id: provided by backend service client_secret: provided by backend service """ self.default_rules = None super().__init__(**kwargs) self.client_id = client_id self.client_secret = client_secret self.session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret)
def __init__(self, client_id=None, client_secret=None, **kwargs): """ Initialize REST backend. client_id: provided by backend service client_secret: provided by backend service """ ProctoringBackendProvider.__init__(self) self.client_id = client_id self.client_secret = client_secret self.default_rules = None for key, value in kwargs.items(): setattr(self, key, value) self.session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret)
def test_automatic_auth(self, client_base_url, custom_oauth_uri, expected_oauth_url): """ Test that the JWT token is automatically set """ client_session = OAuthAPIClient(client_base_url, self.client_id, self.client_secret) client_session.oauth_uri = custom_oauth_uri self._mock_auth_api(expected_oauth_url, 200, {'access_token': 'abcd', 'expires_in': 60}) self._mock_auth_api(self.base_url + '/endpoint', 200, {'status': 'ok'}) response = client_session.post(self.base_url + '/endpoint', data={'test': 'ok'}) self.assertIn('client_id=%s' % self.client_id, responses.calls[0].request.body) self.assertEqual(client_session.auth.token, 'abcd') self.assertEqual(response.json()['status'], 'ok')
def lms_api_client(self): if not self.lms_url: return None return OAuthAPIClient(self.lms_url.strip('/'), self.oauth2_client_id, self.oauth2_client_secret, timeout=settings.OAUTH_API_TIMEOUT)
def _request_financial_assistance(method, url, params=None, data=None): """ An internal function containing common functionality among financial assistance utility function to call edx-financial-assistance backend with appropriate method, url, params and data. """ financial_assistance_configuration = FinancialAssistanceConfiguration.current() if financial_assistance_configuration.enabled: oauth_application = Application.objects.get(user=financial_assistance_configuration.get_service_user()) client = OAuthAPIClient( settings.LMS_ROOT_URL, oauth_application.client_id, oauth_application.client_secret ) return client.request( method, f"{financial_assistance_configuration.api_base_url}{url}", params=params, data=data ) else: return False, 'Financial Assistance configuration is not enabled'
def oauth_api_client(self): # Does not need to be on the Partner model, but is here for historical reasons and this client is usually used # along with URLs from this model. So might as well have it here for convenience. return OAuthAPIClient( settings.BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, settings.BACKEND_SERVICE_EDX_OAUTH2_KEY, settings.BACKEND_SERVICE_EDX_OAUTH2_SECRET, timeout=settings.OAUTH_API_TIMEOUT, )
def enroll_in_course(course_id, email, send_email=True): """ Auto-enroll email in course. Uses the bulk enrollment API, defined in lms/djangoapps/bulk_enroll """ # Raises ValidationError if invalid validate_email(email) client = OAuthAPIClient( settings.SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT, settings.SOCIAL_AUTH_EDX_OAUTH2_KEY, settings.SOCIAL_AUTH_EDX_OAUTH2_SECRET, ) bulk_enroll_url = EDX_BULK_ENROLLMENT_API_PATH % settings.SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT # noqa: E501 # The bulk enrollment API allows us to enroll multiple identifiers # at once, using a comma-separated list for the courses and # identifiers parameters. We deliberately want to process # enrollments one by one, so we use a single request for each # course/identifier combination. request_params = { "auto_enroll": True, "email_students": send_email, "action": "enroll", "courses": course_id, "identifiers": email, } logger.debug("Sending POST request " "to %s with parameters %s" % (bulk_enroll_url, request_params)) response = client.post(bulk_enroll_url, request_params) # Throw an exception if we get anything other than HTTP 200 back # from the API (the only other status we might be getting back # from the bulk enrollment API is HTTP 400). response.raise_for_status() # If all is well, log the response at the debug level. logger.debug("Received response from %s: %s " % (bulk_enroll_url, response.json()))
def create_video_pipeline_api_client(api_client_id, api_client_secret): """ Returns an API client which can be used to make Video Pipeline API requests. Arguments: api_client_id(unicode): Video pipeline client id. api_client_secret(unicode): Video pipeline client secret. """ return OAuthAPIClient(settings.LMS_ROOT_URL, api_client_id, api_client_secret)
def oauth_api_client(self): """ This client is authenticated with the configured oauth settings and automatically cached. Returns: requests.Session: API client """ return OAuthAPIClient( settings.BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, settings.BACKEND_SERVICE_EDX_OAUTH2_KEY, settings.BACKEND_SERVICE_EDX_OAUTH2_SECRET, )
def test_shared_client_credential_jwt_access_token(self): """ Test that get_and_cache_jwt_oauth_access_token returns the same access token used by the OAuthAPIClient. """ body = {'access_token': "my-token", 'expires_in': 1000} now = datetime.datetime.utcnow() expected_return = ('my-token', now + datetime.timedelta(seconds=1000)) with freeze_time(now): self._mock_auth_api(OAUTH_URL, 200, body=body) actual_return = EdxRestApiClient.get_and_cache_jwt_oauth_access_token( OAUTH_URL, 'client_id', 'client_secret') self.assertEqual(actual_return, expected_return) self.assertEqual(len(responses.calls), 1) # ensure OAuthAPIClient uses the same cached auth token without re-requesting the token from the server oauth_client = OAuthAPIClient(OAUTH_URL, 'client_id', 'client_secret') self._mock_auth_api(URL, 200, {'status': 'ok'}) oauth_client.post(URL, data={'test': 'ok'}) self.assertEqual(oauth_client.auth.token, actual_return[0]) self.assertEqual(len(responses.calls), 2) self.assertEqual(URL, responses.calls[1][0].url)
def update_val(self, image_keys): """ Update a course video in edxval database for auto generated images. """ if len(image_keys) > 0: for course_id in self.video_object.course_url: data = { 'course_id': course_id, 'edx_video_id': self.video_object.val_id, 'generated_images': image_keys } client = OAuthAPIClient(self.settings['oauth2_provider_url'], self.settings['oauth2_client_id'], self.settings['oauth2_client_secret']) response = client.request( 'POST', self.settings['val_video_images_url'], json=data) if not response.ok: logger.error(': {id} {message}'.format( id=self.video_object.val_id, message=response.content))
def load_course_structure_from_CMS(course_code): """Uses an OAuthAPIClient to recover the course structure from the CMS backend Arguments: course_code string course-v1:coursename Returns: JSON text response with course tree """ client = OAuthAPIClient(settings.BACKEND_LMS_BASE_URL, settings.BACKEND_SERVICE_EDX_OAUTH2_KEY, settings.BACKEND_SERVICE_EDX_OAUTH2_SECRET) client.headers.update({ "Accept": "application/json, text/javascript, */*; q=0.01", "Content-Type": "application/json" }) response = client.get("{}/course/{}?format=concise".format( settings.BACKEND_CMS_BASE_URL, course_code)) if response.status_code != 200: raise Exception("Request to CMS failed for {}".format(course_code), str(response.text)) return response.text
def __init__(self, video_proto, val_status, **kwargs): """VAL Data""" self.val_status = val_status self.platform_course_url = kwargs.get('platform_course_url', []) """VEDA Data""" self.video_proto = video_proto self.video_object = kwargs.get('video_object', None) self.encode_profile = kwargs.get('encode_profile', None) """if sending urls""" self.endpoint_url = kwargs.get('endpoint_url', None) self.encode_data = [] self.val_profile = None """Generated""" self.val_data = None self.headers = None """Credentials""" self.auth_dict = kwargs.get('CONFIG_DATA', self._AUTH()) self.oauth2_provider_url = self.auth_dict['oauth2_provider_url'] self.oauth2_client_id = self.auth_dict['oauth2_client_id'] self.oauth2_client_secret = self.auth_dict['oauth2_client_secret'] self.oauth2_client = OAuthAPIClient( self.auth_dict['oauth2_provider_url'], self.auth_dict['oauth2_client_id'], self.auth_dict['oauth2_client_secret'])
def enroll_in_course( course_id, email, send_email=settings.WEBHOOK_RECEIVER_SEND_ENROLLMENT_EMAIL, auto_enroll=settings.WEBHOOK_RECEIVER_AUTO_ENROLL): """ Auto-enroll email in course. Uses the bulk enrollment API, defined in lms/djangoapps/bulk_enroll """ # Raises ValidationError if invalid validate_email(email) client = OAuthAPIClient( settings.WEBHOOK_RECEIVER_LMS_BASE_URL, settings.WEBHOOK_RECEIVER_EDX_OAUTH2_KEY, settings.WEBHOOK_RECEIVER_EDX_OAUTH2_SECRET, ) bulk_enroll_url = EDX_BULK_ENROLLMENT_API_PATH % settings.WEBHOOK_RECEIVER_LMS_BASE_URL # noqa: E501 # The bulk enrollment API allows us to enroll multiple identifiers # at once, using a comma-separated list for the courses and # identifiers parameters. We deliberately want to process # enrollments one by one, so we use a single request for each # course/identifier combination. request_params = { "auto_enroll": auto_enroll, "email_students": send_email, "action": "enroll", "courses": course_id, "identifiers": email, } logger.debug("Sending POST request " "to %s with parameters %s" % (bulk_enroll_url, request_params)) response = client.post(bulk_enroll_url, request_params) # Throw an exception if we get any error back from the API. # Apart from an HTTP 200, we might also get: # # HTTP 400: if we've sent a malformed request (for example, one # with a course ID in a format that Open edX can't # parse) # HTTP 401: if our authentication token has expired # HTTP 403: if our auth token is linked to a user ID that lacks # staff credentials in one of the courses we want to # enroll the learner in # HTTP 404: if we've specified a course ID that does not exist # (although it does follow the format that Open edX expects) # HTTP 500: in case of a server-side issue if response.status_code >= 400: logger.error("POST request to %s with parameters %s " "returned HTTP %s" % (bulk_enroll_url, request_params, response.status_code)) response.raise_for_status() # If all is well, log the response at the debug level. logger.debug("Received response from %s: %s " % (bulk_enroll_url, response.json()))
def __init__(self) -> None: self.client = OAuthAPIClient( settings.SOCIAL_AUTH_EDX_OAUTH2_URL_ROOT.strip("/"), self.oauth2_client_id, self.oauth2_client_secret, )
parser = argparse.ArgumentParser( description='run the mockprock server', epilog= 'Retrieve the mockprock client id and secret from the LMS Django admin, and start the server with those arguments' ) parser.add_argument("client_id", type=str, help="oauth client id", nargs="?") parser.add_argument("client_secret", type=str, help="oauth client secret", nargs="?") parser.add_argument('-l', dest='lms_host', type=str, help='LMS host', default='http://host.docker.internal:18000') args = parser.parse_args() if not (args.client_id and args.client_secret): parser.print_help() time.sleep(2) import webbrowser webbrowser.open('%s/admin/oauth2_provider/application/' % args.lms_host) sys.exit(1) app.client = OAuthAPIClient(args.lms_host, args.client_id, args.client_secret) app.run(host='0.0.0.0', port=11136)
class VALAPICall(object): def __init__(self, video_proto, val_status, **kwargs): """VAL Data""" self.val_status = val_status self.platform_course_url = kwargs.get('platform_course_url', []) """VEDA Data""" self.video_proto = video_proto self.video_object = kwargs.get('video_object', None) self.encode_profile = kwargs.get('encode_profile', None) """if sending urls""" self.endpoint_url = kwargs.get('endpoint_url', None) self.encode_data = [] self.val_profile = None """Generated""" self.val_data = None self.headers = None """Credentials""" self.auth_dict = kwargs.get('CONFIG_DATA', self._AUTH()) self.oauth2_provider_url = self.auth_dict['oauth2_provider_url'] self.oauth2_client_id = self.auth_dict['oauth2_client_id'] self.oauth2_client_secret = self.auth_dict['oauth2_client_secret'] self.oauth2_client = OAuthAPIClient( self.auth_dict['oauth2_provider_url'], self.auth_dict['oauth2_client_id'], self.auth_dict['oauth2_client_secret']) def _AUTH(self): return get_config() def call(self): if not self.auth_dict: return None """ Errors covered in other methods """ if self.video_object: self.send_object_data() return if self.video_proto is not None: self.send_val_data() def send_object_data(self): """ Rather than rewrite the protocol to fit the veda models, we'll shoehorn the model into the VideoProto model """ self.video_proto = VideoProto() self.video_proto.s3_filename = self.video_object.studio_id self.video_proto.veda_id = self.video_object.edx_id self.video_proto.client_title = self.video_object.client_title if self.video_proto.client_title is None: self.video_proto.client_title = '' self.video_proto.duration = self.video_object.video_orig_duration self.send_val_data() def send_val_data(self): """ VAL is very tetchy -- it needs a great deal of specific info or it will fail """ ''' sending_data = { encoded_videos = [{ url="https://testurl.mp4", file_size=8499040, bitrate=131, profile="override", }, {...},], client_video_id = "This is a VEDA-VAL Test", courses = [ "TEST", "..." ], duration = 517.82, edx_video_id = "TESTID", status = "transcode_active" } ## "POST" for new objects to 'video' root url ## "PUT" for extant objects to video/id -- cannot send duplicate course records ''' if self.video_proto.s3_filename is None or \ len(self.video_proto.s3_filename) == 0: self.video_proto.val_id = self.video_proto.veda_id else: self.video_proto.val_id = self.video_proto.s3_filename if self.val_status != 'invalid_token': self.video_object = Video.objects.filter( edx_id=self.video_proto.veda_id).latest() """ Data Cleaning """ if self.video_proto.platform_course_url is None: self.video_proto.platform_course_url = [] if not isinstance(self.video_proto.platform_course_url, list): self.video_proto.platform_course_url = [ self.video_proto.platform_course_url ] try: self.video_object.video_orig_duration except NameError: self.video_object.video_orig_duration = 0 self.video_object.duration = 0.0 except AttributeError: pass if not isinstance(self.video_proto.duration, float) and self.val_status != 'invalid_token': self.video_proto.duration = Output._seconds_from_string( duration=self.video_object.video_orig_duration) """ Sort out courses """ val_courses = [] if self.val_status != 'invalid_token': for f in self.video_object.inst_class.local_storedir.split(','): if f.strip() not in val_courses and len(f.strip()) > 0: val_courses.append({f.strip(): None}) for g in self.video_proto.platform_course_url: if g.strip() not in val_courses: val_courses.append({g.strip(): None}) self.val_data = { 'client_video_id': self.video_proto.client_title, 'duration': self.video_proto.duration, 'edx_video_id': self.video_proto.val_id, 'courses': val_courses } r1 = self.oauth2_client.request( 'GET', '/'.join( (self.auth_dict['val_api_url'], self.video_proto.val_id))) if r1.status_code != 200 and r1.status_code != 404: LOGGER.error('[API] : VAL Communication error %d', r1.status_code) return if r1.status_code == 404: self.send_404() elif r1.status_code == 200: val_api_return = ast.literal_eval(r1.text.replace('null', 'None')) self.send_200(val_api_return) """ Update Status """ LOGGER.info('[INGEST] send_val_data : video ID : %s', self.video_proto.veda_id) url_query = URL.objects.filter(videoID=Video.objects.filter( edx_id=self.video_proto.veda_id)) for u in url_query: URL.objects.filter(pk=u.pk).update(val_input=True) def profile_determiner(self, val_api_return): """ Determine VAL profile data, from return/encode submix """ # Defend against old/deprecated encodes if self.encode_profile: try: self.auth_dict['val_profile_dict'][self.encode_profile] except KeyError: return if self.endpoint_url: for p in self.auth_dict['val_profile_dict'][self.encode_profile]: self.encode_data.append( dict(url=self.endpoint_url, file_size=self.video_proto.filesize, bitrate=int(self.video_proto.bitrate.split(' ')[0]), profile=p)) test_list = [] if self.video_proto.veda_id: url_query = URL.objects.filter(videoID=Video.objects.filter( edx_id=self.video_proto.veda_id).latest()) for u in url_query: final = URL.objects.filter(encode_profile=u.encode_profile, videoID=u.videoID).latest() if final.encode_profile.product_spec == 'review': pass else: try: self.auth_dict['val_profile_dict'][ final.encode_profile.product_spec] except KeyError: continue for p in self.auth_dict['val_profile_dict'][ final.encode_profile.product_spec]: test_list.append( dict(url=str(final.encode_url), file_size=final.encode_size, bitrate=int( final.encode_bitdepth.split(' ')[0]), profile=str(p))) for t in test_list: if t['profile'] not in [g['profile'] for g in self.encode_data]: self.encode_data.append(t) if len(val_api_return) == 0: return """ All URL Records Deleted (for some reason) """ if len(self.encode_data) == 0: return for i in val_api_return['encoded_videos']: if i['profile'] not in [g['profile'] for g in self.encode_data]: self.encode_data.append(i) return @staticmethod def should_update_status(encode_list, val_status): """ Check if we need to update video status in val Arguments: encode_list (list): list of video encodes val_status (unicode): val status """ if len(encode_list) == 0 and val_status in FILE_COMPLETE_STATUSES: return False return True def send_404(self): """ Generate new VAL ID """ self.profile_determiner(val_api_return=[]) self.val_data['status'] = self.val_status if self.should_update_status(self.encode_data, self.val_status) is False: return None sending_data = dict(encoded_videos=self.encode_data, **self.val_data) r2 = self.oauth2_client.request('POST', '/'.join( (self.auth_dict['val_api_url'], '')), json=sending_data) if r2.status_code > 299: LOGGER.error('[API] : VAL POST {code}'.format(code=r2.status_code)) def send_200(self, val_api_return): """ VAL ID is previously extant just update --- VAL will not allow duped studio urls to be sent, so we must scrub the data """ for retrieved_course in val_api_return['courses']: for course in list(self.val_data['courses']): if list(retrieved_course.keys()).sort() == list( course.keys()).sort(): self.val_data['courses'].remove(course) self.profile_determiner(val_api_return=val_api_return) self.val_data['status'] = self.val_status """ Double check for profiles in case of overwrite """ sending_data = dict(encoded_videos=self.encode_data, **self.val_data) """ Make Request, finally """ if self.should_update_status(self.encode_data, self.val_status) is False: return None r4 = self.oauth2_client.request('PUT', '/'.join( (self.auth_dict['val_api_url'], self.video_proto.val_id)), json=sending_data) LOGGER.info('[API] {id} : {status} sent to VAL {code}'.format( id=self.video_proto.val_id, status=self.val_status, code=r4.status_code)) if r4.status_code > 299: LOGGER.error( '[API] : VAL PUT : {status}'.format(status=r4.status_code)) def update_val_transcript(self, video_id, lang_code, name, transcript_format, provider): """ Update status for a completed transcript. """ post_data = { 'video_id': video_id, 'name': name, 'provider': provider, 'language_code': lang_code, 'file_format': transcript_format, } response = self.oauth2_client.request( 'POST', self.auth_dict['val_transcript_create_url'], json=post_data) if not response.ok: LOGGER.error( '[API] : VAL update_val_transcript failed -- video_id=%s -- provider=% -- status=%s -- content=%s', video_id, provider, response.status_code, response.content, ) def update_video_status(self, video_id, status): """ Update video transcript status. """ val_data = {'edx_video_id': video_id, 'status': status} response = self.oauth2_client.request( 'PATCH', self.auth_dict['val_video_transcript_status_url'], json=val_data) if not response.ok: LOGGER.error( '[API] : VAL Update_video_status failed -- video_id=%s -- status=%s -- text=%s', video_id, response.status_code, response.text)
class BaseRestProctoringProvider(ProctoringBackendProvider): """ Base class for official REST API proctoring service. Subclasses must override base_url and may override the other url properties """ base_url = None token_expiration_time = 60 needs_oauth = True has_dashboard = True supports_onboarding = True passing_statuses = (SoftwareSecureReviewStatus.clean, ) @property def exam_attempt_url(self): "Returns exam attempt url" return self.base_url + u'/api/v1/exam/{exam_id}/attempt/{attempt_id}/' @property def create_exam_attempt_url(self): "Returns the create exam url" return self.base_url + u'/api/v1/exam/{exam_id}/attempt/' @property def create_exam_url(self): "Returns create exam url" return self.base_url + u'/api/v1/exam/' @property def exam_url(self): "Returns exam url" return self.base_url + u'/api/v1/exam/{exam_id}/' @property def config_url(self): "Returns proctor config url" return self.base_url + u'/api/v1/config/' @property def instructor_url(self): "Returns the instructor dashboard url" return self.base_url + u'/api/v1/instructor/{client_id}/?jwt={jwt}' @property def user_info_url(self): "Returns the user info url" return self.base_url + u'/api/v1/user/{user_id}/' @property def proctoring_instructions(self): "Returns the (optional) proctoring instructions" return [] def __init__(self, client_id=None, client_secret=None, **kwargs): """ Initialize REST backend. client_id: provided by backend service client_secret: provided by backend service """ ProctoringBackendProvider.__init__(self) self.client_id = client_id self.client_secret = client_secret self.default_rules = None for key, value in kwargs.items(): setattr(self, key, value) self.session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) def get_javascript(self): """ Returns the url of the javascript bundle into which the provider's JS will be loaded """ # use the defined npm_module name, or else the python package name package = getattr(self, 'npm_module', self.__class__.__module__.split('.')[0]) js_url = '' try: bundle_chunks = get_files(package, config="WORKERS") # still necessary to check, since webpack_loader can be # configured to ignore all matching packages if bundle_chunks: js_url = bundle_chunks[0]["url"] except WebpackBundleLookupError: warnings.warn( u'Could not find webpack bundle for proctoring backend {package}.' u' Check whether webpack is configured to build such a bundle'. format(package=package)) except BaseWebpackLoaderException: warnings.warn( u'Could not find webpack bundle for proctoring backend {package}.' .format(package=package)) except IOError as err: warnings.warn( u'Webpack stats file corresponding to WebWorkers not found: {}' .format(str(err))) # if the Javascript URL is not an absolute URL (i.e. doesn't have a scheme), prepend # the LMS Root URL to it, if it is defined, to make it an absolute URL if not urlparse(js_url).scheme: if hasattr(settings, 'LMS_ROOT_URL'): js_url = settings.LMS_ROOT_URL + js_url return js_url def get_software_download_url(self): """ Returns the URL that the user needs to go to in order to download the corresponding desktop software """ return self.get_proctoring_config().get('download_url', None) def get_proctoring_config(self): """ Returns the metadata and configuration options for the proctoring service """ url = self.config_url log.debug(u'Requesting config from %r', url) response = self.session.get( url, headers=self._get_language_headers()).json() return response def get_exam(self, exam): """ Returns the exam metadata stored by the proctoring service """ url = self.exam_url.format(exam_id=exam['id']) log.debug(u'Requesting exam from %r', url) response = self.session.get(url).json() return response def get_attempt(self, attempt): """ Returns the attempt object from the backend """ response = self._make_attempt_request( attempt['proctored_exam']['external_id'], attempt['external_id'], method='GET') # If the class has instructions defined, use them. # Otherwise, the instructions should be returned by this # API request. Subclasses should wrap each instruction with gettext response[ 'instructions'] = self.proctoring_instructions or response.get( 'instructions', []) return response def register_exam_attempt(self, exam, context): """ Called when the exam attempt has been created but not started """ url = self.create_exam_attempt_url.format(exam_id=exam['external_id']) payload = context payload['status'] = 'created' # attempt code isn't needed in this API payload.pop('attempt_code', False) log.debug(u'Creating exam attempt for %r at %r', exam['external_id'], url) response = self.session.post(url, json=payload) if response.status_code != 200: raise BackendProviderCannotRegisterAttempt(response.content, response.status_code) status_code = response.status_code response = response.json() log.debug(response) onboarding_status = response.get('status', None) if onboarding_status in ProctoredExamStudentAttemptStatus.onboarding_errors: raise BackendProviderOnboardingException(onboarding_status) attempt_id = response.get('id', None) if not attempt_id: raise BackendProviderSentNoAttemptID(response, status_code) return attempt_id def start_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to establish a new proctored exam """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.started, method='PATCH') return response.get('status') def stop_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to finish a proctored exam """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.submitted, method='PATCH') return response.get('status') def remove_exam_attempt(self, exam, attempt): """ Removes the exam attempt on the backend provider's server """ response = self._make_attempt_request(exam, attempt, method='DELETE') return response.get('status', None) == 'deleted' def mark_erroneous_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to mark an unfinished exam to be in error """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.error, method='PATCH') return response.get('status') def on_review_callback(self, attempt, payload): """ Called when the reviewing 3rd party service posts back the results """ # REST backends should convert the payload into the expected data structure return payload def on_exam_saved(self, exam): """ Called after an exam is saved. """ if self.default_rules and not exam.get('rules', None): # allows the platform to define a default configuration exam['rules'] = self.default_rules external_id = exam.get('external_id', None) if external_id: url = self.exam_url.format(exam_id=external_id) else: url = self.create_exam_url log.info(u'Saving exam to %r', url) response = None try: response = self.session.post(url, json=exam) data = response.json() except Exception as exc: # pylint: disable=broad-except if response: # pylint: disable=no-member content = exc.response.content if hasattr( exc, 'response') else response.content else: content = None log.exception(u'failed to save exam. %r', content) data = {} return data.get('id') def get_instructor_url(self, course_id, user, exam_id=None, attempt_id=None, show_configuration_dashboard=False): """ Return a URL to the instructor dashboard course_id: str user: dict of {id, full_name, email} for the instructor or reviewer exam_id: str optional exam external id attempt_id: str optional exam attempt external id """ exp = time.time() + self.token_expiration_time token = { 'course_id': course_id, 'user': user, 'iss': self.client_id, 'jti': uuid.uuid4().hex, 'exp': exp } if exam_id: token['exam_id'] = exam_id if show_configuration_dashboard: token['config'] = True if attempt_id: token['attempt_id'] = attempt_id encoded = jwt.encode(token, self.client_secret).decode('utf-8') url = self.instructor_url.format(client_id=self.client_id, jwt=encoded) log.debug(u'Created instructor url for %r %r %r', course_id, exam_id, attempt_id) return url def retire_user(self, user_id): url = self.user_info_url.format(user_id=user_id) try: response = self.session.delete(url) data = response.json() assert data in (True, False) except Exception as exc: # pylint: disable=broad-except # pylint: disable=no-member content = exc.response.content if hasattr( exc, 'response') else response.content raise BackendProviderCannotRetireUser(content) return data def _get_language_headers(self): """ Returns a dictionary of the Accept-Language headers """ # This import is here because developers writing backends which subclass this class # may want to import this module and use the other methods, without having to run in the context # of django settings, etc. from django.utils.translation import get_language # pylint: disable=import-outside-toplevel current_lang = get_language() default_lang = settings.LANGUAGE_CODE lang_header = default_lang if current_lang and current_lang != default_lang: lang_header = '{};{}'.format(current_lang, default_lang) return {'Accept-Language': lang_header} def _make_attempt_request(self, exam, attempt, method='POST', status=None, **payload): """ Calls backend attempt API """ if not attempt: return {} if status: payload['status'] = status else: payload = None url = self.exam_attempt_url.format(exam_id=exam, attempt_id=attempt) headers = {} if method == 'GET': headers.update(self._get_language_headers()) log.debug(u'Making %r attempt request at %r', method, url) response = self.session.request(method, url, json=payload, headers=headers) try: data = response.json() except ValueError: log.exception(u"Decoding attempt %r -> %r", attempt, response.content) data = {} return data
def send_val_data(self): """ VAL is very tetchy -- it needs a great deal of specific info or it will fail """ ''' sending_data = { encoded_videos = [{ url="https://testurl.mp4", file_size=8499040, bitrate=131, profile="override", }, {...},], client_video_id = "This is a VEDA-VAL Test", courses = [ "TEST", "..." ], duration = 517.82, edx_video_id = "TESTID", status = "transcode_active" } ## "POST" for new objects to 'video' root url ## "PUT" for extant objects to video/id -- cannot send duplicate course records ''' # in case non-studio side upload if self.VideoObject.val_id is None or len( self.VideoObject.val_id) == 0: self.VideoObject.val_id = self.VideoObject.veda_id val_data = { 'client_video_id': self.VideoObject.val_id, 'duration': self.VideoObject.mezz_duration, 'edx_video_id': self.VideoObject.val_id, } if not isinstance(self.VideoObject.course_url, list): self.VideoObject.course_url = [self.VideoObject.course_url] client = OAuthAPIClient(settings['oauth2_provider_url'], settings['oauth2_client_id'], settings['oauth2_client_secret']) r1 = client.request( 'GET', '/'.join( (settings['val_api_url'], self.VideoObject.val_id, ''))) if r1.status_code != 200 and r1.status_code != 404: # Total API Failure logger.error('VAL Communication error %d', r1.status_code) return None if r1.status_code == 404: # Generate new VAL ID (shouldn't happen, but whatever) val_data['encoded_videos'] = [] val_data['courses'] = self.VideoObject.course_url val_data['status'] = self.val_video_status # Final Connection r2 = client.request('POST', settings['val_api_url'], json=val_data) if r2.status_code > 299: logger.error('VAL POST error %d', r2.status_code) return None elif r1.status_code == 200: # ID is previously extant val_api_return = ast.literal_eval(r1.text) # extract course ids, courses will be a list of dicts, [{'course_id': 'image_name'}] course_ids = reduce(operator.concat, (list(d.keys()) for d in val_api_return['courses'])) # VAL will not allow duped studio urls to be sent, so # we must scrub the data for course_id in self.VideoObject.course_url: if course_id in course_ids: self.VideoObject.course_url.remove(course_id) val_data['courses'] = self.VideoObject.course_url # Double check for profiles in case of overwrite val_data['encoded_videos'] = [] # add back in the encodes for e in val_api_return['encoded_videos']: val_data['encoded_videos'].append(e) # Determine Status val_data['status'] = self.val_video_status # Make Request, finally r2 = client.request('PUT', '/'.join((settings['val_api_url'], self.VideoObject.val_id)), json=val_data) if r2.status_code > 299: logger.error('VAL PUT error %d', r2.status_code) return None
class BaseRestProctoringProvider(ProctoringBackendProvider): """ Base class for official REST API proctoring service. Subclasses must override base_url and may override the other url properties """ base_url = None token_expiration_time = 60 needs_oauth = True has_dashboard = True supports_onboarding = True passing_statuses = (SoftwareSecureReviewStatus.clean,) @property def exam_attempt_url(self): "Returns exam attempt url" return self.base_url + u'/api/v1/exam/{exam_id}/attempt/{attempt_id}/' @property def create_exam_attempt_url(self): "Returns the create exam url" return self.base_url + u'/api/v1/exam/{exam_id}/attempt/' @property def create_exam_url(self): "Returns create exam url" return self.base_url + u'/api/v1/exam/' @property def exam_url(self): "Returns exam url" return self.base_url + u'/api/v1/exam/{exam_id}/' @property def config_url(self): "Returns proctor config url" return self.base_url + u'/api/v1/config/' @property def instructor_url(self): "Returns the instructor dashboard url" return self.base_url + u'/api/v1/instructor/{client_id}/?jwt={jwt}' @property def user_info_url(self): "Returns the user info url" return self.base_url + u'/api/v1/user/{user_id}/' @property def proctoring_instructions(self): "Returns the (optional) proctoring instructions" return [] def __init__(self, client_id=None, client_secret=None, **kwargs): """ Initialize REST backend. client_id: provided by backend service client_secret: provided by backend service """ ProctoringBackendProvider.__init__(self) self.client_id = client_id self.client_secret = client_secret self.default_rules = None for key, value in kwargs.items(): setattr(self, key, value) self.session = OAuthAPIClient(self.base_url, self.client_id, self.client_secret) def get_javascript(self): """ Returns the url of the javascript bundle into which the provider's JS will be loaded """ # use the defined npm_module name, or else the python package name package = getattr(self, 'npm_module', self.__class__.__module__.split('.')[0]) js_url = '' try: bundle_chunks = get_files(package, config="WORKERS") # still necessary to check, since webpack_loader can be # configured to ignore all matching packages if bundle_chunks: js_url = bundle_chunks[0]["url"] except WebpackBundleLookupError: warnings.warn( u'Could not find webpack bundle for proctoring backend {package}.' u' Check whether webpack is configured to build such a bundle'.format( package=package ) ) except BaseWebpackLoaderException: warnings.warn( u'Could not find webpack bundle for proctoring backend {package}.'.format( package=package ) ) except IOError as err: warnings.warn( u'Webpack stats file corresponding to WebWorkers not found: {}' .format(str(err)) ) # if the Javascript URL is not an absolute URL (i.e. doesn't have a scheme), prepend # the LMS Root URL to it, if it is defined, to make it an absolute URL if not urlparse(js_url).scheme: if hasattr(settings, 'LMS_ROOT_URL'): js_url = settings.LMS_ROOT_URL + js_url return js_url def get_software_download_url(self): """ Returns the URL that the user needs to go to in order to download the corresponding desktop software """ return self.get_proctoring_config().get('download_url', None) def get_proctoring_config(self): """ Returns the metadata and configuration options for the proctoring service """ url = self.config_url log.debug('Requesting config from %r', url) response = self.session.get(url, headers=self._get_language_headers()).json() return response def get_exam(self, exam): """ Returns the exam metadata stored by the proctoring service """ url = self.exam_url.format(exam_id=exam['id']) log.debug('Requesting exam from %r', url) response = self.session.get(url).json() return response def get_attempt(self, attempt): """ Returns the attempt object from the backend """ response = self._make_attempt_request( attempt['proctored_exam']['external_id'], attempt['external_id'], method='GET') # If the class has instructions defined, use them. # Otherwise, the instructions should be returned by this # API request. Subclasses should wrap each instruction with gettext response['instructions'] = self.proctoring_instructions or response.get('instructions', []) return response def register_exam_attempt(self, exam, context): """ Called when the exam attempt has been created but not started """ url = self.create_exam_attempt_url.format(exam_id=exam['external_id']) payload = context payload['status'] = 'created' # attempt code isn't needed in this API payload.pop('attempt_code', False) log.debug('Creating exam attempt for %r at %r', exam['external_id'], url) response = self.session.post(url, json=payload) if response.status_code != 200: raise BackendProviderCannotRegisterAttempt(response.content) response = response.json() log.debug(response) onboarding_status = response.get('status', None) if onboarding_status in ProctoredExamStudentAttemptStatus.onboarding_errors: raise BackendProviderOnboardingException(onboarding_status) return response['id'] def start_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to establish a new proctored exam """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.started, method='PATCH') return response.get('status') def stop_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to finish a proctored exam """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.submitted, method='PATCH') return response.get('status') def remove_exam_attempt(self, exam, attempt): """ Removes the exam attempt on the backend provider's server """ response = self._make_attempt_request( exam, attempt, method='DELETE') return response.get('status', None) == 'deleted' def mark_erroneous_exam_attempt(self, exam, attempt): """ Method that is responsible for communicating with the backend provider to mark an unfinished exam to be in error """ response = self._make_attempt_request( exam, attempt, status=ProctoredExamStudentAttemptStatus.error, method='PATCH') return response.get('status') def on_review_callback(self, attempt, payload): """ Called when the reviewing 3rd party service posts back the results """ # REST backends should convert the payload into the expected data structure return payload def on_exam_saved(self, exam): """ Called after an exam is saved. """ if self.default_rules and not exam.get('rules', None): # allows the platform to define a default configuration exam['rules'] = self.default_rules external_id = exam.get('external_id', None) if external_id: url = self.exam_url.format(exam_id=external_id) else: url = self.create_exam_url log.info('Saving exam to %r', url) response = None try: response = self.session.post(url, json=exam) data = response.json() except Exception as exc: # pylint: disable=broad-except # pylint: disable=no-member content = exc.response.content if hasattr(exc, 'response') else response.content log.exception('failed to save exam. %r', content) data = {} return data.get('id') def get_instructor_url(self, course_id, user, exam_id=None, attempt_id=None, show_configuration_dashboard=False): """ Return a URL to the instructor dashboard course_id: str user: dict of {id, full_name, email} for the instructor or reviewer exam_id: str optional exam external id attempt_id: str optional exam attempt external id """ exp = time.time() + self.token_expiration_time token = { 'course_id': course_id, 'user': user, 'iss': self.client_id, 'jti': uuid.uuid4().hex, 'exp': exp } if exam_id: token['exam_id'] = exam_id if show_configuration_dashboard: token['config'] = True if attempt_id: token['attempt_id'] = attempt_id encoded = jwt.encode(token, self.client_secret) url = self.instructor_url.format(client_id=self.client_id, jwt=encoded) log.debug('Created instructor url for %r %r %r', course_id, exam_id, attempt_id) return url def retire_user(self, user_id): url = self.user_info_url.format(user_id=user_id) try: response = self.session.delete(url) data = response.json() assert data in (True, False) except Exception as exc: # pylint: disable=broad-except # pylint: disable=no-member content = exc.response.content if hasattr(exc, 'response') else response.content raise BackendProviderCannotRetireUser(content) return data def _get_language_headers(self): """ Returns a dictionary of the Accept-Language headers """ # This import is here because developers writing backends which subclass this class # may want to import this module and use the other methods, without having to run in the context # of django settings, etc. from django.utils.translation import get_language current_lang = get_language() default_lang = settings.LANGUAGE_CODE lang_header = default_lang if current_lang and current_lang != default_lang: lang_header = '{};{}'.format(current_lang, default_lang) return {'Accept-Language': lang_header} def _make_attempt_request(self, exam, attempt, method='POST', status=None, **payload): """ Calls backend attempt API """ if not attempt: return {} if status: payload['status'] = status else: payload = None url = self.exam_attempt_url.format(exam_id=exam, attempt_id=attempt) headers = {} if method == 'GET': headers.update(self._get_language_headers()) log.debug('Making %r attempt request at %r', method, url) response = self.session.request(method, url, json=payload, headers=headers) try: data = response.json() except ValueError: log.exception("Decoding attempt %r -> %r", attempt, response.content) data = {} return data
def lms_api_client(self): if not self.lms_url: return None return OAuthAPIClient(self.lms_url.strip('/'), self.oidc_key, self.oidc_secret)
def __init__(self): assert self.api_url_root self._client = OAuthAPIClient(OAUTH_ACCESS_TOKEN_URL, OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET)