def _test_download_wrong_auth_http_request(self): ''' Here, instead of a GET request, we have it reject other types of http requests ''' download_info = [{ 'resource_pk': 1, 'owner': 2 }, { 'resource_pk': 2, 'owner': 2 }] # make a mock request and add a session dictionary, which is required by the method state = 'abc123' code = 'mycode' mock_request = mock.MagicMock() mock_request.method = 'POST' mock_request.session = { 'session_state': state, 'download_info': download_info, 'download_destination': self.destination } mock_request.GET = {'state': state} downloader_cls = downloaders.get_downloader(self.destination) with self.assertRaises(MethodNotAllowed): downloader_cls.finish_authentication_and_start_download( mock_request)
def _test_warn_of_conflict_case1(self): ''' Here, we pretend that a user has previously started a download that is still going. Then they try to download that same file again (and also add a new one). Here, we check that we block appropriately. ''' downloader_cls = downloaders.get_downloader(self.destination) # create a Transfer that is ongoing: tc = TransferCoordinator.objects.create() resource_pk = 1 resource = Resource.objects.get(pk=resource_pk) t = Transfer.objects.create( download=True, resource = resource, destination = 'dropbox', coordinator = tc, originator = self.regular_user ) # prep the download info as is usually performed: originator_pk = self.regular_user.pk resource_pks = [1,2] # note that we are requesting the same transfer with pk=1 download_info, errors = downloader_cls.check_format(resource_pks, originator_pk) self.assertTrue(len(download_info) == 1) self.assertTrue(len(errors) == 1)
def _test_warn_of_conflict_case1(self): ''' Here, we pretend that a user has previously started a download that is still going. Then they try to download that same file again (and also add a new one). Here, we check that we block appropriately. ''' downloader_cls = downloaders.get_downloader(self.destination) # get two Resources owned by the regular user: owner = self.regular_user all_user_resources = Resource.objects.filter( owner=owner, is_active=True, originated_from_upload=False) # create a Transfer that is ongoing: tc = TransferCoordinator.objects.create() resource = all_user_resources[0] t = Transfer.objects.create(download=True, resource=resource, destination='dropbox', coordinator=tc, originator=self.regular_user) # prep the download info as is usually performed: originator_pk = self.regular_user.pk other_resource = all_user_resources[1] resource_pks = [resource.pk, other_resource.pk ] # note that we are requesting the same transfer download_info, errors = downloader_cls.check_format( resource_pks, originator_pk) self.assertTrue(len(download_info) == 1) self.assertTrue(len(errors) == 1)
def _test_simultaneous_download_by_two_originators(self): ''' Here, we pretend that two users (e.g. an admin and a regular user) are trying to download the same resource at the same time This should be allowed. ''' downloader_cls = downloaders.get_downloader(self.destination) # get two Resources owned by the regular user: owner = self.regular_user all_user_resources = Resource.objects.filter( owner=owner, is_active=True, originated_from_upload=False) # create a Transfer that is ongoing: tc = TransferCoordinator.objects.create() resource = all_user_resources[0] resource_pk = resource.pk t = Transfer.objects.create(download=True, resource=resource, destination='dropbox', coordinator=tc, originator=self.regular_user) # prep the download info as is usually performed: originator_pk = self.admin_user.pk other_resource = all_user_resources[1] resource_pks = [resource.pk, other_resource.pk ] # note that we are requesting the same transfer download_info, errors = downloader_cls.check_format( resource_pks, originator_pk) self.assertTrue(len(download_info) == 2) self.assertTrue(len(errors) == 0)
def drive_code_exchange_test(self, request): user = request.user # ensure we have the correct user for the test: if user.email == settings.LIVE_TEST_CONFIG_PARAMS['test_email']: # need to ensure we have the Resource already in the database: try: r = Resource.objects.get( path=settings.LIVE_TEST_CONFIG_PARAMS['file_to_transfer'], size=settings. LIVE_TEST_CONFIG_PARAMS['file_size_in_bytes'], owner=user) download_info = [ { 'resource_pk': r.pk, 'originator': user.pk, 'destination': settings.GOOGLE_DRIVE }, ] downloader_cls = downloaders.get_downloader( settings.GOOGLE_DRIVE) request.session['download_info'] = download_info request.session['download_destination'] = settings.GOOGLE_DRIVE callback_url = reverse( settings.LIVE_TEST_CONFIG_PARAMS['drive_transfer_callback'] ) with mock.patch.dict(downloaders.settings.CONFIG_PARAMS, {'drive_callback': callback_url}): return downloader_cls.authenticate(request) except Exception as ex: print('Could not find!') return HttpResponse('Could not find the test resource')
def _test_simultaneous_download_by_two_originators(self): ''' Here, we pretend that two users (e.g. an admin and a regular user) are trying to download the same resource at the same time This should be allowed. ''' downloader_cls = downloaders.get_downloader(self.destination) # create a Transfer that is ongoing: tc = TransferCoordinator.objects.create() resource_pk = 1 resource = Resource.objects.get(pk=resource_pk) t = Transfer.objects.create( download=True, resource = resource, destination = 'dropbox', coordinator = tc, originator = self.regular_user ) # prep the download info as is usually performed: originator_pk = self.admin_user.pk resource_pks = [1,2] # note that we are requesting the same transfer with pk=1 download_info, errors = downloader_cls.check_format(resource_pks, originator_pk) self.assertTrue(len(download_info) == 2) self.assertTrue(len(errors) == 0)
def _test_download_auth_error_exchanging_code(self): ''' Here we intentionally omit the 'code' key in the dictionary "returned" by the OAuth2 service. Thus, error ''' download_info = [{ 'resource_pk': 1, 'owner': 2 }, { 'resource_pk': 2, 'owner': 2 }] # make a mock request and add a session dictionary, which is required by the method state = 'abc123' code = 'mycode' mock_request = mock.MagicMock() mock_request.method = 'GET' mock_request.session = { 'session_state': state, 'download_info': download_info, 'download_destination': self.destination } mock_request.GET = {'state': state} downloader_cls = downloaders.get_downloader(self.destination) with self.assertRaises(exceptions.RequestError): downloader_cls.finish_authentication_and_start_download( mock_request)
def download(download_info, download_destination): ''' download_info is a list, with each entry a dictionary. Each of those dictionaries has keys which are specific to the upload source ''' downloader_cls = downloaders.get_downloader(download_destination) downloader = downloader_cls(download_info) downloader.download()
def drive_code_exchange_test(self, request): downloader_cls = downloaders.get_downloader(settings.GOOGLE_DRIVE) request.session['download_info'] = [] request.session['download_destination'] = settings.DROPBOX callback_url = reverse( settings.LIVE_TEST_CONFIG_PARAMS['drive_oauth_callback']) with mock.patch.dict(downloaders.settings.CONFIG_PARAMS, {'drive_callback': callback_url}): return downloader_cls.authenticate(request)
def drive_token_exchange_test(self, request, mock_tasks): downloader_cls = downloaders.get_downloader(settings.GOOGLE_DRIVE) callback_url = reverse( settings.LIVE_TEST_CONFIG_PARAMS['drive_oauth_callback']) with mock.patch.dict(downloaders.settings.CONFIG_PARAMS, {'drive_callback': callback_url}): response = downloader_cls.finish_authentication_and_start_download( request) self.assertEqual(response.status_code, 200) return response
def _test_admin_requests_transfer_of_other_user_resources(self): ''' Here, we test that an admin can transfer others resource without issue ''' downloader_cls = downloaders.get_downloader(self.destination) originator_pk = self.admin_user.pk resource_pks = [1, 2, 3] expected_return = [{ 'resource_pk': x, 'originator': originator_pk, 'destination': self.destination } for x in resource_pks] download_info, errors = downloader_cls.check_format( resource_pks, originator_pk) self.assertEqual(download_info, expected_return)
def _test_dropbox_downloader_on_google_params(self, mock_build): ''' This test takes a properly formatted request and checks that the database objects have been properly created. ''' #mock_os.environ['GCLOUD'] = '/mock/bin/gcloud' downloader_cls = downloaders.get_downloader(self.destination) originator = self.regular_user originator_pk = originator.pk all_resources = Resource.objects.filter(owner=originator, is_active=True, originated_from_upload=False) resource_subset = [all_resources[0], all_resources[1]] resource_pks = [x.pk for x in resource_subset] # prep the download info as is usually performed: download_info = [{ 'resource_pk': x, 'originator': originator_pk, 'destination': self.destination, 'access_token': 'abc123' } for x in resource_pks] # instantiate the class, but mock out the launcher. # Recall the launcher is the class that actually creates the worker VMs, which # we do not want to do as part of the test downloader = downloader_cls(download_info) m = mock.MagicMock() downloader.launcher = m # now that the launcher has been mocked, call the download method which # constructs the request to start a new VM. It is difficult to check that, so # we only check that the proper database objects have been created downloader.download() self.assertTrue(m.go.called) self.assertEqual(len(resource_pks), m.go.call_count) # check database objects: all_transfers = Transfer.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 2) self.assertTrue(len(all_tc) == 1) self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertFalse( all_tc[0].completed) # the transfer coord is also not completed
def _test_reformats_request_for_download(self): ''' Here, we assure that the request is reformatted to eventually create a proper download. ''' downloader_cls = downloaders.get_downloader(self.destination) originator_pk = 2 resource_pks = [1,2] expected_return = [ { 'resource_pk':x, 'originator':originator_pk, 'destination':self.destination } for x in resource_pks] download_info, errors = downloader_cls.check_format(resource_pks, originator_pk) self.assertEqual(download_info, expected_return)
def test_download_initial_auth(self, mock_redirect, mock_hashlib, mock_os): ''' Here, we check that the proper request is constructed for the Dropbox OAuth2 service ''' # setup elements on the mocks: # first, we need to mock out the random state we create. # in the code, we have `state = hashlib.sha256(os.urandom(1024)).hexdigest()` # so we have to mock all those elements state = 'abc123' mock_hex = mock.MagicMock() mock_hex.hexdigest.return_value = state mock_os.urandom.return_value = 100 # doesn't matter what this is, since it does not go to a 'real' method mock_hashlib.sha256.return_value = mock_hex downloader_cls = downloaders.get_downloader(self.destination) # make a mock request and add a session dictionary, which is required by the method mock_request = mock.MagicMock() mock_request.session = {} settings.CONFIG_PARAMS[ 'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize' settings.CONFIG_PARAMS['drive_callback'] = 'dropbox/callback/' settings.CONFIG_PARAMS['drive_client_id'] = 'mockclient' scope = 'some scope' settings.CONFIG_PARAMS['drive_scope'] = scope response_type = 'code' # construct the callback URL for Dropbox to use: current_site = Site.objects.get_current() domain = current_site.domain code_callback_url = 'https://%s%s' % ( domain, settings.CONFIG_PARAMS['drive_callback']) expected_url = "{code_request_uri}?response_type={response_type}&client_id={client_id}&redirect_uri={redirect_uri}&scope={scope}&state={state}".format( code_request_uri=settings.CONFIG_PARAMS['drive_auth_endpoint'], response_type=response_type, client_id=settings.CONFIG_PARAMS['drive_client_id'], redirect_uri=code_callback_url, scope=scope, state=state) downloader_cls.authenticate(mock_request) mock_redirect.assert_called_once_with(expected_url) self.assertEqual(mock_request.session['session_state'], state)
def post(self, request, *args, **kwargs): # Parse the submitted data: data = request.data try: json_str = data['data'] # j is a json-format string data = json.loads(json_str) resource_pks = data['resource_pks'] resource_pks = [x for x in resource_pks if x ] # remove any None that may have gotten through download_destination = data['destination'] except KeyError as ex: raise exceptions.RequestError(''' Missing required information for initiating transfer. ''') # Here, we first do a spot-check on the data that was passed, BEFORE we invoke any asynchronous methods. # We prepare/massage the necessary data for the upload here, and then pass a simple dictionary to the asynchronous # method call. We do this since it is easiest to pass a simple native dictionary to celery. user_pk = request.user.pk try: # Depending on which download destination was requested (and which compute environment we are in), grab the proper class: downloader_cls = _downloaders.get_downloader(download_destination) # Check that the upload data has the required format to work with this uploader implementation: download_info, error_messages = downloader_cls.check_format( resource_pks, user_pk) if len(error_messages) > 0: return Response({'errors': error_messages}, status=409) else: # stash the download info, since we will be redirecting through an authentication flow request.session['download_info'] = download_info request.session['download_destination'] = download_destination return Response({'success': True}) except exceptions.ExceptionWithMessage as ex: raise exceptions.RequestError(ex.message) except Exception as ex: response = exception_handler(ex, None) return Response({'message': 'thanks'})
def _test_reformats_request_for_download(self): ''' Here, we assure that the request is reformatted to eventually create a proper download. ''' downloader_cls = downloaders.get_downloader(self.destination) originator_pk = self.regular_user.pk all_resources = Resource.objects.filter(owner=originator_pk, is_active=True, originated_from_upload=False) resource_subset = [all_resources[0], all_resources[1]] resource_pks = [x.pk for x in resource_subset] expected_return = [{ 'resource_pk': x, 'originator': originator_pk, 'destination': self.destination } for x in resource_pks] download_info, errors = downloader_cls.check_format( resource_pks, originator_pk) self.assertEqual(download_info, expected_return)
def get(self, request, *args, **kwargs): download_destination = request.session['download_destination'] downloader_cls = _downloaders.get_downloader(download_destination) return downloader_cls.authenticate(request)
def test_download_finish_auth(self, mock_tasks, \ mock_httplib, \ mock_google_credentials_module, \ mock_build): reguser = get_user_model().objects.get( email=settings.REGULAR_TEST_EMAIL) resources = Resource.objects.filter(owner=reguser, originated_from_upload=False) download_info = [] for r in resources: download_info.append({'resource_pk': r.pk, 'owner': reguser.pk}) mock_parser = mock.MagicMock() content = b'{"access_token": "foo"}' # a json-format string mock_parser.request.return_value = (None, content) mock_httplib.Http.return_value = mock_parser mock_credentials_obj = mock.MagicMock() mock_google_credentials_module.Credentials.return_value = mock_credentials_obj mock_service = mock.MagicMock() quota_dict = {'limit': 1e10, 'usage': 1000} about_dict = {'storageQuota': quota_dict} mock_service.about.return_value.get.return_value.execute.return_value = about_dict mock_build.return_value = mock_service mock_download = mock.MagicMock() mock_tasks.download = mock_download # make a mock request and add a session dictionary, which is required by the method state = 'abc123' code = 'mycode' mock_request = mock.MagicMock() mock_request.method = 'GET' mock_request.session = { 'session_state': state, 'download_info': download_info, 'download_destination': self.destination } mock_request.GET = {'code': code, 'state': state} token_url = 'https://fake-auth.com/oauth2/token' callback_url = 'google_drive/callback/' client_id = 'mockclient' secret = 'somesecret' scope = 'some scope' settings.CONFIG_PARAMS[ 'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize' settings.CONFIG_PARAMS['drive_token_endpoint'] = token_url settings.CONFIG_PARAMS['drive_callback'] = callback_url settings.CONFIG_PARAMS['drive_client_id'] = client_id settings.CONFIG_PARAMS['drive_secret'] = secret current_site = Site.objects.get_current() domain = current_site.domain full_callback_url = 'https://%s%s' % ( domain, settings.CONFIG_PARAMS['drive_callback']) headers = {'content-type': 'application/x-www-form-urlencoded'} expected_params = urllib.parse.urlencode({ 'code': code, 'redirect_uri': full_callback_url, 'client_id': client_id, 'client_secret': secret, 'grant_type': 'authorization_code' }) downloader_cls = downloaders.get_downloader(self.destination) downloader_cls.finish_authentication_and_start_download(mock_request) [x.update({'access_token': 'foo'}) for x in download_info] mock_parser.request.assert_called_once_with(token_url, method='POST', body=expected_params, headers=headers) mock_download.delay.assert_called_once_with(download_info, self.destination)
def test_download_finish_auth(self, mock_tasks, mock_httplib, mock_dropbox_mod): reguser = get_user_model().objects.get( email=settings.REGULAR_TEST_EMAIL) resources = Resource.objects.filter(owner=reguser, originated_from_upload=False) download_info = [] for r in resources: download_info.append({'resource_pk': r.pk, 'owner': reguser.pk}) mock_parser = mock.MagicMock() content = b'{"access_token": "foo"}' # a json-format string mock_parser.request.return_value = (None, content) mock_httplib.Http.return_value = mock_parser mock_dropbox_obj = mock.MagicMock() mock_individual = mock.MagicMock(allocated=1e10) mock_team = mock.MagicMock(allocated=1e10) mock_allocation = mock.MagicMock() mock_allocation.is_team.return_value = False mock_allocation.get_individual.return_value = mock_individual mock_allocation.get_team.return_value = mock_team mock_space_usage = mock.MagicMock(allocation=mock_allocation, used=1000) mock_dropbox_obj.users_get_space_usage.return_value = mock_space_usage mock_dropbox_mod.Dropbox.return_value = mock_dropbox_obj mock_download = mock.MagicMock() mock_tasks.download = mock_download # make a mock request and add a session dictionary, which is required by the method state = 'abc123' code = 'mycode' mock_request = mock.MagicMock() mock_request.method = 'GET' mock_request.session = { 'session_state': state, 'download_info': download_info, 'download_destination': self.destination } mock_request.GET = {'code': code, 'state': state} token_url = 'https://fake-auth.com/oauth2/token' callback_url = '/dropbox/callback/' client_id = 'mockclient' secret = 'somesecret' settings.CONFIG_PARAMS[ 'dropbox_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize' settings.CONFIG_PARAMS['dropbox_token_endpoint'] = token_url settings.CONFIG_PARAMS['dropbox_callback'] = callback_url settings.CONFIG_PARAMS['dropbox_client_id'] = client_id settings.CONFIG_PARAMS['dropbox_secret'] = secret headers = {'content-type': 'application/x-www-form-urlencoded'} current_site = Site.objects.get_current() domain = current_site.domain full_callback_url = 'https://%s%s' % ( domain, settings.CONFIG_PARAMS['dropbox_callback']) expected_params = urllib.parse.urlencode({ 'code': code, 'redirect_uri': full_callback_url, 'client_id': client_id, 'client_secret': secret, 'grant_type': 'authorization_code' }) downloader_cls = downloaders.get_downloader(self.destination) downloader_cls.finish_authentication_and_start_download(mock_request) [x.update({'access_token': 'foo'}) for x in download_info] mock_parser.request.assert_called_once_with(token_url, method='POST', body=expected_params, headers=headers) mock_download.delay.assert_called_once_with(download_info, self.destination)
def test_download_finish_auth(self, mock_tasks, mock_httplib): download_info = [{ 'resource_pk': 1, 'owner': 2 }, { 'resource_pk': 2, 'owner': 2 }] mock_parser = mock.MagicMock() content = b'{"access_token": "foo"}' # a json-format string mock_parser.request.return_value = (None, content) mock_httplib.Http.return_value = mock_parser mock_download = mock.MagicMock() mock_tasks.download = mock_download # make a mock request and add a session dictionary, which is required by the method state = 'abc123' code = 'mycode' mock_request = mock.MagicMock() mock_request.method = 'GET' mock_request.session = { 'session_state': state, 'download_info': download_info, 'download_destination': self.destination } mock_request.GET = {'code': code, 'state': state} token_url = 'https://fake-auth.com/oauth2/token' callback_url = 'google_drive/callback/' client_id = 'mockclient' secret = 'somesecret' scope = 'some scope' settings.CONFIG_PARAMS[ 'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize' settings.CONFIG_PARAMS['drive_token_endpoint'] = token_url settings.CONFIG_PARAMS['drive_callback'] = callback_url settings.CONFIG_PARAMS['drive_client_id'] = client_id settings.CONFIG_PARAMS['drive_secret'] = secret current_site = Site.objects.get_current() domain = current_site.domain full_callback_url = 'https://%s%s' % ( domain, settings.CONFIG_PARAMS['drive_callback']) headers = {'content-type': 'application/x-www-form-urlencoded'} expected_params = urllib.parse.urlencode({ 'code': code, 'redirect_uri': full_callback_url, 'client_id': client_id, 'client_secret': secret, 'grant_type': 'authorization_code' }) downloader_cls = downloaders.get_downloader(self.destination) downloader_cls.finish_authentication_and_start_download(mock_request) [x.update({'access_token': 'foo'}) for x in download_info] mock_parser.request.assert_called_once_with(token_url, method='POST', body=expected_params, headers=headers) mock_download.delay.assert_called_once_with(download_info, self.destination)