def _test_download_wrong_auth_http_request(self):
'''
Here, instead of a GET request, we have it reject other types of http requests
'''
download_info = [{
'resource_pk': 1,
'owner': 2
}, {
'resource_pk': 2,
'owner': 2
}]
# make a mock request and add a session dictionary, which is required by the method
state = 'abc123'
code = 'mycode'
mock_request = mock.MagicMock()
mock_request.method = 'POST'
mock_request.session = {
'session_state': state,
'download_info': download_info,
'download_destination': self.destination
}
mock_request.GET = {'state': state}
downloader_cls = downloaders.get_downloader(self.destination)
with self.assertRaises(MethodNotAllowed):
downloader_cls.finish_authentication_and_start_download(
mock_request)
def _test_warn_of_conflict_case1(self):
'''
Here, we pretend that a user has previously started a download that is still going.
Then they try to download that same file again (and also add a new one). Here, we check that we block appropriately.
'''
downloader_cls = downloaders.get_downloader(self.destination)
# create a Transfer that is ongoing:
tc = TransferCoordinator.objects.create()
resource_pk = 1
resource = Resource.objects.get(pk=resource_pk)
t = Transfer.objects.create(
download=True,
resource = resource,
destination = 'dropbox',
coordinator = tc,
originator = self.regular_user
)
# prep the download info as is usually performed:
originator_pk = self.regular_user.pk
resource_pks = [1,2] # note that we are requesting the same transfer with pk=1
download_info, errors = downloader_cls.check_format(resource_pks, originator_pk)
self.assertTrue(len(download_info) == 1)
self.assertTrue(len(errors) == 1)
def _test_warn_of_conflict_case1(self):
'''
Here, we pretend that a user has previously started a download that is still going.
Then they try to download that same file again (and also add a new one). Here, we check that we block appropriately.
'''
downloader_cls = downloaders.get_downloader(self.destination)
# get two Resources owned by the regular user:
owner = self.regular_user
all_user_resources = Resource.objects.filter(
owner=owner, is_active=True, originated_from_upload=False)
# create a Transfer that is ongoing:
tc = TransferCoordinator.objects.create()
resource = all_user_resources[0]
t = Transfer.objects.create(download=True,
resource=resource,
destination='dropbox',
coordinator=tc,
originator=self.regular_user)
# prep the download info as is usually performed:
originator_pk = self.regular_user.pk
other_resource = all_user_resources[1]
resource_pks = [resource.pk, other_resource.pk
] # note that we are requesting the same transfer
download_info, errors = downloader_cls.check_format(
resource_pks, originator_pk)
self.assertTrue(len(download_info) == 1)
self.assertTrue(len(errors) == 1)
def _test_simultaneous_download_by_two_originators(self):
'''
Here, we pretend that two users (e.g. an admin and a regular user) are trying to download the same resource at the same time
This should be allowed.
'''
downloader_cls = downloaders.get_downloader(self.destination)
# get two Resources owned by the regular user:
owner = self.regular_user
all_user_resources = Resource.objects.filter(
owner=owner, is_active=True, originated_from_upload=False)
# create a Transfer that is ongoing:
tc = TransferCoordinator.objects.create()
resource = all_user_resources[0]
resource_pk = resource.pk
t = Transfer.objects.create(download=True,
resource=resource,
destination='dropbox',
coordinator=tc,
originator=self.regular_user)
# prep the download info as is usually performed:
originator_pk = self.admin_user.pk
other_resource = all_user_resources[1]
resource_pks = [resource.pk, other_resource.pk
] # note that we are requesting the same transfer
download_info, errors = downloader_cls.check_format(
resource_pks, originator_pk)
self.assertTrue(len(download_info) == 2)
self.assertTrue(len(errors) == 0)
def drive_code_exchange_test(self, request):
user = request.user
# ensure we have the correct user for the test:
if user.email == settings.LIVE_TEST_CONFIG_PARAMS['test_email']:
# need to ensure we have the Resource already in the database:
try:
r = Resource.objects.get(
path=settings.LIVE_TEST_CONFIG_PARAMS['file_to_transfer'],
size=settings.
LIVE_TEST_CONFIG_PARAMS['file_size_in_bytes'],
owner=user)
download_info = [
{
'resource_pk': r.pk,
'originator': user.pk,
'destination': settings.GOOGLE_DRIVE
},
]
downloader_cls = downloaders.get_downloader(
settings.GOOGLE_DRIVE)
request.session['download_info'] = download_info
request.session['download_destination'] = settings.GOOGLE_DRIVE
callback_url = reverse(
settings.LIVE_TEST_CONFIG_PARAMS['drive_transfer_callback']
)
with mock.patch.dict(downloaders.settings.CONFIG_PARAMS,
{'drive_callback': callback_url}):
return downloader_cls.authenticate(request)
except Exception as ex:
print('Could not find!')
return HttpResponse('Could not find the test resource')
def _test_simultaneous_download_by_two_originators(self):
'''
Here, we pretend that two users (e.g. an admin and a regular user) are trying to download the same resource at the same time
This should be allowed.
'''
downloader_cls = downloaders.get_downloader(self.destination)
# create a Transfer that is ongoing:
tc = TransferCoordinator.objects.create()
resource_pk = 1
resource = Resource.objects.get(pk=resource_pk)
t = Transfer.objects.create(
download=True,
resource = resource,
destination = 'dropbox',
coordinator = tc,
originator = self.regular_user
)
# prep the download info as is usually performed:
originator_pk = self.admin_user.pk
resource_pks = [1,2] # note that we are requesting the same transfer with pk=1
download_info, errors = downloader_cls.check_format(resource_pks, originator_pk)
self.assertTrue(len(download_info) == 2)
self.assertTrue(len(errors) == 0)
def _test_download_auth_error_exchanging_code(self):
'''
Here we intentionally omit the 'code' key in the dictionary
"returned" by the OAuth2 service. Thus, error
'''
download_info = [{
'resource_pk': 1,
'owner': 2
}, {
'resource_pk': 2,
'owner': 2
}]
# make a mock request and add a session dictionary, which is required by the method
state = 'abc123'
code = 'mycode'
mock_request = mock.MagicMock()
mock_request.method = 'GET'
mock_request.session = {
'session_state': state,
'download_info': download_info,
'download_destination': self.destination
}
mock_request.GET = {'state': state}
downloader_cls = downloaders.get_downloader(self.destination)
with self.assertRaises(exceptions.RequestError):
downloader_cls.finish_authentication_and_start_download(
mock_request)
def download(download_info, download_destination):
'''
download_info is a list, with each entry a dictionary.
Each of those dictionaries has keys which are specific to the upload source
'''
downloader_cls = downloaders.get_downloader(download_destination)
downloader = downloader_cls(download_info)
downloader.download()
def drive_code_exchange_test(self, request):
downloader_cls = downloaders.get_downloader(settings.GOOGLE_DRIVE)
request.session['download_info'] = []
request.session['download_destination'] = settings.DROPBOX
callback_url = reverse(
settings.LIVE_TEST_CONFIG_PARAMS['drive_oauth_callback'])
with mock.patch.dict(downloaders.settings.CONFIG_PARAMS,
{'drive_callback': callback_url}):
return downloader_cls.authenticate(request)
def drive_token_exchange_test(self, request, mock_tasks):
downloader_cls = downloaders.get_downloader(settings.GOOGLE_DRIVE)
callback_url = reverse(
settings.LIVE_TEST_CONFIG_PARAMS['drive_oauth_callback'])
with mock.patch.dict(downloaders.settings.CONFIG_PARAMS,
{'drive_callback': callback_url}):
response = downloader_cls.finish_authentication_and_start_download(
request)
self.assertEqual(response.status_code, 200)
return response
def _test_admin_requests_transfer_of_other_user_resources(self):
'''
Here, we test that an admin can transfer others resource without issue
'''
downloader_cls = downloaders.get_downloader(self.destination)
originator_pk = self.admin_user.pk
resource_pks = [1, 2, 3]
expected_return = [{
'resource_pk': x,
'originator': originator_pk,
'destination': self.destination
} for x in resource_pks]
download_info, errors = downloader_cls.check_format(
resource_pks, originator_pk)
self.assertEqual(download_info, expected_return)
def _test_dropbox_downloader_on_google_params(self, mock_build):
'''
This test takes a properly formatted request and checks that the database objects have been properly
created.
'''
#mock_os.environ['GCLOUD'] = '/mock/bin/gcloud'
downloader_cls = downloaders.get_downloader(self.destination)
originator = self.regular_user
originator_pk = originator.pk
all_resources = Resource.objects.filter(owner=originator,
is_active=True,
originated_from_upload=False)
resource_subset = [all_resources[0], all_resources[1]]
resource_pks = [x.pk for x in resource_subset]
# prep the download info as is usually performed:
download_info = [{
'resource_pk': x,
'originator': originator_pk,
'destination': self.destination,
'access_token': 'abc123'
} for x in resource_pks]
# instantiate the class, but mock out the launcher.
# Recall the launcher is the class that actually creates the worker VMs, which
# we do not want to do as part of the test
downloader = downloader_cls(download_info)
m = mock.MagicMock()
downloader.launcher = m
# now that the launcher has been mocked, call the download method which
# constructs the request to start a new VM. It is difficult to check that, so
# we only check that the proper database objects have been created
downloader.download()
self.assertTrue(m.go.called)
self.assertEqual(len(resource_pks), m.go.call_count)
# check database objects:
all_transfers = Transfer.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 2)
self.assertTrue(len(all_tc) == 1)
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertFalse(
all_tc[0].completed) # the transfer coord is also not completed
def _test_reformats_request_for_download(self):
'''
Here, we assure that the request is reformatted to eventually create
a proper download.
'''
downloader_cls = downloaders.get_downloader(self.destination)
originator_pk = 2
resource_pks = [1,2]
expected_return = [
{
'resource_pk':x,
'originator':originator_pk,
'destination':self.destination
} for x in resource_pks]
download_info, errors = downloader_cls.check_format(resource_pks, originator_pk)
self.assertEqual(download_info, expected_return)
def test_download_initial_auth(self, mock_redirect, mock_hashlib, mock_os):
'''
Here, we check that the proper request is constructed for the Dropbox OAuth2 service
'''
# setup elements on the mocks:
# first, we need to mock out the random state we create.
# in the code, we have `state = hashlib.sha256(os.urandom(1024)).hexdigest()`
# so we have to mock all those elements
state = 'abc123'
mock_hex = mock.MagicMock()
mock_hex.hexdigest.return_value = state
mock_os.urandom.return_value = 100 # doesn't matter what this is, since it does not go to a 'real' method
mock_hashlib.sha256.return_value = mock_hex
downloader_cls = downloaders.get_downloader(self.destination)
# make a mock request and add a session dictionary, which is required by the method
mock_request = mock.MagicMock()
mock_request.session = {}
settings.CONFIG_PARAMS[
'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize'
settings.CONFIG_PARAMS['drive_callback'] = 'dropbox/callback/'
settings.CONFIG_PARAMS['drive_client_id'] = 'mockclient'
scope = 'some scope'
settings.CONFIG_PARAMS['drive_scope'] = scope
response_type = 'code'
# construct the callback URL for Dropbox to use:
current_site = Site.objects.get_current()
domain = current_site.domain
code_callback_url = 'https://%s%s' % (
domain, settings.CONFIG_PARAMS['drive_callback'])
expected_url = "{code_request_uri}?response_type={response_type}&client_id={client_id}&redirect_uri={redirect_uri}&scope={scope}&state={state}".format(
code_request_uri=settings.CONFIG_PARAMS['drive_auth_endpoint'],
response_type=response_type,
client_id=settings.CONFIG_PARAMS['drive_client_id'],
redirect_uri=code_callback_url,
scope=scope,
state=state)
downloader_cls.authenticate(mock_request)
mock_redirect.assert_called_once_with(expected_url)
self.assertEqual(mock_request.session['session_state'], state)
def post(self, request, *args, **kwargs):
# Parse the submitted data:
data = request.data
try:
json_str = data['data'] # j is a json-format string
data = json.loads(json_str)
resource_pks = data['resource_pks']
resource_pks = [x for x in resource_pks if x
] # remove any None that may have gotten through
download_destination = data['destination']
except KeyError as ex:
raise exceptions.RequestError('''
Missing required information for initiating transfer.
''')
# Here, we first do a spot-check on the data that was passed, BEFORE we invoke any asynchronous methods.
# We prepare/massage the necessary data for the upload here, and then pass a simple dictionary to the asynchronous
# method call. We do this since it is easiest to pass a simple native dictionary to celery.
user_pk = request.user.pk
try:
# Depending on which download destination was requested (and which compute environment we are in), grab the proper class:
downloader_cls = _downloaders.get_downloader(download_destination)
# Check that the upload data has the required format to work with this uploader implementation:
download_info, error_messages = downloader_cls.check_format(
resource_pks, user_pk)
if len(error_messages) > 0:
return Response({'errors': error_messages}, status=409)
else:
# stash the download info, since we will be redirecting through an authentication flow
request.session['download_info'] = download_info
request.session['download_destination'] = download_destination
return Response({'success': True})
except exceptions.ExceptionWithMessage as ex:
raise exceptions.RequestError(ex.message)
except Exception as ex:
response = exception_handler(ex, None)
return Response({'message': 'thanks'})
def _test_reformats_request_for_download(self):
'''
Here, we assure that the request is reformatted to eventually create
a proper download.
'''
downloader_cls = downloaders.get_downloader(self.destination)
originator_pk = self.regular_user.pk
all_resources = Resource.objects.filter(owner=originator_pk,
is_active=True,
originated_from_upload=False)
resource_subset = [all_resources[0], all_resources[1]]
resource_pks = [x.pk for x in resource_subset]
expected_return = [{
'resource_pk': x,
'originator': originator_pk,
'destination': self.destination
} for x in resource_pks]
download_info, errors = downloader_cls.check_format(
resource_pks, originator_pk)
self.assertEqual(download_info, expected_return)
def get(self, request, *args, **kwargs):
download_destination = request.session['download_destination']
downloader_cls = _downloaders.get_downloader(download_destination)
return downloader_cls.authenticate(request)
def test_download_finish_auth(self, mock_tasks, \
mock_httplib, \
mock_google_credentials_module, \
mock_build):
reguser = get_user_model().objects.get(
email=settings.REGULAR_TEST_EMAIL)
resources = Resource.objects.filter(owner=reguser,
originated_from_upload=False)
download_info = []
for r in resources:
download_info.append({'resource_pk': r.pk, 'owner': reguser.pk})
mock_parser = mock.MagicMock()
content = b'{"access_token": "foo"}' # a json-format string
mock_parser.request.return_value = (None, content)
mock_httplib.Http.return_value = mock_parser
mock_credentials_obj = mock.MagicMock()
mock_google_credentials_module.Credentials.return_value = mock_credentials_obj
mock_service = mock.MagicMock()
quota_dict = {'limit': 1e10, 'usage': 1000}
about_dict = {'storageQuota': quota_dict}
mock_service.about.return_value.get.return_value.execute.return_value = about_dict
mock_build.return_value = mock_service
mock_download = mock.MagicMock()
mock_tasks.download = mock_download
# make a mock request and add a session dictionary, which is required by the method
state = 'abc123'
code = 'mycode'
mock_request = mock.MagicMock()
mock_request.method = 'GET'
mock_request.session = {
'session_state': state,
'download_info': download_info,
'download_destination': self.destination
}
mock_request.GET = {'code': code, 'state': state}
token_url = 'https://fake-auth.com/oauth2/token'
callback_url = 'google_drive/callback/'
client_id = 'mockclient'
secret = 'somesecret'
scope = 'some scope'
settings.CONFIG_PARAMS[
'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize'
settings.CONFIG_PARAMS['drive_token_endpoint'] = token_url
settings.CONFIG_PARAMS['drive_callback'] = callback_url
settings.CONFIG_PARAMS['drive_client_id'] = client_id
settings.CONFIG_PARAMS['drive_secret'] = secret
current_site = Site.objects.get_current()
domain = current_site.domain
full_callback_url = 'https://%s%s' % (
domain, settings.CONFIG_PARAMS['drive_callback'])
headers = {'content-type': 'application/x-www-form-urlencoded'}
expected_params = urllib.parse.urlencode({
'code':
code,
'redirect_uri':
full_callback_url,
'client_id':
client_id,
'client_secret':
secret,
'grant_type':
'authorization_code'
})
downloader_cls = downloaders.get_downloader(self.destination)
downloader_cls.finish_authentication_and_start_download(mock_request)
[x.update({'access_token': 'foo'}) for x in download_info]
mock_parser.request.assert_called_once_with(token_url,
method='POST',
body=expected_params,
headers=headers)
mock_download.delay.assert_called_once_with(download_info,
self.destination)
def test_download_finish_auth(self, mock_tasks, mock_httplib,
mock_dropbox_mod):
reguser = get_user_model().objects.get(
email=settings.REGULAR_TEST_EMAIL)
resources = Resource.objects.filter(owner=reguser,
originated_from_upload=False)
download_info = []
for r in resources:
download_info.append({'resource_pk': r.pk, 'owner': reguser.pk})
mock_parser = mock.MagicMock()
content = b'{"access_token": "foo"}' # a json-format string
mock_parser.request.return_value = (None, content)
mock_httplib.Http.return_value = mock_parser
mock_dropbox_obj = mock.MagicMock()
mock_individual = mock.MagicMock(allocated=1e10)
mock_team = mock.MagicMock(allocated=1e10)
mock_allocation = mock.MagicMock()
mock_allocation.is_team.return_value = False
mock_allocation.get_individual.return_value = mock_individual
mock_allocation.get_team.return_value = mock_team
mock_space_usage = mock.MagicMock(allocation=mock_allocation,
used=1000)
mock_dropbox_obj.users_get_space_usage.return_value = mock_space_usage
mock_dropbox_mod.Dropbox.return_value = mock_dropbox_obj
mock_download = mock.MagicMock()
mock_tasks.download = mock_download
# make a mock request and add a session dictionary, which is required by the method
state = 'abc123'
code = 'mycode'
mock_request = mock.MagicMock()
mock_request.method = 'GET'
mock_request.session = {
'session_state': state,
'download_info': download_info,
'download_destination': self.destination
}
mock_request.GET = {'code': code, 'state': state}
token_url = 'https://fake-auth.com/oauth2/token'
callback_url = '/dropbox/callback/'
client_id = 'mockclient'
secret = 'somesecret'
settings.CONFIG_PARAMS[
'dropbox_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize'
settings.CONFIG_PARAMS['dropbox_token_endpoint'] = token_url
settings.CONFIG_PARAMS['dropbox_callback'] = callback_url
settings.CONFIG_PARAMS['dropbox_client_id'] = client_id
settings.CONFIG_PARAMS['dropbox_secret'] = secret
headers = {'content-type': 'application/x-www-form-urlencoded'}
current_site = Site.objects.get_current()
domain = current_site.domain
full_callback_url = 'https://%s%s' % (
domain, settings.CONFIG_PARAMS['dropbox_callback'])
expected_params = urllib.parse.urlencode({
'code':
code,
'redirect_uri':
full_callback_url,
'client_id':
client_id,
'client_secret':
secret,
'grant_type':
'authorization_code'
})
downloader_cls = downloaders.get_downloader(self.destination)
downloader_cls.finish_authentication_and_start_download(mock_request)
[x.update({'access_token': 'foo'}) for x in download_info]
mock_parser.request.assert_called_once_with(token_url,
method='POST',
body=expected_params,
headers=headers)
mock_download.delay.assert_called_once_with(download_info,
self.destination)
def test_download_finish_auth(self, mock_tasks, mock_httplib):
download_info = [{
'resource_pk': 1,
'owner': 2
}, {
'resource_pk': 2,
'owner': 2
}]
mock_parser = mock.MagicMock()
content = b'{"access_token": "foo"}' # a json-format string
mock_parser.request.return_value = (None, content)
mock_httplib.Http.return_value = mock_parser
mock_download = mock.MagicMock()
mock_tasks.download = mock_download
# make a mock request and add a session dictionary, which is required by the method
state = 'abc123'
code = 'mycode'
mock_request = mock.MagicMock()
mock_request.method = 'GET'
mock_request.session = {
'session_state': state,
'download_info': download_info,
'download_destination': self.destination
}
mock_request.GET = {'code': code, 'state': state}
token_url = 'https://fake-auth.com/oauth2/token'
callback_url = 'google_drive/callback/'
client_id = 'mockclient'
secret = 'somesecret'
scope = 'some scope'
settings.CONFIG_PARAMS[
'drive_auth_endpoint'] = 'https://fake-auth.com/oauth2/authorize'
settings.CONFIG_PARAMS['drive_token_endpoint'] = token_url
settings.CONFIG_PARAMS['drive_callback'] = callback_url
settings.CONFIG_PARAMS['drive_client_id'] = client_id
settings.CONFIG_PARAMS['drive_secret'] = secret
current_site = Site.objects.get_current()
domain = current_site.domain
full_callback_url = 'https://%s%s' % (
domain, settings.CONFIG_PARAMS['drive_callback'])
headers = {'content-type': 'application/x-www-form-urlencoded'}
expected_params = urllib.parse.urlencode({
'code':
code,
'redirect_uri':
full_callback_url,
'client_id':
client_id,
'client_secret':
secret,
'grant_type':
'authorization_code'
})
downloader_cls = downloaders.get_downloader(self.destination)
downloader_cls.finish_authentication_and_start_download(mock_request)
[x.update({'access_token': 'foo'}) for x in download_info]
mock_parser.request.assert_called_once_with(token_url,
method='POST',
body=expected_params,
headers=headers)
mock_download.delay.assert_called_once_with(download_info,
self.destination)
