def test_dropbox_uploader_on_google_params_single(self):
source = settings.DROPBOX
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader)
upload_info = {
'source_path': 'https://dropbox-link.com/1',
'name': 'f1.txt',
'owner': 2
}
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
uploader = uploader_cls(upload_info)
m = mock.MagicMock()
uploader.launcher = m
uploader.upload()
self.assertTrue(m.go.called)
self.assertEqual(1, m.go.call_count)
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 1)
self.assertTrue(len(all_resources) == 1)
self.assertTrue(len(all_tc) == 1)
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertFalse(
all_tc[0].completed) # the transfer coord is also not completed
def test_drive_uploader_on_google_params_single(self):
source = settings.GOOGLE_DRIVE
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDriveUploader)
upload_info = {
'file_id': 'abc123',
'name': 'f1.txt',
'drive_token': 'fooToken',
'owner': 2
}
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
uploader = uploader_cls(upload_info)
m = mock.MagicMock()
uploader.launcher = m
uploader.upload()
self.assertTrue(m.go.called)
self.assertEqual(1, m.go.call_count)
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 1)
self.assertTrue(len(all_resources) == 1)
self.assertTrue(len(all_tc) == 1)
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertFalse(
all_tc[0].completed) # the transfer coord is also not completed
def test_dropbox_uploader_on_google_disk_sizing(self):
source = settings.DROPBOX
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader)
upload_info = []
filesize = 100e9 # 100GB
upload_info.append({
'source_path': 'https://dropbox-link.com/1',
'name': 'f1.txt',
'owner': 2,
'size_in_bytes': filesize
})
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
uploader = uploader_cls(upload_info)
uploader.config_params['disk_size_factor'] = 3
m = mock.MagicMock()
uploader.launcher = m
uploader.upload()
self.assertTrue(m.go.called)
self.assertEqual(1, m.go.call_count)
call_arg = m.go.call_args
the_call = call_arg.call_list()[0]
import re
target = '--boot-disk-size=300GB'
matches = re.findall(target, str(the_call))
self.assertEqual(len(matches), 1)
def upload(upload_info, upload_source):
'''
upload_info is a list, with each entry a dictionary.
Each of those dictionaries has keys which are specific to the upload source
'''
uploader_cls = uploaders.get_uploader(upload_source)
uploader = uploader_cls(upload_info)
uploader.upload()
def test_drive_uploader_on_google_params(self):
'''
This test takes a properly formatted request and checks that the database objects have been properly
created.
'''
source = settings.GOOGLE_DRIVE
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDriveUploader)
# prep the upload info as is usually performed:
upload_info = []
upload_info.append({
'file_id': 'abc123',
'name': 'f1.txt',
'drive_token': 'fooToken',
'owner': 2
})
upload_info.append({
'file_id': 'def123',
'name': 'f2.txt',
'drive_token': 'fooToken',
'owner': 2
})
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
# instantiate the class, but mock out the launcher.
# Recall the launcher is the class that actually creates the worker VMs, which
# we do not want to do as part of the test
uploader = uploader_cls(upload_info)
m = mock.MagicMock()
uploader.launcher = m
# now that the launcher has been mocked, call the upload method which
# constructs the request to start a new VM. It is difficult to check that, so
# we only check that the proper database objects have been created
uploader.upload()
self.assertTrue(m.go.called)
self.assertEqual(2, m.go.call_count)
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 2)
self.assertTrue(len(all_resources) == 2)
self.assertTrue(len(all_tc) == 1)
self.assertTrue(
all([x.source == settings.GOOGLE_DRIVE for x in all_resources]))
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertFalse(
all_tc[0].completed) # the transfer coord is also not completed
def test_uploader_mod_returns_correct_uploader_implementation(self):
# first try dropbox uploader on google:
settings.CONFIG_PARAMS['cloud_environment'] = settings.GOOGLE
source = settings.DROPBOX
result_cls = uploaders.get_uploader(source)
self.assertEqual(result_cls, uploaders.GoogleDropboxUploader)
settings.CONFIG_PARAMS['cloud_environment'] = settings.GOOGLE
source = settings.GOOGLE_DRIVE
result_cls = uploaders.get_uploader(source)
self.assertEqual(result_cls, uploaders.GoogleDriveUploader)
settings.CONFIG_PARAMS['cloud_environment'] = settings.AWS
source = settings.DROPBOX
result_cls = uploaders.get_uploader(source)
self.assertEqual(result_cls, uploaders.AWSDropboxUploader)
settings.CONFIG_PARAMS['cloud_environment'] = settings.AWS
source = settings.GOOGLE_DRIVE
result_cls = uploaders.get_uploader(source)
self.assertEqual(result_cls, uploaders.AWSDriveUploader)
def test_reject_long_name_in_google(self):
source = settings.DROPBOX
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader)
upload_info = []
long_name = 'x' * 2000 + '.txt'
upload_info.append({
'source_path': 'https://dropbox-link.com/1',
'name': long_name,
'owner': 2
})
upload_info.append({
'source_path': 'https://dropbox-link.com/2',
'name': 'f2.txt',
'owner': 2
})
with self.assertRaises(exceptions.FilenameException):
upload_info, error_messages = uploader_cls.check_format(
upload_info, 2)
def post(self, request, *args, **kwargs):
data = request.data
try:
upload_source = data['upload_source'] # (dropbox, drive, etc)
upload_info = data['upload_info']
upload_info = json.loads(upload_info)
#raise Exception('temp!');
except KeyError as ex:
raise exceptions.RequestError(
'The request JSON body did not contain the required data (%s).'
% ex)
# Here, we first do a spot-check on the data that was passed, BEFORE we invoke any asynchronous methods.
# We prepare/massage the necessary data for the upload here, and then pass a simple dictionary to the asynchronous
# method call. We do this since it is easiest to pass a simple native dictionary to celery.
user_pk = request.user.pk
try:
# Depending on which upload method was requested (and which compute environment we are in), grab the proper class:
uploader_cls = _uploaders.get_uploader(upload_source)
# Check that the upload data has the required format to work with this uploader implementation:
upload_info, error_messages = uploader_cls.check_format(
upload_info, user_pk)
if len(error_messages) > 0:
return Response({'errors': error_messages})
elif len(upload_info) > 0:
# call async method:
transfer_tasks.upload.delay(upload_info, upload_source)
return Response({})
else: # no errors, but also nothing to do...
return Response({})
except exceptions.ExceptionWithMessage as ex:
raise exceptions.RequestError(ex.message)
except Exception as ex:
print('Exception: %s' % ex)
response = exception_handler(ex, None)
return response
def test_warn_of_conflict_case3(self, mock_datetime):
'''
Here, we initiate two transfers. We mock one being completed, and THEN the user uploads another to the same
as an overwrite. We want to allow this.
'''
source = settings.DROPBOX
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader)
# prep the upload info as is usually performed:
upload_info = []
upload_info.append({
'source_path': 'https://dropbox-link.com/1',
'name': 'f1.txt',
'owner': 2
})
upload_info.append({
'source_path': 'https://dropbox-link.com/2',
'name': 'f2.txt',
'owner': 2
})
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
self.assertEqual(len(upload_info), 2)
self.assertEqual(len(error_messages), 0)
# instantiate the class, but mock out the launcher.
# Recall the launcher is the class that actually creates the worker VMs, which
# we do not want to do as part of the test
uploader = uploader_cls(upload_info)
m = mock.MagicMock()
uploader.launcher = m
# mock launch of transfers, which creates the database objects
uploader.upload()
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 2)
self.assertTrue(len(all_resources) == 2)
self.assertTrue(
len(all_tc) == 1
) # this means no NEW coordinators were created for the 'empty' case
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertTrue(all([not tc.completed for tc in all_tc
])) # no transfer_coordinators are complete
# make the transfers 'complete' and set the Resources to 'active'
for t in all_transfers:
t.completed = True
t.save()
for tc in all_tc:
tc.completed = True
tc.save()
for r in all_resources:
r.is_active = True
r.save()
# prep the upload info as is usually performed:
# setup the mock to return
d = datetime.datetime(2019, 4, 15, 21, 3, 7, 0)
mock_datetime.datetime.now.return_value = d
expected_stamp = d.strftime('%m%d%Y-%H%M%S')
expected_filename = 'f2.' + expected_stamp + '.txt'
additional_uploads = []
additional_uploads.append({
'source_path': 'https://dropbox-link.com/2',
'name': 'f2.txt',
'owner': 2
}) # same as above
processed_uploads, error_messages = uploader_cls.check_format(
additional_uploads, 2)
self.assertEqual(len(processed_uploads), 1)
self.assertEqual(len(error_messages), 0)
self.assertEqual(processed_uploads[0]['name'], expected_filename)
uploader = uploader_cls(processed_uploads)
m2 = mock.MagicMock()
uploader.launcher = m2
uploader.upload()
self.assertTrue(m2.go.called)
self.assertEqual(1, m2.go.call_count)
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 3)
self.assertTrue(len(all_resources) == 3)
self.assertTrue(len(all_tc) == 2)
# count complete and incomplete:
self.assertTrue(sum([not x.completed for x in all_transfers]) == 1)
self.assertTrue(sum([x.completed for x in all_transfers]) == 2)
self.assertTrue(sum([not tc.completed for tc in all_tc]) == 1)
self.assertTrue(sum([tc.completed for tc in all_tc]) == 1)
def test_warn_of_conflict_case2(self):
'''
Here, we pretend that a user has previously started an upload that is still going.
Then they try to upload the same files again
'''
source = settings.DROPBOX
uploader_cls = uploaders.get_uploader(source)
self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader)
# prep the upload info as is usually performed:
upload_info = []
upload_info.append({
'source_path': 'https://dropbox-link.com/1',
'name': 'f1.txt',
'owner': 2
})
upload_info.append({
'source_path': 'https://dropbox-link.com/2',
'name': 'f2.txt',
'owner': 2
})
upload_info, error_messages = uploader_cls.check_format(upload_info, 2)
self.assertEqual(len(upload_info), 2)
self.assertEqual(len(error_messages), 0)
# instantiate the class, but mock out the launcher.
# Recall the launcher is the class that actually creates the worker VMs, which
# we do not want to do as part of the test
uploader = uploader_cls(upload_info)
m = mock.MagicMock()
uploader.launcher = m
# mock launch of transfers, which creates the database objects
uploader.upload()
# prep the upload info as is usually performed:
additional_uploads = []
additional_uploads.append({
'source_path': 'https://dropbox-link.com/2',
'name': 'f2.txt',
'owner': 2
}) # same as above- so reject!
additional_uploads.append({
'source_path': 'https://dropbox-link.com/1',
'name': 'f1.txt',
'owner': 2
}) # same as above- reject
processed_uploads, error_messages = uploader_cls.check_format(
additional_uploads, 2)
self.assertEqual(len(processed_uploads), 0)
self.assertEqual(len(error_messages), 2)
uploader = uploader_cls(processed_uploads)
m2 = mock.MagicMock()
uploader.launcher = m2
uploader.upload()
self.assertFalse(m2.go.called)
# check database objects:
all_transfers = Transfer.objects.all()
all_resources = Resource.objects.all()
all_tc = TransferCoordinator.objects.all()
self.assertTrue(len(all_transfers) == 2)
self.assertTrue(len(all_resources) == 2)
self.assertTrue(
len(all_tc) == 1
) # this means no NEW coordinators were created for the 'empty' case
self.assertTrue(all([not x.completed for x in all_transfers
])) # no transfer is complete
self.assertTrue(all([not tc.completed for tc in all_tc
])) # no transfer_coordinators are complete
