def test_dropbox_uploader_on_google_params_single(self): source = settings.DROPBOX uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader) upload_info = { 'source_path': 'https://dropbox-link.com/1', 'name': 'f1.txt', 'owner': 2 } upload_info, error_messages = uploader_cls.check_format(upload_info, 2) uploader = uploader_cls(upload_info) m = mock.MagicMock() uploader.launcher = m uploader.upload() self.assertTrue(m.go.called) self.assertEqual(1, m.go.call_count) # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 1) self.assertTrue(len(all_resources) == 1) self.assertTrue(len(all_tc) == 1) self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertFalse( all_tc[0].completed) # the transfer coord is also not completed
def test_drive_uploader_on_google_params_single(self): source = settings.GOOGLE_DRIVE uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDriveUploader) upload_info = { 'file_id': 'abc123', 'name': 'f1.txt', 'drive_token': 'fooToken', 'owner': 2 } upload_info, error_messages = uploader_cls.check_format(upload_info, 2) uploader = uploader_cls(upload_info) m = mock.MagicMock() uploader.launcher = m uploader.upload() self.assertTrue(m.go.called) self.assertEqual(1, m.go.call_count) # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 1) self.assertTrue(len(all_resources) == 1) self.assertTrue(len(all_tc) == 1) self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertFalse( all_tc[0].completed) # the transfer coord is also not completed
def test_dropbox_uploader_on_google_disk_sizing(self): source = settings.DROPBOX uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader) upload_info = [] filesize = 100e9 # 100GB upload_info.append({ 'source_path': 'https://dropbox-link.com/1', 'name': 'f1.txt', 'owner': 2, 'size_in_bytes': filesize }) upload_info, error_messages = uploader_cls.check_format(upload_info, 2) uploader = uploader_cls(upload_info) uploader.config_params['disk_size_factor'] = 3 m = mock.MagicMock() uploader.launcher = m uploader.upload() self.assertTrue(m.go.called) self.assertEqual(1, m.go.call_count) call_arg = m.go.call_args the_call = call_arg.call_list()[0] import re target = '--boot-disk-size=300GB' matches = re.findall(target, str(the_call)) self.assertEqual(len(matches), 1)
def upload(upload_info, upload_source): ''' upload_info is a list, with each entry a dictionary. Each of those dictionaries has keys which are specific to the upload source ''' uploader_cls = uploaders.get_uploader(upload_source) uploader = uploader_cls(upload_info) uploader.upload()
def test_drive_uploader_on_google_params(self): ''' This test takes a properly formatted request and checks that the database objects have been properly created. ''' source = settings.GOOGLE_DRIVE uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDriveUploader) # prep the upload info as is usually performed: upload_info = [] upload_info.append({ 'file_id': 'abc123', 'name': 'f1.txt', 'drive_token': 'fooToken', 'owner': 2 }) upload_info.append({ 'file_id': 'def123', 'name': 'f2.txt', 'drive_token': 'fooToken', 'owner': 2 }) upload_info, error_messages = uploader_cls.check_format(upload_info, 2) # instantiate the class, but mock out the launcher. # Recall the launcher is the class that actually creates the worker VMs, which # we do not want to do as part of the test uploader = uploader_cls(upload_info) m = mock.MagicMock() uploader.launcher = m # now that the launcher has been mocked, call the upload method which # constructs the request to start a new VM. It is difficult to check that, so # we only check that the proper database objects have been created uploader.upload() self.assertTrue(m.go.called) self.assertEqual(2, m.go.call_count) # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 2) self.assertTrue(len(all_resources) == 2) self.assertTrue(len(all_tc) == 1) self.assertTrue( all([x.source == settings.GOOGLE_DRIVE for x in all_resources])) self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertFalse( all_tc[0].completed) # the transfer coord is also not completed
def test_uploader_mod_returns_correct_uploader_implementation(self): # first try dropbox uploader on google: settings.CONFIG_PARAMS['cloud_environment'] = settings.GOOGLE source = settings.DROPBOX result_cls = uploaders.get_uploader(source) self.assertEqual(result_cls, uploaders.GoogleDropboxUploader) settings.CONFIG_PARAMS['cloud_environment'] = settings.GOOGLE source = settings.GOOGLE_DRIVE result_cls = uploaders.get_uploader(source) self.assertEqual(result_cls, uploaders.GoogleDriveUploader) settings.CONFIG_PARAMS['cloud_environment'] = settings.AWS source = settings.DROPBOX result_cls = uploaders.get_uploader(source) self.assertEqual(result_cls, uploaders.AWSDropboxUploader) settings.CONFIG_PARAMS['cloud_environment'] = settings.AWS source = settings.GOOGLE_DRIVE result_cls = uploaders.get_uploader(source) self.assertEqual(result_cls, uploaders.AWSDriveUploader)
def test_reject_long_name_in_google(self): source = settings.DROPBOX uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader) upload_info = [] long_name = 'x' * 2000 + '.txt' upload_info.append({ 'source_path': 'https://dropbox-link.com/1', 'name': long_name, 'owner': 2 }) upload_info.append({ 'source_path': 'https://dropbox-link.com/2', 'name': 'f2.txt', 'owner': 2 }) with self.assertRaises(exceptions.FilenameException): upload_info, error_messages = uploader_cls.check_format( upload_info, 2)
def post(self, request, *args, **kwargs): data = request.data try: upload_source = data['upload_source'] # (dropbox, drive, etc) upload_info = data['upload_info'] upload_info = json.loads(upload_info) #raise Exception('temp!'); except KeyError as ex: raise exceptions.RequestError( 'The request JSON body did not contain the required data (%s).' % ex) # Here, we first do a spot-check on the data that was passed, BEFORE we invoke any asynchronous methods. # We prepare/massage the necessary data for the upload here, and then pass a simple dictionary to the asynchronous # method call. We do this since it is easiest to pass a simple native dictionary to celery. user_pk = request.user.pk try: # Depending on which upload method was requested (and which compute environment we are in), grab the proper class: uploader_cls = _uploaders.get_uploader(upload_source) # Check that the upload data has the required format to work with this uploader implementation: upload_info, error_messages = uploader_cls.check_format( upload_info, user_pk) if len(error_messages) > 0: return Response({'errors': error_messages}) elif len(upload_info) > 0: # call async method: transfer_tasks.upload.delay(upload_info, upload_source) return Response({}) else: # no errors, but also nothing to do... return Response({}) except exceptions.ExceptionWithMessage as ex: raise exceptions.RequestError(ex.message) except Exception as ex: print('Exception: %s' % ex) response = exception_handler(ex, None) return response
def test_warn_of_conflict_case3(self, mock_datetime): ''' Here, we initiate two transfers. We mock one being completed, and THEN the user uploads another to the same as an overwrite. We want to allow this. ''' source = settings.DROPBOX uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader) # prep the upload info as is usually performed: upload_info = [] upload_info.append({ 'source_path': 'https://dropbox-link.com/1', 'name': 'f1.txt', 'owner': 2 }) upload_info.append({ 'source_path': 'https://dropbox-link.com/2', 'name': 'f2.txt', 'owner': 2 }) upload_info, error_messages = uploader_cls.check_format(upload_info, 2) self.assertEqual(len(upload_info), 2) self.assertEqual(len(error_messages), 0) # instantiate the class, but mock out the launcher. # Recall the launcher is the class that actually creates the worker VMs, which # we do not want to do as part of the test uploader = uploader_cls(upload_info) m = mock.MagicMock() uploader.launcher = m # mock launch of transfers, which creates the database objects uploader.upload() # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 2) self.assertTrue(len(all_resources) == 2) self.assertTrue( len(all_tc) == 1 ) # this means no NEW coordinators were created for the 'empty' case self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertTrue(all([not tc.completed for tc in all_tc ])) # no transfer_coordinators are complete # make the transfers 'complete' and set the Resources to 'active' for t in all_transfers: t.completed = True t.save() for tc in all_tc: tc.completed = True tc.save() for r in all_resources: r.is_active = True r.save() # prep the upload info as is usually performed: # setup the mock to return d = datetime.datetime(2019, 4, 15, 21, 3, 7, 0) mock_datetime.datetime.now.return_value = d expected_stamp = d.strftime('%m%d%Y-%H%M%S') expected_filename = 'f2.' + expected_stamp + '.txt' additional_uploads = [] additional_uploads.append({ 'source_path': 'https://dropbox-link.com/2', 'name': 'f2.txt', 'owner': 2 }) # same as above processed_uploads, error_messages = uploader_cls.check_format( additional_uploads, 2) self.assertEqual(len(processed_uploads), 1) self.assertEqual(len(error_messages), 0) self.assertEqual(processed_uploads[0]['name'], expected_filename) uploader = uploader_cls(processed_uploads) m2 = mock.MagicMock() uploader.launcher = m2 uploader.upload() self.assertTrue(m2.go.called) self.assertEqual(1, m2.go.call_count) # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 3) self.assertTrue(len(all_resources) == 3) self.assertTrue(len(all_tc) == 2) # count complete and incomplete: self.assertTrue(sum([not x.completed for x in all_transfers]) == 1) self.assertTrue(sum([x.completed for x in all_transfers]) == 2) self.assertTrue(sum([not tc.completed for tc in all_tc]) == 1) self.assertTrue(sum([tc.completed for tc in all_tc]) == 1)
def test_warn_of_conflict_case2(self): ''' Here, we pretend that a user has previously started an upload that is still going. Then they try to upload the same files again ''' source = settings.DROPBOX uploader_cls = uploaders.get_uploader(source) self.assertEqual(uploader_cls, uploaders.GoogleDropboxUploader) # prep the upload info as is usually performed: upload_info = [] upload_info.append({ 'source_path': 'https://dropbox-link.com/1', 'name': 'f1.txt', 'owner': 2 }) upload_info.append({ 'source_path': 'https://dropbox-link.com/2', 'name': 'f2.txt', 'owner': 2 }) upload_info, error_messages = uploader_cls.check_format(upload_info, 2) self.assertEqual(len(upload_info), 2) self.assertEqual(len(error_messages), 0) # instantiate the class, but mock out the launcher. # Recall the launcher is the class that actually creates the worker VMs, which # we do not want to do as part of the test uploader = uploader_cls(upload_info) m = mock.MagicMock() uploader.launcher = m # mock launch of transfers, which creates the database objects uploader.upload() # prep the upload info as is usually performed: additional_uploads = [] additional_uploads.append({ 'source_path': 'https://dropbox-link.com/2', 'name': 'f2.txt', 'owner': 2 }) # same as above- so reject! additional_uploads.append({ 'source_path': 'https://dropbox-link.com/1', 'name': 'f1.txt', 'owner': 2 }) # same as above- reject processed_uploads, error_messages = uploader_cls.check_format( additional_uploads, 2) self.assertEqual(len(processed_uploads), 0) self.assertEqual(len(error_messages), 2) uploader = uploader_cls(processed_uploads) m2 = mock.MagicMock() uploader.launcher = m2 uploader.upload() self.assertFalse(m2.go.called) # check database objects: all_transfers = Transfer.objects.all() all_resources = Resource.objects.all() all_tc = TransferCoordinator.objects.all() self.assertTrue(len(all_transfers) == 2) self.assertTrue(len(all_resources) == 2) self.assertTrue( len(all_tc) == 1 ) # this means no NEW coordinators were created for the 'empty' case self.assertTrue(all([not x.completed for x in all_transfers ])) # no transfer is complete self.assertTrue(all([not tc.completed for tc in all_tc ])) # no transfer_coordinators are complete