def testListSuccessful(self): """Tests successful List call.""" responses = [ self.storage_messages.Objects( items=[ self.storage_messages.Object(name='subdir/file_' + six.text_type(next_page_token) + '_' + six.text_type(i)) for i in range(5) ], nextPageToken=next_page_token) for next_page_token in ['foo', 'bar', None] ] expected_list_response = [] for response in responses: expected_list_response += response.items env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectList(self.test_gcs_bucket, 'subdir/', responses=responses) actual_list_response = storage_util.List( env_ref, 'subdir', release_track=self.track) six.assertCountEqual(self, expected_list_response, actual_list_response)
def testDeleteSuccessful(self, exec_mock): """Tests successful Delete call.""" target = 'c/d' env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) fake_exec = kubectl_util.FakeExec() exec_mock.side_effect = fake_exec fake_exec.AddCallback( 0, self.MakeGsutilExecCallback( ['-m', 'rm', '-r', '{}/subdir/{}'.format(self.test_gcs_bucket_path, target)])) self.ExpectObjectGet( gcs_util.ObjectReference(self.test_gcs_bucket, 'subdir/')) storage_util.Delete(env_ref, target, 'subdir', release_track=self.track) fake_exec.Verify()
def testExportDestDoesNotExist(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) with self.AssertRaisesExceptionMatches( command_util.Error, 'Destination for export must be a directory.'): storage_util.Export( env_ref, 'dags/source', '/my/fake/path')
def testExportSourceIsFileDestIsLocal(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectGet(self.object_refs[0]) self._ExpectCopyFileFromGcs(self.object_refs[0]) storage_util.Export( env_ref, 'dags/source/a.txt', self.export_dir)
def testDeleteTargetIsFile(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectGet(self.object_refs[0]) self.ExpectObjectDelete(self.object_refs[0]) self._ExpectEnsureSubdirExists() storage_util.Delete(env_ref, 'source/a.txt', 'dags')
def testImportSourceDoesNotExist(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) with self.AssertRaisesExceptionMatches( command_util.Error, 'Source for import does not exist.'): storage_util.Import( env_ref, '/my/fake/path', 'dags/subdir')
def testImportSourceIsLocalFile(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectInsert( gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt'), file_size=0) storage_util.Import(env_ref, self.file, 'dags/subdir/')
def testExportLocalDestinationIsNotDirectory(self, isdir_mock, exec_mock): self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) with self.AssertRaisesExceptionMatches( command_util.Error, 'must be a directory'): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) storage_util.Export( env_ref, 'subdir/*', 'subdir', release_track=self.track)
def testExportSourceIsFileDestIsGcs(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectGet(self.object_refs[0]) self.ExpectCopy( self.object_refs[0], gcs_util.ObjectReference(self.other_bucket, 'dest/a.txt')) storage_util.Export( env_ref, 'dags/source/a.txt', self.other_bucket_path + '/dest')
def testImportSourceIsGcsObject(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) other_bucket_object = gcs_util.ObjectReference( self.other_bucket, 'dags/source/a.txt') self.ExpectObjectGet(other_bucket_object) self.ExpectCopy( other_bucket_object, gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt')) storage_util.Import(env_ref, other_bucket_object.ToUrl(), 'dags/subdir/')
def testDeleteTargetIsDir(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self._ExpectFailedGet(self.test_gcs_bucket, 'dags/source/b') self.ExpectObjectList( self.test_gcs_bucket, 'dags/source/b/', responses=[self.storage_messages.Objects(items=[self.objects[1]])]) self.ExpectObjectDelete(self.object_refs[1]) self._ExpectEnsureSubdirExists() storage_util.Delete(env_ref, 'source/b', 'dags')
def testExportSourceIsDirDestIsLocal(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.list_response.items.append( self.storage_messages.Object(name='dags/')) self.ExpectObjectList( self.test_gcs_bucket, 'dags/', responses=[self.list_response]) for obj_ref in self.object_refs: self._ExpectCopyFileFromGcs(obj_ref) storage_util.Export( env_ref, 'dags/*', self.export_dir)
def testDeleteStar(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) # Star has a special meaning and tells the delete function to not try # and get the object. This is necessary because subdirs in the GCS buckets # are created as objects to ensure they exist. self.ExpectObjectList( self.test_gcs_bucket, 'dags/', responses=[self.list_response]) for obj_ref in self.object_refs: self.ExpectObjectDelete(obj_ref) self._ExpectEnsureSubdirExists() storage_util.Delete(env_ref, '*', 'dags')
def testImportSourceIsDirWithGcloudIgnore(self): self.Touch(self.temp_path, os.path.join(self.source_dir, '.gcloudignore'), contents='.gcloudignore\nc.txt\n') env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectInsert( gcs_util.ObjectReference( self.test_gcs_bucket, 'dags/subdir/source/a.txt'), file_size=0) storage_util.Import( env_ref, os.path.join(self.temp_path, self.source_dir), 'dags/subdir/')
def testImportSourceIsGcsDirectoryWithAsterisk(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectList( self.other_bucket, 'dags/source/', responses=[self.list_response]) self.ExpectCopy( gcs_util.ObjectReference(self.other_bucket, 'dags/source/a.txt'), gcs_util.ObjectReference( self.test_gcs_bucket, 'dags/subdir/a.txt')) self.ExpectCopy( gcs_util.ObjectReference(self.other_bucket, 'dags/source/b/c.txt'), gcs_util.ObjectReference( self.test_gcs_bucket, 'dags/subdir/b/c.txt')) storage_util.Import( env_ref, self.other_bucket_path + '/dags/source/*', 'dags/subdir/')
def testListStorageListFails(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.ExpectObjectList( self.test_gcs_bucket, 'subdir/', exception=http_error.MakeHttpError( code=403, message='PERMISSION_DENIED')) with self.AssertRaisesExceptionMatches( storage_api.ListBucketError, r'403 Could not list bucket [{}]: PERMISSION_DENIED'.format( self.test_gcs_bucket)): list(storage_util.List(env_ref, 'subdir', release_track=self.track))
def testExportSourceIsDirDestAlreadyExists(self): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) self.list_response.items.append( self.storage_messages.Object(name='dags/')) self.ExpectObjectList( self.test_gcs_bucket, 'dags/', responses=[self.list_response]) for obj_ref in self.object_refs: self._ExpectCopyFileFromGcs(obj_ref) # Create a file that will be overwritten. self.file = self.Touch( self.temp_path, os.path.join('dest', self.source_dir, 'a.txt'), makedirs=True) storage_util.Export( env_ref, 'dags/*', self.export_dir)
def testExportSuccessful(self, isdir_mock, exec_mock): """Tests successful Export call.""" source = 'c/d' env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) fake_exec = kubectl_util.FakeExec() exec_mock.side_effect = fake_exec fake_exec.AddCallback( 0, self.MakeGsutilExecCallback( ['-m', 'cp', '-r'] + [posixpath.join(self.test_gcs_bucket_path, source)] + ['subdir'])) storage_util.Export(env_ref, source, 'subdir', release_track=self.track) fake_exec.Verify()
def testExportGcsDestinationHasSlashAdded(self, isdir_mock, exec_mock): """Tests that a trailing slash is automatically added to gs:// dests.""" source = 'c/d' env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) self.ExpectEnvironmentGet( self.TEST_PROJECT, self.TEST_LOCATION, self.TEST_ENVIRONMENT_ID, response=self.MakeEnvironmentWithBucket()) fake_exec = kubectl_util.FakeExec() exec_mock.side_effect = fake_exec fake_exec.AddCallback( 0, self.MakeGsutilExecCallback( ['-m', 'cp', '-r'] + [posixpath.join(self.test_gcs_bucket_path, source)] + ['gs://subdir/'])) storage_util.Export( env_ref, source, 'gs://subdir', release_track=self.track) fake_exec.Verify()
def _Callback(): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) storage_util.List(env_ref, 'subdir', release_track=self.track)