def testListSuccessful(self):
"""Tests successful List call."""
responses = [
self.storage_messages.Objects(
items=[
self.storage_messages.Object(name='subdir/file_' +
six.text_type(next_page_token) +
'_' + six.text_type(i))
for i in range(5)
],
nextPageToken=next_page_token)
for next_page_token in ['foo', 'bar', None]
]
expected_list_response = []
for response in responses:
expected_list_response += response.items
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectList(self.test_gcs_bucket, 'subdir/', responses=responses)
actual_list_response = storage_util.List(
env_ref, 'subdir', release_track=self.track)
six.assertCountEqual(self, expected_list_response, actual_list_response)
def testDeleteSuccessful(self, exec_mock):
"""Tests successful Delete call."""
target = 'c/d'
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
fake_exec = kubectl_util.FakeExec()
exec_mock.side_effect = fake_exec
fake_exec.AddCallback(
0,
self.MakeGsutilExecCallback(
['-m', 'rm', '-r',
'{}/subdir/{}'.format(self.test_gcs_bucket_path, target)]))
self.ExpectObjectGet(
gcs_util.ObjectReference(self.test_gcs_bucket, 'subdir/'))
storage_util.Delete(env_ref, target, 'subdir', release_track=self.track)
fake_exec.Verify()
def testExportDestDoesNotExist(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
with self.AssertRaisesExceptionMatches(
command_util.Error, 'Destination for export must be a directory.'):
storage_util.Export(
env_ref, 'dags/source', '/my/fake/path')
def testExportSourceIsFileDestIsLocal(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectGet(self.object_refs[0])
self._ExpectCopyFileFromGcs(self.object_refs[0])
storage_util.Export(
env_ref, 'dags/source/a.txt', self.export_dir)
def testDeleteTargetIsFile(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectGet(self.object_refs[0])
self.ExpectObjectDelete(self.object_refs[0])
self._ExpectEnsureSubdirExists()
storage_util.Delete(env_ref, 'source/a.txt', 'dags')
def testImportSourceDoesNotExist(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
with self.AssertRaisesExceptionMatches(
command_util.Error, 'Source for import does not exist.'):
storage_util.Import(
env_ref, '/my/fake/path', 'dags/subdir')
def testImportSourceIsLocalFile(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectInsert(
gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt'),
file_size=0)
storage_util.Import(env_ref, self.file, 'dags/subdir/')
def testExportLocalDestinationIsNotDirectory(self, isdir_mock, exec_mock):
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
with self.AssertRaisesExceptionMatches(
command_util.Error, 'must be a directory'):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
storage_util.Export(
env_ref, 'subdir/*', 'subdir', release_track=self.track)
def testExportSourceIsFileDestIsGcs(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectGet(self.object_refs[0])
self.ExpectCopy(
self.object_refs[0],
gcs_util.ObjectReference(self.other_bucket, 'dest/a.txt'))
storage_util.Export(
env_ref, 'dags/source/a.txt', self.other_bucket_path + '/dest')
def testImportSourceIsGcsObject(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
other_bucket_object = gcs_util.ObjectReference(
self.other_bucket, 'dags/source/a.txt')
self.ExpectObjectGet(other_bucket_object)
self.ExpectCopy(
other_bucket_object,
gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt'))
storage_util.Import(env_ref, other_bucket_object.ToUrl(), 'dags/subdir/')
def testDeleteTargetIsDir(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self._ExpectFailedGet(self.test_gcs_bucket, 'dags/source/b')
self.ExpectObjectList(
self.test_gcs_bucket, 'dags/source/b/',
responses=[self.storage_messages.Objects(items=[self.objects[1]])])
self.ExpectObjectDelete(self.object_refs[1])
self._ExpectEnsureSubdirExists()
storage_util.Delete(env_ref, 'source/b', 'dags')
def testExportSourceIsDirDestIsLocal(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.list_response.items.append(
self.storage_messages.Object(name='dags/'))
self.ExpectObjectList(
self.test_gcs_bucket, 'dags/', responses=[self.list_response])
for obj_ref in self.object_refs:
self._ExpectCopyFileFromGcs(obj_ref)
storage_util.Export(
env_ref, 'dags/*', self.export_dir)
def testDeleteStar(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
# Star has a special meaning and tells the delete function to not try
# and get the object. This is necessary because subdirs in the GCS buckets
# are created as objects to ensure they exist.
self.ExpectObjectList(
self.test_gcs_bucket, 'dags/', responses=[self.list_response])
for obj_ref in self.object_refs:
self.ExpectObjectDelete(obj_ref)
self._ExpectEnsureSubdirExists()
storage_util.Delete(env_ref, '*', 'dags')
def testImportSourceIsDirWithGcloudIgnore(self):
self.Touch(self.temp_path,
os.path.join(self.source_dir, '.gcloudignore'),
contents='.gcloudignore\nc.txt\n')
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectInsert(
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/source/a.txt'),
file_size=0)
storage_util.Import(
env_ref, os.path.join(self.temp_path, self.source_dir), 'dags/subdir/')
def testImportSourceIsGcsDirectoryWithAsterisk(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectList(
self.other_bucket, 'dags/source/', responses=[self.list_response])
self.ExpectCopy(
gcs_util.ObjectReference(self.other_bucket, 'dags/source/a.txt'),
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/a.txt'))
self.ExpectCopy(
gcs_util.ObjectReference(self.other_bucket, 'dags/source/b/c.txt'),
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/b/c.txt'))
storage_util.Import(
env_ref, self.other_bucket_path + '/dags/source/*', 'dags/subdir/')
def testListStorageListFails(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectList(
self.test_gcs_bucket,
'subdir/',
exception=http_error.MakeHttpError(
code=403, message='PERMISSION_DENIED'))
with self.AssertRaisesExceptionMatches(
storage_api.ListBucketError,
r'403 Could not list bucket [{}]: PERMISSION_DENIED'.format(
self.test_gcs_bucket)):
list(storage_util.List(env_ref, 'subdir', release_track=self.track))
def testExportSourceIsDirDestAlreadyExists(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.list_response.items.append(
self.storage_messages.Object(name='dags/'))
self.ExpectObjectList(
self.test_gcs_bucket, 'dags/', responses=[self.list_response])
for obj_ref in self.object_refs:
self._ExpectCopyFileFromGcs(obj_ref)
# Create a file that will be overwritten.
self.file = self.Touch(
self.temp_path,
os.path.join('dest', self.source_dir, 'a.txt'),
makedirs=True)
storage_util.Export(
env_ref, 'dags/*', self.export_dir)
def testExportSuccessful(self, isdir_mock, exec_mock):
"""Tests successful Export call."""
source = 'c/d'
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
fake_exec = kubectl_util.FakeExec()
exec_mock.side_effect = fake_exec
fake_exec.AddCallback(
0,
self.MakeGsutilExecCallback(
['-m', 'cp', '-r']
+ [posixpath.join(self.test_gcs_bucket_path, source)]
+ ['subdir']))
storage_util.Export(env_ref, source, 'subdir', release_track=self.track)
fake_exec.Verify()
def testExportGcsDestinationHasSlashAdded(self, isdir_mock, exec_mock):
"""Tests that a trailing slash is automatically added to gs:// dests."""
source = 'c/d'
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
fake_exec = kubectl_util.FakeExec()
exec_mock.side_effect = fake_exec
fake_exec.AddCallback(
0,
self.MakeGsutilExecCallback(
['-m', 'cp', '-r']
+ [posixpath.join(self.test_gcs_bucket_path, source)]
+ ['gs://subdir/']))
storage_util.Export(
env_ref, source, 'gs://subdir', release_track=self.track)
fake_exec.Verify()
def _Callback(): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) storage_util.List(env_ref, 'subdir', release_track=self.track)
