def Run(self, args):
env_ref = args.CONCEPTS.environment.Parse()
gcs_subdir = Import.SUBDIR_BASE
if args.destination:
gcs_subdir = posixpath.join(gcs_subdir,
args.destination.strip(posixpath.sep))
gcs_subdir = posixpath.join(gcs_subdir, '')
return storage_util.Import(env_ref, [args.source], gcs_subdir)
def Run(self, args):
storage_util.WarnIfWildcardIsPresent(args.source, '--source')
env_ref = args.CONCEPTS.environment.Parse()
gcs_subdir = Import.SUBDIR_BASE
if args.destination:
gcs_subdir = posixpath.join(gcs_subdir,
args.destination.strip(posixpath.sep))
gcs_subdir = posixpath.join(gcs_subdir, '')
return storage_util.Import(
env_ref, args.source, gcs_subdir, release_track=self.ReleaseTrack())
def testImportSourceDoesNotExist(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
with self.AssertRaisesExceptionMatches(
command_util.Error, 'Source for import does not exist.'):
storage_util.Import(
env_ref, '/my/fake/path', 'dags/subdir')
def testImportSourceIsLocalFile(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectInsert(
gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt'),
file_size=0)
storage_util.Import(env_ref, self.file, 'dags/subdir/')
def testImportSourceIsGcsObject(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
other_bucket_object = gcs_util.ObjectReference(
self.other_bucket, 'dags/source/a.txt')
self.ExpectObjectGet(other_bucket_object)
self.ExpectCopy(
other_bucket_object,
gcs_util.ObjectReference(self.test_gcs_bucket, 'dags/subdir/a.txt'))
storage_util.Import(env_ref, other_bucket_object.ToUrl(), 'dags/subdir/')
def testImportSourceIsDirWithGcloudIgnore(self):
self.Touch(self.temp_path,
os.path.join(self.source_dir, '.gcloudignore'),
contents='.gcloudignore\nc.txt\n')
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectInsert(
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/source/a.txt'),
file_size=0)
storage_util.Import(
env_ref, os.path.join(self.temp_path, self.source_dir), 'dags/subdir/')
def testImportSourceIsGcsDirectoryWithAsterisk(self):
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
self.ExpectObjectList(
self.other_bucket, 'dags/source/', responses=[self.list_response])
self.ExpectCopy(
gcs_util.ObjectReference(self.other_bucket, 'dags/source/a.txt'),
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/a.txt'))
self.ExpectCopy(
gcs_util.ObjectReference(self.other_bucket, 'dags/source/b/c.txt'),
gcs_util.ObjectReference(
self.test_gcs_bucket, 'dags/subdir/b/c.txt'))
storage_util.Import(
env_ref, self.other_bucket_path + '/dags/source/*', 'dags/subdir/')
def testImportSuccessful(self, exec_mock):
"""Tests successful Import call."""
source = 'c/d'
env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME)
self.ExpectEnvironmentGet(
self.TEST_PROJECT,
self.TEST_LOCATION,
self.TEST_ENVIRONMENT_ID,
response=self.MakeEnvironmentWithBucket())
fake_exec = kubectl_util.FakeExec()
exec_mock.side_effect = fake_exec
fake_exec.AddCallback(
0,
self.MakeGsutilExecCallback(
['-m', 'cp', '-r']
+ [source]
+ [self.test_gcs_bucket_path + '/subdir/']))
storage_util.Import(env_ref, source, 'subdir/', release_track=self.track)
fake_exec.Verify()
def _Callback(): env_ref = parsers.ParseEnvironment(self.TEST_ENVIRONMENT_NAME) storage_util.Import(env_ref, '', 'subdir', release_track=self.track)
