def testMissingRecursive(self):
with self.assertRaisesRegex(
copying.RecursionError,
r'Source path matches directories but --recursive was not specified.'
):
copier = copying.CopyTaskGenerator()
copier.GetCopyTasks([self._Abs('some')],
self._Abs('gs://bucket1/'))
def testWildcardDestination(self):
dest = self._Abs('foo/*')
with self.assertRaisesRegex(
copying.WildcardError,
r'Destination \[{dest}\] cannot contain wildcards.'.format(
dest=re.escape(dest.path))):
copier = copying.CopyTaskGenerator()
copier.GetCopyTasks([], dest)
def testMultiSourceNoDirDest(self):
with self.assertRaisesRegex(
copying.Error,
r'When copying multiple sources, destination must be a directory '
r'\(a path ending with a slash\).'):
copier = copying.CopyTaskGenerator()
copier.GetCopyTasks(
[self._Abs('some/file'),
self._Abs('another/file')], paths.Path('gs://bucket1/o'))
def testNoLocalCopy(self):
with self.assertRaisesRegex(
copying.LocationMismatchError,
r'When destination is a local path, all sources must be remote paths.'
):
copier = copying.CopyTaskGenerator()
copier.GetCopyTasks(
[self._Abs('some/file'),
self._Abs('gs://bucket1/file')], self._Abs('foo'))
def Run(self, args):
sources = [paths.Path(p) for p in args.source]
dest = paths.Path(args.destination)
copier = copying.CopyTaskGenerator()
tasks = copier.GetCopyTasks(sources, dest, recursive=args.recursive)
storage_parallel.ExecuteTasks(tasks,
num_threads=args.num_threads,
progress_bar_label='Copying Files')
log.status.write('Copied [{}] file{}.\n'.format(
len(tasks), 's' if len(tasks) > 1 else ''))
def testSingleSource(self, source, dest, expected):
copier = copying.CopyTaskGenerator()
if source.startswith('gs://') or dest.startswith('gs://'):
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
self.bucket1_resp)
tasks = copier.GetCopyTasks([self._Abs(source)], self._Abs(dest))
expected = [e.format(root=self.root_path, s=os.sep) for e in expected]
self.assertEqual(expected, [six.text_type(t) for t in tasks])
def testCopyUnderFile(self, source, dest, target):
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
self.bucket1_resp)
source = self._Abs(source)
dest = self._Abs(dest)
with self.assertRaisesRegex(
copying.DestinationDirectoryExistsError,
r'Cannot copy \[{source}\] to \[{target}\]: '
r'\[{dest}\] exists and is a file.'.format(
source=re.escape(source.path),
target=re.escape(target),
dest=re.escape(dest.path))):
copier = copying.CopyTaskGenerator()
copier.GetCopyTasks([source], dest, recursive=True)
def testCopyDestExists(self, source, dest, error):
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
self.bucket1_resp)
source = self._Abs(source)
dest = self._Abs(dest)
copier = copying.CopyTaskGenerator()
if error:
with self.assertRaisesRegex(
copying.DestinationDirectoryExistsError,
r'Cannot copy \[{source}\] to \[{dest}\]: '
r'The destination already exists.'.format(
source=re.escape(source.path),
dest=re.escape(dest.path))):
copier.GetCopyTasks([source], dest, recursive=True)
else:
tasks = copier.GetCopyTasks([source], dest, recursive=True)
self.assertEqual(1, len(tasks))
def testInterestingPathsRecursiveError(self, bad_path):
source = self._Abs('gs://bucket1/')
dest = self._Abs('some/dir/')
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
self.messages.Objects(
items=[self.messages.Object(name=bad_path + '/file')]))
copier = copying.CopyTaskGenerator()
bad_source = source.Join(bad_path).Join('file')
bad_dest = dest.Join('bucket1').Join(bad_path).Join('file')
with self.assertRaisesRegex(
copying.InvalidDestinationError,
r'Cannot copy \[{source}\] to \[{dest}\] because of "." or ".." in the '
r'path.'.format(source=re.escape(bad_source.path),
dest=re.escape(bad_dest.path))):
copier.GetCopyTasks([source], dest, recursive=True)
def testDirDest(self, sources, dest, expected):
bucket1 = [s for s in sources if 'bucket1' in s]
if bucket1 or 'bucket1' in dest:
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
self.bucket1_resp)
bucket2 = [s for s in sources if 'bucket2' in s]
if bucket2 or 'bucket2' in dest:
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket2'),
self.messages.Objects(items=[]))
copier = copying.CopyTaskGenerator()
sources = [self._Abs(p) for p in sources]
dest = self._Abs(dest)
tasks = copier.GetCopyTasks(sources, dest)
expected = [e.format(root=self.root_path, s=os.sep) for e in expected]
self.assertCountEqual(expected, [six.text_type(t) for t in tasks])
def testInterestingPathsError(self, source, dest):
copier = copying.CopyTaskGenerator()
if source.startswith('gs://') or dest.startswith('gs://'):
bucket1_resp = self.messages.Objects(items=[
self.messages.Object(name='file'),
self.messages.Object(name='./file'),
self.messages.Object(name='../file'),
])
self.client.objects.List.Expect(
self.messages.StorageObjectsListRequest(bucket='bucket1'),
bucket1_resp)
source = self._Abs(source)
dest = self._Abs(dest)
with self.assertRaisesRegex(
copying.InvalidDestinationError,
r'Cannot copy \[{source}\] to \[{dest}\] because of "." or ".." in the '
r'path.'.format(source=re.escape(source.path),
dest=re.escape(dest.path))):
copier.GetCopyTasks([source], dest)