def testMissingRecursive(self): with self.assertRaisesRegex( copying.RecursionError, r'Source path matches directories but --recursive was not specified.' ): copier = copying.CopyTaskGenerator() copier.GetCopyTasks([self._Abs('some')], self._Abs('gs://bucket1/'))
def testWildcardDestination(self): dest = self._Abs('foo/*') with self.assertRaisesRegex( copying.WildcardError, r'Destination \[{dest}\] cannot contain wildcards.'.format( dest=re.escape(dest.path))): copier = copying.CopyTaskGenerator() copier.GetCopyTasks([], dest)
def testMultiSourceNoDirDest(self): with self.assertRaisesRegex( copying.Error, r'When copying multiple sources, destination must be a directory ' r'\(a path ending with a slash\).'): copier = copying.CopyTaskGenerator() copier.GetCopyTasks( [self._Abs('some/file'), self._Abs('another/file')], paths.Path('gs://bucket1/o'))
def testNoLocalCopy(self): with self.assertRaisesRegex( copying.LocationMismatchError, r'When destination is a local path, all sources must be remote paths.' ): copier = copying.CopyTaskGenerator() copier.GetCopyTasks( [self._Abs('some/file'), self._Abs('gs://bucket1/file')], self._Abs('foo'))
def Run(self, args): sources = [paths.Path(p) for p in args.source] dest = paths.Path(args.destination) copier = copying.CopyTaskGenerator() tasks = copier.GetCopyTasks(sources, dest, recursive=args.recursive) storage_parallel.ExecuteTasks(tasks, num_threads=args.num_threads, progress_bar_label='Copying Files') log.status.write('Copied [{}] file{}.\n'.format( len(tasks), 's' if len(tasks) > 1 else ''))
def testSingleSource(self, source, dest, expected): copier = copying.CopyTaskGenerator() if source.startswith('gs://') or dest.startswith('gs://'): self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), self.bucket1_resp) tasks = copier.GetCopyTasks([self._Abs(source)], self._Abs(dest)) expected = [e.format(root=self.root_path, s=os.sep) for e in expected] self.assertEqual(expected, [six.text_type(t) for t in tasks])
def testCopyUnderFile(self, source, dest, target): self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), self.bucket1_resp) source = self._Abs(source) dest = self._Abs(dest) with self.assertRaisesRegex( copying.DestinationDirectoryExistsError, r'Cannot copy \[{source}\] to \[{target}\]: ' r'\[{dest}\] exists and is a file.'.format( source=re.escape(source.path), target=re.escape(target), dest=re.escape(dest.path))): copier = copying.CopyTaskGenerator() copier.GetCopyTasks([source], dest, recursive=True)
def testCopyDestExists(self, source, dest, error): self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), self.bucket1_resp) source = self._Abs(source) dest = self._Abs(dest) copier = copying.CopyTaskGenerator() if error: with self.assertRaisesRegex( copying.DestinationDirectoryExistsError, r'Cannot copy \[{source}\] to \[{dest}\]: ' r'The destination already exists.'.format( source=re.escape(source.path), dest=re.escape(dest.path))): copier.GetCopyTasks([source], dest, recursive=True) else: tasks = copier.GetCopyTasks([source], dest, recursive=True) self.assertEqual(1, len(tasks))
def testInterestingPathsRecursiveError(self, bad_path): source = self._Abs('gs://bucket1/') dest = self._Abs('some/dir/') self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), self.messages.Objects( items=[self.messages.Object(name=bad_path + '/file')])) copier = copying.CopyTaskGenerator() bad_source = source.Join(bad_path).Join('file') bad_dest = dest.Join('bucket1').Join(bad_path).Join('file') with self.assertRaisesRegex( copying.InvalidDestinationError, r'Cannot copy \[{source}\] to \[{dest}\] because of "." or ".." in the ' r'path.'.format(source=re.escape(bad_source.path), dest=re.escape(bad_dest.path))): copier.GetCopyTasks([source], dest, recursive=True)
def testDirDest(self, sources, dest, expected): bucket1 = [s for s in sources if 'bucket1' in s] if bucket1 or 'bucket1' in dest: self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), self.bucket1_resp) bucket2 = [s for s in sources if 'bucket2' in s] if bucket2 or 'bucket2' in dest: self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket2'), self.messages.Objects(items=[])) copier = copying.CopyTaskGenerator() sources = [self._Abs(p) for p in sources] dest = self._Abs(dest) tasks = copier.GetCopyTasks(sources, dest) expected = [e.format(root=self.root_path, s=os.sep) for e in expected] self.assertCountEqual(expected, [six.text_type(t) for t in tasks])
def testInterestingPathsError(self, source, dest): copier = copying.CopyTaskGenerator() if source.startswith('gs://') or dest.startswith('gs://'): bucket1_resp = self.messages.Objects(items=[ self.messages.Object(name='file'), self.messages.Object(name='./file'), self.messages.Object(name='../file'), ]) self.client.objects.List.Expect( self.messages.StorageObjectsListRequest(bucket='bucket1'), bucket1_resp) source = self._Abs(source) dest = self._Abs(dest) with self.assertRaisesRegex( copying.InvalidDestinationError, r'Cannot copy \[{source}\] to \[{dest}\] because of "." or ".." in the ' r'path.'.format(source=re.escape(source.path), dest=re.escape(dest.path))): copier.GetCopyTasks([source], dest)