def testMakeDestinationPathsMap(self): source_paths = [ '/foo', '/a/foo', '/a/b/foo', '/c/foo', ] expected = { '/foo': '/x/foo', '/a/foo': '/x/a/foo', '/a/b/foo': '/x/a/b/foo', '/c/foo': '/x/c/foo', } destination_map = utils.make_destination_paths_map(source_paths, '/x') self.assertEqual(expected, destination_map) # Test strip_prefix. source_paths = [ '/a/foo', '/a/b/foo', ] expected = { '/a/foo': '/x/foo', '/a/b/foo': '/x/b/foo', } destination_map = utils.make_destination_paths_map( source_paths, destination_dir_path='/x', strip_prefix='/a') self.assertEqual(expected, destination_map) # With trailing slashes should be equivalent. destination_map = utils.make_destination_paths_map( source_paths, destination_dir_path='/x/', strip_prefix='/a/') self.assertEqual(expected, destination_map) # Error handling. self.assertRaises(ValueError, utils.make_destination_paths_map, '/a/b', '/x') self.assertRaises(ValueError, utils.make_destination_paths_map, ['/a/b'], '/x/', strip_prefix='/fake')
def testMakeDestinationPathsMap(self): source_paths = [ '/foo', '/a/foo', '/a/b/foo', '/c/foo', ] expected = { '/foo': '/x/foo', '/a/foo': '/x/a/foo', '/a/b/foo': '/x/a/b/foo', '/c/foo': '/x/c/foo', } destination_map = utils.make_destination_paths_map(source_paths, '/x') self.assertEqual(expected, destination_map) # Test strip_prefix. source_paths = [ '/a/foo', '/a/b/foo', ] expected = { '/a/foo': '/x/foo', '/a/b/foo': '/x/b/foo', } destination_map = utils.make_destination_paths_map( source_paths, destination_dir_path='/x', strip_prefix='/a') self.assertEqual(expected, destination_map) # With trailing slashes should be equivalent. destination_map = utils.make_destination_paths_map( source_paths, destination_dir_path='/x/', strip_prefix='/a/') self.assertEqual(expected, destination_map) # Error handling. self.assertRaises(ValueError, utils.make_destination_paths_map, '/a/b', '/x') self.assertRaises( ValueError, utils.make_destination_paths_map, ['/a/b'], '/x/', strip_prefix='/fake')
def _move_or_copy_to(self, dir_path, namespace=None, is_move=False, strip_prefix=None, timeout=None, result_files=None, failed_files=None, max_workers=DEFAULT_MAX_WORKERS, **kwargs): """This encapsulate repeated logic for copy_to and move_to methods.""" utils.validate_dir_path(dir_path) destination_map = utils.make_destination_paths_map( self.keys(), destination_dir_path=dir_path, strip_prefix=strip_prefix) future_results = [] if not futures: raise ImportError( 'Moving or copying files requires the Python futures library: ' 'http://code.google.com/p/pythonfutures/') with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: for source_path, destination_path in destination_map.iteritems(): source_file = self[source_path] destination_file = File(destination_path, namespace=namespace, **kwargs) if result_files is not None: result_files.update( Files(files=[destination_file], namespace=namespace)) file_method = source_file.move_to if is_move else source_file.copy_to future = executor.submit(file_method, destination_file) future_results.append(future) futures.wait(future_results, timeout=timeout) errors = [] for future in future_results: try: future.result() except (CopyFileError, MoveFileError) as e: if failed_files is not None: # No namespace here, since this is a source file. failed_files.update(Files(files=[e.titan_file])) # Remove the failed file from successfully copied files collection. if result_files is not None: del result_files[e.titan_file.path] logging.exception('Operation failed:') errors.append(e) # Important: clear the in-context cache since we changed state in threads. ndb.get_context().clear_cache() if errors: raise CopyFilesError( 'Failed to copy files: \n%s' % '\n'.join([str(e) for e in errors]))