def testMakeDestinationPathsMap(self):
source_paths = [
'/foo',
'/a/foo',
'/a/b/foo',
'/c/foo',
]
expected = {
'/foo': '/x/foo',
'/a/foo': '/x/a/foo',
'/a/b/foo': '/x/a/b/foo',
'/c/foo': '/x/c/foo',
}
destination_map = utils.make_destination_paths_map(source_paths, '/x')
self.assertEqual(expected, destination_map)
# Test strip_prefix.
source_paths = [
'/a/foo',
'/a/b/foo',
]
expected = {
'/a/foo': '/x/foo',
'/a/b/foo': '/x/b/foo',
}
destination_map = utils.make_destination_paths_map(
source_paths, destination_dir_path='/x', strip_prefix='/a')
self.assertEqual(expected, destination_map)
# With trailing slashes should be equivalent.
destination_map = utils.make_destination_paths_map(
source_paths, destination_dir_path='/x/', strip_prefix='/a/')
self.assertEqual(expected, destination_map)
# Error handling.
self.assertRaises(ValueError, utils.make_destination_paths_map, '/a/b',
'/x')
self.assertRaises(ValueError,
utils.make_destination_paths_map, ['/a/b'],
'/x/',
strip_prefix='/fake')
def testMakeDestinationPathsMap(self):
source_paths = [
'/foo',
'/a/foo',
'/a/b/foo',
'/c/foo',
]
expected = {
'/foo': '/x/foo',
'/a/foo': '/x/a/foo',
'/a/b/foo': '/x/a/b/foo',
'/c/foo': '/x/c/foo',
}
destination_map = utils.make_destination_paths_map(source_paths, '/x')
self.assertEqual(expected, destination_map)
# Test strip_prefix.
source_paths = [
'/a/foo',
'/a/b/foo',
]
expected = {
'/a/foo': '/x/foo',
'/a/b/foo': '/x/b/foo',
}
destination_map = utils.make_destination_paths_map(
source_paths, destination_dir_path='/x', strip_prefix='/a')
self.assertEqual(expected, destination_map)
# With trailing slashes should be equivalent.
destination_map = utils.make_destination_paths_map(
source_paths, destination_dir_path='/x/', strip_prefix='/a/')
self.assertEqual(expected, destination_map)
# Error handling.
self.assertRaises(ValueError, utils.make_destination_paths_map, '/a/b', '/x')
self.assertRaises(
ValueError,
utils.make_destination_paths_map, ['/a/b'], '/x/', strip_prefix='/fake')
def _move_or_copy_to(self, dir_path, namespace=None, is_move=False,
strip_prefix=None, timeout=None, result_files=None,
failed_files=None, max_workers=DEFAULT_MAX_WORKERS,
**kwargs):
"""This encapsulate repeated logic for copy_to and move_to methods."""
utils.validate_dir_path(dir_path)
destination_map = utils.make_destination_paths_map(
self.keys(), destination_dir_path=dir_path, strip_prefix=strip_prefix)
future_results = []
if not futures:
raise ImportError(
'Moving or copying files requires the Python futures library: '
'http://code.google.com/p/pythonfutures/')
with futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
for source_path, destination_path in destination_map.iteritems():
source_file = self[source_path]
destination_file = File(destination_path, namespace=namespace, **kwargs)
if result_files is not None:
result_files.update(
Files(files=[destination_file], namespace=namespace))
file_method = source_file.move_to if is_move else source_file.copy_to
future = executor.submit(file_method, destination_file)
future_results.append(future)
futures.wait(future_results, timeout=timeout)
errors = []
for future in future_results:
try:
future.result()
except (CopyFileError, MoveFileError) as e:
if failed_files is not None:
# No namespace here, since this is a source file.
failed_files.update(Files(files=[e.titan_file]))
# Remove the failed file from successfully copied files collection.
if result_files is not None:
del result_files[e.titan_file.path]
logging.exception('Operation failed:')
errors.append(e)
# Important: clear the in-context cache since we changed state in threads.
ndb.get_context().clear_cache()
if errors:
raise CopyFilesError(
'Failed to copy files: \n%s' % '\n'.join([str(e) for e in errors]))
