def recurse(self):
"""Recursively synchronise everything."""
source_dir_list = self.source_transport.listdir(self.source)
dest = FileObject(self.destination_transport, self.destination)
# If the source is a file, rather than a directory, just copy it. We know for sure that
# it exists from the checks we did before, so the "False" return value can't be because
# of that.
if not source_dir_list:
# If the destination ends in a slash or is an actual directory:
if self.destination.endswith("/") or dest.isdir:
if not dest.isdir:
self.destination_transport.mkdir(dest.url)
# Splice the source filename onto the destination URL.
dest_url = url_split(dest.url)
dest_url.file = url_split(self.source,
uses_hostname=self.source_transport.uses_hostname,
split_filename=True).file
dest_url = url_join(dest_url)
else:
dest_url = self.destination
self.compare_and_copy(
FileObject(self.source_transport, self.source, {"isdir": False}),
FileObject(self.destination_transport, dest_url, {"isdir": False}),
)
return
# If source is a directory...
directory_stack = [FileObject(self.source_transport, self.source, {"isdir": True})]
# Depth-first tree traversal.
while directory_stack:
# TODO: Rethink the assumption that a file cannot have the same name as a directory.
item = directory_stack.pop()
log.debug("URL %s is %sa directory." % \
(item.url, not item.isdir and "not " or ""))
if item.isdir:
# Don't skip the first directory.
if not self.config.recursive and item.url != self.source:
log.info("Skipping directory %s..." % item)
continue
# Obtain a directory list.
new_dir_list = []
for new_file in reversed(self.source_transport.listdir(item.url)):
if self.include_file(new_file):
new_dir_list.append(new_file)
else:
log.debug("Skipping %s..." % (new_file))
dest = url_splice(self.source, item.url, self.destination)
dest = FileObject(self.destination_transport, dest)
log.debug("Comparing directories %s and %s..." % (item.url, dest.url))
self.compare_directories(item, new_dir_list, dest.url)
directory_stack.extend(new_dir_list)
else:
dest_url = url_splice(self.source, item.url, self.destination)
log.debug("Destination URL is %s." % dest_url)
dest = FileObject(self.destination_transport, dest_url)
self.compare_and_copy(item, dest)
def test_url_splice(self):
"""Test url_splice."""
tests = (
(
("file://*****:*****@myhost:21/test/",
"ftp://*****:*****@myhost:21/test/file",
"file://otherhost:21/test;someparams"),
"file://otherhost:21/test/file;someparams",
),
)
for test, expected_output in tests:
self.assertEqual(urlfunctions.url_splice(*test), expected_output)
def test_url_splice(self):
"""Test url_splice."""
tests = (
(("file://*****:*****@myhost:21/test/",
"ftp://*****:*****@myhost:21/test/file",
"file://otherhost:21/test;someparams"),
"file://otherhost:21/test/file;someparams",
),
)
for test, expected_output in tests:
self.assertEqual(urlfunctions.url_splice(*test), expected_output)
def recurse(self):
"""Recursively synchronise everything."""
source_dir_list = self.source_transport.listdir(self.source)
dest = FileObject(self.destination_transport, self.destination)
# If the source is a file, rather than a directory, just copy it. We know for sure that
# it exists from the checks we did before, so the "False" return value can't be because
# of that.
if not source_dir_list:
# If the destination ends in a slash or is an actual directory:
if self.destination.endswith("/") or dest.isdir:
if not dest.isdir:
self.destination_transport.mkdir(dest.url)
# Splice the source filename onto the destination URL.
dest_url = url_split(dest.url)
dest_url.file = url_split(
self.source,
uses_hostname=self.source_transport.uses_hostname,
split_filename=True).file
dest_url = url_join(dest_url)
else:
dest_url = self.destination
self.compare_and_copy(
FileObject(self.source_transport, self.source,
{"isdir": False}),
FileObject(self.destination_transport, dest_url,
{"isdir": False}),
)
return
# If source is a directory...
directory_stack = [
FileObject(self.source_transport, self.source, {"isdir": True})
]
# Depth-first tree traversal.
while directory_stack:
# TODO: Rethink the assumption that a file cannot have the same name as a directory.
item = directory_stack.pop()
log.debug("URL %s is %sa directory." % \
(item.url, not item.isdir and "not " or ""))
if item.isdir:
# Don't skip the first directory.
if not self.config.recursive and item.url != self.source:
log.info("Skipping directory %s..." % item)
continue
# Obtain a directory list.
new_dir_list = []
for new_file in reversed(
self.source_transport.listdir(item.url)):
if self.include_file(new_file):
new_dir_list.append(new_file)
else:
log.debug("Skipping %s..." % (new_file))
dest = url_splice(self.source, item.url, self.destination)
dest = FileObject(self.destination_transport, dest)
log.debug("Comparing directories %s and %s..." %
(item.url, dest.url))
self.compare_directories(item, new_dir_list, dest.url)
directory_stack.extend(new_dir_list)
else:
dest_url = url_splice(self.source, item.url, self.destination)
log.debug("Destination URL is %s." % dest_url)
dest = FileObject(self.destination_transport, dest_url)
self.compare_and_copy(item, dest)
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url,
source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname, True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(
self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url,
item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url, source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname,
True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url, item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)