def _get_filename(self, url, remove_slash=True):
"""Retrieve the local filename from a given URL."""
# Remove the trailing slash as a convention unless specified otherwise.
if remove_slash:
urlfunctions.append_slash(url, False)
filename = urlfunctions.url_split(url).path
if filename == "":
filename = "/"
return filename
def _get_filename(self, url, remove_slash=True):
"""Retrieve the local filename from a given URL."""
# Remove the trailing slash as a convention unless specified otherwise.
if remove_slash:
urlfunctions.append_slash(url, False)
filename = urlfunctions.url_split(url).path
if filename == "":
filename = "/"
return filename
def test_append_slash(self):
"""Test append_slash."""
tests = (
(("file:///home/user/", True), "file:///home/user/"),
(("file:///home/user/", False), "file:///home/user"),
(("file:///home/user/", True), "file:///home/user/"),
(("file:///home/user", False), "file:///home/user"),
)
for test, expected_output in tests:
self.assertEqual(urlfunctions.append_slash(*test), expected_output)
def test_append_slash(self):
"""Test append_slash."""
tests = (
(("file:///home/user/", True), "file:///home/user/"),
(("file:///home/user/", False), "file:///home/user"),
(("file:///home/user/", True), "file:///home/user/"),
(("file:///home/user", False), "file:///home/user"),
)
for test, expected_output in tests:
self.assertEqual(urlfunctions.append_slash(*test), expected_output)
def listdir(self, url):
"""Retrieve a directory listing of the given location.
Returns a list of (url, attribute_dict) tuples if the
given URL is a directory, False otherwise.
"""
# Add a slash so we don't have to remove it from the start of the subpaths.
url = urlfunctions.append_slash(url)
filename = self._get_filename(url, False)
files = set()
for key in self._filesystem:
# Check the length to prevent returning the directory itself.
if key.startswith(filename) and len(key) > len(filename):
subpath = key[len(filename):]
if "/" not in subpath:
# Add the subpath in the set as is, because there are no lower levels.
files.add(subpath)
else:
files.add(subpath[:subpath.find("/")])
return [FileObject(self, url + x,) for x in files]
def listdir(self, url):
"""Retrieve a directory listing of the given location.
Returns a list of (url, attribute_dict) tuples if the
given URL is a directory, False otherwise.
"""
# Add a slash so we don't have to remove it from the start of the subpaths.
url = urlfunctions.append_slash(url)
filename = self._get_filename(url, False)
files = set()
for key in self._filesystem:
# Check the length to prevent returning the directory itself.
if key.startswith(filename) and len(key) > len(filename):
subpath = key[len(filename):]
if "/" not in subpath:
# Add the subpath in the set as is, because there are no lower levels.
files.add(subpath)
else:
files.add(subpath[:subpath.find("/")])
return [FileObject(
self,
url + x,
) for x in files]
def listdir(self, url):
"""Retrieve a directory listing of the given location.
Returns a list of (url, attribute_dict) tuples if the given URL is a directory,
False otherwise. URLs should be absolute, including protocol, etc.
attribute_dict is a dictionary of {key: value} pairs for any applicable
attributes from ("size", "mtime", "atime", "ctime", "isdir").
"""
url = urlfunctions.append_slash(url, True)
try:
dir_list = self._connection.listdir_attr(self._get_filename(url))
except IOError:
return False
file_list = []
for item in dir_list:
file_list.append(FileObject(self, url + item.filename,
{"size": item.st_size,
"mtime": item.st_mtime,
"atime": item.st_atime,
"perms": item.st_mode,
"owner": item.st_uid,
"group": item.st_gid,
}))
return file_list
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url,
source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname, True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(
self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url,
item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes
| self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)
def compare_directories(self, source, source_dir_list, dest_dir_url):
"""Compare the source's directory list with the destination's and perform any actions
necessary, such as deleting files or creating directories."""
dest_dir_list = self.destination_transport.listdir(dest_dir_url)
if not dest_dir_list:
if not self.config.dry_run:
self.destination_transport.mkdir(dest_dir_url)
# Populate the item's attributes for the remote directory so we can set them.
attribute_set = self.max_evaluation_attributes & \
self.destination_transport.setattr_attributes
attribute_set = attribute_set | self.config.requested_attributes
attribute_set = attribute_set ^ self.config.exclude_attributes
source.populate_attributes(attribute_set)
self.set_destination_attributes(dest_dir_url, source.attributes)
dest_dir_list = []
# Construct a dictionary of {filename: FileObject} items.
dest_paths = dict([(url_split(append_slash(x.url, False),
self.destination_transport.uses_hostname,
True).file, x) for x in dest_dir_list])
create_dirs = []
for item in source_dir_list:
# Remove slashes so the splitter can get the filename.
url = url_split(append_slash(item.url, False),
self.source_transport.uses_hostname,
True).file
# If the file exists and both the source and destination are of the same type...
if url in dest_paths and dest_paths[url].isdir == item.isdir:
# ...if it's a directory, set its attributes as well...
if dest_paths[url].isdir:
log.info("Setting attributes for %s..." % url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_paths[url].url, item.attributes)
# ...and remove it from the list.
del dest_paths[url]
else:
# If an item is in the source but not the destination tree...
if item.isdir and self.config.recursive:
# ...create it if it's a directory.
create_dirs.append(item)
if self.config.delete:
for item in dest_paths.values():
if item.isdir:
if self.config.recursive:
log.info("Deleting destination directory %s..." % item)
self.recursively_delete(item)
else:
log.info("Deleting destination file %s..." % item)
self.destination_transport.remove(item.url)
if self.config.dry_run:
return
# Create directories after we've deleted everything else because sometimes a directory in
# the source might have the same name as a file, so we need to delete files first.
for item in create_dirs:
dest_url = url_splice(self.source, item.url, self.destination)
self.destination_transport.mkdir(dest_url)
item.populate_attributes(self.max_evaluation_attributes |
self.config.requested_attributes)
self.set_destination_attributes(dest_url, item.attributes)