def _get_filename(self, url, remove_slash=True): """Retrieve the local filename from a given URL.""" # Remove the trailing slash as a convention unless specified otherwise. if remove_slash: urlfunctions.append_slash(url, False) filename = urlfunctions.url_split(url).path if filename == "": filename = "/" return filename
def test_append_slash(self): """Test append_slash.""" tests = ( (("file:///home/user/", True), "file:///home/user/"), (("file:///home/user/", False), "file:///home/user"), (("file:///home/user/", True), "file:///home/user/"), (("file:///home/user", False), "file:///home/user"), ) for test, expected_output in tests: self.assertEqual(urlfunctions.append_slash(*test), expected_output)
def listdir(self, url): """Retrieve a directory listing of the given location. Returns a list of (url, attribute_dict) tuples if the given URL is a directory, False otherwise. """ # Add a slash so we don't have to remove it from the start of the subpaths. url = urlfunctions.append_slash(url) filename = self._get_filename(url, False) files = set() for key in self._filesystem: # Check the length to prevent returning the directory itself. if key.startswith(filename) and len(key) > len(filename): subpath = key[len(filename):] if "/" not in subpath: # Add the subpath in the set as is, because there are no lower levels. files.add(subpath) else: files.add(subpath[:subpath.find("/")]) return [FileObject(self, url + x,) for x in files]
def listdir(self, url): """Retrieve a directory listing of the given location. Returns a list of (url, attribute_dict) tuples if the given URL is a directory, False otherwise. """ # Add a slash so we don't have to remove it from the start of the subpaths. url = urlfunctions.append_slash(url) filename = self._get_filename(url, False) files = set() for key in self._filesystem: # Check the length to prevent returning the directory itself. if key.startswith(filename) and len(key) > len(filename): subpath = key[len(filename):] if "/" not in subpath: # Add the subpath in the set as is, because there are no lower levels. files.add(subpath) else: files.add(subpath[:subpath.find("/")]) return [FileObject( self, url + x, ) for x in files]
def listdir(self, url): """Retrieve a directory listing of the given location. Returns a list of (url, attribute_dict) tuples if the given URL is a directory, False otherwise. URLs should be absolute, including protocol, etc. attribute_dict is a dictionary of {key: value} pairs for any applicable attributes from ("size", "mtime", "atime", "ctime", "isdir"). """ url = urlfunctions.append_slash(url, True) try: dir_list = self._connection.listdir_attr(self._get_filename(url)) except IOError: return False file_list = [] for item in dir_list: file_list.append(FileObject(self, url + item.filename, {"size": item.st_size, "mtime": item.st_mtime, "atime": item.st_atime, "perms": item.st_mode, "owner": item.st_uid, "group": item.st_gid, })) return file_list
def compare_directories(self, source, source_dir_list, dest_dir_url): """Compare the source's directory list with the destination's and perform any actions necessary, such as deleting files or creating directories.""" dest_dir_list = self.destination_transport.listdir(dest_dir_url) if not dest_dir_list: if not self.config.dry_run: self.destination_transport.mkdir(dest_dir_url) # Populate the item's attributes for the remote directory so we can set them. attribute_set = self.max_evaluation_attributes & \ self.destination_transport.setattr_attributes attribute_set = attribute_set | self.config.requested_attributes attribute_set = attribute_set ^ self.config.exclude_attributes source.populate_attributes(attribute_set) self.set_destination_attributes(dest_dir_url, source.attributes) dest_dir_list = [] # Construct a dictionary of {filename: FileObject} items. dest_paths = dict([(url_split(append_slash(x.url, False), self.destination_transport.uses_hostname, True).file, x) for x in dest_dir_list]) create_dirs = [] for item in source_dir_list: # Remove slashes so the splitter can get the filename. url = url_split(append_slash(item.url, False), self.source_transport.uses_hostname, True).file # If the file exists and both the source and destination are of the same type... if url in dest_paths and dest_paths[url].isdir == item.isdir: # ...if it's a directory, set its attributes as well... if dest_paths[url].isdir: log.info("Setting attributes for %s..." % url) item.populate_attributes( self.max_evaluation_attributes | self.config.requested_attributes) self.set_destination_attributes(dest_paths[url].url, item.attributes) # ...and remove it from the list. del dest_paths[url] else: # If an item is in the source but not the destination tree... if item.isdir and self.config.recursive: # ...create it if it's a directory. create_dirs.append(item) if self.config.delete: for item in dest_paths.values(): if item.isdir: if self.config.recursive: log.info("Deleting destination directory %s..." % item) self.recursively_delete(item) else: log.info("Deleting destination file %s..." % item) self.destination_transport.remove(item.url) if self.config.dry_run: return # Create directories after we've deleted everything else because sometimes a directory in # the source might have the same name as a file, so we need to delete files first. for item in create_dirs: dest_url = url_splice(self.source, item.url, self.destination) self.destination_transport.mkdir(dest_url) item.populate_attributes(self.max_evaluation_attributes | self.config.requested_attributes) self.set_destination_attributes(dest_url, item.attributes)
def compare_directories(self, source, source_dir_list, dest_dir_url): """Compare the source's directory list with the destination's and perform any actions necessary, such as deleting files or creating directories.""" dest_dir_list = self.destination_transport.listdir(dest_dir_url) if not dest_dir_list: if not self.config.dry_run: self.destination_transport.mkdir(dest_dir_url) # Populate the item's attributes for the remote directory so we can set them. attribute_set = self.max_evaluation_attributes & \ self.destination_transport.setattr_attributes attribute_set = attribute_set | self.config.requested_attributes attribute_set = attribute_set ^ self.config.exclude_attributes source.populate_attributes(attribute_set) self.set_destination_attributes(dest_dir_url, source.attributes) dest_dir_list = [] # Construct a dictionary of {filename: FileObject} items. dest_paths = dict([(url_split(append_slash(x.url, False), self.destination_transport.uses_hostname, True).file, x) for x in dest_dir_list]) create_dirs = [] for item in source_dir_list: # Remove slashes so the splitter can get the filename. url = url_split(append_slash(item.url, False), self.source_transport.uses_hostname, True).file # If the file exists and both the source and destination are of the same type... if url in dest_paths and dest_paths[url].isdir == item.isdir: # ...if it's a directory, set its attributes as well... if dest_paths[url].isdir: log.info("Setting attributes for %s..." % url) item.populate_attributes(self.max_evaluation_attributes | self.config.requested_attributes) self.set_destination_attributes(dest_paths[url].url, item.attributes) # ...and remove it from the list. del dest_paths[url] else: # If an item is in the source but not the destination tree... if item.isdir and self.config.recursive: # ...create it if it's a directory. create_dirs.append(item) if self.config.delete: for item in dest_paths.values(): if item.isdir: if self.config.recursive: log.info("Deleting destination directory %s..." % item) self.recursively_delete(item) else: log.info("Deleting destination file %s..." % item) self.destination_transport.remove(item.url) if self.config.dry_run: return # Create directories after we've deleted everything else because sometimes a directory in # the source might have the same name as a file, so we need to delete files first. for item in create_dirs: dest_url = url_splice(self.source, item.url, self.destination) self.destination_transport.mkdir(dest_url) item.populate_attributes(self.max_evaluation_attributes | self.config.requested_attributes) self.set_destination_attributes(dest_url, item.attributes)