Esempio n. 1
0
    def recurse(self):
        """Recursively synchronise everything."""
        source_dir_list = self.source_transport.listdir(self.source)
        dest = FileObject(self.destination_transport, self.destination)
        # If the source is a file, rather than a directory, just copy it. We know for sure that
        # it exists from the checks we did before, so the "False" return value can't be because
        # of that.
        if not source_dir_list:
            # If the destination ends in a slash or is an actual directory:
            if self.destination.endswith("/") or dest.isdir:
                if not dest.isdir:
                    self.destination_transport.mkdir(dest.url)
                # Splice the source filename onto the destination URL.
                dest_url = url_split(dest.url)
                dest_url.file = url_split(self.source,
                                          uses_hostname=self.source_transport.uses_hostname,
                                          split_filename=True).file
                dest_url = url_join(dest_url)
            else:
                dest_url = self.destination
            self.compare_and_copy(
                FileObject(self.source_transport, self.source, {"isdir": False}),
                FileObject(self.destination_transport, dest_url, {"isdir": False}),
                )
            return

        # If source is a directory...
        directory_stack = [FileObject(self.source_transport, self.source, {"isdir": True})]

        # Depth-first tree traversal.
        while directory_stack:
            # TODO: Rethink the assumption that a file cannot have the same name as a directory.
            item = directory_stack.pop()
            log.debug("URL %s is %sa directory." % \
                          (item.url, not item.isdir and "not " or ""))
            if item.isdir:
                # Don't skip the first directory.
                if not self.config.recursive and item.url != self.source:
                    log.info("Skipping directory %s..." % item)
                    continue
                # Obtain a directory list.
                new_dir_list = []
                for new_file in reversed(self.source_transport.listdir(item.url)):
                    if self.include_file(new_file):
                        new_dir_list.append(new_file)
                    else:
                        log.debug("Skipping %s..." % (new_file))
                dest = url_splice(self.source, item.url, self.destination)
                dest = FileObject(self.destination_transport, dest)
                log.debug("Comparing directories %s and %s..." % (item.url, dest.url))
                self.compare_directories(item, new_dir_list, dest.url)
                directory_stack.extend(new_dir_list)
            else:
                dest_url = url_splice(self.source, item.url, self.destination)
                log.debug("Destination URL is %s." % dest_url)
                dest = FileObject(self.destination_transport, dest_url)
                self.compare_and_copy(item, dest)
Esempio n. 2
0
 def test_url_splice(self):
     """Test url_splice."""
     tests = (
         (
             ("file://*****:*****@myhost:21/test/",
              "ftp://*****:*****@myhost:21/test/file",
              "file://otherhost:21/test;someparams"),
             "file://otherhost:21/test/file;someparams",
         ),
     )
     for test, expected_output in tests:
         self.assertEqual(urlfunctions.url_splice(*test), expected_output)
Esempio n. 3
0
 def test_url_splice(self):
     """Test url_splice."""
     tests = (
         (("file://*****:*****@myhost:21/test/",
           "ftp://*****:*****@myhost:21/test/file",
           "file://otherhost:21/test;someparams"),
          "file://otherhost:21/test/file;someparams",
          ),
     )
     for test, expected_output in tests:
         self.assertEqual(urlfunctions.url_splice(*test), expected_output)
Esempio n. 4
0
    def recurse(self):
        """Recursively synchronise everything."""
        source_dir_list = self.source_transport.listdir(self.source)
        dest = FileObject(self.destination_transport, self.destination)
        # If the source is a file, rather than a directory, just copy it. We know for sure that
        # it exists from the checks we did before, so the "False" return value can't be because
        # of that.
        if not source_dir_list:
            # If the destination ends in a slash or is an actual directory:
            if self.destination.endswith("/") or dest.isdir:
                if not dest.isdir:
                    self.destination_transport.mkdir(dest.url)
                # Splice the source filename onto the destination URL.
                dest_url = url_split(dest.url)
                dest_url.file = url_split(
                    self.source,
                    uses_hostname=self.source_transport.uses_hostname,
                    split_filename=True).file
                dest_url = url_join(dest_url)
            else:
                dest_url = self.destination
            self.compare_and_copy(
                FileObject(self.source_transport, self.source,
                           {"isdir": False}),
                FileObject(self.destination_transport, dest_url,
                           {"isdir": False}),
            )
            return

        # If source is a directory...
        directory_stack = [
            FileObject(self.source_transport, self.source, {"isdir": True})
        ]

        # Depth-first tree traversal.
        while directory_stack:
            # TODO: Rethink the assumption that a file cannot have the same name as a directory.
            item = directory_stack.pop()
            log.debug("URL %s is %sa directory." % \
                          (item.url, not item.isdir and "not " or ""))
            if item.isdir:
                # Don't skip the first directory.
                if not self.config.recursive and item.url != self.source:
                    log.info("Skipping directory %s..." % item)
                    continue
                # Obtain a directory list.
                new_dir_list = []
                for new_file in reversed(
                        self.source_transport.listdir(item.url)):
                    if self.include_file(new_file):
                        new_dir_list.append(new_file)
                    else:
                        log.debug("Skipping %s..." % (new_file))
                dest = url_splice(self.source, item.url, self.destination)
                dest = FileObject(self.destination_transport, dest)
                log.debug("Comparing directories %s and %s..." %
                          (item.url, dest.url))
                self.compare_directories(item, new_dir_list, dest.url)
                directory_stack.extend(new_dir_list)
            else:
                dest_url = url_splice(self.source, item.url, self.destination)
                log.debug("Destination URL is %s." % dest_url)
                dest = FileObject(self.destination_transport, dest_url)
                self.compare_and_copy(item, dest)
Esempio n. 5
0
    def compare_directories(self, source, source_dir_list, dest_dir_url):
        """Compare the source's directory list with the destination's and perform any actions
           necessary, such as deleting files or creating directories."""
        dest_dir_list = self.destination_transport.listdir(dest_dir_url)
        if not dest_dir_list:
            if not self.config.dry_run:
                self.destination_transport.mkdir(dest_dir_url)
                # Populate the item's attributes for the remote directory so we can set them.
                attribute_set = self.max_evaluation_attributes & \
                                self.destination_transport.setattr_attributes
                attribute_set = attribute_set | self.config.requested_attributes
                attribute_set = attribute_set ^ self.config.exclude_attributes
                source.populate_attributes(attribute_set)

                self.set_destination_attributes(dest_dir_url,
                                                source.attributes)
            dest_dir_list = []
        # Construct a dictionary of {filename: FileObject} items.
        dest_paths = dict([(url_split(append_slash(x.url, False),
                                      self.destination_transport.uses_hostname,
                                      True).file, x) for x in dest_dir_list])
        create_dirs = []
        for item in source_dir_list:
            # Remove slashes so the splitter can get the filename.
            url = url_split(append_slash(item.url, False),
                            self.source_transport.uses_hostname, True).file
            # If the file exists and both the source and destination are of the same type...
            if url in dest_paths and dest_paths[url].isdir == item.isdir:
                # ...if it's a directory, set its attributes as well...
                if dest_paths[url].isdir:
                    log.info("Setting attributes for %s..." % url)
                    item.populate_attributes(
                        self.max_evaluation_attributes
                        | self.config.requested_attributes)
                    self.set_destination_attributes(dest_paths[url].url,
                                                    item.attributes)
                # ...and remove it from the list.
                del dest_paths[url]
            else:
                # If an item is in the source but not the destination tree...
                if item.isdir and self.config.recursive:
                    # ...create it if it's a directory.
                    create_dirs.append(item)

        if self.config.delete:
            for item in dest_paths.values():
                if item.isdir:
                    if self.config.recursive:
                        log.info("Deleting destination directory %s..." % item)
                        self.recursively_delete(item)
                else:
                    log.info("Deleting destination file %s..." % item)
                    self.destination_transport.remove(item.url)

        if self.config.dry_run:
            return

        # Create directories after we've deleted everything else because sometimes a directory in
        # the source might have the same name as a file, so we need to delete files first.
        for item in create_dirs:
            dest_url = url_splice(self.source, item.url, self.destination)
            self.destination_transport.mkdir(dest_url)
            item.populate_attributes(self.max_evaluation_attributes
                                     | self.config.requested_attributes)
            self.set_destination_attributes(dest_url, item.attributes)
Esempio n. 6
0
    def compare_directories(self, source, source_dir_list, dest_dir_url):
        """Compare the source's directory list with the destination's and perform any actions
           necessary, such as deleting files or creating directories."""
        dest_dir_list = self.destination_transport.listdir(dest_dir_url)
        if not dest_dir_list:
            if not self.config.dry_run:
                self.destination_transport.mkdir(dest_dir_url)
                # Populate the item's attributes for the remote directory so we can set them.
                attribute_set = self.max_evaluation_attributes & \
                                self.destination_transport.setattr_attributes
                attribute_set = attribute_set | self.config.requested_attributes
                attribute_set = attribute_set ^ self.config.exclude_attributes
                source.populate_attributes(attribute_set)

                self.set_destination_attributes(dest_dir_url, source.attributes)
            dest_dir_list = []
        # Construct a dictionary of {filename: FileObject} items.
        dest_paths = dict([(url_split(append_slash(x.url, False),
                                      self.destination_transport.uses_hostname,
                                      True).file, x) for x in dest_dir_list])
        create_dirs = []
        for item in source_dir_list:
            # Remove slashes so the splitter can get the filename.
            url = url_split(append_slash(item.url, False),
                            self.source_transport.uses_hostname,
                            True).file
            # If the file exists and both the source and destination are of the same type...
            if url in dest_paths and dest_paths[url].isdir == item.isdir:
                # ...if it's a directory, set its attributes as well...
                if dest_paths[url].isdir:
                    log.info("Setting attributes for %s..." % url)
                    item.populate_attributes(self.max_evaluation_attributes |
                                             self.config.requested_attributes)
                    self.set_destination_attributes(dest_paths[url].url, item.attributes)
                # ...and remove it from the list.
                del dest_paths[url]
            else:
                # If an item is in the source but not the destination tree...
                if item.isdir and self.config.recursive:
                    # ...create it if it's a directory.
                    create_dirs.append(item)

        if self.config.delete:
            for item in dest_paths.values():
                if item.isdir:
                    if self.config.recursive:
                        log.info("Deleting destination directory %s..." % item)
                        self.recursively_delete(item)
                else:
                    log.info("Deleting destination file %s..." % item)
                    self.destination_transport.remove(item.url)

        if self.config.dry_run:
            return

        # Create directories after we've deleted everything else because sometimes a directory in
        # the source might have the same name as a file, so we need to delete files first.
        for item in create_dirs:
            dest_url = url_splice(self.source, item.url, self.destination)
            self.destination_transport.mkdir(dest_url)
            item.populate_attributes(self.max_evaluation_attributes |
                                       self.config.requested_attributes)
            self.set_destination_attributes(dest_url, item.attributes)