Example #1
0
    def create(path):
        """Creates a new SyncFile instance"""

        drive = Drive()

        if drive.is_drivepath(path):
            filepath = drive.normpath(path)
            from libgsync.sync.file.remote import SyncFileRemote
            return SyncFileRemote(filepath)
        filepath = os.path.normpath(path)
        from libgsync.sync.file.local import SyncFileLocal
        return SyncFileLocal(filepath)
Example #2
0
    def create(path):
        """Creates a new SyncFile instance"""

        drive = Drive()

        if drive.is_drivepath(path):
            filepath = drive.normpath(path)

            from libgsync.sync.file.remote import SyncFileRemote
            return SyncFileRemote(filepath)

        else:
            filepath = os.path.normpath(path)

            from libgsync.sync.file.local import SyncFileLocal
            return SyncFileLocal(filepath)
Example #3
0
    def create(path):
        debug("SyncFileFactory.create(%s)" % repr(path))

        drive = Drive()

        if drive.is_drivepath(path):
            filepath = drive.normpath(path)

            debug("Creating SyncFileRemote(%s)" % repr(filepath))

            from libgsync.sync.file.remote import SyncFileRemote
            return SyncFileRemote(filepath)

        else:
            filepath = os.path.normpath(path)

            debug("Creating SyncFileLocal(%s)" % repr(filepath))

            from libgsync.sync.file.local import SyncFileLocal
            return SyncFileLocal(filepath)
Example #4
0
    def create(path):
        debug("SyncFileFactory.create(%s)" % repr(path))

        drive = Drive()

        if drive.is_drivepath(path):
            filepath = drive.normpath(path)

            debug("Creating SyncFileRemote(%s)" % repr(filepath))

            from libgsync.sync.file.remote import SyncFileRemote
            return SyncFileRemote(filepath)

        else:
            filepath = os.path.normpath(path)

            debug("Creating SyncFileLocal(%s)" % repr(filepath))

            from libgsync.sync.file.local import SyncFileLocal
            return SyncFileLocal(filepath)
Example #5
0
class Crawler(object):
    def __init__(self, src, dst):
        self._dev = None
        self._src = None
        self._dst = None

        self._drive = Drive()

        if self._drive.is_drivepath(src):
            self._walkCallback = bind("walk", self._drive)
            self._src = self._drive.normpath(src)
        else:
            self._walkCallback = os.walk
            self._src = os.path.normpath(src)
            st_info = os.stat(self._src)

            if GsyncOptions.one_file_system:
                self._dev = st_info.st_dev

        if self._drive.is_drivepath(dst):
            self._dst = self._drive.normpath(dst)
        else:
            self._dst = os.path.normpath(dst)

        if src[-1] == "/": self._src += "/"
        if dst[-1] == "/": self._dst += "/"

        #super(Crawler, self).__init__(name = "Crawler: %s" % src)
    

    def _devCheck(self, dev, path):
        if dev is not None:
            st_info = os.stat(path)
            if st_info.st_dev != dev:
                debug("Not on same dev: %s" % path)
                return False

        return True


    def _walk(self, path, generator, dev):
        for d, dirs, files in generator(path):
            debug("Walking: %s" % d)

            if not self._devCheck(dev, d):
                debug("Not on same device: %s" % d)
                continue

            if GsyncOptions.dirs or GsyncOptions.recursive:
                # Sync the directory but not its contents
                debug("Synchronising directory: %s" % d)
                self._sync(d)
            else:
                sys.stdout.write("skipping directory %s\n" % d)
                break

            for f in files:
                f = os.path.join(d, f)
                if not self._devCheck(dev, f):
                    continue
                    
                debug("Synchronising file: %s" % f)
                self._sync(f)

            if not GsyncOptions.recursive:
                break


    def run(self):
        srcpath = self._src
        basepath, path = os.path.split(srcpath)

        if self._drive.is_drivepath(self._src):
            basepath = self._drive.normpath(basepath)

        debug("Source srcpath: %s" % srcpath)
        debug("Source basepath: %s" % basepath)
        debug("Source path: %s" % path)

        if GsyncOptions.relative:
            # Supports the foo/./bar notation in rsync.
            path = re.sub(r'^.*/\./', "", path)

        self._sync = Sync(basepath, self._dst)

        debug("Enumerating: %s" % srcpath)

        try:
            self._walk(srcpath, self._walkCallback, self._dev)

        except KeyboardInterrupt, e:
            print("\nInterrupted")
            raise

        except Exception, e:
            debug.exception(e)
            print("Error: %s" % str(e))
Example #6
0
class Crawler(object):
    """
    Crawler class that defines an instance of a crawler that is bound to
    either a local or remote filesystem.
    """

    def __init__(self, src, dst):
        self._dev = None
        self._src = None
        self._dst = None
        self._sync = None

        force_dest_file = GsyncOptions.force_dest_file

        self._drive = Drive()

        if self._drive.is_drivepath(src):
            self._walk_callback = bind("walk", self._drive)
            self._src = self._drive.normpath(src)
            info = self._drive.stat(self._src)

            if info and info.mimeType != MimeTypes.FOLDER:
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True
        else:
            self._walk_callback = os_walk_wrapper
            self._src = os.path.normpath(src)
            st_info = os.stat(self._src)

            if os.path.isfile(self._src):
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True

            if GsyncOptions.one_file_system:
                self._dev = st_info.st_dev

        if self._drive.is_drivepath(dst):
            self._dst = self._drive.normpath(dst)
            info = self._drive.stat(self._dst)

            if info and info.mimeType == MimeTypes.FOLDER:
                debug("Dest is a directory, not forcing dest file: %s" % (repr(
                    self._dst)))
                force_dest_file = False
        else:
            self._dst = os.path.normpath(dst)
            if os.path.isdir(self._dst):
                debug("Dest is a directory, not forcing dest file: %s" % (repr(
                    self._dst)))
                force_dest_file = False

        if src[-1] == "/":
            self._src += "/"

        if dst[-1] == "/":
            self._dst += "/"
            debug("Dest has trailing slash, not forcing dest file: %s" %
                  (self._dst))
            force_dest_file = False

        # Only update if not already set.
        if GsyncOptions.force_dest_file is None:
            debug("force_dest_file = %s" % force_dest_file)
            GsyncOptions.force_dest_file = force_dest_file

        #super(Crawler, self).__init__(name = "Crawler: %s" % src)

    def _dev_check(self, device_id, path):
        """
        Checks if the path provided resides on the device specified by the
        device ID provided.

        @param {int} device_id    The device ID.
        @param {String} path      Path to verify.

        @return {bool} True if the path resides on device with the
                       specified ID.
        """
        if device_id is not None:
            st_info = os.stat(path)
            if st_info.st_dev != device_id:
                debug("Not on same device: %s" % repr(path))
                return False

        return True

    def _walk(self, path, generator, device_id):
        """
        Walks the path provided, calling the generator function on the path,
        which yields the subdirectories and files.  It then iterates these
        lists and calls the sync method '_sync'.

        @param {String} path          Path to walk.
        @param {Function} generator   Generator function to call on path.
        @param {int} device_id        Device ID for the path, None if device
                                      cannot be determined.
        """
        for dirpath, _, files in generator(path):
            debug("Walking: %s" % repr(dirpath))

            if not self._dev_check(device_id, dirpath):
                debug("Not on same device: %s" % repr(dirpath))
                continue

            if not GsyncOptions.force_dest_file:
                if GsyncOptions.dirs or GsyncOptions.recursive:

                    # Sync the directory but not its contents
                    debug("Synchronising directory: %s" % repr(dirpath))
                    self._sync(dirpath)
                else:
                    sys.stdout.write("skipping directory %s\n" % dirpath)
                    break

            for filename in files:
                absfile = os.path.join(dirpath, filename)
                if not self._dev_check(device_id, absfile):
                    continue

                debug("Synchronising file: %s" % repr(absfile))
                self._sync(absfile)

            if not GsyncOptions.recursive:
                break

    def run(self):
        """
        Worker method called synchronously or as part of an asynchronous
        thread or subprocess.
        """
        srcpath = self._src
        basepath, path = os.path.split(srcpath)

        if self._drive.is_drivepath(self._src):
            basepath = self._drive.normpath(basepath)

        debug("Source srcpath: %s" % repr(srcpath))
        debug("Source basepath: %s" % repr(basepath))
        debug("Source path: %s" % repr(path))

        if GsyncOptions.relative:
            # Supports the foo/./bar notation in rsync.
            path = re.sub(r'^.*/\./', "", path)

        self._sync = Sync(basepath, self._dst)

        debug("Enumerating: %s" % repr(srcpath))

        try:
            self._walk(srcpath, self._walk_callback, self._dev)

        except KeyboardInterrupt, ex:
            print("\nInterrupted")
            raise

        except Exception, ex:
            debug.exception(ex)
            print("Error: %s" % repr(ex))
Example #7
0
class Crawler(object):
    """
    Crawler class that defines an instance of a crawler that is bound to
    either a local or remote filesystem.
    """
    def __init__(self, src, dst):
        self._dev = None
        self._src = None
        self._dst = None
        self._sync = None

        force_dest_file = GsyncOptions.force_dest_file

        self._drive = Drive()

        if self._drive.is_drivepath(src):
            self._walk_callback = bind("walk", self._drive)
            self._src = self._drive.normpath(src)
            info = self._drive.stat(self._src)

            if info and info.mimeType != MimeTypes.FOLDER:
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True
        else:
            self._walk_callback = os_walk_wrapper
            self._src = os.path.normpath(src)
            st_info = os.stat(self._src)

            if os.path.isfile(self._src):
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True

            if GsyncOptions.one_file_system:
                self._dev = st_info.st_dev

        if self._drive.is_drivepath(dst):
            self._dst = self._drive.normpath(dst)
            info = self._drive.stat(self._dst)

            if info and info.mimeType == MimeTypes.FOLDER:
                debug("Dest is a directory, not forcing dest file: %s" %
                      (repr(self._dst)))
                force_dest_file = False
        else:
            self._dst = os.path.normpath(dst)
            if os.path.isdir(self._dst):
                debug("Dest is a directory, not forcing dest file: %s" %
                      (repr(self._dst)))
                force_dest_file = False

        if src[-1] == "/":
            self._src += "/"

        if dst[-1] == "/":
            self._dst += "/"
            debug("Dest has trailing slash, not forcing dest file: %s" %
                  (self._dst))
            force_dest_file = False

        # Only update if not already set.
        if GsyncOptions.force_dest_file is None:
            debug("force_dest_file = %s" % force_dest_file)
            GsyncOptions.force_dest_file = force_dest_file

        # super(Crawler, self).__init__(name = "Crawler: %s" % src)

    @staticmethod
    def _dev_check(device_id, path):
        """
        Checks if the path provided resides on the device specified by the
        device ID provided.

        @param {int} device_id    The device ID.
        @param {String} path      Path to verify.

        @return {bool} True if the path resides on device with the
                       specified ID.
        """
        if device_id is not None:
            st_info = os.stat(path)
            if st_info.st_dev != device_id:
                debug("Not on same device: %s" % repr(path))
                return False

        return True

    def _walk(self, path, generator, device_id):
        """
        Walks the path provided, calling the generator function on the path,
        which yields the subdirectories and files.  It then iterates these
        lists and calls the sync method '_sync'.

        @param {String} path          Path to walk.
        @param {Function} generator   Generator function to call on path.
        @param {int} device_id        Device ID for the path, None if device
                                      cannot be determined.
        """
        for dirpath, _, files in generator(path):
            debug("Walking: %s" % repr(dirpath))

            if not self._dev_check(device_id, dirpath):
                debug("Not on same device: %s" % repr(dirpath))
                continue

            if not GsyncOptions.force_dest_file:
                if GsyncOptions.dirs or GsyncOptions.recursive:

                    # Sync the directory but not its contents
                    debug("Synchronising directory: %s" % repr(dirpath))
                    self._sync(dirpath)
                else:
                    sys.stdout.write("skipping directory %s\n" % dirpath)
                    break

            for filename in files:
                absfile = os.path.join(dirpath, filename)
                if not self._dev_check(device_id, absfile):
                    continue

                debug("Synchronising file: %s" % repr(absfile))
                self._sync(absfile)

            if not GsyncOptions.recursive:
                break

    def run(self):
        """
        Worker method called synchronously or as part of an asynchronous
        thread or subprocess.
        """
        srcpath = self._src
        basepath, path = os.path.split(srcpath)

        if self._drive.is_drivepath(self._src):
            basepath = self._drive.normpath(basepath)

        debug("Source srcpath: %s" % repr(srcpath))
        debug("Source basepath: %s" % repr(basepath))
        debug("Source path: %s" % repr(path))

        if GsyncOptions.relative:
            # Supports the foo/./bar notation in rsync.
            path = re.sub(r'^.*/\./', "", path)

        self._sync = Sync(basepath, self._dst)

        debug("Enumerating: %s" % repr(srcpath))

        try:
            self._walk(srcpath, self._walk_callback, self._dev)

        except KeyboardInterrupt:
            print("\nInterrupted")
            raise

        except Exception as ex:
            debug.exception(ex)
            print("Error: %s" % repr(ex))

        finally:
            verbose("sent %d bytes  received %d bytes  %.2f bytes/sec" %
                    (self._sync.total_bytes_sent,
                     self._sync.total_bytes_received, self._sync.rate()))
Example #8
0
class Crawler(object):
    def __init__(self, src, dst):
        self._dev = None
        self._src = None
        self._dst = None
        
        force_dest_file = GsyncOptions.force_dest_file

        self._drive = Drive()

        if self._drive.is_drivepath(src):
            self._walkCallback = bind("walk", self._drive)
            self._src = self._drive.normpath(src)
            info = self._drive.stat(self._src)

            if info and info.mimeType != MimeTypes.FOLDER:
                debug("Source is not a directory, forcing dest file: %s" % (
                    repr(self._src)
                ))
                force_dest_file = True
        else:
            self._walkCallback = os_walk_wrapper
            self._src = os.path.normpath(src)
            st_info = os.stat(self._src)

            if os.path.isfile(self._src):
                debug("Source is not a directory, forcing dest file: %s" % (
                    repr(self._src)
                ))
                force_dest_file = True

            if GsyncOptions.one_file_system:
                self._dev = st_info.st_dev

        if self._drive.is_drivepath(dst):
            self._dst = self._drive.normpath(dst)
            info = self._drive.stat(self._dst)

            if info and info.mimeType == MimeTypes.FOLDER:
                debug("Dest is a directory, not forcing dest file: %s" % (
                    repr(self._dst)
                ))
                force_dest_file = False
        else:
            self._dst = os.path.normpath(dst)
            if os.path.isdir(self._dst):
                debug("Dest is a directory, not forcing dest file: %s" % (
                    repr(self._dst)
                ))
                force_dest_file = False

        if src[-1] == "/":
            self._src += "/"

        if dst[-1] == "/":
            self._dst += "/"
            debug("Dest has trailing slash, not forcing dest file: %s" % (
                self._dst
            ))
            force_dest_file = False

        # Only update if not already set.
        if GsyncOptions.force_dest_file is None:
            debug("force_dest_file = %s" % force_dest_file)
            GsyncOptions.force_dest_file = force_dest_file

        #super(Crawler, self).__init__(name = "Crawler: %s" % src)
    

    def _devCheck(self, dev, path):
        if dev is not None:
            st_info = os.stat(path)
            if st_info.st_dev != dev:
                debug("Not on same dev: %s" % repr(path))
                return False

        return True


    def _walk(self, path, generator, dev):
        for d, dirs, files in generator(path):
            debug("Walking: %s" % repr(d))

            if not self._devCheck(dev, d):
                debug("Not on same device: %s" % repr(d))
                continue

            if not GsyncOptions.force_dest_file:
                if GsyncOptions.dirs or GsyncOptions.recursive:

                    # Sync the directory but not its contents
                    debug("Synchronising directory: %s" % repr(d))
                    self._sync(d)
                else:
                    sys.stdout.write("skipping directory %s\n" % d)
                    break

            for f in files:
                f = os.path.join(d, f)
                if not self._devCheck(dev, f):
                    continue
                    
                debug("Synchronising file: %s" % repr(f))
                self._sync(f)

            if not GsyncOptions.recursive:
                break


    def run(self):
        srcpath = self._src
        basepath, path = os.path.split(srcpath)

        if self._drive.is_drivepath(self._src):
            basepath = self._drive.normpath(basepath)

        debug("Source srcpath: %s" % repr(srcpath))
        debug("Source basepath: %s" % repr(basepath))
        debug("Source path: %s" % repr(path))

        if GsyncOptions.relative:
            # Supports the foo/./bar notation in rsync.
            path = re.sub(r'^.*/\./', "", path)

        self._sync = Sync(basepath, self._dst)

        debug("Enumerating: %s" % repr(srcpath))

        try:
            self._walk(srcpath, self._walkCallback, self._dev)

        except KeyboardInterrupt, e:
            print("\nInterrupted")
            raise

        except Exception, e:
            debug.exception(e)
            print("Error: %s" % repr(e))
Example #9
0
class Crawler(object):
    def __init__(self, src, dst):
        self._dev = None
        self._src = None
        self._dst = None

        force_dest_file = GsyncOptions.force_dest_file

        self._drive = Drive()

        if self._drive.is_drivepath(src):
            self._walkCallback = bind("walk", self._drive)
            self._src = self._drive.normpath(src)
            info = self._drive.stat(self._src)

            if info and info.mimeType != MimeTypes.FOLDER:
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True
        else:
            self._walkCallback = os_walk_wrapper
            self._src = os.path.normpath(src)
            st_info = os.stat(self._src)

            if os.path.isfile(self._src):
                debug("Source is not a directory, forcing dest file: %s" %
                      (repr(self._src)))
                force_dest_file = True

            if GsyncOptions.one_file_system:
                self._dev = st_info.st_dev

        if self._drive.is_drivepath(dst):
            self._dst = self._drive.normpath(dst)
            info = self._drive.stat(self._dst)

            if info and info.mimeType == MimeTypes.FOLDER:
                debug("Dest is a directory, not forcing dest file: %s" %
                      (repr(self._dst)))
                force_dest_file = False
        else:
            self._dst = os.path.normpath(dst)
            if os.path.isdir(self._dst):
                debug("Dest is a directory, not forcing dest file: %s" %
                      (repr(self._dst)))
                force_dest_file = False

        if src[-1] == "/":
            self._src += "/"

        if dst[-1] == "/":
            self._dst += "/"
            debug("Dest has trailing slash, not forcing dest file: %s" %
                  (self._dst))
            force_dest_file = False

        # Only update if not already set.
        if GsyncOptions.force_dest_file is None:
            debug("force_dest_file = %s" % force_dest_file)
            GsyncOptions.force_dest_file = force_dest_file

        #super(Crawler, self).__init__(name = "Crawler: %s" % src)

    def _devCheck(self, dev, path):
        if dev is not None:
            st_info = os.stat(path)
            if st_info.st_dev != dev:
                debug("Not on same dev: %s" % repr(path))
                return False

        return True

    def _walk(self, path, generator, dev):
        for d, dirs, files in generator(path):
            debug("Walking: %s" % repr(d))

            if not self._devCheck(dev, d):
                debug("Not on same device: %s" % repr(d))
                continue

            if not GsyncOptions.force_dest_file:
                if GsyncOptions.dirs or GsyncOptions.recursive:

                    # Sync the directory but not its contents
                    debug("Synchronising directory: %s" % repr(d))
                    self._sync(d)
                else:
                    sys.stdout.write("skipping directory %s\n" % d)
                    break

            for f in files:
                f = os.path.join(d, f)
                if not self._devCheck(dev, f):
                    continue

                debug("Synchronising file: %s" % repr(f))
                self._sync(f)

            if not GsyncOptions.recursive:
                break

    def run(self):
        srcpath = self._src
        basepath, path = os.path.split(srcpath)

        if self._drive.is_drivepath(self._src):
            basepath = self._drive.normpath(basepath)

        debug("Source srcpath: %s" % repr(srcpath))
        debug("Source basepath: %s" % repr(basepath))
        debug("Source path: %s" % repr(path))

        if GsyncOptions.relative:
            # Supports the foo/./bar notation in rsync.
            path = re.sub(r'^.*/\./', "", path)

        self._sync = Sync(basepath, self._dst)

        debug("Enumerating: %s" % repr(srcpath))

        try:
            self._walk(srcpath, self._walkCallback, self._dev)

        except KeyboardInterrupt, e:
            print("\nInterrupted")
            raise

        except Exception, e:
            debug.exception(e)
            print("Error: %s" % repr(e))