示例#1
0
def upload_dir(local, remote, dry_run=False, opts=None):
    env = environment()
    if 'ssh' in env:
        ssh = env['ssh']
        rsync_opts = [
            '--recursive', '--archive', '--compress', '--itemize-changes'
        ]
        if dry_run:
            rsync_opts.append('--dry-run')
        if state['verbose']:
            rsync_opts.append('--verbose')
        # trailing slash is important, see rsync documentation
        src = local + '/'
        abs_remote = os.path.join(ssh['document_root'], remote)
        dest = '{}@{}:{}'.format(ssh['user'], ssh['host'], abs_remote)
        args = rsync_opts + ([] if opts is None else opts) + [src, dest]
        # TODO rsync still prompts for the password
        with shell_env(RSYNC_PASSWORD=ssh['password']):
            fab.local('rsync {}'.format(' '.join(args)))
    else:
        ftp = env['ftp']
        host = ftp_host(env)
        extra_opts = {'ftp_debug': 1 if state['verbose'] else 0}
        url = urlparse(ftp['url'])
        if url.scheme != 'ftp':
            fab.warn('non-ftp url scheme, may not be supported')
        abs_remote = os.path.join(url.path, remote)
        if not host.path.exists(abs_remote):
            fab.warn('remote path is missing: {}'.format(abs_remote))
            if console.confirm('create missing directories?'):
                host.makedirs(abs_remote)
        local_target = FsTarget(local, extra_opts=extra_opts)
        remote_target = FtpTarget(path=abs_remote,
                                  host=url.netloc,
                                  username=ftp['user'],
                                  password=ftp['password'],
                                  extra_opts=extra_opts)
        opts = {
            'force': False,
            # TODO
            'delete_unmatched': False,
            'verbose': 3,
            'execute': True,
            'dry_run': dry_run
        }
        s = UploadSynchronizer(local_target, remote_target, opts)
        try:
            s.run()
        except KeyboardInterrupt:
            fab.warn('aborted by user')
        finally:
            s.local.close()
            s.remote.close()
        stats = s.get_stats()
        pprint(stats)
示例#2
0
    def test_issue_20(self):

        opts = {"verbose": 5}

        local_target = make_target(self.local_url)
        remote_target = make_target(self.remote_url)

        ftp_downloader = DownloadSynchronizer(local_target, remote_target, opts)
        ftp_uploader = UploadSynchronizer(local_target, remote_target, opts)

        write_test_file("local/large1.txt", size=10 * 1000)
        write_test_file("remote/large2.txt", size=10 * 1000)
        ftp_downloader.run()
        ftp_uploader.run()
示例#3
0
    def test_upload_fs_fs(self):
        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
        remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
        opts = {"force": False, "delete": False, "dry_run": False}
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
#        pprint(stats)
        self.assertEqual(stats["local_dirs"], 2)
        self.assertEqual(stats["local_files"], 4) # currently files are not counted, when inside a *new* folder
        self.assertEqual(stats["remote_dirs"], 0)
        self.assertEqual(stats["remote_files"], 0)
        self.assertEqual(stats["files_written"], 6)
        self.assertEqual(stats["dirs_created"], 2)
        self.assertEqual(stats["bytes_written"], 16403)
        # file times are preserved
        self.assertEqual(_get_test_file_date("local/file1.txt"), STAMP_20140101_120000)
        self.assertEqual(_get_test_file_date("remote/file1.txt"), STAMP_20140101_120000)
示例#4
0
    def _transfer_files(self, count, size):
        temp1_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "temp1")
        _empty_folder(temp1_path)  # remove standard test files

        local = FsTarget(temp1_path)
        remote = self.remote

        for i in range(count):
            _write_test_file("temp1/file_%s.txt" % i, size=size)

        # Upload all of /temp1 to remote

        opts = {
            "force": False,
            "delete": False,
            "verbose": 3,
            "dry_run": False
        }
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)
        #        pprint(stats)
        print("Upload %s x %s bytes took %s: %s" %
              (count, size, stats["upload_write_time"],
               stats["upload_rate_str"]),
              file=sys.stderr)

        # Download all of remote to /temp2

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "temp2"))

        opts = {"force": False, "delete": True, "verbose": 3, "dry_run": False}
        s = DownloadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)

        #        pprint(stats)
        print("Download %s x %s bytes took %s: %s" %
              (count, size, stats["download_write_time"],
               stats["download_rate_str"]),
              file=sys.stderr)
示例#5
0
    def _transfer_files(self, count, size):
        temp1_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "local")
        empty_folder(temp1_path)  # remove standard test files

        local = FsTarget(temp1_path)
        remote = self.remote

        for i in range(count):
            write_test_file("local/file_{}.txt".format(i), size=size)

        # Upload all of temp/local to remote

        opts = {"force": False, "delete": False, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)
        #        pprint(stats)
        print(
            "Upload {} x {} bytes took {}: {}".format(
                count, size, stats["upload_write_time"],
                stats["upload_rate_str"]),
            file=sys.stderr,
        )

        # Download all of remote to temp/remote

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"force": False, "delete": True, "verbose": 3}
        s = DownloadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)

        #        pprint(stats)
        print(
            "Download {} x {} bytes took {}: {}".format(
                count, size, stats["download_write_time"],
                stats["download_rate_str"]),
            file=sys.stderr,
        )
示例#6
0
    def sync_to_server(self):
        """Sync local readings to FTP server

        Note: this recreates the download/upload synchronisers each time to
        avoid issues with flags set in these classes (bug in pyftpsync; see
        https://github.com/mar10/pyftpsync/issues/20)
        """

        # FTP upload options
        upload_opts = {"force": True,
                       "resolve": "local",
                       "verbose": 3}

        # create uploader
        ftp_uploader = UploadSynchronizer(self.local_target,
                                          self.remote_target,
                                          upload_opts)

        # upload
        ftp_uploader.run()
示例#7
0
    def _transfer_files(self, count, size):
        temp1_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "local")
        empty_folder(temp1_path)  # remove standard test files

        local = FsTarget(temp1_path)
        remote = self.remote

        for i in range(count):
            write_test_file("local/file_{}.txt".format(i), size=size)

        # Upload all of temp/local to remote

        opts = {"force": False, "delete": False, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)
        #        pprint(stats)
        print(
            "Upload {} x {} bytes took {}: {}".format(
                count, size, stats["upload_write_time"], stats["upload_rate_str"]
            ),
            file=sys.stderr,
        )

        # Download all of remote to temp/remote

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"force": False, "delete": True, "verbose": 3}
        s = DownloadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["files_written"], count)
        self.assertEqual(stats["bytes_written"], count * size)

        #        pprint(stats)
        print(
            "Download {} x {} bytes took {}: {}".format(
                count, size, stats["download_write_time"], stats["download_rate_str"]
            ),
            file=sys.stderr,
        )
示例#8
0
    def test_sync_fs_ftp(self):
        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
        remote = self.remote

        # Upload all of temp/local to remote

        opts = {"force": False, "delete": True, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()

        stats = s.get_stats()
        #         pprint(stats)

        self.assertEqual(stats["local_dirs"], 2)
        # currently files are not counted, when inside a *new* folder:
        self.assertEqual(stats["local_files"], 4)
        self.assertEqual(stats["remote_dirs"], 0)
        self.assertEqual(stats["remote_files"], 0)
        self.assertEqual(stats["files_written"], 6)
        self.assertEqual(stats["dirs_created"], 2)
        self.assertEqual(stats["bytes_written"], 16403)

        # Change one file and upload again

        touch_test_file("local/file1.txt")

        opts = {"force": False, "delete": True, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #         pprint(stats)
        #         assert False

        self.assertEqual(stats["entries_seen"], 18)  # ???
        self.assertEqual(stats["entries_touched"], 1)
        self.assertEqual(stats["files_created"], 0)
        self.assertEqual(stats["files_deleted"], 0)
        self.assertEqual(stats["files_written"], 1)
        self.assertEqual(stats["dirs_created"], 0)
        self.assertEqual(stats["download_files_written"], 0)
        self.assertEqual(stats["upload_files_written"], 1)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 3)

        # Download all from remote to temp/remote

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"force": False, "delete": True, "verbose": 3}
        s = DownloadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["entries_seen"], 8)
        self.assertEqual(stats["entries_touched"], 8)
        #        self.assertEqual(stats["files_created"], 6)
        self.assertEqual(stats["files_deleted"], 0)
        self.assertEqual(stats["files_written"], 6)
        self.assertEqual(stats["dirs_created"], 2)
        self.assertEqual(stats["download_files_written"], 6)
        self.assertEqual(stats["upload_files_written"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 16403)

        # Original file times are preserved, even when retrieved from FTP

        self.assertNotEqual(get_test_file_date("local/file1.txt"),
                            STAMP_20140101_120000)
        self.assertEqual(
            get_test_file_date("local/file1.txt"),
            get_test_file_date("local//file1.txt"),
        )

        self.assertEqual(get_test_file_date("local/file2.txt"),
                         STAMP_20140101_120000)
        self.assertEqual(get_test_file_date("remote//file2.txt"),
                         STAMP_20140101_120000)

        # Synchronize temp/local <=> remote : nothing to do

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))

        opts = {"verbose": 3}
        s = BiDirSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        pprint(stats)
        self.assertEqual(stats["entries_touched"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 0)

        # Synchronize temp/remote <=> remote : nothing to do

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"verbose": 3}
        s = BiDirSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        pprint(stats)
        self.assertEqual(stats["entries_touched"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 0)
示例#9
0
def run():
    """CLI main entry point."""

    # Use print() instead of logging when running in CLI mode:
    set_pyftpsync_logger(None)

    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync",
        parents=[verbose_parser],
    )

    # Note: we want to allow --version to be combined with --verbose. However
    # on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used.
    if check_cli_verbose(3) > 3:
        version_info = "pyftpsync/{} Python/{} {}".format(
            __version__, PYTHON_VERSION, platform.platform()
        )
    else:
        version_info = "{}".format(__version__)

    parser.add_argument("-V", "--version", action="version", version=version_info)

    subparsers = parser.add_subparsers(help="sub-command help")

    # --- Create the parser for the "upload" command ---------------------------

    sp = subparsers.add_parser(
        "upload",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files to remote folder",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite remote files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["local", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove remote files if they don't exist locally",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove remote files if they don't exist locally "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="upload")

    # --- Create the parser for the "download" command -------------------------

    sp = subparsers.add_parser(
        "download",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files from remote folder to local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite local files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove local files if they don't exist on remote target",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove local files if they don't exist on remote target "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="download")

    # --- Create the parser for the "sync" command -----------------------------

    sp = subparsers.add_parser(
        "sync",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="synchronize new and modified files between remote folder and local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["old", "new", "local", "remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )

    sp.set_defaults(command="sync")

    # --- Create the parser for the "run" command -----------------------------

    add_run_parser(subparsers)

    # --- Create the parser for the "scan" command -----------------------------

    add_scan_parser(subparsers)

    # --- Create the parser for the "tree" command -----------------------------

    add_tree_parser(subparsers)

    # --- Parse command line ---------------------------------------------------

    args = parser.parse_args()

    args.verbose -= args.quiet
    del args.quiet

    # print("verbose", args.verbose)

    ftp_debug = 0
    if args.verbose >= 6:
        ftp_debug = 1

    if args.debug:
        if args.verbose < 4:
            parser.error("'--debug' requires verbose level >= 4")
        DEBUG_FLAGS.update(args.debug)

    # Modify the `args` from the `pyftpsync.yaml` config:
    if getattr(args, "command", None) == "run":
        handle_run_command(parser, args)

    if callable(getattr(args, "command", None)):
        # scan_handler
        try:
            return args.command(parser, args)
        except KeyboardInterrupt:
            print("\nAborted by user.", file=sys.stderr)
            sys.exit(3)

    elif not hasattr(args, "command"):
        parser.error(
            "missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')"
        )

    # Post-process and check arguments
    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True

    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})

    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(
        args.remote_target, FsTarget
    ):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "sync":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command '{}'".format(args.command))

    s.is_script = True

    try:
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.", file=sys.stderr)
        sys.exit(3)
    finally:
        # Prevent sporadic exceptions in ftplib, when closing in __del__
        s.local.close()
        s.remote.close()

    stats = s.get_stats()
    if args.verbose >= 5:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print(
            "Wrote {}/{} files in {} directories, skipped: {}.".format(
                stats["files_written"],
                stats["local_files"],
                stats["local_dirs"],
                stats["conflict_files_skipped"],
            ),
            end="",
        )
        if stats["interactive_ask"]:
            print()
        else:
            print(" Elap: {}.".format(stats["elap_str"]))

    return
示例#10
0
def run():
    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync"
        )
    
    qv_group = parser.add_mutually_exclusive_group()
    qv_group.add_argument("--verbose", "-v", action="count", default=3,
                        help="increment verbosity by one (default: %(default)s, range: 0..5)")
    qv_group.add_argument("--quiet", "-q", action="count", default=0,
                        help="decrement verbosity by one")

    parser.add_argument("--version", action="version", version="%s" % __version__)
    parser.add_argument("--progress", "-p", 
                        action="store_true",
                        default=False, 
                        help="show progress info, even if redirected or verbose < 3")
    
    subparsers = parser.add_subparsers(help="sub-command help")
    
    def __add_common_sub_args(parser):
        parser.add_argument("local", 
                            metavar="LOCAL",
#                             required=True,
                            default=".",
                            help="path to local folder (default: %(default)s)")      
        parser.add_argument("remote", 
                            metavar="REMOTE",
                            help="path to remote folder")
#        upload_parser.add_argument("--dry-run", 
#                                   action="store_true",
#                                   help="just simulate and log results; don't change anything")
        parser.add_argument("-x", "--execute", 
                            action="store_false", dest="dry_run", default=True,
                            help="turn off the dry-run mode (which is ON by default), "
                            "that would just print status messages but does "
                            "not change anything")
        parser.add_argument("-f", "--include-files", 
                            help="wildcard for file names (default: all, "
                            "separate multiple values with ',')")
        parser.add_argument("-o", "--omit", 
                            help="wildcard of files and directories to exclude (applied after --include)")
        parser.add_argument("--store-password", 
                                 action="store_true",
                                 help="save password to keyring if login succeeds")
        parser.add_argument("--no-prompt", 
                            action="store_true",
                            help="prevent prompting for missing credentials")
        parser.add_argument("--no-color", 
                            action="store_true",
                            help="prevent use of ansi terminal color codes")    
    
    # Create the parser for the "upload" command
    upload_parser = subparsers.add_parser("upload", 
                                          help="copy new and modified files to remote folder")
    __add_common_sub_args(upload_parser)

    upload_parser.add_argument("--force", 
                               action="store_true",
                               help="overwrite different remote files, even if the target is newer")
    upload_parser.add_argument("--delete", 
                               action="store_true",
                               help="remove remote files if they don't exist locally")
    upload_parser.add_argument("--delete-unmatched", 
                               action="store_true",
                               help="remove remote files if they don't exist locally "
                               "or don't match the current filter (implies '--delete' option)")

    upload_parser.set_defaults(command="upload")
    

    # Create the parser for the "download" command
    download_parser = subparsers.add_parser("download", 
            help="copy new and modified files from remote folder to local target")
    __add_common_sub_args(download_parser)
    
    download_parser.add_argument("--force", 
                                 action="store_true",
                                 help="overwrite different local files, even if the target is newer")
    download_parser.add_argument("--delete", 
                                 action="store_true",
                                 help="remove local files if they don't exist on remote target")
    download_parser.add_argument("--delete-unmatched", 
                                 action="store_true",
                                 help="remove local files if they don't exist on remote target "
                                 "or don't match the current filter (implies '--delete' option)")
    
    download_parser.set_defaults(command="download")
    
    # Create the parser for the "sync" command
    sync_parser = subparsers.add_parser("sync", 
            help="synchronize new and modified files between remote folder and local target")
    __add_common_sub_args(sync_parser)
    
#     sync_parser.add_argument("--store-password", 
#                              action="store_true",
#                              help="save password to keyring if login succeeds")
#     sync_parser.add_argument("--no-prompt", 
#                              action="store_true",
#                              help="prevent prompting for missing credentials")
#     sync_parser.add_argument("--no-color", 
#                              action="store_true",
#                              help="prevent use of ansi terminal color codes")
    sync_parser.add_argument("--resolve", 
                             default="ask",
                             choices=["old", "new", "local", "remote", "skip", "ask"],
                             help="conflict resolving strategy (default: 'ask')")
    
    sync_parser.set_defaults(command="synchronize")
    
    # Parse command line
    args = parser.parse_args()

    if not hasattr(args, "command"):
        parser.error("missing command (choose from 'upload', 'download', 'sync')")

    # Post-process and check arguments
    args.verbose -= args.quiet
    del args.quiet

    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True
    
    ftp_debug = 0
    if args.verbose >= 5:
        ftp_debug = 1 
        
    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
    
    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(args.remote_target, FsTarget):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "synchronize":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command %s" % args.command)

    try:    
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.")
        return

    stats = s.get_stats()
    if args.verbose >= 4:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print("Wrote %s/%s files in %s dirs. Elap: %s" 
              % (stats["files_written"], stats["local_files"], stats["local_dirs"], stats["elap_str"]))
示例#11
0
def run():
    """CLI main entry point."""

    # Use print() instead of logging when running in CLI mode:
    set_pyftpsync_logger(None)

    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync")

    parser.add_argument("-V",
                        "--version",
                        action="version",
                        version="{}".format(__version__))

    subparsers = parser.add_subparsers(help="sub-command help")

    # --- Create the parser for the "upload" command ---------------------------

    sp = subparsers.add_parser(
        "upload", help="copy new and modified files to remote folder")

    sp.add_argument("--force",
                    action="store_true",
                    help="overwrite remote files, even if the target is newer "
                    "(but no conflict was detected)")
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["local", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")
    sp.add_argument("--delete",
                    action="store_true",
                    help="remove remote files if they don't exist locally")
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove remote files if they don't exist locally "
        "or don't match the current filter (implies '--delete' option)")

    add_common_sub_args(sp)
    sp.set_defaults(command="upload")

    # --- Create the parser for the "download" command -------------------------

    sp = subparsers.add_parser(
        "download",
        help="copy new and modified files from remote folder to local target")

    sp.add_argument("--force",
                    action="store_true",
                    help="overwrite local files, even if the target is newer "
                    "(but no conflict was detected)")
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove local files if they don't exist on remote target")
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove local files if they don't exist on remote target "
        "or don't match the current filter (implies '--delete' option)")

    add_common_sub_args(sp)
    sp.set_defaults(command="download")

    # --- Create the parser for the "sync" command -----------------------------

    sp = subparsers.add_parser(
        "sync",
        help=
        "synchronize new and modified files between remote folder and local target"
    )

    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["old", "new", "local", "remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")

    add_common_sub_args(sp)
    sp.set_defaults(command="synchronize")

    # --- Create the parser for the "scan" command -----------------------------

    add_scan_parser(subparsers)

    # --- Parse command line ---------------------------------------------------

    args = parser.parse_args()

    args.verbose -= args.quiet
    del args.quiet

    ftp_debug = 0
    if args.verbose >= 6:
        ftp_debug = 1

    if callable(getattr(args, "command", None)):
        try:
            return getattr(args, "command")(args)
        except KeyboardInterrupt:
            print("\nAborted by user.", file=sys.stderr)
            sys.exit(3)

    elif not hasattr(args, "command"):
        parser.error(
            "missing command (choose from 'upload', 'download', 'sync', 'scan')"
        )

    # Post-process and check arguments
    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True

    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})

    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(
            args.remote_target, FsTarget):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "synchronize":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command {}".format(args.command))

    s.is_script = True

    try:
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.", file=sys.stderr)
        sys.exit(3)
    finally:
        # Prevent sporadic exceptions in ftplib, when closing in __del__
        s.local.close()
        s.remote.close()

    stats = s.get_stats()
    if args.verbose >= 5:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print("Wrote {}/{} files in {} directories, skipped: {}.".format(
            stats["files_written"], stats["local_files"], stats["local_dirs"],
            stats["conflict_files_skipped"]),
              end="")
        if stats["interactive_ask"]:
            print()
        else:
            print(" Elap: {}.".format(stats["elap_str"]))

    return
示例#12
0
from ftpsync.targets import FsTarget
from ftpsync.ftp_target import FtpTarget
from ftpsync.synchronizers import UploadSynchronizer

local = FsTarget('C:\\temp')
user = '******'
passwd = 'pwd'
remote = FtpTarget("/foo", "ftp.svr.com", username=user, password=passwd)
opts = {"force": True, "delete_unmatched": True, "verbose": 3}
s = UploadSynchronizer(local, remote, opts)
s.run()
示例#13
0
def run():
    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync")

    qv_group = parser.add_mutually_exclusive_group()
    qv_group.add_argument(
        "--verbose",
        "-v",
        action="count",
        default=3,
        help="increment verbosity by one (default: %(default)s, range: 0..5)")
    qv_group.add_argument("--quiet",
                          "-q",
                          action="count",
                          default=0,
                          help="decrement verbosity by one")

    parser.add_argument("-V",
                        "--version",
                        action="version",
                        version="%s" % __version__)
    parser.add_argument(
        "--progress",
        "-p",
        action="store_true",
        default=False,
        help="show progress info, even if redirected or verbose < 3")

    subparsers = parser.add_subparsers(help="sub-command help")

    def __add_common_sub_args(parser):
        parser.add_argument(
            "local",
            metavar="LOCAL",
            #                             required=True,
            default=".",
            help="path to local folder (default: %(default)s)")
        parser.add_argument("remote",
                            metavar="REMOTE",
                            help="path to remote folder")
        parser.add_argument(
            "--dry-run",
            action="store_true",
            help="just simulate and log results, but don't change anything")
        # parser.add_argument("-x", "--execute",
        #                     action="store_false", dest="dry_run", default=True,
        #                     help="turn off the dry-run mode (which is ON by default), "
        #                     "that would just print status messages but does "
        #                     "not change anything")
        parser.add_argument("-f",
                            "--include-files",
                            help="wildcard for file names (default: all, "
                            "separate multiple values with ',')")
        parser.add_argument(
            "-o",
            "--omit",
            help=
            "wildcard of files and directories to exclude (applied after --include)"
        )
        parser.add_argument("--store-password",
                            action="store_true",
                            help="save password to keyring if login succeeds")
        parser.add_argument("--no-prompt",
                            action="store_true",
                            help="prevent prompting for missing credentials")
        parser.add_argument("--no-color",
                            action="store_true",
                            help="prevent use of ansi terminal color codes")

    # --- Create the parser for the "upload" command ---------------------------

    upload_parser = subparsers.add_parser(
        "upload", help="copy new and modified files to remote folder")
    __add_common_sub_args(upload_parser)

    upload_parser.add_argument(
        "--force",
        action="store_true",
        help=
        "overwrite remote files, even if the target is newer (but no conflict was detected)"
    )
    upload_parser.add_argument(
        "--resolve",
        default="skip",
        choices=["local", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")
    upload_parser.add_argument(
        "--delete",
        action="store_true",
        help="remove remote files if they don't exist locally")
    upload_parser.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove remote files if they don't exist locally "
        "or don't match the current filter (implies '--delete' option)")

    upload_parser.set_defaults(command="upload")

    # --- Create the parser for the "download" command -------------------------

    download_parser = subparsers.add_parser(
        "download",
        help="copy new and modified files from remote folder to local target")
    __add_common_sub_args(download_parser)

    download_parser.add_argument(
        "--force",
        action="store_true",
        help=
        "overwrite local files, even if the target is newer (but no conflict was detected)"
    )
    download_parser.add_argument(
        "--resolve",
        default="skip",
        choices=["remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")
    download_parser.add_argument(
        "--delete",
        action="store_true",
        help="remove local files if they don't exist on remote target")
    download_parser.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove local files if they don't exist on remote target "
        "or don't match the current filter (implies '--delete' option)")

    download_parser.set_defaults(command="download")

    # --- Create the parser for the "sync" command -----------------------------

    sync_parser = subparsers.add_parser(
        "sync",
        help=
        "synchronize new and modified files between remote folder and local target"
    )
    __add_common_sub_args(sync_parser)

    sync_parser.add_argument(
        "--resolve",
        default="ask",
        choices=["old", "new", "local", "remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')")

    sync_parser.set_defaults(command="synchronize")

    # --- Create the parser for the "scan" command -----------------------------

    scan_parser = add_scan_parser(subparsers)

    # --- Parse command line ---------------------------------------------------

    args = parser.parse_args()

    args.verbose -= args.quiet
    del args.quiet

    ftp_debug = 0
    if args.verbose >= 5:
        ftp_debug = 1

    if callable(getattr(args, "command", None)):
        return getattr(args, "command")(args)
    elif not hasattr(args, "command"):
        parser.error(
            "missing command (choose from 'upload', 'download', 'sync')")

    # Post-process and check arguments
    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True

    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})

    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(
            args.remote_target, FsTarget):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "synchronize":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command %s" % args.command)

    try:
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.")
        return
    finally:
        # prevent sporadic exceptions in ftplib, when closing in __del__
        s.local.close()
        s.remote.close()

    stats = s.get_stats()
    if args.verbose >= 4:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print("Wrote %s/%s files in %s dirs. Elap: %s" %
              (stats["files_written"], stats["local_files"],
               stats["local_dirs"], stats["elap_str"]))

    return
示例#14
0
    def test_sync_fs_ftp(self):
        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
        remote = self.remote

        # Upload all of temp/local to remote

        opts = {"force": False, "delete": True, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()

        stats = s.get_stats()
        #         pprint(stats)

        self.assertEqual(stats["local_dirs"], 2)
        # currently files are not counted, when inside a *new* folder:
        self.assertEqual(stats["local_files"], 4)
        self.assertEqual(stats["remote_dirs"], 0)
        self.assertEqual(stats["remote_files"], 0)
        self.assertEqual(stats["files_written"], 6)
        self.assertEqual(stats["dirs_created"], 2)
        self.assertEqual(stats["bytes_written"], 16403)

        # Change one file and upload again

        touch_test_file("local/file1.txt")

        opts = {"force": False, "delete": True, "verbose": 3}
        s = UploadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #         pprint(stats)
        #         assert False

        self.assertEqual(stats["entries_seen"], 18)  # ???
        self.assertEqual(stats["entries_touched"], 1)
        self.assertEqual(stats["files_created"], 0)
        self.assertEqual(stats["files_deleted"], 0)
        self.assertEqual(stats["files_written"], 1)
        self.assertEqual(stats["dirs_created"], 0)
        self.assertEqual(stats["download_files_written"], 0)
        self.assertEqual(stats["upload_files_written"], 1)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 3)

        # Download all from remote to temp/remote

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"force": False, "delete": True, "verbose": 3}
        s = DownloadSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        #        pprint(stats)

        self.assertEqual(stats["entries_seen"], 8)
        self.assertEqual(stats["entries_touched"], 8)
        #        self.assertEqual(stats["files_created"], 6)
        self.assertEqual(stats["files_deleted"], 0)
        self.assertEqual(stats["files_written"], 6)
        self.assertEqual(stats["dirs_created"], 2)
        self.assertEqual(stats["download_files_written"], 6)
        self.assertEqual(stats["upload_files_written"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 16403)

        # Original file times are preserved, even when retrieved from FTP

        self.assertNotEqual(
            get_test_file_date("local/file1.txt"), STAMP_20140101_120000
        )
        self.assertEqual(
            get_test_file_date("local/file1.txt"),
            get_test_file_date("local//file1.txt"),
        )

        self.assertEqual(get_test_file_date("local/file2.txt"), STAMP_20140101_120000)
        self.assertEqual(get_test_file_date("remote//file2.txt"), STAMP_20140101_120000)

        # Synchronize temp/local <=> remote : nothing to do

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))

        opts = {"verbose": 3}
        s = BiDirSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        pprint(stats)
        self.assertEqual(stats["entries_touched"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 0)

        # Synchronize temp/remote <=> remote : nothing to do

        local = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))

        opts = {"verbose": 3}
        s = BiDirSynchronizer(local, remote, opts)
        s.run()
        stats = s.get_stats()
        pprint(stats)
        self.assertEqual(stats["entries_touched"], 0)
        self.assertEqual(stats["conflict_files"], 0)
        self.assertEqual(stats["bytes_written"], 0)
示例#15
0
def run():
    """CLI main entry point."""

    # Use print() instead of logging when running in CLI mode:
    set_pyftpsync_logger(None)

    parser = argparse.ArgumentParser(
        description="Synchronize folders over FTP.",
        epilog="See also https://github.com/mar10/pyftpsync",
        parents=[verbose_parser],
    )

    # Note: we want to allow --version to be combined with --verbose. However
    # on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used.
    if check_cli_verbose(3) > 3:
        version_info = "pyftpsync/{} Python/{} {}".format(
            __version__, PYTHON_VERSION, platform.platform()
        )
    else:
        version_info = "{}".format(__version__)

    parser.add_argument("-V", "--version", action="version", version=version_info)

    subparsers = parser.add_subparsers(help="sub-command help")

    # --- Create the parser for the "upload" command ---------------------------

    sp = subparsers.add_parser(
        "upload",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files to remote folder",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite remote files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["local", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove remote files if they don't exist locally",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove remote files if they don't exist locally "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="upload")

    # --- Create the parser for the "download" command -------------------------

    sp = subparsers.add_parser(
        "download",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="copy new and modified files from remote folder to local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--force",
        action="store_true",
        help="overwrite local files, even if the target is newer "
        "(but no conflict was detected)",
    )
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )
    sp.add_argument(
        "--delete",
        action="store_true",
        help="remove local files if they don't exist on remote target",
    )
    sp.add_argument(
        "--delete-unmatched",
        action="store_true",
        help="remove local files if they don't exist on remote target "
        "or don't match the current filter (implies '--delete' option)",
    )

    sp.set_defaults(command="download")

    # --- Create the parser for the "sync" command -----------------------------

    sp = subparsers.add_parser(
        "sync",
        parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
        help="synchronize new and modified files between remote folder and local target",
    )

    sp.add_argument(
        "local",
        metavar="LOCAL",
        default=".",
        help="path to local folder (default: %(default)s)",
    )
    sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
    sp.add_argument(
        "--resolve",
        default="ask",
        choices=["old", "new", "local", "remote", "skip", "ask"],
        help="conflict resolving strategy (default: '%(default)s')",
    )

    sp.set_defaults(command="sync")

    # --- Create the parser for the "run" command -----------------------------

    add_run_parser(subparsers)

    # --- Create the parser for the "scan" command -----------------------------

    add_scan_parser(subparsers)

    # --- Parse command line ---------------------------------------------------

    args = parser.parse_args()

    args.verbose -= args.quiet
    del args.quiet

    # print("verbose", args.verbose)

    ftp_debug = 0
    if args.verbose >= 6:
        ftp_debug = 1

    # Modify the `args` from the `pyftpsync.yaml` config:
    if getattr(args, "command", None) == "run":
        handle_run_command(parser, args)

    if callable(getattr(args, "command", None)):
        # scan_handler
        try:
            return args.command(parser, args)
        except KeyboardInterrupt:
            print("\nAborted by user.", file=sys.stderr)
            sys.exit(3)

    elif not hasattr(args, "command"):
        parser.error(
            "missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')"
        )

    # Post-process and check arguments
    if hasattr(args, "delete_unmatched") and args.delete_unmatched:
        args.delete = True

    args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})

    if args.remote == ".":
        parser.error("'.' is expected to be the local target (not remote)")
    args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
    if not isinstance(args.local_target, FsTarget) and isinstance(
        args.remote_target, FsTarget
    ):
        parser.error("a file system target is expected to be local")

    # Let the command handler do its thing
    opts = namespace_to_dict(args)
    if args.command == "upload":
        s = UploadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "download":
        s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
    elif args.command == "sync":
        s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
    else:
        parser.error("unknown command '{}'".format(args.command))

    s.is_script = True

    try:
        s.run()
    except KeyboardInterrupt:
        print("\nAborted by user.", file=sys.stderr)
        sys.exit(3)
    finally:
        # Prevent sporadic exceptions in ftplib, when closing in __del__
        s.local.close()
        s.remote.close()

    stats = s.get_stats()
    if args.verbose >= 5:
        pprint(stats)
    elif args.verbose >= 1:
        if args.dry_run:
            print("(DRY-RUN) ", end="")
        print(
            "Wrote {}/{} files in {} directories, skipped: {}.".format(
                stats["files_written"],
                stats["local_files"],
                stats["local_dirs"],
                stats["conflict_files_skipped"],
            ),
            end="",
        )
        if stats["interactive_ask"]:
            print()
        else:
            print(" Elap: {}.".format(stats["elap_str"]))

    return