def test_issue_24(self): if not self.use_ftp_target: raise SkipTest("Only FTP targets.") # empty_folder() empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "local")) empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote")) local_target = make_target(self.local_url) remote_target = make_target(self.remote_url) SIZE = 1000 * 1000 write_test_file("local/large1.txt", size=SIZE) write_test_file("remote/large2.txt", size=SIZE) opts = { "verbose": 5, "match": "large*.txt", } synchronizer = DownloadSynchronizer(local_target, remote_target, opts) assert is_test_file("local/large1.txt") assert is_test_file("remote/large2.txt") assert not is_test_file("remote/large1.txt") assert not is_test_file("local/large2.txt") synchronizer.run() # Expect that large2 was downloaded assert not is_test_file("remote/large1.txt") assert get_test_file_size("local/large2.txt") == SIZE
def test_make_target(self): t = make_target("sftp://ftp.example.com/target/folder") self.assertTrue(isinstance(t, SFTPTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.username, None) # scheme is case-insensitive t = make_target("SFTP://ftp.example.com/target/folder") self.assertTrue(isinstance(t, SFTPTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.username, None) # pass credentials with URL url = "user:[email protected]/target/folder" t = make_target("sftp://" + url) self.assertTrue(isinstance(t, SFTPTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.username, "user") self.assertEqual(t.password, "secret") self.assertEqual(t.root_dir, "/target/folder") url = "[email protected]:[email protected]/target/folder" t = make_target("sftp://" + url) self.assertTrue(isinstance(t, SFTPTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.username, "*****@*****.**") self.assertEqual(t.password, "secret") self.assertEqual(t.root_dir, "/target/folder")
def test_issue_24(self): if not self.use_ftp_target: raise SkipTest("Only FTP targets.") # empty_folder() empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "local")) empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote")) local_target = make_target(self.local_url) remote_target = make_target(self.remote_url) SIZE = 1000 * 1000 write_test_file("local/large1.txt", size=SIZE) write_test_file("remote/large2.txt", size=SIZE) opts = {"verbose": 5, "match": "large*.txt"} synchronizer = DownloadSynchronizer(local_target, remote_target, opts) assert is_test_file("local/large1.txt") assert is_test_file("remote/large2.txt") assert not is_test_file("remote/large1.txt") assert not is_test_file("local/large2.txt") synchronizer.run() # Expect that large2 was downloaded assert not is_test_file("remote/large1.txt") assert get_test_file_size("local/large2.txt") == SIZE
def do_sync(): from ftpsync.synchronizers import BiDirSynchronizer from ftpsync.targets import make_target local_target = make_target("~/test_pyftpsync_dl") # local_target = make_target("c:/tmp/test_pyftpsync") remote_target = make_target("ftp://wwwendt.de/test_pyftpsync", {"ftp_debug": False}) opts = {"verbose": 6, "dry_run": False, "resolve": "local"} s = BiDirSynchronizer(local_target, remote_target, opts) s.run()
def test_issue_20(self): opts = {"verbose": 5} local_target = make_target(self.local_url) remote_target = make_target(self.remote_url) ftp_downloader = DownloadSynchronizer(local_target, remote_target, opts) ftp_uploader = UploadSynchronizer(local_target, remote_target, opts) write_test_file("local/large1.txt", size=10 * 1000) write_test_file("remote/large2.txt", size=10 * 1000) ftp_downloader.run() ftp_uploader.run()
def setUp(self): # Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder" # TODO: some of those tests are still relevant self.skipTest("Not yet implemented.") ftp_url = PYFTPSYNC_TEST_FTP_URL if not ftp_url: self.skipTest("Must configure an FTP target " "(environment variable PYFTPSYNC_TEST_FTP_URL)") self.assertTrue("/test" in ftp_url or "/temp" in ftp_url, "FTP target path must include '/test' or '/temp'") # Create temp/local folder with files and empty temp/remote folder prepare_fixtures_1() # print(ftp_url) parts = urlparse(ftp_url, allow_fragments=False) self.assertIn(parts.scheme.lower(), ["ftp", "ftps"]) # print(parts) # self.creds = parts.username, parts.password # self.HOST = parts.netloc.split("@", 1)[1] self.PATH = parts.path # user, passwd = get_stored_credentials("pyftpsync.pw", self.HOST) self.remote = make_target(ftp_url) self.remote.open() # This check is already performed in the constructor: # self.assertEqual(self.remote.pwd(), self.PATH) # Delete all files in remote target folder, except for LOCK file self.remote._rmdir_impl(".", keep_root_folder=True, predicate=lambda n: n != DirMetadata.LOCK_FILE_NAME)
def _make_remote_target(cls): """Return the remote target instance, depending on `use_ftp_target`.""" if cls.use_ftp_target: check_ftp_test_connection(PYFTPSYNC_TEST_FOLDER, PYFTPSYNC_TEST_FTP_URL) remote = make_target(PYFTPSYNC_TEST_FTP_URL) else: remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote")) return remote
def test_make_target(self): for scheme in ["ftp", "ftps"]: tls = True if scheme == "ftps" else False t = make_target(scheme + "://ftp.example.com/target/folder") self.assertTrue(isinstance(t, FtpTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.username, None) self.assertEqual(t.tls, tls) # scheme is case-insensitive t = make_target(scheme.upper() + "://ftp.example.com/target/folder") self.assertTrue(isinstance(t, FtpTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.username, None) self.assertEqual(t.tls, tls) # pass credentials with URL url = "user:[email protected]/target/folder" t = make_target(scheme + "://" + url) self.assertTrue(isinstance(t, FtpTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.username, "user") self.assertEqual(t.password, "secret") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.tls, tls) url = "[email protected]:[email protected]/target/folder" t = make_target(scheme + "://" + url) self.assertTrue(isinstance(t, FtpTarget)) self.assertEqual(t.host, "ftp.example.com") self.assertEqual(t.username, "*****@*****.**") self.assertEqual(t.password, "secret") self.assertEqual(t.root_dir, "/target/folder") self.assertEqual(t.tls, tls) # unsupported schemes self.assertRaises(ValueError, make_target, "ftpa://ftp.example.com/test") self.assertRaises(ValueError, make_target, "http://example.com/test") self.assertRaises(ValueError, make_target, "https://example.com/test")
def scan_handler(args): """Implement `cleanup` sub-command.""" target = make_target(args.target, {"ftp_debug": args.verbose > 5}) target.readonly = True root_depth = target.root_dir.count("/") start = time.time() dir_count = 1 file_count = 0 processed_files = set() try: target.open() for e in target.walk(): is_dir = isinstance(e, DirectoryEntry) indent = " " * (target.cur_dir.count("/") - root_depth) if is_dir: dir_count += 1 else: file_count += 1 if args.list: if is_dir: print(indent, "[{e.name}]".format(e=e)) else: delta = e.mtime_org - e.mtime if delta: prefix = "+" if delta > 0 else "" print( indent, "{e.name:<40} {e.dt_modified} (system: {prefix}{delta})" .format(e=e, prefix=prefix, delta=timedelta(seconds=delta))) else: print(indent, "{e.name:<40} {e.dt_modified}".format(e=e)) if args.remove_meta and target.cur_dir_meta and target.cur_dir_meta.was_read: fspec = target.cur_dir_meta.get_full_path() if fspec not in processed_files: processed_files.add(fspec) print("DELETE {}".format(fspec)) if args.remove_locks and not is_dir and e.name == DirMetadata.LOCK_FILE_NAME: fspec = e.get_rel_path() print("DELETE {}".format(fspec)) finally: target.close() print("Scanning {:,} files in {:,} dirs took {:02.2f} seconds.".format( file_count, dir_count, time.time() - start))
def setUp(self): if not DO_BENCHMARKS: self.skipTest("DO_BENCHMARKS is not set.") # Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder" ftp_url = PYFTPSYNC_TEST_FTP_URL if not ftp_url: self.skipTest("Must configure an FTP target " "(environment variable PYFTPSYNC_TEST_FTP_URL)") self.assertTrue("/test" in ftp_url or "/temp" in ftp_url, "FTP target path must include '/test' or '/temp'") # Create temp/local folder with files and empty temp/remote folder prepare_fixtures_1() self.remote = make_target(ftp_url) self.remote.open() # Delete all files in remote target folder self.remote._rmdir_impl(".", keep_root=True)
def setUp(self): # Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder" # TODO: some of those tests are still relevant self.skipTest("Not yet implemented.") ftp_url = PYFTPSYNC_TEST_FTP_URL if not ftp_url: self.skipTest( "Must configure an FTP target " "(environment variable PYFTPSYNC_TEST_FTP_URL)" ) self.assertTrue( "/test" in ftp_url or "/temp" in ftp_url, "FTP target path must include '/test' or '/temp'", ) # Create temp/local folder with files and empty temp/remote folder prepare_fixtures_1() # print(ftp_url) parts = urlparse(ftp_url, allow_fragments=False) self.assertIn(parts.scheme.lower(), ["ftp", "ftps"]) # print(parts) # self.creds = parts.username, parts.password # self.HOST = parts.netloc.split("@", 1)[1] self.PATH = parts.path # user, passwd = get_stored_credentials("pyftpsync.pw", self.HOST) self.remote = make_target(ftp_url) self.remote.open() # This check is already performed in the constructor: # self.assertEqual(self.remote.pwd(), self.PATH) # Delete all files in remote target folder, except for LOCK file self.remote._rmdir_impl( ".", keep_root_folder=True, predicate=lambda n: n != DirMetadata.LOCK_FILE_NAME, )
def setUp(self): if not DO_BENCHMARKS: self.skipTest("DO_BENCHMARKS is not set.") # Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder" ftp_url = PYFTPSYNC_TEST_FTP_URL if not ftp_url: self.skipTest( "Must configure an FTP target " "(environment variable PYFTPSYNC_TEST_FTP_URL)" ) self.assertTrue( "/test" in ftp_url or "/temp" in ftp_url, "FTP target path must include '/test' or '/temp'", ) # Create temp/local folder with files and empty temp/remote folder prepare_fixtures_1() self.remote = make_target(ftp_url) self.remote.open() # Delete all files in remote target folder self.remote._rmdir_impl(".", keep_root_folder=True)
def tree_handler(parser, args): """Implement `scan` sub-command.""" opts = namespace_to_dict(args) opts.update({"ftp_debug": args.verbose >= 6}) target = make_target(args.target, opts) target.readonly = True start = time.time() dir_count = 1 file_count = 0 opts = namespace_to_dict(args) process_options(opts) def _pred(entry): """Walker predicate that check match/exclude options.""" if not match_path(entry, opts): return False try: target.open() print("[{}]".format(target.root_dir)) for path, entry in target.walk_tree(sort=args.sort, files=args.files, pred=_pred): name = entry.name if entry.is_dir(): dir_count += 1 line = "{}[{}]".format(path, name) else: file_count += 1 line = "{}{:<20} {}".format(path, name, entry.as_string()) print(line) finally: target.close() print("Scanning {:,} files in {:,} directories took {:02.2f} seconds.". format(file_count, dir_count, time.time() - start))
def scan_handler(parser, args): """Implement `scan` sub-command.""" opts = namespace_to_dict(args) opts.update({"ftp_debug": args.verbose >= 6}) target = make_target(args.target, opts) target.readonly = True root_depth = target.root_dir.count("/") start = time.time() dir_count = 1 file_count = 0 processed_files = set() opts = namespace_to_dict(args) process_options(opts) def _pred(entry): """Walker predicate that check match/exclude options.""" if not match_path(entry, opts): return False try: target.open() for e in target.walk(recursive=args.recursive, pred=_pred): is_dir = isinstance(e, DirectoryEntry) indent = " " * (target.cur_dir.count("/") - root_depth) if is_dir: dir_count += 1 else: file_count += 1 if args.list: if is_dir: print(indent, "[{e.name}]".format(e=e)) else: delta = e.mtime_org - e.mtime dt_modified = pretty_stamp(e.mtime) if delta: prefix = "+" if delta > 0 else "" print( indent, "{e.name:<40} {dt_modified} (system: {prefix}{delta})".format( e=e, prefix=prefix, delta=timedelta(seconds=delta), dt_modified=dt_modified, ), ) else: print( indent, "{e.name:<40} {dt_modified}".format( e=e, dt_modified=dt_modified ), ) if ( args.remove_meta and target.cur_dir_meta and target.cur_dir_meta.was_read ): fspec = target.cur_dir_meta.get_full_path() if fspec not in processed_files: processed_files.add(fspec) print("DELETE {}".format(fspec)) if ( args.remove_locks and not is_dir and e.name == DirMetadata.LOCK_FILE_NAME ): fspec = e.get_rel_path() print("DELETE {}".format(fspec)) finally: target.close() print( "Scanning {:,} files in {:,} directories took {:02.2f} seconds.".format( file_count, dir_count, time.time() - start ) )
def scan_handler(args): """Implement `cleanup` sub-command.""" opts = namespace_to_dict(args) opts.update({ "ftp_debug": args.verbose >= 6, }) target = make_target(args.target, opts) target.readonly = True root_depth = target.root_dir.count("/") start = time.time() dir_count = 1 file_count = 0 processed_files = set() opts = namespace_to_dict(args) process_options(opts) def _pred(entry): """Walker predicate that check match/exclude options.""" if not match_path(entry, opts): return False try: target.open() for e in target.walk(recursive=args.recursive, pred=_pred): is_dir = isinstance(e, DirectoryEntry) indent = " " * (target.cur_dir.count("/") - root_depth) if is_dir: dir_count += 1 else: file_count += 1 if args.list: if is_dir: print(indent, "[{e.name}]".format(e=e)) else: delta = e.mtime_org - e.mtime dt_modified = pretty_stamp(e.mtime) if delta: prefix = "+" if delta > 0 else "" print( indent, "{e.name:<40} {dt_modified} (system: {prefix}{delta})" .format(e=e, prefix=prefix, delta=timedelta(seconds=delta), dt_modified=dt_modified)) else: print( indent, "{e.name:<40} {dt_modified}".format( e=e, dt_modified=dt_modified)) if args.remove_meta and target.cur_dir_meta and target.cur_dir_meta.was_read: fspec = target.cur_dir_meta.get_full_path() if fspec not in processed_files: processed_files.add(fspec) print("DELETE {}".format(fspec)) if args.remove_locks and not is_dir and e.name == DirMetadata.LOCK_FILE_NAME: fspec = e.get_rel_path() print("DELETE {}".format(fspec)) finally: target.close() print("Scanning {:,} files in {:,} directories took {:02.2f} seconds.". format(file_count, dir_count, time.time() - start))
def run(): """CLI main entry point.""" # Use print() instead of logging when running in CLI mode: set_pyftpsync_logger(None) parser = argparse.ArgumentParser( description="Synchronize folders over FTP.", epilog="See also https://github.com/mar10/pyftpsync", parents=[verbose_parser], ) # Note: we want to allow --version to be combined with --verbose. However # on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used. if check_cli_verbose(3) > 3: version_info = "pyftpsync/{} Python/{} {}".format( __version__, PYTHON_VERSION, platform.platform() ) else: version_info = "{}".format(__version__) parser.add_argument("-V", "--version", action="version", version=version_info) subparsers = parser.add_subparsers(help="sub-command help") # --- Create the parser for the "upload" command --------------------------- sp = subparsers.add_parser( "upload", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="copy new and modified files to remote folder", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--force", action="store_true", help="overwrite remote files, even if the target is newer " "(but no conflict was detected)", ) sp.add_argument( "--resolve", default="ask", choices=["local", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.add_argument( "--delete", action="store_true", help="remove remote files if they don't exist locally", ) sp.add_argument( "--delete-unmatched", action="store_true", help="remove remote files if they don't exist locally " "or don't match the current filter (implies '--delete' option)", ) sp.set_defaults(command="upload") # --- Create the parser for the "download" command ------------------------- sp = subparsers.add_parser( "download", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="copy new and modified files from remote folder to local target", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--force", action="store_true", help="overwrite local files, even if the target is newer " "(but no conflict was detected)", ) sp.add_argument( "--resolve", default="ask", choices=["remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.add_argument( "--delete", action="store_true", help="remove local files if they don't exist on remote target", ) sp.add_argument( "--delete-unmatched", action="store_true", help="remove local files if they don't exist on remote target " "or don't match the current filter (implies '--delete' option)", ) sp.set_defaults(command="download") # --- Create the parser for the "sync" command ----------------------------- sp = subparsers.add_parser( "sync", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="synchronize new and modified files between remote folder and local target", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--resolve", default="ask", choices=["old", "new", "local", "remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.set_defaults(command="sync") # --- Create the parser for the "run" command ----------------------------- add_run_parser(subparsers) # --- Create the parser for the "scan" command ----------------------------- add_scan_parser(subparsers) # --- Create the parser for the "tree" command ----------------------------- add_tree_parser(subparsers) # --- Parse command line --------------------------------------------------- args = parser.parse_args() args.verbose -= args.quiet del args.quiet # print("verbose", args.verbose) ftp_debug = 0 if args.verbose >= 6: ftp_debug = 1 if args.debug: if args.verbose < 4: parser.error("'--debug' requires verbose level >= 4") DEBUG_FLAGS.update(args.debug) # Modify the `args` from the `pyftpsync.yaml` config: if getattr(args, "command", None) == "run": handle_run_command(parser, args) if callable(getattr(args, "command", None)): # scan_handler try: return args.command(parser, args) except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) elif not hasattr(args, "command"): parser.error( "missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')" ) # Post-process and check arguments if hasattr(args, "delete_unmatched") and args.delete_unmatched: args.delete = True args.local_target = make_target(args.local, {"ftp_debug": ftp_debug}) if args.remote == ".": parser.error("'.' is expected to be the local target (not remote)") args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug}) if not isinstance(args.local_target, FsTarget) and isinstance( args.remote_target, FsTarget ): parser.error("a file system target is expected to be local") # Let the command handler do its thing opts = namespace_to_dict(args) if args.command == "upload": s = UploadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "download": s = DownloadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "sync": s = BiDirSynchronizer(args.local_target, args.remote_target, opts) else: parser.error("unknown command '{}'".format(args.command)) s.is_script = True try: s.run() except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) finally: # Prevent sporadic exceptions in ftplib, when closing in __del__ s.local.close() s.remote.close() stats = s.get_stats() if args.verbose >= 5: pprint(stats) elif args.verbose >= 1: if args.dry_run: print("(DRY-RUN) ", end="") print( "Wrote {}/{} files in {} directories, skipped: {}.".format( stats["files_written"], stats["local_files"], stats["local_dirs"], stats["conflict_files_skipped"], ), end="", ) if stats["interactive_ask"]: print() else: print(" Elap: {}.".format(stats["elap_str"])) return
def run(): parser = argparse.ArgumentParser( description="Synchronize folders over FTP.", epilog="See also https://github.com/mar10/pyftpsync") qv_group = parser.add_mutually_exclusive_group() qv_group.add_argument( "--verbose", "-v", action="count", default=3, help="increment verbosity by one (default: %(default)s, range: 0..5)") qv_group.add_argument("--quiet", "-q", action="count", default=0, help="decrement verbosity by one") parser.add_argument("-V", "--version", action="version", version="%s" % __version__) parser.add_argument( "--progress", "-p", action="store_true", default=False, help="show progress info, even if redirected or verbose < 3") subparsers = parser.add_subparsers(help="sub-command help") def __add_common_sub_args(parser): parser.add_argument( "local", metavar="LOCAL", # required=True, default=".", help="path to local folder (default: %(default)s)") parser.add_argument("remote", metavar="REMOTE", help="path to remote folder") parser.add_argument( "--dry-run", action="store_true", help="just simulate and log results, but don't change anything") # parser.add_argument("-x", "--execute", # action="store_false", dest="dry_run", default=True, # help="turn off the dry-run mode (which is ON by default), " # "that would just print status messages but does " # "not change anything") parser.add_argument("-f", "--include-files", help="wildcard for file names (default: all, " "separate multiple values with ',')") parser.add_argument( "-o", "--omit", help= "wildcard of files and directories to exclude (applied after --include)" ) parser.add_argument("--store-password", action="store_true", help="save password to keyring if login succeeds") parser.add_argument("--no-prompt", action="store_true", help="prevent prompting for missing credentials") parser.add_argument("--no-color", action="store_true", help="prevent use of ansi terminal color codes") # --- Create the parser for the "upload" command --------------------------- upload_parser = subparsers.add_parser( "upload", help="copy new and modified files to remote folder") __add_common_sub_args(upload_parser) upload_parser.add_argument( "--force", action="store_true", help= "overwrite remote files, even if the target is newer (but no conflict was detected)" ) upload_parser.add_argument( "--resolve", default="skip", choices=["local", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") upload_parser.add_argument( "--delete", action="store_true", help="remove remote files if they don't exist locally") upload_parser.add_argument( "--delete-unmatched", action="store_true", help="remove remote files if they don't exist locally " "or don't match the current filter (implies '--delete' option)") upload_parser.set_defaults(command="upload") # --- Create the parser for the "download" command ------------------------- download_parser = subparsers.add_parser( "download", help="copy new and modified files from remote folder to local target") __add_common_sub_args(download_parser) download_parser.add_argument( "--force", action="store_true", help= "overwrite local files, even if the target is newer (but no conflict was detected)" ) download_parser.add_argument( "--resolve", default="skip", choices=["remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") download_parser.add_argument( "--delete", action="store_true", help="remove local files if they don't exist on remote target") download_parser.add_argument( "--delete-unmatched", action="store_true", help="remove local files if they don't exist on remote target " "or don't match the current filter (implies '--delete' option)") download_parser.set_defaults(command="download") # --- Create the parser for the "sync" command ----------------------------- sync_parser = subparsers.add_parser( "sync", help= "synchronize new and modified files between remote folder and local target" ) __add_common_sub_args(sync_parser) sync_parser.add_argument( "--resolve", default="ask", choices=["old", "new", "local", "remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") sync_parser.set_defaults(command="synchronize") # --- Create the parser for the "scan" command ----------------------------- scan_parser = add_scan_parser(subparsers) # --- Parse command line --------------------------------------------------- args = parser.parse_args() args.verbose -= args.quiet del args.quiet ftp_debug = 0 if args.verbose >= 5: ftp_debug = 1 if callable(getattr(args, "command", None)): return getattr(args, "command")(args) elif not hasattr(args, "command"): parser.error( "missing command (choose from 'upload', 'download', 'sync')") # Post-process and check arguments if hasattr(args, "delete_unmatched") and args.delete_unmatched: args.delete = True args.local_target = make_target(args.local, {"ftp_debug": ftp_debug}) if args.remote == ".": parser.error("'.' is expected to be the local target (not remote)") args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug}) if not isinstance(args.local_target, FsTarget) and isinstance( args.remote_target, FsTarget): parser.error("a file system target is expected to be local") # Let the command handler do its thing opts = namespace_to_dict(args) if args.command == "upload": s = UploadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "download": s = DownloadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "synchronize": s = BiDirSynchronizer(args.local_target, args.remote_target, opts) else: parser.error("unknown command %s" % args.command) try: s.run() except KeyboardInterrupt: print("\nAborted by user.") return finally: # prevent sporadic exceptions in ftplib, when closing in __del__ s.local.close() s.remote.close() stats = s.get_stats() if args.verbose >= 4: pprint(stats) elif args.verbose >= 1: if args.dry_run: print("(DRY-RUN) ", end="") print("Wrote %s/%s files in %s dirs. Elap: %s" % (stats["files_written"], stats["local_files"], stats["local_dirs"], stats["elap_str"])) return
def run(): parser = argparse.ArgumentParser( description="Synchronize folders over FTP.", epilog="See also https://github.com/mar10/pyftpsync" ) qv_group = parser.add_mutually_exclusive_group() qv_group.add_argument("--verbose", "-v", action="count", default=3, help="increment verbosity by one (default: %(default)s, range: 0..5)") qv_group.add_argument("--quiet", "-q", action="count", default=0, help="decrement verbosity by one") parser.add_argument("--version", action="version", version="%s" % __version__) parser.add_argument("--progress", "-p", action="store_true", default=False, help="show progress info, even if redirected or verbose < 3") subparsers = parser.add_subparsers(help="sub-command help") def __add_common_sub_args(parser): parser.add_argument("local", metavar="LOCAL", # required=True, default=".", help="path to local folder (default: %(default)s)") parser.add_argument("remote", metavar="REMOTE", help="path to remote folder") # upload_parser.add_argument("--dry-run", # action="store_true", # help="just simulate and log results; don't change anything") parser.add_argument("-x", "--execute", action="store_false", dest="dry_run", default=True, help="turn off the dry-run mode (which is ON by default), " "that would just print status messages but does " "not change anything") parser.add_argument("-f", "--include-files", help="wildcard for file names (default: all, " "separate multiple values with ',')") parser.add_argument("-o", "--omit", help="wildcard of files and directories to exclude (applied after --include)") parser.add_argument("--store-password", action="store_true", help="save password to keyring if login succeeds") parser.add_argument("--no-prompt", action="store_true", help="prevent prompting for missing credentials") parser.add_argument("--no-color", action="store_true", help="prevent use of ansi terminal color codes") # Create the parser for the "upload" command upload_parser = subparsers.add_parser("upload", help="copy new and modified files to remote folder") __add_common_sub_args(upload_parser) upload_parser.add_argument("--force", action="store_true", help="overwrite different remote files, even if the target is newer") upload_parser.add_argument("--delete", action="store_true", help="remove remote files if they don't exist locally") upload_parser.add_argument("--delete-unmatched", action="store_true", help="remove remote files if they don't exist locally " "or don't match the current filter (implies '--delete' option)") upload_parser.set_defaults(command="upload") # Create the parser for the "download" command download_parser = subparsers.add_parser("download", help="copy new and modified files from remote folder to local target") __add_common_sub_args(download_parser) download_parser.add_argument("--force", action="store_true", help="overwrite different local files, even if the target is newer") download_parser.add_argument("--delete", action="store_true", help="remove local files if they don't exist on remote target") download_parser.add_argument("--delete-unmatched", action="store_true", help="remove local files if they don't exist on remote target " "or don't match the current filter (implies '--delete' option)") download_parser.set_defaults(command="download") # Create the parser for the "sync" command sync_parser = subparsers.add_parser("sync", help="synchronize new and modified files between remote folder and local target") __add_common_sub_args(sync_parser) # sync_parser.add_argument("--store-password", # action="store_true", # help="save password to keyring if login succeeds") # sync_parser.add_argument("--no-prompt", # action="store_true", # help="prevent prompting for missing credentials") # sync_parser.add_argument("--no-color", # action="store_true", # help="prevent use of ansi terminal color codes") sync_parser.add_argument("--resolve", default="ask", choices=["old", "new", "local", "remote", "skip", "ask"], help="conflict resolving strategy (default: 'ask')") sync_parser.set_defaults(command="synchronize") # Parse command line args = parser.parse_args() if not hasattr(args, "command"): parser.error("missing command (choose from 'upload', 'download', 'sync')") # Post-process and check arguments args.verbose -= args.quiet del args.quiet if hasattr(args, "delete_unmatched") and args.delete_unmatched: args.delete = True ftp_debug = 0 if args.verbose >= 5: ftp_debug = 1 args.local_target = make_target(args.local, {"ftp_debug": ftp_debug}) if args.remote == ".": parser.error("'.' is expected to be the local target (not remote)") args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug}) if not isinstance(args.local_target, FsTarget) and isinstance(args.remote_target, FsTarget): parser.error("a file system target is expected to be local") # Let the command handler do its thing opts = namespace_to_dict(args) if args.command == "upload": s = UploadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "download": s = DownloadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "synchronize": s = BiDirSynchronizer(args.local_target, args.remote_target, opts) else: parser.error("unknown command %s" % args.command) try: s.run() except KeyboardInterrupt: print("\nAborted by user.") return stats = s.get_stats() if args.verbose >= 4: pprint(stats) elif args.verbose >= 1: if args.dry_run: print("(DRY-RUN) ", end="") print("Wrote %s/%s files in %s dirs. Elap: %s" % (stats["files_written"], stats["local_files"], stats["local_dirs"], stats["elap_str"]))
def run(): """CLI main entry point.""" # Use print() instead of logging when running in CLI mode: set_pyftpsync_logger(None) parser = argparse.ArgumentParser( description="Synchronize folders over FTP.", epilog="See also https://github.com/mar10/pyftpsync") parser.add_argument("-V", "--version", action="version", version="{}".format(__version__)) subparsers = parser.add_subparsers(help="sub-command help") # --- Create the parser for the "upload" command --------------------------- sp = subparsers.add_parser( "upload", help="copy new and modified files to remote folder") sp.add_argument("--force", action="store_true", help="overwrite remote files, even if the target is newer " "(but no conflict was detected)") sp.add_argument( "--resolve", default="ask", choices=["local", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") sp.add_argument("--delete", action="store_true", help="remove remote files if they don't exist locally") sp.add_argument( "--delete-unmatched", action="store_true", help="remove remote files if they don't exist locally " "or don't match the current filter (implies '--delete' option)") add_common_sub_args(sp) sp.set_defaults(command="upload") # --- Create the parser for the "download" command ------------------------- sp = subparsers.add_parser( "download", help="copy new and modified files from remote folder to local target") sp.add_argument("--force", action="store_true", help="overwrite local files, even if the target is newer " "(but no conflict was detected)") sp.add_argument( "--resolve", default="ask", choices=["remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") sp.add_argument( "--delete", action="store_true", help="remove local files if they don't exist on remote target") sp.add_argument( "--delete-unmatched", action="store_true", help="remove local files if they don't exist on remote target " "or don't match the current filter (implies '--delete' option)") add_common_sub_args(sp) sp.set_defaults(command="download") # --- Create the parser for the "sync" command ----------------------------- sp = subparsers.add_parser( "sync", help= "synchronize new and modified files between remote folder and local target" ) sp.add_argument( "--resolve", default="ask", choices=["old", "new", "local", "remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')") add_common_sub_args(sp) sp.set_defaults(command="synchronize") # --- Create the parser for the "scan" command ----------------------------- add_scan_parser(subparsers) # --- Parse command line --------------------------------------------------- args = parser.parse_args() args.verbose -= args.quiet del args.quiet ftp_debug = 0 if args.verbose >= 6: ftp_debug = 1 if callable(getattr(args, "command", None)): try: return getattr(args, "command")(args) except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) elif not hasattr(args, "command"): parser.error( "missing command (choose from 'upload', 'download', 'sync', 'scan')" ) # Post-process and check arguments if hasattr(args, "delete_unmatched") and args.delete_unmatched: args.delete = True args.local_target = make_target(args.local, {"ftp_debug": ftp_debug}) if args.remote == ".": parser.error("'.' is expected to be the local target (not remote)") args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug}) if not isinstance(args.local_target, FsTarget) and isinstance( args.remote_target, FsTarget): parser.error("a file system target is expected to be local") # Let the command handler do its thing opts = namespace_to_dict(args) if args.command == "upload": s = UploadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "download": s = DownloadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "synchronize": s = BiDirSynchronizer(args.local_target, args.remote_target, opts) else: parser.error("unknown command {}".format(args.command)) s.is_script = True try: s.run() except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) finally: # Prevent sporadic exceptions in ftplib, when closing in __del__ s.local.close() s.remote.close() stats = s.get_stats() if args.verbose >= 5: pprint(stats) elif args.verbose >= 1: if args.dry_run: print("(DRY-RUN) ", end="") print("Wrote {}/{} files in {} directories, skipped: {}.".format( stats["files_written"], stats["local_files"], stats["local_dirs"], stats["conflict_files_skipped"]), end="") if stats["interactive_ask"]: print() else: print(" Elap: {}.".format(stats["elap_str"])) return
def run(): """CLI main entry point.""" # Use print() instead of logging when running in CLI mode: set_pyftpsync_logger(None) parser = argparse.ArgumentParser( description="Synchronize folders over FTP.", epilog="See also https://github.com/mar10/pyftpsync", parents=[verbose_parser], ) # Note: we want to allow --version to be combined with --verbose. However # on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used. if check_cli_verbose(3) > 3: version_info = "pyftpsync/{} Python/{} {}".format( __version__, PYTHON_VERSION, platform.platform() ) else: version_info = "{}".format(__version__) parser.add_argument("-V", "--version", action="version", version=version_info) subparsers = parser.add_subparsers(help="sub-command help") # --- Create the parser for the "upload" command --------------------------- sp = subparsers.add_parser( "upload", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="copy new and modified files to remote folder", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--force", action="store_true", help="overwrite remote files, even if the target is newer " "(but no conflict was detected)", ) sp.add_argument( "--resolve", default="ask", choices=["local", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.add_argument( "--delete", action="store_true", help="remove remote files if they don't exist locally", ) sp.add_argument( "--delete-unmatched", action="store_true", help="remove remote files if they don't exist locally " "or don't match the current filter (implies '--delete' option)", ) sp.set_defaults(command="upload") # --- Create the parser for the "download" command ------------------------- sp = subparsers.add_parser( "download", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="copy new and modified files from remote folder to local target", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--force", action="store_true", help="overwrite local files, even if the target is newer " "(but no conflict was detected)", ) sp.add_argument( "--resolve", default="ask", choices=["remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.add_argument( "--delete", action="store_true", help="remove local files if they don't exist on remote target", ) sp.add_argument( "--delete-unmatched", action="store_true", help="remove local files if they don't exist on remote target " "or don't match the current filter (implies '--delete' option)", ) sp.set_defaults(command="download") # --- Create the parser for the "sync" command ----------------------------- sp = subparsers.add_parser( "sync", parents=[verbose_parser, common_parser, matcher_parser, creds_parser], help="synchronize new and modified files between remote folder and local target", ) sp.add_argument( "local", metavar="LOCAL", default=".", help="path to local folder (default: %(default)s)", ) sp.add_argument("remote", metavar="REMOTE", help="path to remote folder") sp.add_argument( "--resolve", default="ask", choices=["old", "new", "local", "remote", "skip", "ask"], help="conflict resolving strategy (default: '%(default)s')", ) sp.set_defaults(command="sync") # --- Create the parser for the "run" command ----------------------------- add_run_parser(subparsers) # --- Create the parser for the "scan" command ----------------------------- add_scan_parser(subparsers) # --- Parse command line --------------------------------------------------- args = parser.parse_args() args.verbose -= args.quiet del args.quiet # print("verbose", args.verbose) ftp_debug = 0 if args.verbose >= 6: ftp_debug = 1 # Modify the `args` from the `pyftpsync.yaml` config: if getattr(args, "command", None) == "run": handle_run_command(parser, args) if callable(getattr(args, "command", None)): # scan_handler try: return args.command(parser, args) except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) elif not hasattr(args, "command"): parser.error( "missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')" ) # Post-process and check arguments if hasattr(args, "delete_unmatched") and args.delete_unmatched: args.delete = True args.local_target = make_target(args.local, {"ftp_debug": ftp_debug}) if args.remote == ".": parser.error("'.' is expected to be the local target (not remote)") args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug}) if not isinstance(args.local_target, FsTarget) and isinstance( args.remote_target, FsTarget ): parser.error("a file system target is expected to be local") # Let the command handler do its thing opts = namespace_to_dict(args) if args.command == "upload": s = UploadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "download": s = DownloadSynchronizer(args.local_target, args.remote_target, opts) elif args.command == "sync": s = BiDirSynchronizer(args.local_target, args.remote_target, opts) else: parser.error("unknown command '{}'".format(args.command)) s.is_script = True try: s.run() except KeyboardInterrupt: print("\nAborted by user.", file=sys.stderr) sys.exit(3) finally: # Prevent sporadic exceptions in ftplib, when closing in __del__ s.local.close() s.remote.close() stats = s.get_stats() if args.verbose >= 5: pprint(stats) elif args.verbose >= 1: if args.dry_run: print("(DRY-RUN) ", end="") print( "Wrote {}/{} files in {} directories, skipped: {}.".format( stats["files_written"], stats["local_files"], stats["local_dirs"], stats["conflict_files_skipped"], ), end="", ) if stats["interactive_ask"]: print() else: print(" Elap: {}.".format(stats["elap_str"])) return