def test_issue_24(self):
if not self.use_ftp_target:
raise SkipTest("Only FTP targets.")
# empty_folder()
empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
local_target = make_target(self.local_url)
remote_target = make_target(self.remote_url)
SIZE = 1000 * 1000
write_test_file("local/large1.txt", size=SIZE)
write_test_file("remote/large2.txt", size=SIZE)
opts = {
"verbose": 5,
"match": "large*.txt",
}
synchronizer = DownloadSynchronizer(local_target, remote_target, opts)
assert is_test_file("local/large1.txt")
assert is_test_file("remote/large2.txt")
assert not is_test_file("remote/large1.txt")
assert not is_test_file("local/large2.txt")
synchronizer.run()
# Expect that large2 was downloaded
assert not is_test_file("remote/large1.txt")
assert get_test_file_size("local/large2.txt") == SIZE
def test_make_target(self):
t = make_target("sftp://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, SFTPTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
# scheme is case-insensitive
t = make_target("SFTP://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, SFTPTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
# pass credentials with URL
url = "user:[email protected]/target/folder"
t = make_target("sftp://" + url)
self.assertTrue(isinstance(t, SFTPTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "user")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
url = "[email protected]:[email protected]/target/folder"
t = make_target("sftp://" + url)
self.assertTrue(isinstance(t, SFTPTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "*****@*****.**")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
def test_issue_24(self):
if not self.use_ftp_target:
raise SkipTest("Only FTP targets.")
# empty_folder()
empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "local"))
empty_folder(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
local_target = make_target(self.local_url)
remote_target = make_target(self.remote_url)
SIZE = 1000 * 1000
write_test_file("local/large1.txt", size=SIZE)
write_test_file("remote/large2.txt", size=SIZE)
opts = {"verbose": 5, "match": "large*.txt"}
synchronizer = DownloadSynchronizer(local_target, remote_target, opts)
assert is_test_file("local/large1.txt")
assert is_test_file("remote/large2.txt")
assert not is_test_file("remote/large1.txt")
assert not is_test_file("local/large2.txt")
synchronizer.run()
# Expect that large2 was downloaded
assert not is_test_file("remote/large1.txt")
assert get_test_file_size("local/large2.txt") == SIZE
def do_sync():
from ftpsync.synchronizers import BiDirSynchronizer
from ftpsync.targets import make_target
local_target = make_target("~/test_pyftpsync_dl")
# local_target = make_target("c:/tmp/test_pyftpsync")
remote_target = make_target("ftp://wwwendt.de/test_pyftpsync", {"ftp_debug": False})
opts = {"verbose": 6, "dry_run": False, "resolve": "local"}
s = BiDirSynchronizer(local_target, remote_target, opts)
s.run()
def do_sync():
from ftpsync.synchronizers import BiDirSynchronizer
from ftpsync.targets import make_target
local_target = make_target("~/test_pyftpsync_dl")
# local_target = make_target("c:/tmp/test_pyftpsync")
remote_target = make_target("ftp://wwwendt.de/test_pyftpsync",
{"ftp_debug": False})
opts = {"verbose": 6, "dry_run": False, "resolve": "local"}
s = BiDirSynchronizer(local_target, remote_target, opts)
s.run()
def test_issue_20(self):
opts = {"verbose": 5}
local_target = make_target(self.local_url)
remote_target = make_target(self.remote_url)
ftp_downloader = DownloadSynchronizer(local_target, remote_target, opts)
ftp_uploader = UploadSynchronizer(local_target, remote_target, opts)
write_test_file("local/large1.txt", size=10 * 1000)
write_test_file("remote/large2.txt", size=10 * 1000)
ftp_downloader.run()
ftp_uploader.run()
def setUp(self):
# Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder"
# TODO: some of those tests are still relevant
self.skipTest("Not yet implemented.")
ftp_url = PYFTPSYNC_TEST_FTP_URL
if not ftp_url:
self.skipTest("Must configure an FTP target "
"(environment variable PYFTPSYNC_TEST_FTP_URL)")
self.assertTrue("/test" in ftp_url or "/temp" in ftp_url,
"FTP target path must include '/test' or '/temp'")
# Create temp/local folder with files and empty temp/remote folder
prepare_fixtures_1()
# print(ftp_url)
parts = urlparse(ftp_url, allow_fragments=False)
self.assertIn(parts.scheme.lower(), ["ftp", "ftps"])
# print(parts)
# self.creds = parts.username, parts.password
# self.HOST = parts.netloc.split("@", 1)[1]
self.PATH = parts.path
# user, passwd = get_stored_credentials("pyftpsync.pw", self.HOST)
self.remote = make_target(ftp_url)
self.remote.open()
# This check is already performed in the constructor:
# self.assertEqual(self.remote.pwd(), self.PATH)
# Delete all files in remote target folder, except for LOCK file
self.remote._rmdir_impl(".", keep_root_folder=True,
predicate=lambda n: n != DirMetadata.LOCK_FILE_NAME)
def _make_remote_target(cls):
"""Return the remote target instance, depending on `use_ftp_target`."""
if cls.use_ftp_target:
check_ftp_test_connection(PYFTPSYNC_TEST_FOLDER, PYFTPSYNC_TEST_FTP_URL)
remote = make_target(PYFTPSYNC_TEST_FTP_URL)
else:
remote = FsTarget(os.path.join(PYFTPSYNC_TEST_FOLDER, "remote"))
return remote
def test_make_target(self):
for scheme in ["ftp", "ftps"]:
tls = True if scheme == "ftps" else False
t = make_target(scheme + "://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
self.assertEqual(t.tls, tls)
# scheme is case-insensitive
t = make_target(scheme.upper() +
"://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
self.assertEqual(t.tls, tls)
# pass credentials with URL
url = "user:[email protected]/target/folder"
t = make_target(scheme + "://" + url)
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "user")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.tls, tls)
url = "[email protected]:[email protected]/target/folder"
t = make_target(scheme + "://" + url)
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "*****@*****.**")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.tls, tls)
# unsupported schemes
self.assertRaises(ValueError, make_target,
"ftpa://ftp.example.com/test")
self.assertRaises(ValueError, make_target, "http://example.com/test")
self.assertRaises(ValueError, make_target, "https://example.com/test")
def test_make_target(self):
for scheme in ["ftp", "ftps"]:
tls = True if scheme == "ftps" else False
t = make_target(scheme + "://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
self.assertEqual(t.tls, tls)
# scheme is case-insensitive
t = make_target(scheme.upper() + "://ftp.example.com/target/folder")
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.username, None)
self.assertEqual(t.tls, tls)
# pass credentials with URL
url = "user:[email protected]/target/folder"
t = make_target(scheme + "://" + url)
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "user")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.tls, tls)
url = "[email protected]:[email protected]/target/folder"
t = make_target(scheme + "://" + url)
self.assertTrue(isinstance(t, FtpTarget))
self.assertEqual(t.host, "ftp.example.com")
self.assertEqual(t.username, "*****@*****.**")
self.assertEqual(t.password, "secret")
self.assertEqual(t.root_dir, "/target/folder")
self.assertEqual(t.tls, tls)
# unsupported schemes
self.assertRaises(ValueError, make_target, "ftpa://ftp.example.com/test")
self.assertRaises(ValueError, make_target, "http://example.com/test")
self.assertRaises(ValueError, make_target, "https://example.com/test")
def scan_handler(args):
"""Implement `cleanup` sub-command."""
target = make_target(args.target, {"ftp_debug": args.verbose > 5})
target.readonly = True
root_depth = target.root_dir.count("/")
start = time.time()
dir_count = 1
file_count = 0
processed_files = set()
try:
target.open()
for e in target.walk():
is_dir = isinstance(e, DirectoryEntry)
indent = " " * (target.cur_dir.count("/") - root_depth)
if is_dir:
dir_count += 1
else:
file_count += 1
if args.list:
if is_dir:
print(indent, "[{e.name}]".format(e=e))
else:
delta = e.mtime_org - e.mtime
if delta:
prefix = "+" if delta > 0 else ""
print(
indent,
"{e.name:<40} {e.dt_modified} (system: {prefix}{delta})"
.format(e=e,
prefix=prefix,
delta=timedelta(seconds=delta)))
else:
print(indent,
"{e.name:<40} {e.dt_modified}".format(e=e))
if args.remove_meta and target.cur_dir_meta and target.cur_dir_meta.was_read:
fspec = target.cur_dir_meta.get_full_path()
if fspec not in processed_files:
processed_files.add(fspec)
print("DELETE {}".format(fspec))
if args.remove_locks and not is_dir and e.name == DirMetadata.LOCK_FILE_NAME:
fspec = e.get_rel_path()
print("DELETE {}".format(fspec))
finally:
target.close()
print("Scanning {:,} files in {:,} dirs took {:02.2f} seconds.".format(
file_count, dir_count,
time.time() - start))
def setUp(self):
if not DO_BENCHMARKS:
self.skipTest("DO_BENCHMARKS is not set.")
# Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder"
ftp_url = PYFTPSYNC_TEST_FTP_URL
if not ftp_url:
self.skipTest("Must configure an FTP target "
"(environment variable PYFTPSYNC_TEST_FTP_URL)")
self.assertTrue("/test" in ftp_url or "/temp" in ftp_url,
"FTP target path must include '/test' or '/temp'")
# Create temp/local folder with files and empty temp/remote folder
prepare_fixtures_1()
self.remote = make_target(ftp_url)
self.remote.open()
# Delete all files in remote target folder
self.remote._rmdir_impl(".", keep_root=True)
def setUp(self):
# Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder"
# TODO: some of those tests are still relevant
self.skipTest("Not yet implemented.")
ftp_url = PYFTPSYNC_TEST_FTP_URL
if not ftp_url:
self.skipTest(
"Must configure an FTP target "
"(environment variable PYFTPSYNC_TEST_FTP_URL)"
)
self.assertTrue(
"/test" in ftp_url or "/temp" in ftp_url,
"FTP target path must include '/test' or '/temp'",
)
# Create temp/local folder with files and empty temp/remote folder
prepare_fixtures_1()
# print(ftp_url)
parts = urlparse(ftp_url, allow_fragments=False)
self.assertIn(parts.scheme.lower(), ["ftp", "ftps"])
# print(parts)
# self.creds = parts.username, parts.password
# self.HOST = parts.netloc.split("@", 1)[1]
self.PATH = parts.path
# user, passwd = get_stored_credentials("pyftpsync.pw", self.HOST)
self.remote = make_target(ftp_url)
self.remote.open()
# This check is already performed in the constructor:
# self.assertEqual(self.remote.pwd(), self.PATH)
# Delete all files in remote target folder, except for LOCK file
self.remote._rmdir_impl(
".",
keep_root_folder=True,
predicate=lambda n: n != DirMetadata.LOCK_FILE_NAME,
)
def setUp(self):
if not DO_BENCHMARKS:
self.skipTest("DO_BENCHMARKS is not set.")
# Remote URL, e.g. "ftps://*****:*****@example.com/my/test/folder"
ftp_url = PYFTPSYNC_TEST_FTP_URL
if not ftp_url:
self.skipTest(
"Must configure an FTP target "
"(environment variable PYFTPSYNC_TEST_FTP_URL)"
)
self.assertTrue(
"/test" in ftp_url or "/temp" in ftp_url,
"FTP target path must include '/test' or '/temp'",
)
# Create temp/local folder with files and empty temp/remote folder
prepare_fixtures_1()
self.remote = make_target(ftp_url)
self.remote.open()
# Delete all files in remote target folder
self.remote._rmdir_impl(".", keep_root_folder=True)
def tree_handler(parser, args):
"""Implement `scan` sub-command."""
opts = namespace_to_dict(args)
opts.update({"ftp_debug": args.verbose >= 6})
target = make_target(args.target, opts)
target.readonly = True
start = time.time()
dir_count = 1
file_count = 0
opts = namespace_to_dict(args)
process_options(opts)
def _pred(entry):
"""Walker predicate that check match/exclude options."""
if not match_path(entry, opts):
return False
try:
target.open()
print("[{}]".format(target.root_dir))
for path, entry in target.walk_tree(sort=args.sort,
files=args.files,
pred=_pred):
name = entry.name
if entry.is_dir():
dir_count += 1
line = "{}[{}]".format(path, name)
else:
file_count += 1
line = "{}{:<20} {}".format(path, name, entry.as_string())
print(line)
finally:
target.close()
print("Scanning {:,} files in {:,} directories took {:02.2f} seconds.".
format(file_count, dir_count,
time.time() - start))
def scan_handler(parser, args):
"""Implement `scan` sub-command."""
opts = namespace_to_dict(args)
opts.update({"ftp_debug": args.verbose >= 6})
target = make_target(args.target, opts)
target.readonly = True
root_depth = target.root_dir.count("/")
start = time.time()
dir_count = 1
file_count = 0
processed_files = set()
opts = namespace_to_dict(args)
process_options(opts)
def _pred(entry):
"""Walker predicate that check match/exclude options."""
if not match_path(entry, opts):
return False
try:
target.open()
for e in target.walk(recursive=args.recursive, pred=_pred):
is_dir = isinstance(e, DirectoryEntry)
indent = " " * (target.cur_dir.count("/") - root_depth)
if is_dir:
dir_count += 1
else:
file_count += 1
if args.list:
if is_dir:
print(indent, "[{e.name}]".format(e=e))
else:
delta = e.mtime_org - e.mtime
dt_modified = pretty_stamp(e.mtime)
if delta:
prefix = "+" if delta > 0 else ""
print(
indent,
"{e.name:<40} {dt_modified} (system: {prefix}{delta})".format(
e=e,
prefix=prefix,
delta=timedelta(seconds=delta),
dt_modified=dt_modified,
),
)
else:
print(
indent,
"{e.name:<40} {dt_modified}".format(
e=e, dt_modified=dt_modified
),
)
if (
args.remove_meta
and target.cur_dir_meta
and target.cur_dir_meta.was_read
):
fspec = target.cur_dir_meta.get_full_path()
if fspec not in processed_files:
processed_files.add(fspec)
print("DELETE {}".format(fspec))
if (
args.remove_locks
and not is_dir
and e.name == DirMetadata.LOCK_FILE_NAME
):
fspec = e.get_rel_path()
print("DELETE {}".format(fspec))
finally:
target.close()
print(
"Scanning {:,} files in {:,} directories took {:02.2f} seconds.".format(
file_count, dir_count, time.time() - start
)
)
def scan_handler(args):
"""Implement `cleanup` sub-command."""
opts = namespace_to_dict(args)
opts.update({
"ftp_debug": args.verbose >= 6,
})
target = make_target(args.target, opts)
target.readonly = True
root_depth = target.root_dir.count("/")
start = time.time()
dir_count = 1
file_count = 0
processed_files = set()
opts = namespace_to_dict(args)
process_options(opts)
def _pred(entry):
"""Walker predicate that check match/exclude options."""
if not match_path(entry, opts):
return False
try:
target.open()
for e in target.walk(recursive=args.recursive, pred=_pred):
is_dir = isinstance(e, DirectoryEntry)
indent = " " * (target.cur_dir.count("/") - root_depth)
if is_dir:
dir_count += 1
else:
file_count += 1
if args.list:
if is_dir:
print(indent, "[{e.name}]".format(e=e))
else:
delta = e.mtime_org - e.mtime
dt_modified = pretty_stamp(e.mtime)
if delta:
prefix = "+" if delta > 0 else ""
print(
indent,
"{e.name:<40} {dt_modified} (system: {prefix}{delta})"
.format(e=e,
prefix=prefix,
delta=timedelta(seconds=delta),
dt_modified=dt_modified))
else:
print(
indent, "{e.name:<40} {dt_modified}".format(
e=e, dt_modified=dt_modified))
if args.remove_meta and target.cur_dir_meta and target.cur_dir_meta.was_read:
fspec = target.cur_dir_meta.get_full_path()
if fspec not in processed_files:
processed_files.add(fspec)
print("DELETE {}".format(fspec))
if args.remove_locks and not is_dir and e.name == DirMetadata.LOCK_FILE_NAME:
fspec = e.get_rel_path()
print("DELETE {}".format(fspec))
finally:
target.close()
print("Scanning {:,} files in {:,} directories took {:02.2f} seconds.".
format(file_count, dir_count,
time.time() - start))
def run():
"""CLI main entry point."""
# Use print() instead of logging when running in CLI mode:
set_pyftpsync_logger(None)
parser = argparse.ArgumentParser(
description="Synchronize folders over FTP.",
epilog="See also https://github.com/mar10/pyftpsync",
parents=[verbose_parser],
)
# Note: we want to allow --version to be combined with --verbose. However
# on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used.
if check_cli_verbose(3) > 3:
version_info = "pyftpsync/{} Python/{} {}".format(
__version__, PYTHON_VERSION, platform.platform()
)
else:
version_info = "{}".format(__version__)
parser.add_argument("-V", "--version", action="version", version=version_info)
subparsers = parser.add_subparsers(help="sub-command help")
# --- Create the parser for the "upload" command ---------------------------
sp = subparsers.add_parser(
"upload",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="copy new and modified files to remote folder",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--force",
action="store_true",
help="overwrite remote files, even if the target is newer "
"(but no conflict was detected)",
)
sp.add_argument(
"--resolve",
default="ask",
choices=["local", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.add_argument(
"--delete",
action="store_true",
help="remove remote files if they don't exist locally",
)
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove remote files if they don't exist locally "
"or don't match the current filter (implies '--delete' option)",
)
sp.set_defaults(command="upload")
# --- Create the parser for the "download" command -------------------------
sp = subparsers.add_parser(
"download",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="copy new and modified files from remote folder to local target",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--force",
action="store_true",
help="overwrite local files, even if the target is newer "
"(but no conflict was detected)",
)
sp.add_argument(
"--resolve",
default="ask",
choices=["remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.add_argument(
"--delete",
action="store_true",
help="remove local files if they don't exist on remote target",
)
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove local files if they don't exist on remote target "
"or don't match the current filter (implies '--delete' option)",
)
sp.set_defaults(command="download")
# --- Create the parser for the "sync" command -----------------------------
sp = subparsers.add_parser(
"sync",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="synchronize new and modified files between remote folder and local target",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--resolve",
default="ask",
choices=["old", "new", "local", "remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.set_defaults(command="sync")
# --- Create the parser for the "run" command -----------------------------
add_run_parser(subparsers)
# --- Create the parser for the "scan" command -----------------------------
add_scan_parser(subparsers)
# --- Create the parser for the "tree" command -----------------------------
add_tree_parser(subparsers)
# --- Parse command line ---------------------------------------------------
args = parser.parse_args()
args.verbose -= args.quiet
del args.quiet
# print("verbose", args.verbose)
ftp_debug = 0
if args.verbose >= 6:
ftp_debug = 1
if args.debug:
if args.verbose < 4:
parser.error("'--debug' requires verbose level >= 4")
DEBUG_FLAGS.update(args.debug)
# Modify the `args` from the `pyftpsync.yaml` config:
if getattr(args, "command", None) == "run":
handle_run_command(parser, args)
if callable(getattr(args, "command", None)):
# scan_handler
try:
return args.command(parser, args)
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
elif not hasattr(args, "command"):
parser.error(
"missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')"
)
# Post-process and check arguments
if hasattr(args, "delete_unmatched") and args.delete_unmatched:
args.delete = True
args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
if args.remote == ".":
parser.error("'.' is expected to be the local target (not remote)")
args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
if not isinstance(args.local_target, FsTarget) and isinstance(
args.remote_target, FsTarget
):
parser.error("a file system target is expected to be local")
# Let the command handler do its thing
opts = namespace_to_dict(args)
if args.command == "upload":
s = UploadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "download":
s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "sync":
s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
else:
parser.error("unknown command '{}'".format(args.command))
s.is_script = True
try:
s.run()
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
finally:
# Prevent sporadic exceptions in ftplib, when closing in __del__
s.local.close()
s.remote.close()
stats = s.get_stats()
if args.verbose >= 5:
pprint(stats)
elif args.verbose >= 1:
if args.dry_run:
print("(DRY-RUN) ", end="")
print(
"Wrote {}/{} files in {} directories, skipped: {}.".format(
stats["files_written"],
stats["local_files"],
stats["local_dirs"],
stats["conflict_files_skipped"],
),
end="",
)
if stats["interactive_ask"]:
print()
else:
print(" Elap: {}.".format(stats["elap_str"]))
return
def run():
parser = argparse.ArgumentParser(
description="Synchronize folders over FTP.",
epilog="See also https://github.com/mar10/pyftpsync")
qv_group = parser.add_mutually_exclusive_group()
qv_group.add_argument(
"--verbose",
"-v",
action="count",
default=3,
help="increment verbosity by one (default: %(default)s, range: 0..5)")
qv_group.add_argument("--quiet",
"-q",
action="count",
default=0,
help="decrement verbosity by one")
parser.add_argument("-V",
"--version",
action="version",
version="%s" % __version__)
parser.add_argument(
"--progress",
"-p",
action="store_true",
default=False,
help="show progress info, even if redirected or verbose < 3")
subparsers = parser.add_subparsers(help="sub-command help")
def __add_common_sub_args(parser):
parser.add_argument(
"local",
metavar="LOCAL",
# required=True,
default=".",
help="path to local folder (default: %(default)s)")
parser.add_argument("remote",
metavar="REMOTE",
help="path to remote folder")
parser.add_argument(
"--dry-run",
action="store_true",
help="just simulate and log results, but don't change anything")
# parser.add_argument("-x", "--execute",
# action="store_false", dest="dry_run", default=True,
# help="turn off the dry-run mode (which is ON by default), "
# "that would just print status messages but does "
# "not change anything")
parser.add_argument("-f",
"--include-files",
help="wildcard for file names (default: all, "
"separate multiple values with ',')")
parser.add_argument(
"-o",
"--omit",
help=
"wildcard of files and directories to exclude (applied after --include)"
)
parser.add_argument("--store-password",
action="store_true",
help="save password to keyring if login succeeds")
parser.add_argument("--no-prompt",
action="store_true",
help="prevent prompting for missing credentials")
parser.add_argument("--no-color",
action="store_true",
help="prevent use of ansi terminal color codes")
# --- Create the parser for the "upload" command ---------------------------
upload_parser = subparsers.add_parser(
"upload", help="copy new and modified files to remote folder")
__add_common_sub_args(upload_parser)
upload_parser.add_argument(
"--force",
action="store_true",
help=
"overwrite remote files, even if the target is newer (but no conflict was detected)"
)
upload_parser.add_argument(
"--resolve",
default="skip",
choices=["local", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
upload_parser.add_argument(
"--delete",
action="store_true",
help="remove remote files if they don't exist locally")
upload_parser.add_argument(
"--delete-unmatched",
action="store_true",
help="remove remote files if they don't exist locally "
"or don't match the current filter (implies '--delete' option)")
upload_parser.set_defaults(command="upload")
# --- Create the parser for the "download" command -------------------------
download_parser = subparsers.add_parser(
"download",
help="copy new and modified files from remote folder to local target")
__add_common_sub_args(download_parser)
download_parser.add_argument(
"--force",
action="store_true",
help=
"overwrite local files, even if the target is newer (but no conflict was detected)"
)
download_parser.add_argument(
"--resolve",
default="skip",
choices=["remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
download_parser.add_argument(
"--delete",
action="store_true",
help="remove local files if they don't exist on remote target")
download_parser.add_argument(
"--delete-unmatched",
action="store_true",
help="remove local files if they don't exist on remote target "
"or don't match the current filter (implies '--delete' option)")
download_parser.set_defaults(command="download")
# --- Create the parser for the "sync" command -----------------------------
sync_parser = subparsers.add_parser(
"sync",
help=
"synchronize new and modified files between remote folder and local target"
)
__add_common_sub_args(sync_parser)
sync_parser.add_argument(
"--resolve",
default="ask",
choices=["old", "new", "local", "remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
sync_parser.set_defaults(command="synchronize")
# --- Create the parser for the "scan" command -----------------------------
scan_parser = add_scan_parser(subparsers)
# --- Parse command line ---------------------------------------------------
args = parser.parse_args()
args.verbose -= args.quiet
del args.quiet
ftp_debug = 0
if args.verbose >= 5:
ftp_debug = 1
if callable(getattr(args, "command", None)):
return getattr(args, "command")(args)
elif not hasattr(args, "command"):
parser.error(
"missing command (choose from 'upload', 'download', 'sync')")
# Post-process and check arguments
if hasattr(args, "delete_unmatched") and args.delete_unmatched:
args.delete = True
args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
if args.remote == ".":
parser.error("'.' is expected to be the local target (not remote)")
args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
if not isinstance(args.local_target, FsTarget) and isinstance(
args.remote_target, FsTarget):
parser.error("a file system target is expected to be local")
# Let the command handler do its thing
opts = namespace_to_dict(args)
if args.command == "upload":
s = UploadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "download":
s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "synchronize":
s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
else:
parser.error("unknown command %s" % args.command)
try:
s.run()
except KeyboardInterrupt:
print("\nAborted by user.")
return
finally:
# prevent sporadic exceptions in ftplib, when closing in __del__
s.local.close()
s.remote.close()
stats = s.get_stats()
if args.verbose >= 4:
pprint(stats)
elif args.verbose >= 1:
if args.dry_run:
print("(DRY-RUN) ", end="")
print("Wrote %s/%s files in %s dirs. Elap: %s" %
(stats["files_written"], stats["local_files"],
stats["local_dirs"], stats["elap_str"]))
return
def run():
parser = argparse.ArgumentParser(
description="Synchronize folders over FTP.",
epilog="See also https://github.com/mar10/pyftpsync"
)
qv_group = parser.add_mutually_exclusive_group()
qv_group.add_argument("--verbose", "-v", action="count", default=3,
help="increment verbosity by one (default: %(default)s, range: 0..5)")
qv_group.add_argument("--quiet", "-q", action="count", default=0,
help="decrement verbosity by one")
parser.add_argument("--version", action="version", version="%s" % __version__)
parser.add_argument("--progress", "-p",
action="store_true",
default=False,
help="show progress info, even if redirected or verbose < 3")
subparsers = parser.add_subparsers(help="sub-command help")
def __add_common_sub_args(parser):
parser.add_argument("local",
metavar="LOCAL",
# required=True,
default=".",
help="path to local folder (default: %(default)s)")
parser.add_argument("remote",
metavar="REMOTE",
help="path to remote folder")
# upload_parser.add_argument("--dry-run",
# action="store_true",
# help="just simulate and log results; don't change anything")
parser.add_argument("-x", "--execute",
action="store_false", dest="dry_run", default=True,
help="turn off the dry-run mode (which is ON by default), "
"that would just print status messages but does "
"not change anything")
parser.add_argument("-f", "--include-files",
help="wildcard for file names (default: all, "
"separate multiple values with ',')")
parser.add_argument("-o", "--omit",
help="wildcard of files and directories to exclude (applied after --include)")
parser.add_argument("--store-password",
action="store_true",
help="save password to keyring if login succeeds")
parser.add_argument("--no-prompt",
action="store_true",
help="prevent prompting for missing credentials")
parser.add_argument("--no-color",
action="store_true",
help="prevent use of ansi terminal color codes")
# Create the parser for the "upload" command
upload_parser = subparsers.add_parser("upload",
help="copy new and modified files to remote folder")
__add_common_sub_args(upload_parser)
upload_parser.add_argument("--force",
action="store_true",
help="overwrite different remote files, even if the target is newer")
upload_parser.add_argument("--delete",
action="store_true",
help="remove remote files if they don't exist locally")
upload_parser.add_argument("--delete-unmatched",
action="store_true",
help="remove remote files if they don't exist locally "
"or don't match the current filter (implies '--delete' option)")
upload_parser.set_defaults(command="upload")
# Create the parser for the "download" command
download_parser = subparsers.add_parser("download",
help="copy new and modified files from remote folder to local target")
__add_common_sub_args(download_parser)
download_parser.add_argument("--force",
action="store_true",
help="overwrite different local files, even if the target is newer")
download_parser.add_argument("--delete",
action="store_true",
help="remove local files if they don't exist on remote target")
download_parser.add_argument("--delete-unmatched",
action="store_true",
help="remove local files if they don't exist on remote target "
"or don't match the current filter (implies '--delete' option)")
download_parser.set_defaults(command="download")
# Create the parser for the "sync" command
sync_parser = subparsers.add_parser("sync",
help="synchronize new and modified files between remote folder and local target")
__add_common_sub_args(sync_parser)
# sync_parser.add_argument("--store-password",
# action="store_true",
# help="save password to keyring if login succeeds")
# sync_parser.add_argument("--no-prompt",
# action="store_true",
# help="prevent prompting for missing credentials")
# sync_parser.add_argument("--no-color",
# action="store_true",
# help="prevent use of ansi terminal color codes")
sync_parser.add_argument("--resolve",
default="ask",
choices=["old", "new", "local", "remote", "skip", "ask"],
help="conflict resolving strategy (default: 'ask')")
sync_parser.set_defaults(command="synchronize")
# Parse command line
args = parser.parse_args()
if not hasattr(args, "command"):
parser.error("missing command (choose from 'upload', 'download', 'sync')")
# Post-process and check arguments
args.verbose -= args.quiet
del args.quiet
if hasattr(args, "delete_unmatched") and args.delete_unmatched:
args.delete = True
ftp_debug = 0
if args.verbose >= 5:
ftp_debug = 1
args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
if args.remote == ".":
parser.error("'.' is expected to be the local target (not remote)")
args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
if not isinstance(args.local_target, FsTarget) and isinstance(args.remote_target, FsTarget):
parser.error("a file system target is expected to be local")
# Let the command handler do its thing
opts = namespace_to_dict(args)
if args.command == "upload":
s = UploadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "download":
s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "synchronize":
s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
else:
parser.error("unknown command %s" % args.command)
try:
s.run()
except KeyboardInterrupt:
print("\nAborted by user.")
return
stats = s.get_stats()
if args.verbose >= 4:
pprint(stats)
elif args.verbose >= 1:
if args.dry_run:
print("(DRY-RUN) ", end="")
print("Wrote %s/%s files in %s dirs. Elap: %s"
% (stats["files_written"], stats["local_files"], stats["local_dirs"], stats["elap_str"]))
def run():
"""CLI main entry point."""
# Use print() instead of logging when running in CLI mode:
set_pyftpsync_logger(None)
parser = argparse.ArgumentParser(
description="Synchronize folders over FTP.",
epilog="See also https://github.com/mar10/pyftpsync")
parser.add_argument("-V",
"--version",
action="version",
version="{}".format(__version__))
subparsers = parser.add_subparsers(help="sub-command help")
# --- Create the parser for the "upload" command ---------------------------
sp = subparsers.add_parser(
"upload", help="copy new and modified files to remote folder")
sp.add_argument("--force",
action="store_true",
help="overwrite remote files, even if the target is newer "
"(but no conflict was detected)")
sp.add_argument(
"--resolve",
default="ask",
choices=["local", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
sp.add_argument("--delete",
action="store_true",
help="remove remote files if they don't exist locally")
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove remote files if they don't exist locally "
"or don't match the current filter (implies '--delete' option)")
add_common_sub_args(sp)
sp.set_defaults(command="upload")
# --- Create the parser for the "download" command -------------------------
sp = subparsers.add_parser(
"download",
help="copy new and modified files from remote folder to local target")
sp.add_argument("--force",
action="store_true",
help="overwrite local files, even if the target is newer "
"(but no conflict was detected)")
sp.add_argument(
"--resolve",
default="ask",
choices=["remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
sp.add_argument(
"--delete",
action="store_true",
help="remove local files if they don't exist on remote target")
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove local files if they don't exist on remote target "
"or don't match the current filter (implies '--delete' option)")
add_common_sub_args(sp)
sp.set_defaults(command="download")
# --- Create the parser for the "sync" command -----------------------------
sp = subparsers.add_parser(
"sync",
help=
"synchronize new and modified files between remote folder and local target"
)
sp.add_argument(
"--resolve",
default="ask",
choices=["old", "new", "local", "remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')")
add_common_sub_args(sp)
sp.set_defaults(command="synchronize")
# --- Create the parser for the "scan" command -----------------------------
add_scan_parser(subparsers)
# --- Parse command line ---------------------------------------------------
args = parser.parse_args()
args.verbose -= args.quiet
del args.quiet
ftp_debug = 0
if args.verbose >= 6:
ftp_debug = 1
if callable(getattr(args, "command", None)):
try:
return getattr(args, "command")(args)
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
elif not hasattr(args, "command"):
parser.error(
"missing command (choose from 'upload', 'download', 'sync', 'scan')"
)
# Post-process and check arguments
if hasattr(args, "delete_unmatched") and args.delete_unmatched:
args.delete = True
args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
if args.remote == ".":
parser.error("'.' is expected to be the local target (not remote)")
args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
if not isinstance(args.local_target, FsTarget) and isinstance(
args.remote_target, FsTarget):
parser.error("a file system target is expected to be local")
# Let the command handler do its thing
opts = namespace_to_dict(args)
if args.command == "upload":
s = UploadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "download":
s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "synchronize":
s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
else:
parser.error("unknown command {}".format(args.command))
s.is_script = True
try:
s.run()
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
finally:
# Prevent sporadic exceptions in ftplib, when closing in __del__
s.local.close()
s.remote.close()
stats = s.get_stats()
if args.verbose >= 5:
pprint(stats)
elif args.verbose >= 1:
if args.dry_run:
print("(DRY-RUN) ", end="")
print("Wrote {}/{} files in {} directories, skipped: {}.".format(
stats["files_written"], stats["local_files"], stats["local_dirs"],
stats["conflict_files_skipped"]),
end="")
if stats["interactive_ask"]:
print()
else:
print(" Elap: {}.".format(stats["elap_str"]))
return
def run():
"""CLI main entry point."""
# Use print() instead of logging when running in CLI mode:
set_pyftpsync_logger(None)
parser = argparse.ArgumentParser(
description="Synchronize folders over FTP.",
epilog="See also https://github.com/mar10/pyftpsync",
parents=[verbose_parser],
)
# Note: we want to allow --version to be combined with --verbose. However
# on Py2, argparse makes sub-commands mandatory, unless `action="version"` is used.
if check_cli_verbose(3) > 3:
version_info = "pyftpsync/{} Python/{} {}".format(
__version__, PYTHON_VERSION, platform.platform()
)
else:
version_info = "{}".format(__version__)
parser.add_argument("-V", "--version", action="version", version=version_info)
subparsers = parser.add_subparsers(help="sub-command help")
# --- Create the parser for the "upload" command ---------------------------
sp = subparsers.add_parser(
"upload",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="copy new and modified files to remote folder",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--force",
action="store_true",
help="overwrite remote files, even if the target is newer "
"(but no conflict was detected)",
)
sp.add_argument(
"--resolve",
default="ask",
choices=["local", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.add_argument(
"--delete",
action="store_true",
help="remove remote files if they don't exist locally",
)
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove remote files if they don't exist locally "
"or don't match the current filter (implies '--delete' option)",
)
sp.set_defaults(command="upload")
# --- Create the parser for the "download" command -------------------------
sp = subparsers.add_parser(
"download",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="copy new and modified files from remote folder to local target",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--force",
action="store_true",
help="overwrite local files, even if the target is newer "
"(but no conflict was detected)",
)
sp.add_argument(
"--resolve",
default="ask",
choices=["remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.add_argument(
"--delete",
action="store_true",
help="remove local files if they don't exist on remote target",
)
sp.add_argument(
"--delete-unmatched",
action="store_true",
help="remove local files if they don't exist on remote target "
"or don't match the current filter (implies '--delete' option)",
)
sp.set_defaults(command="download")
# --- Create the parser for the "sync" command -----------------------------
sp = subparsers.add_parser(
"sync",
parents=[verbose_parser, common_parser, matcher_parser, creds_parser],
help="synchronize new and modified files between remote folder and local target",
)
sp.add_argument(
"local",
metavar="LOCAL",
default=".",
help="path to local folder (default: %(default)s)",
)
sp.add_argument("remote", metavar="REMOTE", help="path to remote folder")
sp.add_argument(
"--resolve",
default="ask",
choices=["old", "new", "local", "remote", "skip", "ask"],
help="conflict resolving strategy (default: '%(default)s')",
)
sp.set_defaults(command="sync")
# --- Create the parser for the "run" command -----------------------------
add_run_parser(subparsers)
# --- Create the parser for the "scan" command -----------------------------
add_scan_parser(subparsers)
# --- Parse command line ---------------------------------------------------
args = parser.parse_args()
args.verbose -= args.quiet
del args.quiet
# print("verbose", args.verbose)
ftp_debug = 0
if args.verbose >= 6:
ftp_debug = 1
# Modify the `args` from the `pyftpsync.yaml` config:
if getattr(args, "command", None) == "run":
handle_run_command(parser, args)
if callable(getattr(args, "command", None)):
# scan_handler
try:
return args.command(parser, args)
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
elif not hasattr(args, "command"):
parser.error(
"missing command (choose from 'upload', 'download', 'run', 'sync', 'scan')"
)
# Post-process and check arguments
if hasattr(args, "delete_unmatched") and args.delete_unmatched:
args.delete = True
args.local_target = make_target(args.local, {"ftp_debug": ftp_debug})
if args.remote == ".":
parser.error("'.' is expected to be the local target (not remote)")
args.remote_target = make_target(args.remote, {"ftp_debug": ftp_debug})
if not isinstance(args.local_target, FsTarget) and isinstance(
args.remote_target, FsTarget
):
parser.error("a file system target is expected to be local")
# Let the command handler do its thing
opts = namespace_to_dict(args)
if args.command == "upload":
s = UploadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "download":
s = DownloadSynchronizer(args.local_target, args.remote_target, opts)
elif args.command == "sync":
s = BiDirSynchronizer(args.local_target, args.remote_target, opts)
else:
parser.error("unknown command '{}'".format(args.command))
s.is_script = True
try:
s.run()
except KeyboardInterrupt:
print("\nAborted by user.", file=sys.stderr)
sys.exit(3)
finally:
# Prevent sporadic exceptions in ftplib, when closing in __del__
s.local.close()
s.remote.close()
stats = s.get_stats()
if args.verbose >= 5:
pprint(stats)
elif args.verbose >= 1:
if args.dry_run:
print("(DRY-RUN) ", end="")
print(
"Wrote {}/{} files in {} directories, skipped: {}.".format(
stats["files_written"],
stats["local_files"],
stats["local_dirs"],
stats["conflict_files_skipped"],
),
end="",
)
if stats["interactive_ask"]:
print()
else:
print(" Elap: {}.".format(stats["elap_str"]))
return
