def _print_pair_diff(self, pair):
any_entry = pair.any_entry
has_meta = any_entry.get_sync_info("m") is not None
# write("pair", pair)
# print("pair.local", pair.local)
# print("pair.remote", pair.remote)
write(
(
VT_ERASE_LINE
+ ansi_code("Fore.LIGHTRED_EX")
+ "CONFLICT: {!r} was modified on both targets since last sync ({})."
+ ansi_code("Style.RESET_ALL")
).format(any_entry.get_rel_path(), _ts(any_entry.get_sync_info("u")))
)
if has_meta:
write(
" Original modification time: {}, size: {:,d} bytes.".format(
_ts(any_entry.get_sync_info("m")), any_entry.get_sync_info("s")
)
)
else:
write(" (No meta data available.)")
write(" Local: {}".format(pair.local.as_string() if pair.local else "n.a."))
write(
" Remote: {}".format(
pair.remote.as_string(pair.local) if pair.remote else "n.a."
)
)
def _log_action(self, action, status, symbol, entry, min_level=3):
if self.verbose < min_level:
return
if len(symbol) > 1 and symbol[0] in (">", "<"):
symbol = (
" " + symbol
) # make sure direction characters are aligned at 2nd column
color = ""
final = ""
if not self.options.get("no_color"):
# CM = self.COLOR_MAP
# color = CM.get((action, status),
# CM.get(("*", status),
# CM.get((action, "*"),
# "")))
if action in ("copy", "restore"):
if "<" in symbol:
if status == "new":
color = ansi_code("Fore.GREEN") + ansi_code(
"Style.BRIGHT")
else:
color = ansi_code("Fore.GREEN")
else:
if status == "new":
color = ansi_code("Fore.CYAN") + ansi_code(
"Style.BRIGHT")
else:
color = ansi_code("Fore.CYAN")
elif action == "delete":
color = ansi_code("Fore.RED")
elif status == "conflict":
color = ansi_code("Fore.LIGHTRED_EX")
elif action == "skip" or status == "equal" or status == "visit":
color = ansi_code("Fore.LIGHTBLACK_EX")
final = ansi_code("Style.RESET_ALL")
if colorama:
# Clear line"ESC [ mode K" mode 0:to-right, 2:all
final += colorama.ansi.clear_line(0)
else:
final += " " * 10
prefix = ""
if self.dry_run:
prefix = DRY_RUN_PREFIX
if action and status:
tag = ("{} {}".format(action, status)).upper()
else:
assert status
tag = ("{}".format(status)).upper()
name = entry.get_rel_path()
if entry.is_dir():
name = "[{}]".format(name)
write("{}{}{:<16} {:^3} {}{}".format(prefix, color, tag, symbol, name,
final))
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".
format(self.root_dir))
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}"
.format(self.cur_dir))
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None):
# FTP does not support deletion of non-empty directories.
self.check_write(dir_name)
names = []
nlst_res = self.ftp.nlst(dir_name)
# write("rmdir(%s): %s" % (dir_name, nlst_res))
for name in nlst_res:
if "/" in name:
name = os.path.basename(name)
if name in (".", ".."):
continue
if predicate and not predicate(name):
continue
names.append(name)
if len(names) > 0:
self.ftp.cwd(dir_name)
try:
for name in names:
try:
# try to delete this as a file
self.ftp.delete(name)
except ftplib.all_errors as _e:
write(
" ftp.delete({}) failed: {}, trying rmdir()...".
format(name, _e))
# assume <name> is a folder
self.rmdir(name)
finally:
if dir_name != ".":
self.ftp.cwd("..")
# write("ftp.rmd(%s)..." % (dir_name, ))
if not keep_root_folder:
self.ftp.rmd(dir_name)
return
def test_logging(self):
import logging
import logging.handlers
import os
# Create and use a custom logger
custom_logger = logging.getLogger("pyftpsync_test")
log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log")
handler = logging.handlers.WatchedFileHandler(log_path)
# formatter = logging.Formatter(logging.BASIC_FORMAT)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
custom_logger.addHandler(handler)
set_pyftpsync_logger(custom_logger)
custom_logger.setLevel(logging.DEBUG)
print("print 1")
write("write info 1")
write_error("write error 1")
custom_logger.setLevel(logging.WARNING)
write("write info 2")
write_error("write error 2")
handler.flush()
log_data = read_test_file("pyftpsync.log")
assert "print 1" not in log_data
assert "write info 1" in log_data
assert "write error 1" in log_data
assert "write info 2" not in log_data, "Loglevel honored"
assert "write error 2" in log_data
# Cleanup properly (log file would be locked otherwise)
custom_logger.removeHandler(handler)
handler.close()
def _probe_lock_file(self, reported_mtime):
"""Called by get_dir"""
delta = reported_mtime - self.lock_data["lock_time"]
# delta2 = reported_mtime - self.lock_write_time
self.server_time_ofs = delta
if self.get_option("verbose", 3) >= 4:
write("Server time offset: {:.2f} seconds.".format(delta))
def _probe_lock_file(self, reported_mtime):
"""Called by get_dir"""
delta = reported_mtime - self.lock_data["lock_time"]
# delta2 = reported_mtime - self.lock_write_time
self.server_time_ofs = delta
if self.get_option("verbose", 3) >= 4:
write("Server time offset: {:.2f} seconds.".format(delta))
def close(self):
if not self.connected:
return
if self.get_option("verbose", 3) >= 5:
write("Closing target {}.".format(self))
self.connected = False
self.readonly = False # issue #20
self._rlock.release()
def __init__(
self,
path,
host,
port=0,
username=None,
password=None,
tls=False,
timeout=None,
extra_opts=None,
):
"""Create FTP target with host, initial path, optional credentials and options.
Args:
path (str): root path on FTP server, relative to *host*
host (str): hostname of FTP server
port (int): FTP port (defaults to 21)
username (str):
password (str):
tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
timeout (int): the timeout to set against the ftp socket (seconds)
extra_opts (dict):
"""
self.encoding = _get_encoding_opt(None, extra_opts, "utf-8")
# path = self.to_unicode(path)
path = path or "/"
assert compat.is_native(path)
super(FtpTarget, self).__init__(path, extra_opts)
if tls:
try:
self.ftp = ftplib.FTP_TLS()
except AttributeError:
write("Python 2.7/3.2+ required for FTPS (TLS).")
raise
else:
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
self.host = host
self.port = port or 0
self.username = username
self.password = password
self.tls = tls
self.timeout = timeout
#: dict: written to ftp target root folder before synchronization starts.
#: set to False, if write failed. Default: None
self.lock_data = None
self.lock_write_time = None
self.feat_response = None
self.syst_response = None
self.is_unix = None
#: True if server reports FEAT UTF8
self.support_utf8 = None
#: Time difference between <local upload time> and the mtime that the server reports afterwards.
#: The value is added to the 'u' time stored in meta data.
#: (This is only a rough estimation, derived from the lock-file.)
self.server_time_ofs = None
self.ftp_socket_connected = False
self.support_set_time = False
def __init__(
self,
path,
host,
port=0,
username=None,
password=None,
tls=False,
timeout=None,
extra_opts=None,
):
"""Create FTP target with host, initial path, optional credentials and options.
Args:
path (str): root path on FTP server, relative to *host*
host (str): hostname of FTP server
port (int): FTP port (defaults to 21)
username (str):
password (str):
tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
timeout (int): the timeout to set against the ftp socket (seconds)
extra_opts (dict):
"""
self.encoding = _get_encoding_opt(None, extra_opts, "utf-8")
# path = self.to_unicode(path)
path = path or "/"
assert is_native(path)
super(FTPTarget, self).__init__(path, extra_opts)
if tls:
try:
self.ftp = ftplib.FTP_TLS()
except AttributeError:
write("Python 2.7/3.2+ required for FTPS (TLS).")
raise
else:
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
self.host = host
self.port = port or 0
self.username = username
self.password = password
self.tls = tls
self.timeout = timeout
#: dict: written to ftp target root folder before synchronization starts.
#: set to False, if write failed. Default: None
self.lock_data = None
self.lock_write_time = None
self.feat_response = None
self.syst_response = None
self.is_unix = None
#: True if server reports FEAT UTF8
self.support_utf8 = None
#: Time difference between <local upload time> and the mtime that the server reports afterwards.
#: The value is added to the 'u' time stored in meta data.
#: (This is only a rough estimation, derived from the lock-file.)
self.server_time_ofs = None
self.ftp_socket_connected = False
self.support_set_time = False
def _log_action(self, action, status, symbol, entry, min_level=3):
if self.verbose < min_level:
return
if len(symbol) > 1 and symbol[0] in (">", "<"):
symbol = (
" " + symbol
) # make sure direction characters are aligned at 2nd column
color = ""
final = ""
if not self.options.get("no_color"):
# CM = self.COLOR_MAP
# color = CM.get((action, status),
# CM.get(("*", status),
# CM.get((action, "*"),
# "")))
if action in ("copy", "restore"):
if "<" in symbol:
if status == "new":
color = ansi_code("Fore.GREEN") + ansi_code("Style.BRIGHT")
else:
color = ansi_code("Fore.GREEN")
else:
if status == "new":
color = ansi_code("Fore.CYAN") + ansi_code("Style.BRIGHT")
else:
color = ansi_code("Fore.CYAN")
elif action == "delete":
color = ansi_code("Fore.RED")
elif status == "conflict":
color = ansi_code("Fore.LIGHTRED_EX")
elif action == "skip" or status == "equal" or status == "visit":
color = ansi_code("Fore.LIGHTBLACK_EX")
final = ansi_code("Style.RESET_ALL")
if colorama:
# Clear line"ESC [ mode K" mode 0:to-right, 2:all
final += colorama.ansi.clear_line(0)
else:
final += " " * 10
prefix = ""
if self.dry_run:
prefix = DRY_RUN_PREFIX
if action and status:
tag = ("{} {}".format(action, status)).upper()
else:
assert status
tag = ("{}".format(status)).upper()
name = entry.get_rel_path()
if entry.is_dir():
name = "[{}]".format(name)
write("{}{}{:<16} {:^3} {}{}".format(prefix, color, tag, symbol, name, final))
def flush(self):
"""Write self to .pyftpsync-meta.json."""
# We DO write meta files even on read-only targets, but not in dry-run mode
# if self.target.readonly:
# write("DirMetadata.flush(%s): read-only; nothing to do" % self.target)
# return
assert self.path == self.target.cur_dir
if self.target.dry_run:
# write("DirMetadata.flush(%s): dry-run; nothing to do" % self.target)
pass
elif self.was_read and len(self.list) == 0 and len(
self.peer_sync) == 0:
write("Remove empty meta data file: {}".format(self.target))
self.target.remove_file(self.filename)
elif not self.modified_list and not self.modified_sync:
# write("DirMetadata.flush(%s): unmodified; nothing to do" % self.target)
pass
else:
self.dir[
"_disclaimer"] = "Generated by https://github.com/mar10/pyftpsync"
self.dir["_time_str"] = pretty_stamp(time.time())
self.dir["_file_version"] = self.VERSION
self.dir["_version"] = __version__
self.dir["_time"] = time.mktime(time.gmtime())
# We always save utf-8 encoded.
# `ensure_ascii` would escape all bytes >127 as `\x12` or `\u1234`,
# which makes it hard to read, so we set it to false.
# `sort_keys` converts binary keys to unicode using utf-8, so we
# must make sure that we don't pass cp1225 or other encoded data.
data = self.dir
opts = {"indent": 4, "sort_keys": True, "ensure_ascii": False}
# if compat.PY2:
# # The `encoding` arg defaults to utf-8 on Py2 and was removed in Py3
# # opts["encoding"] = "utf-8"
# # Python 2 has problems with mixed keys (str/unicode)
# data = decode_dict_keys(data, "utf-8")
if not self.PRETTY:
opts["indent"] = None
opts["separators"] = (",", ":")
s = json.dumps(data, **opts)
self.target.write_text(self.filename, s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_written",
len(s))
self.modified_list = False
self.modified_sync = False
def on_error(self, e, pair):
"""Called for pairs that don't match `match` and `exclude` filters."""
RED = ansi_code("Fore.LIGHTRED_EX")
R = ansi_code("Style.RESET_ALL")
# any_entry = pair.any_entry
write((RED + "ERROR: {}\n {}" + R).format(e, pair))
# Return True to ignore this error (instead of raising and terminating the app)
if "[Errno 92] Illegal byte sequence" in "{}".format(e) and compat.PY2:
write(RED + "This _may_ be solved by using Python 3." + R)
# return True
return False
def on_error(self, e, pair):
"""Called for pairs that don't match `match` and `exclude` filters."""
RED = ansi_code("Fore.LIGHTRED_EX")
R = ansi_code("Style.RESET_ALL")
# any_entry = pair.any_entry
write((RED + "ERROR: {}\n {}" + R).format(e, pair))
# Return True to ignore this error (instead of raising and terminating the app)
if "[Errno 92] Illegal byte sequence" in "{}".format(e) and compat.PY2:
write(RED + "This _may_ be solved by using Python 3." + R)
# return True
return False
def flush(self):
"""Write self to .pyftpsync-meta.json."""
# We DO write meta files even on read-only targets, but not in dry-run mode
# if self.target.readonly:
# write("DirMetadata.flush(%s): read-only; nothing to do" % self.target)
# return
assert self.path == self.target.cur_dir
if self.target.dry_run:
# write("DirMetadata.flush(%s): dry-run; nothing to do" % self.target)
pass
elif self.was_read and len(self.list) == 0 and len(self.peer_sync) == 0:
write("Remove empty meta data file: {}".format(self.target))
self.target.remove_file(self.filename)
elif not self.modified_list and not self.modified_sync:
# write("DirMetadata.flush(%s): unmodified; nothing to do" % self.target)
pass
else:
self.dir["_disclaimer"] = "Generated by https://github.com/mar10/pyftpsync"
self.dir["_time_str"] = pretty_stamp(time.time())
self.dir["_file_version"] = self.VERSION
self.dir["_version"] = __version__
self.dir["_time"] = time.mktime(time.gmtime())
# We always save utf-8 encoded.
# `ensure_ascii` would escape all bytes >127 as `\x12` or `\u1234`,
# which makes it hard to read, so we set it to false.
# `sort_keys` converts binary keys to unicode using utf-8, so we
# must make sure that we don't pass cp1225 or other encoded data.
data = self.dir
opts = {"indent": 4, "sort_keys": True, "ensure_ascii": False}
if compat.PY2:
# The `encoding` arg defaults to utf-8 on Py2 and was removed in Py3
# opts["encoding"] = "utf-8"
# Python 2 has problems with mixed keys (str/unicode)
data = decode_dict_keys(data, "utf-8")
if not self.PRETTY:
opts["indent"] = None
opts["separators"] = (",", ":")
s = json.dumps(data, **opts)
self.target.write_text(self.filename, s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_written", len(s))
self.modified_list = False
self.modified_sync = False
def run(self):
start = time.time()
info_strings = self.get_info_strings()
if self.verbose >= 3:
write(
"{} {}\n{:>20} {}".format(
info_strings[0].capitalize(),
self.local.get_base_name(),
info_strings[1],
self.remote.get_base_name(),
)
)
write(
"Encoding local: {}, remote: {}".format(
self.local.encoding, self.remote.encoding
)
)
try:
self.local.synchronizer = self.remote.synchronizer = self
self.local.peer = self.remote
self.remote.peer = self.local
if self.dry_run:
self.local.readonly = True
self.local.dry_run = True
self.remote.readonly = True
self.remote.dry_run = True
if not self.local.connected:
self.local.open()
if not self.remote.connected:
self.remote.open()
res = self._sync_dir()
finally:
self.local.synchronizer = self.remote.synchronizer = None
self.local.peer = self.remote.peer = None
self.close()
stats = self._stats
stats["elap_secs"] = time.time() - start
stats["elap_str"] = "%0.2f sec" % stats["elap_secs"]
def _add(rate, size, time):
if stats.get(time) and stats.get(size):
stats[rate] = "%0.2f kB/sec" % (0.001 * stats[size] / stats[time])
_add("upload_rate_str", "upload_bytes_written", "upload_write_time")
_add("download_rate_str", "download_bytes_written", "download_write_time")
return res
def run(self):
start = time.time()
info_strings = self.get_info_strings()
if self.verbose >= 3:
write(
"{} {}\n{:>20} {}".format(
info_strings[0].capitalize(),
self.local.get_base_name(),
info_strings[1],
self.remote.get_base_name(),
)
)
write(
"Encoding local: {}, remote: {}".format(
self.local.encoding, self.remote.encoding
)
)
try:
self.local.synchronizer = self.remote.synchronizer = self
self.local.peer = self.remote
self.remote.peer = self.local
if self.dry_run:
self.local.readonly = True
self.local.dry_run = True
self.remote.readonly = True
self.remote.dry_run = True
if not self.local.connected:
self.local.open()
if not self.remote.connected:
self.remote.open()
res = self._sync_dir()
finally:
self.local.synchronizer = self.remote.synchronizer = None
self.local.peer = self.remote.peer = None
self.close()
stats = self._stats
stats["elap_secs"] = time.time() - start
stats["elap_str"] = "{:0.2f} sec".format(stats["elap_secs"])
def _add(rate, size, time):
if stats.get(time) and stats.get(size):
stats[rate] = "{:0.2f} kB/sec".format(0.001 * stats[size] / stats[time])
_add("upload_rate_str", "upload_bytes_written", "upload_write_time")
_add("download_rate_str", "download_bytes_written", "download_write_time")
return res
def _compare_file(self, local, remote):
"""Byte compare two files (early out on first difference)."""
assert isinstance(local, FileEntry) and isinstance(remote, FileEntry)
if not local or not remote:
write(" Files cannot be compared ({} != {}).".format(local, remote))
return False
elif local.size != remote.size:
write(
" Files are different (size {:,d} != {:,d}).".format(
local.size, remote.size
)
)
return False
with local.target.open_readable(
local.name
) as fp_src, remote.target.open_readable(remote.name) as fp_dest:
res, ofs = byte_compare(fp_src, fp_dest)
if not res:
write(" Files are different at offset {:,d}.".format(ofs))
else:
write(" Files are equal.")
return res
def _compare_file(self, local, remote):
"""Byte compare two files (early out on first difference)."""
assert isinstance(local, FileEntry) and isinstance(remote, FileEntry)
if not local or not remote:
write(" Files cannot be compared ({} != {}).".format(local, remote))
return False
elif local.size != remote.size:
write(
" Files are different (size {:,d} != {:,d}).".format(
local.size, remote.size
)
)
return False
with local.target.open_readable(
local.name
) as fp_src, remote.target.open_readable(remote.name) as fp_dest:
res, ofs = byte_compare(fp_src, fp_dest)
if not res:
write(" Files are different at offset {:,d}.".format(ofs))
else:
write(" Files are equal.")
return res
def read(self):
"""Initialize self from .pyftpsync-meta.json file."""
assert self.path == self.target.cur_dir
try:
self.modified_list = False
self.modified_sync = False
is_valid_file = False
s = self.target.read_text(self.filename)
# print("s", s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_read", len(s))
self.was_read = True # True if a file exists (even invalid)
self.dir = json.loads(s)
# import pprint
# print("dir")
# print(pprint.pformat(self.dir))
self.dir = make_native_dict_keys(self.dir)
# print(pprint.pformat(self.dir))
self.list = self.dir["mtimes"]
self.peer_sync = self.dir["peer_sync"]
is_valid_file = True
# write"DirMetadata: read(%s)" % (self.filename, ), self.dir)
# except IncompatibleMetadataVersion:
# raise # We want version errors to terminate the app
except Exception as e:
write_error("Could not read meta info {}: {!r}".format(self, e))
# If the version is incompatible, we stop, unless:
# if --migrate is set, we simply ignore this file (and probably replace it
# with a current version)
if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION:
if not self.target or not self.target.get_option("migrate"):
raise IncompatibleMetadataVersion(
"Invalid meta data version: {} (expected {}).\n"
"Consider passing --migrate to discard old data.".format(
self.dir.get("_file_version"), self.VERSION
)
)
#
write(
"Migrating meta data version from {} to {} (discarding old): {}".format(
self.dir.get("_file_version"), self.VERSION, self.filename
)
)
self.list = {}
self.peer_sync = {}
return
def override_operation(self, operation, reason):
"""Re-Classify entry pair."""
# prev_class = (self.local_classification, self.remote_classification)
prev_op = self.operation
assert operation != prev_op
assert operation in PAIR_OPERATIONS
if "classify" in DEBUG_FLAGS:
write(
"override_operation {} -> {} (reason: '{}')".format(
self, operation, reason
),
debug=True,
)
self.operation = operation
self.re_class_reason = reason
def override_operation(self, operation, reason):
"""Re-Classify entry pair."""
prev_class = (self.local_classification, self.remote_classification)
prev_op = self.operation
assert operation != prev_op
assert operation in PAIR_OPERATIONS
if self.any_entry.target.synchronizer.verbose > 3:
write(
"override_operation({}, {}) -> {} ({})".format(
prev_class, prev_op, operation, reason
),
debug=True,
)
self.operation = operation
self.re_class_reason = reason
def on_error(self, exc, pair):
"""Called for pairs that don't match `match` and `exclude` filters."""
# any_entry = pair.any_entry
msg = "{red}ERROR: {exc}\n {pair}{reset}".format(
exc=exc,
pair=pair,
red=ansi_code("Fore.LIGHTRED_EX"),
reset=ansi_code("Style.RESET_ALL"),
)
write(msg)
# Return True to ignore this error (instead of raising and terminating the app)
# if "[Errno 92] Illegal byte sequence" in "{}".format(e) and compat.PY2:
# write(RED + "This _may_ be solved by using Python 3." + R)
# # return True
return False
def read(self):
"""Initialize self from .pyftpsync-meta.json file."""
assert self.path == self.target.cur_dir
try:
self.modified_list = False
self.modified_sync = False
is_valid_file = False
s = self.target.read_text(self.filename)
# print("s", s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_read", len(s))
self.was_read = True # True if a file exists (even invalid)
self.dir = json.loads(s)
# import pprint
# print("dir")
# print(pprint.pformat(self.dir))
self.dir = make_native_dict_keys(self.dir)
# print(pprint.pformat(self.dir))
self.list = self.dir["mtimes"]
self.peer_sync = self.dir["peer_sync"]
is_valid_file = True
# write"DirMetadata: read(%s)" % (self.filename, ), self.dir)
# except IncompatibleMetadataVersion:
# raise # We want version errors to terminate the app
except Exception as e:
write_error("Could not read meta info {}: {!r}".format(self, e))
# If the version is incompatible, we stop, unless:
# if --migrate is set, we simply ignore this file (and probably replace it
# with a current version)
if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION:
if not self.target or not self.target.get_option("migrate"):
raise IncompatibleMetadataVersion(
"Invalid meta data version: {} (expected {}).\n"
"Consider passing --migrate to discard old data.".format(
self.dir.get("_file_version"), self.VERSION))
#
write(
"Migrating meta data version from {} to {} (i.e. discarding it): {}"
.format(self.dir.get("_file_version"), self.VERSION,
self.filename))
self.list = {}
self.peer_sync = {}
# This will remove .pyftpsync-meta.json:
self.flush()
return
def __init__(self,
path,
host,
port=0,
username=None,
password=None,
tls=False,
timeout=None,
extra_opts=None):
"""Create FTP target with host, initial path, optional credentials and options.
Args:
path (str): root path on FTP server, relative to *host*
host (str): hostname of FTP server
port (int): FTP port (defaults to 21)
username (str):
password (str):
tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
timeout (int): the timeout to set against the ftp socket (seconds)
extra_opts (dict):
"""
path = path or "/"
super(FtpTarget, self).__init__(path, extra_opts)
if tls:
try:
self.ftp = ftplib.FTP_TLS()
except AttributeError:
write("Python 2.7/3.2+ required for FTPS (TLS).")
raise
else:
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
self.host = host
self.port = port or 0
self.username = username
self.password = password
self.tls = tls
self.timeout = timeout
#: dict: written to ftp target root folder before synchronization starts.
#: set to False, if write failed. Default: None
self.lock_data = None
self.is_unix = None
self.time_zone_ofs = None
self.clock_ofs = None
self.ftp_socket_connected = False
self.support_set_time = False
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".format(
self.root_dir
)
)
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}".format(
self.cur_dir
)
)
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
try:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
except Exception as e:
# I have seen '226 Closing data connection' responses here,
# probably when a previous command threw another error.
# However here, 2xx response should be Ok(?):
# A 226 reply code is sent by the server before closing the
# data connection after successfully processing the previous client command
if e.args[0][:3] == "226":
write_error("Ignoring 226 response for ftp.delete() lockfile")
else:
raise
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _ftp_pwd(self):
"""Variant of `self.ftp.pwd()` that supports encoding-fallback.
Returns:
Current working directory as native string.
"""
try:
return self.ftp.pwd()
except UnicodeEncodeError:
if compat.PY2 or self.ftp.encoding != "utf-8":
raise # should not happen, since Py2 does not try to encode
# TODO: this is NOT THREAD-SAFE!
prev_encoding = self.ftp.encoding
try:
write("ftp.pwd() failed with utf-8: trying Cp1252...", warning=True)
return self.ftp.pwd()
finally:
self.ftp.encoding = prev_encoding
def classify(self, peer_dir_meta):
"""Classify this entry as 'new', 'unmodified', or 'modified'."""
assert self.classification is None
peer_entry_meta = None
if peer_dir_meta:
# Metadata is generally available, so we can detect 'new' or 'modified'
peer_entry_meta = peer_dir_meta.get(self.name, False)
if self.is_dir():
# Directories are considered 'unmodified' (would require deep traversal
# to check otherwise)
if peer_entry_meta:
self.classification = "unmodified"
else:
self.classification = "new"
elif peer_entry_meta:
# File entries can be classified as modified/unmodified
self.ps_size = peer_entry_meta.get("s")
self.ps_mtime = peer_entry_meta.get("m")
self.ps_utime = peer_entry_meta.get("u")
if (
self.size == self.ps_size
and FileEntry._eps_compare(self.mtime, self.ps_mtime) == 0
):
self.classification = "unmodified"
else:
self.classification = "modified"
else:
# A new file entry
self.classification = "new"
else:
# No metadata available:
if self.is_dir():
# Directories are considered 'unmodified' (would require deep traversal
# to check otherwise)
self.classification = "unmodified"
else:
# That's all we know, but EntryPair.classify() may adjust this
self.classification = "existing"
if PRINT_CLASSIFICATIONS:
write("classify {}".format(self))
assert self.classification in ENTRY_CLASSIFICATIONS
return self.classification
def _ftp_pwd(self):
"""Variant of `self.ftp.pwd()` that supports encoding-fallback.
Returns:
Current working directory as native string.
"""
try:
return self.ftp.pwd()
except UnicodeEncodeError:
if self.ftp.encoding != "utf-8":
raise # should not happen, since Py2 does not try to encode
# TODO: this is NOT THREAD-SAFE!
prev_encoding = self.ftp.encoding
try:
write("ftp.pwd() failed with utf-8: trying Cp1252...",
warning=True)
return self.ftp.pwd()
finally:
self.ftp.encoding = prev_encoding
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".
format(self.root_dir))
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}"
.format(self.cur_dir))
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
try:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
except Exception as e:
# I have seen '226 Closing data connection' responses here,
# probably when a previous command threw another error.
# However here, 2xx response should be Ok(?):
# A 226 reply code is sent by the server before closing the
# data connection after successfully processing the previous client command
if e.args[0][:3] == "226":
write_error(
"Ignoring 226 response for ftp.delete() lockfile")
else:
raise
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None):
# FTP does not support deletion of non-empty directories.
assert compat.is_native(dir_name)
self.check_write(dir_name)
names = []
nlst_res = self._ftp_nlst(dir_name)
# nlst_res = self.ftp.nlst(dir_name)
# write("rmdir(%s): %s" % (dir_name, nlst_res))
for name in nlst_res:
# name = self.re_encode_to_native(name)
if "/" in name:
name = os.path.basename(name)
if name in (".", ".."):
continue
if predicate and not predicate(name):
continue
names.append(name)
if len(names) > 0:
self.ftp.cwd(dir_name)
try:
for name in names:
try:
# try to delete this as a file
self.ftp.delete(name)
except ftplib.all_errors as _e:
write(
" ftp.delete({}) failed: {}, trying rmdir()...".format(
name, _e
)
)
# assume <name> is a folder
self.rmdir(name)
finally:
if dir_name != ".":
self.ftp.cwd("..")
# write("ftp.rmd(%s)..." % (dir_name, ))
if not keep_root_folder:
self.ftp.rmd(dir_name)
return
def classify(self, peer_dir_meta):
"""Classify entry pair."""
assert self.operation is None
# Note: We pass False if the entry is not listed in the metadata.
# We pass None if we don't have metadata all.
peer_entry_meta = peer_dir_meta.get(self.name, False) if peer_dir_meta else None
if self.local:
self.local.classify(peer_dir_meta)
self.local_classification = self.local.classification
elif peer_entry_meta:
self.local_classification = "deleted"
else:
self.local_classification = "missing"
if self.remote:
self.remote.classify(peer_dir_meta)
self.remote_classification = self.remote.classification
elif peer_entry_meta:
self.remote_classification = "deleted"
else:
self.remote_classification = "missing"
c_pair = (self.local_classification, self.remote_classification)
self.operation = operation_map.get(c_pair)
if not self.operation:
raise RuntimeError(
"Undefined operation for pair classification {}".format(c_pair)
)
if "classify" in DEBUG_FLAGS:
write(
"Classified pair {}, meta={}".format(self, peer_entry_meta),
debug=True,
)
# if not entry.meta:
# assert self.classification in PAIR_CLASSIFICATIONS
assert self.operation in PAIR_OPERATIONS
return self.operation
def classify(self, peer_dir_meta):
"""Classify entry pair."""
assert self.operation is None
# write("CLASSIFIY", self, peer_dir_meta)
# Note: We pass False if the entry is not listed in the metadata.
# We pass None if we don't have metadata all.
peer_entry_meta = peer_dir_meta.get(self.name, False) if peer_dir_meta else None
# write("=>", self, peer_entry_meta)
if self.local:
self.local.classify(peer_dir_meta)
self.local_classification = self.local.classification
elif peer_entry_meta:
self.local_classification = "deleted"
else:
self.local_classification = "missing"
if self.remote:
self.remote.classify(peer_dir_meta)
self.remote_classification = self.remote.classification
elif peer_entry_meta:
self.remote_classification = "deleted"
else:
self.remote_classification = "missing"
c_pair = (self.local_classification, self.remote_classification)
self.operation = operation_map.get(c_pair)
if not self.operation:
raise RuntimeError(
"Undefined operation for pair classification {}".format(c_pair)
)
if PRINT_CLASSIFICATIONS:
write("classify {}".format(self))
# if not entry.meta:
# assert self.classification in PAIR_CLASSIFICATIONS
assert self.operation in PAIR_OPERATIONS
return self.operation
def test_logging(self):
import logging
import logging.handlers
import os
# Create and use a custom logger
custom_logger = logging.getLogger("pyftpsync_test")
log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log")
handler = logging.handlers.WatchedFileHandler(log_path)
# formatter = logging.Formatter(logging.BASIC_FORMAT)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
handler.setFormatter(formatter)
custom_logger.addHandler(handler)
set_pyftpsync_logger(custom_logger)
custom_logger.setLevel(logging.DEBUG)
print("print 1")
write("write info 1")
write_error("write error 1")
custom_logger.setLevel(logging.WARNING)
write("write info 2")
write_error("write error 2")
handler.flush()
log_data = read_test_file("pyftpsync.log")
assert "print 1" not in log_data
assert "write info 1" in log_data
assert "write error 1" in log_data
assert "write info 2" not in log_data, "Loglevel honored"
assert "write error 2" in log_data
# Cleanup properly (log file would be locked otherwise)
custom_logger.removeHandler(handler)
handler.close()
def _print_pair_diff(self, pair):
RED = ansi_code("Fore.LIGHTRED_EX")
# M = ansi_code("Style.BRIGHT") + ansi_code("Style.UNDERLINE")
R = ansi_code("Style.RESET_ALL")
any_entry = pair.any_entry
has_meta = any_entry.get_sync_info("m") is not None
# write("pair", pair)
# print("pair.local", pair.local)
# print("pair.remote", pair.remote)
write(
(
VT_ERASE_LINE
+ RED
+ "CONFLICT: {!r} was modified on both targets since last sync ({})."
+ R
).format(any_entry.get_rel_path(), _ts(any_entry.get_sync_info("u")))
)
if has_meta:
write(
" Original modification time: {}, size: {:,d} bytes.".format(
_ts(any_entry.get_sync_info("m")), any_entry.get_sync_info("s")
)
)
else:
write(" (No meta data available.)")
write(" Local: {}".format(pair.local.as_string() if pair.local else "n.a."))
write(
" Remote: {}".format(
pair.remote.as_string(pair.local) if pair.remote else "n.a."
)
)
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:".format(
encoding
),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:", name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f")
)
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S")
)
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (
name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir
):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type)
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
def open(self):
assert not self.ftp_socket_connected
super(FtpTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
if self.timeout:
self.ftp.connect(self.host, self.port, self.timeout)
else:
# Py2.7 uses -999 as default for `timeout`, Py3 uses None
self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(
self.host, options, force_user=self.username
)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if verbose >= 4:
write(
"Login as '{}'.".format(
self.username if self.username else "anonymous"
)
)
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error(
"Could not login to {}@{}: {}".format(self.username, self.host, e)
)
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.syst_response = self.ftp.sendcmd("SYST")
if verbose >= 5:
write("SYST: '{}'.".format(self.syst_response.replace("\n", " ")))
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
# TODO: case sensitivity?
except Exception as e:
write("SYST command failed: '{}'".format(e))
try:
self.feat_response = self.ftp.sendcmd("FEAT")
self.support_utf8 = "UTF8" in self.feat_response
if verbose >= 5:
write("FEAT: '{}'.".format(self.feat_response.replace("\n", " ")))
except Exception as e:
write("FEAT command failed: '{}'".format(e))
if self.encoding == "utf-8":
if not self.support_utf8 and verbose >= 4:
write(
"Server does not list utf-8 as supported feature (using it anyway).",
warning=True,
)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00
# Note: this RFC is inactive, expired, and failed on Strato
self.ftp.sendcmd("OPTS UTF-8")
if verbose >= 4:
write("Sent 'OPTS UTF-8'.")
except Exception as e:
if verbose >= 4:
write("Could not send 'OPTS UTF-8': '{}'".format(e), warning=True)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/rfc2389
# https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608
# Note: this was accepted on Strato
self.ftp.sendcmd("OPTS UTF8 ON")
if verbose >= 4:
write("Sent 'OPTS UTF8 ON'.")
except Exception as e:
write("Could not send 'OPTS UTF8 ON': '{}'".format(e), warning=True)
if hasattr(self.ftp, "encoding"):
# Python 3 encodes using latin-1 by default(!)
# (In Python 2 ftp.encoding does not exist, but ascii is used)
if self.encoding != codecs.lookup(self.ftp.encoding).name:
write(
"Setting FTP encoding to {} (was {}).".format(
self.encoding, self.ftp.encoding
)
)
self.ftp.encoding = self.encoding
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".format(
self.root_dir, e
)
)
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".format(
self.root_dir, pwd
)
)
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
def get_dir(self):
entry_list = []
entry_map = {}
local_var = {
"has_meta": False
} # pass local variables outside func scope
encoding = self.encoding
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
# We use a custom wrapper here, so we can implement a codding fall back:
self._ftp_retrlines_native("MLSD", _addline, encoding)
# self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command.")
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_var["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error("Could not read meta info {}: {}".format(
self.cur_dir_meta, e))
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {})."
.format(n, entry_map[n].size, meta.get("s")))
missing.append(n)
elif (entry_map[n].mtime -
upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write("Removing meta entry {} (modified {} > {}).".
format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
))
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def _get_dir_impl(self):
entry_list = []
entry_map = {}
has_meta = False
attr_list = self.sftp.listdir_attr()
for de in attr_list:
is_dir = stat.S_ISDIR(de.st_mode)
name = de.filename
entry = None
if is_dir:
if name not in (".", ".."):
entry = DirectoryEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
elif name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
has_meta = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. Compare reported mtime with
# local upload time
self._probe_lock_file(de.st_mtime)
else:
entry = FileEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if has_meta:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error(
"Could not read meta info {}: {}".format(self.cur_dir_meta, e)
)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the SFTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {}).".format(
n, entry_map[n].size, meta.get("s")
)
)
missing.append(n)
elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (modified {} > {}).".format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
)
)
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by SFTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on SFTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
# print("entry_list", entry_list)
return entry_list
def handle_run_command(parser, args):
"""Implement `run` sub-command."""
MAX_LEVELS = 15
# --- Look for `pyftpsync.yaml` in current folder and parents ---
cur_level = 0
cur_folder = os.getcwd()
config_path = None
while cur_level < MAX_LEVELS:
path = os.path.join(cur_folder, CONFIG_FILE_NAME)
# print("Searching for {}...".format(path))
if os.path.isfile(path):
config_path = path
break
parent = os.path.dirname(cur_folder)
if parent == cur_folder:
break
cur_folder = parent
cur_level += 1
if not config_path:
parser.error(
"Could not locate `.pyftpsync.yaml` in {} or {} parent folders.".
format(os.getcwd(), cur_level))
# --- Parse `pyftpsync.yaml` and set `args` attributes ---
try:
with open(config_path, "rb") as f:
config = yaml.safe_load(f)
except Exception as e:
parser.error("Error parsing {}: {}".format(config_path, e))
# write_error("Error parsing {}: {}".format(config_path, e))
# raise
# print(config)
if "tasks" not in config:
parser.error("Missing option `tasks` in {}".format(config_path))
common_config = config.get("common_config", {})
default_task = config.get("default_task", "default")
task_name = args.task or default_task
if task_name not in config["tasks"]:
parser.error("Missing option `tasks.{}` in {}".format(
task_name, config_path))
task = config["tasks"][task_name]
write("Running task '{}' from {}".format(task_name, config_path))
common_config.update(task)
task = common_config
# write("task", task)
# --- Check task syntax ---
task_args = set(task.keys())
missing_args = MANDATORY_TASK_ARGS.difference(task_args)
if missing_args:
parser.error("Missing mandatory options: tasks.{}.{}".format(
task_name, ", ".join(missing_args)))
allowed_args = KNOWN_TASK_ARGS.union(MANDATORY_TASK_ARGS)
invalid_args = task_args.difference(allowed_args)
if invalid_args:
parser.error("Invalid options: tasks.{}.{}".format(
task_name, ", ".join(invalid_args)))
# write("args", args)
for name in allowed_args:
val = task.get(name, None) # default)
if val is None:
continue # option not specified in yaml
# Override yaml entry by command line
cmd_val = getattr(args, name, None)
# write("check --{}: {} => {}".format(name, val, cmd_val))
if cmd_val != val:
override = False
if name in OVERRIDABLE_BOOL_ARGS and cmd_val:
override = True
elif name in {"here", "root"} and (args.here or args.root):
override = True
elif name == "verbose" and cmd_val != 3:
override = True
if override:
write("Yaml entry overriden by --{}: {} => {}".format(
name, val, cmd_val))
continue
setattr(args, name, val)
# --- Figure out local target path ---
cur_folder = os.getcwd()
root_folder = os.path.dirname(config_path)
path_ofs = os.path.relpath(os.getcwd(), root_folder)
if cur_level == 0 or args.root:
path_ofs = ""
args.local = root_folder
elif args.here:
write("Using sub-branch {sub} of {root}".format(root=root_folder,
sub=path_ofs))
args.local = cur_folder
args.remote = os.path.join(args.remote, path_ofs)
else:
parser.error(
"`.pyftpsync.yaml` configuration was found in a parent directory. "
"Please pass an additional argument to clarify:\n"
" --root: synchronize whole project ({root})\n"
" --here: synchronize sub branch ({root}/{sub})".format(
root=root_folder, sub=path_ofs))
def handle_run_command(parser, args):
"""Implement `run` sub-command."""
MAX_LEVELS = 15
# --- Look for `pyftpsync.yaml` in current folder and parents ---
cur_level = 0
cur_folder = os.getcwd()
config_path = None
while cur_level < MAX_LEVELS:
path = os.path.join(cur_folder, CONFIG_FILE_NAME)
# print("Searching for {}...".format(path))
if os.path.isfile(path):
config_path = path
break
parent = os.path.dirname(cur_folder)
if parent == cur_folder:
break
cur_folder = parent
cur_level += 1
if not config_path:
parser.error(
"Could not locate `.pyftpsync.yaml` in {} or {} parent folders.".format(
os.getcwd(), cur_level
)
)
# --- Parse `pyftpsync.yaml` and set `args` attributes ---
try:
with open(config_path, "rb") as f:
config = yaml.safe_load(f)
except Exception as e:
parser.error("Error parsing {}: {}".format(config_path, e))
# write_error("Error parsing {}: {}".format(config_path, e))
# raise
# print(config)
if "tasks" not in config:
parser.error("Missing option `tasks` in {}".format(config_path))
common_config = config.get("common_config", {})
default_task = config.get("default_task", "default")
task_name = args.task or default_task
if task_name not in config["tasks"]:
parser.error("Missing option `tasks.{}` in {}".format(task_name, config_path))
task = config["tasks"][task_name]
write("Running task '{}' from {}".format(task_name, config_path))
common_config.update(task)
task = common_config
# write("task", task)
# --- Check task syntax ---
task_args = set(task.keys())
missing_args = MANDATORY_TASK_ARGS.difference(task_args)
if missing_args:
parser.error(
"Missing mandatory options: tasks.{}.{}".format(
task_name, ", ".join(missing_args)
)
)
allowed_args = KNOWN_TASK_ARGS.union(MANDATORY_TASK_ARGS)
invalid_args = task_args.difference(allowed_args)
if invalid_args:
parser.error(
"Invalid options: tasks.{}.{}".format(task_name, ", ".join(invalid_args))
)
# write("args", args)
for name in allowed_args:
val = task.get(name, None) # default)
if val is None:
continue # option not specified in yaml
# Override yaml entry by command line
cmd_val = getattr(args, name, None)
# write("check --{}: {} => {}".format(name, val, cmd_val))
if cmd_val != val:
override = False
if name in OVERRIDABLE_BOOL_ARGS and cmd_val:
override = True
elif name in {"here", "root"} and (args.here or args.root):
override = True
elif name == "verbose" and cmd_val != 3:
override = True
if override:
write(
"Yaml entry overriden by --{}: {} => {}".format(name, val, cmd_val)
)
continue
setattr(args, name, val)
# --- Figure out local target path ---
cur_folder = os.getcwd()
root_folder = os.path.dirname(config_path)
path_ofs = os.path.relpath(os.getcwd(), root_folder)
if cur_level == 0 or args.root:
path_ofs = ""
args.local = root_folder
elif args.here:
write("Using sub-branch {sub} of {root}".format(root=root_folder, sub=path_ofs))
args.local = cur_folder
args.remote = os.path.join(args.remote, path_ofs)
else:
parser.error(
"`.pyftpsync.yaml` configuration was found in a parent directory. "
"Please pass an additional argument to clarify:\n"
" --root: synchronize whole project ({root})\n"
" --here: synchronize sub branch ({root}/{sub})".format(
root=root_folder, sub=path_ofs
)
)
def open(self):
assert not self.ftp_socket_connected
super(FTPTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
self.ftp.connect(self.host, self.port, self.timeout)
# if self.timeout:
# self.ftp.connect(self.host, self.port, self.timeout)
# else:
# # Py2.7 uses -999 as default for `timeout`, Py3 uses None
# self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(self.host,
options,
force_user=self.username)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if verbose >= 4:
write("Login as '{}'.".format(
self.username if self.username else "anonymous"))
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error("Could not login to {}@{}: {}".format(
self.username, self.host, e))
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.syst_response = self.ftp.sendcmd("SYST")
if verbose >= 5:
write("SYST: '{}'.".format(
self.syst_response.replace("\n", " ")))
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
# TODO: case sensitivity?
except Exception as e:
write("SYST command failed: '{}'".format(e))
try:
self.feat_response = self.ftp.sendcmd("FEAT")
self.support_utf8 = "UTF8" in self.feat_response
if verbose >= 5:
write("FEAT: '{}'.".format(
self.feat_response.replace("\n", " ")))
except Exception as e:
write("FEAT command failed: '{}'".format(e))
if self.encoding == "utf-8":
if not self.support_utf8 and verbose >= 4:
write(
"Server does not list utf-8 as supported feature (using it anyway).",
warning=True,
)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00
# Note: this RFC is inactive, expired, and failed on Strato
self.ftp.sendcmd("OPTS UTF-8")
if verbose >= 4:
write("Sent 'OPTS UTF-8'.")
except Exception as e:
if verbose >= 4:
write("Could not send 'OPTS UTF-8': '{}'".format(e),
warning=True)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/rfc2389
# https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608
# Note: this was accepted on Strato
self.ftp.sendcmd("OPTS UTF8 ON")
if verbose >= 4:
write("Sent 'OPTS UTF8 ON'.")
except Exception as e:
write("Could not send 'OPTS UTF8 ON': '{}'".format(e),
warning=True)
if hasattr(self.ftp, "encoding"):
# Python 3 encodes using latin-1 by default(!)
# (In Python 2 ftp.encoding does not exist, but ascii is used)
if self.encoding != codecs.lookup(self.ftp.encoding).name:
write("Setting FTP encoding to {} (was {}).".format(
self.encoding, self.ftp.encoding))
self.ftp.encoding = self.encoding
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".
format(self.root_dir, e))
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".
format(self.root_dir, pwd))
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return
def get_dir(self):
entry_list = []
entry_map = {}
local_res = {
"has_meta": False
} # pass local variables outside func scope
def _addline(line):
data, _, name = line.partition("; ")
res_type = size = mtime = unique = None
fields = data.split(";")
# http://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_res["has_meta"] = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command.")
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_res["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error("Could not read meta info {}: {}".format(
self.cur_dir_meta, e))
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
if (entry_map[n].size == meta.get("s")
and FileEntry._eps_compare(entry_map[n].mtime,
upload_time) <= 0):
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
# entry_map[n].dt_modified = datetime.fromtimestamp(meta["m"])
else:
# Discard stored meta-data if
# 1. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
# or
# 2. the reported files size is different than the
# size we stored in the meta-data
if self.get_option("verbose", 3) >= 5:
write(
("META: Removing outdated meta entry {}\n" +
" modified after upload ({} > {}), or\n"
" cur. size ({}) != meta size ({})"
).format(n, time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
entry_map[n].size, meta.get("s")))
missing.append(n)
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def get_dir(self):
entry_list = []
entry_map = {}
local_var = {"has_meta": False} # pass local variables outside func scope
encoding = self.encoding
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:".format(
encoding
),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:", name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f")
)
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S")
)
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (
name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir
):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type)
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
# We use a custom wrapper here, so we can implement a codding fall back:
self._ftp_retrlines_native("MLSD", _addline, encoding)
# self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command."
)
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_var["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error(
"Could not read meta info {}: {}".format(self.cur_dir_meta, e)
)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {}).".format(
n, entry_map[n].size, meta.get("s")
)
)
missing.append(n)
elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (modified {} > {}).".format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
)
)
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def open(self):
assert not self.ftp_socket_connected
super(FtpTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
if self.timeout:
self.ftp.connect(self.host, self.port, self.timeout)
else:
# Py2.7 uses -999 as default for `timeout`, Py3 uses None
self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(self.host,
options,
force_user=self.username)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if self.get_option("verbose", 3) >= 4:
write("Login as '{}'.".format(
self.username if self.username else "anonymous"))
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error("Could not login to {}@{}: {}".format(
self.username, self.host, e))
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".
format(self.root_dir, e))
pwd = self.ftp.pwd()
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".
format(self.root_dir, pwd))
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
# TODO: case sensitivity?
# resp = self.ftp.sendcmd("system")
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
self._lock()
return
def open(self):
assert not self.ftp_socket_connected
super(SFTPTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
verify_host_keys = not self.get_option("no_verify_host_keys", False)
if self.get_option("ftp_active", False):
raise RuntimeError("SFTP does not have active/passive mode.")
if verbose <= 3:
logging.getLogger("paramiko.transport").setLevel(logging.WARNING)
write("Connecting {}:*** to sftp://{}".format(self.username, self.host))
cnopts = pysftp.CnOpts()
cnopts.log = self.get_option("ftp_debug", False)
if not verify_host_keys:
cnopts.hostkeys = None
if self.username is None or self.password is None:
creds = get_credentials_for_url(
self.host, options, force_user=self.username
)
if creds:
self.username, self.password = creds
assert self.sftp is None
while True:
try:
self.sftp = pysftp.Connection(
self.host,
username=self.username,
password=self.password,
port=self.port,
cnopts=cnopts,
)
break
except paramiko.ssh_exception.AuthenticationException as e:
write_error(
"Could not login to {}@{}: {}".format(self.username, self.host, e)
)
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
except paramiko.ssh_exception.SSHException as e:
write_error(
"{exc}: Try `ssh-keyscan HOST` to add it "
"(or pass `--no-verify-host-keys` if you don't care about security).".format(
exc=e
)
)
raise
if verbose >= 4:
write(
"Login as '{}'.".format(self.username if self.username else "anonymous")
)
if self.sftp.logfile:
write("Logging to {}".format(self.sftp.logfile))
self.sftp.timeout = self.timeout
self.ftp_socket_connected = True
try:
self.sftp.cwd(self.root_dir)
except IOError as e:
# if not e.args[0].startswith("550"):
# raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".format(
self.root_dir, e
)
)
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".format(
self.root_dir, pwd
)
)
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return