def test_logging(self):
import logging
import logging.handlers
import os
# Create and use a custom logger
custom_logger = logging.getLogger("pyftpsync_test")
log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log")
handler = logging.handlers.WatchedFileHandler(log_path)
# formatter = logging.Formatter(logging.BASIC_FORMAT)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
custom_logger.addHandler(handler)
set_pyftpsync_logger(custom_logger)
custom_logger.setLevel(logging.DEBUG)
print("print 1")
write("write info 1")
write_error("write error 1")
custom_logger.setLevel(logging.WARNING)
write("write info 2")
write_error("write error 2")
handler.flush()
log_data = read_test_file("pyftpsync.log")
assert "print 1" not in log_data
assert "write info 1" in log_data
assert "write error 1" in log_data
assert "write info 2" not in log_data, "Loglevel honored"
assert "write error 2" in log_data
# Cleanup properly (log file would be locked otherwise)
custom_logger.removeHandler(handler)
handler.close()
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".
format(self.root_dir))
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}"
.format(self.cur_dir))
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _paramiko_py3compat_u_wrapper(s, encoding="utf8"):
try:
return SFTPTarget._paramiko_py3compat_u(s, encoding)
except UnicodeDecodeError:
write_error(
"Failed to decode {} using {}. Trying cp1252...".format(s, encoding)
)
s = s.decode("cp1252")
return s
def close(self):
if self.lock_data:
self._unlock(closing=True)
if self.ftp_socket_connected:
try:
self.ftp.quit()
except (CompatConnectionError, EOFError) as e:
write_error("ftp.quit() failed: {}".format(e))
self.ftp_socket_connected = False
super(FtpTarget, self).close()
def close(self):
if self.lock_data:
self._unlock(closing=True)
if self.ftp_socket_connected:
try:
self.ftp.quit()
except (CompatConnectionError, EOFError) as e:
write_error("ftp.quit() failed: {}".format(e))
self.ftp_socket_connected = False
super(FtpTarget, self).close()
def _lock(self, break_existing=False):
"""Write a special file to the target root folder."""
# write("_lock")
data = {"lock_time": time.time(), "lock_holder": None}
try:
assert self.cur_dir == self.root_dir
self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data))
self.lock_data = data
self.lock_write_time = time.time()
except Exception as e:
write_error("Could not write lock file: {}".format(e))
# Set to False, so we don't try to remove later
self.lock_data = False
def read(self):
"""Initialize self from .pyftpsync-meta.json file."""
assert self.path == self.target.cur_dir
try:
self.modified_list = False
self.modified_sync = False
is_valid_file = False
s = self.target.read_text(self.filename)
# print("s", s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_read", len(s))
self.was_read = True # True if a file exists (even invalid)
self.dir = json.loads(s)
# import pprint
# print("dir")
# print(pprint.pformat(self.dir))
self.dir = make_native_dict_keys(self.dir)
# print(pprint.pformat(self.dir))
self.list = self.dir["mtimes"]
self.peer_sync = self.dir["peer_sync"]
is_valid_file = True
# write"DirMetadata: read(%s)" % (self.filename, ), self.dir)
# except IncompatibleMetadataVersion:
# raise # We want version errors to terminate the app
except Exception as e:
write_error("Could not read meta info {}: {!r}".format(self, e))
# If the version is incompatible, we stop, unless:
# if --migrate is set, we simply ignore this file (and probably replace it
# with a current version)
if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION:
if not self.target or not self.target.get_option("migrate"):
raise IncompatibleMetadataVersion(
"Invalid meta data version: {} (expected {}).\n"
"Consider passing --migrate to discard old data.".format(
self.dir.get("_file_version"), self.VERSION
)
)
#
write(
"Migrating meta data version from {} to {} (discarding old): {}".format(
self.dir.get("_file_version"), self.VERSION, self.filename
)
)
self.list = {}
self.peer_sync = {}
return
def read(self):
"""Initialize self from .pyftpsync-meta.json file."""
assert self.path == self.target.cur_dir
try:
self.modified_list = False
self.modified_sync = False
is_valid_file = False
s = self.target.read_text(self.filename)
# print("s", s)
if self.target.synchronizer:
self.target.synchronizer._inc_stat("meta_bytes_read", len(s))
self.was_read = True # True if a file exists (even invalid)
self.dir = json.loads(s)
# import pprint
# print("dir")
# print(pprint.pformat(self.dir))
self.dir = make_native_dict_keys(self.dir)
# print(pprint.pformat(self.dir))
self.list = self.dir["mtimes"]
self.peer_sync = self.dir["peer_sync"]
is_valid_file = True
# write"DirMetadata: read(%s)" % (self.filename, ), self.dir)
# except IncompatibleMetadataVersion:
# raise # We want version errors to terminate the app
except Exception as e:
write_error("Could not read meta info {}: {!r}".format(self, e))
# If the version is incompatible, we stop, unless:
# if --migrate is set, we simply ignore this file (and probably replace it
# with a current version)
if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION:
if not self.target or not self.target.get_option("migrate"):
raise IncompatibleMetadataVersion(
"Invalid meta data version: {} (expected {}).\n"
"Consider passing --migrate to discard old data.".format(
self.dir.get("_file_version"), self.VERSION))
#
write(
"Migrating meta data version from {} to {} (i.e. discarding it): {}"
.format(self.dir.get("_file_version"), self.VERSION,
self.filename))
self.list = {}
self.peer_sync = {}
# This will remove .pyftpsync-meta.json:
self.flush()
return
def _lock(self, break_existing=False):
"""Write a special file to the target root folder."""
# write("_lock")
data = {"lock_time": time.time(), "lock_holder": None}
try:
assert self.cur_dir == self.root_dir
self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data))
self.lock_data = data
except Exception as e:
errmsg = "{}".format(e)
write_error("Could not write lock file: {}".format(errmsg))
if errmsg.startswith("550") and self.ftp.passiveserver:
try:
self.ftp.makepasv()
except Exception:
write_error(
"The server probably requires FTP Active mode. "
"Try passing the --ftp-active option.")
# Set to False, so we don't try to remove later
self.lock_data = False
def test_logging(self):
import logging
import logging.handlers
import os
# Create and use a custom logger
custom_logger = logging.getLogger("pyftpsync_test")
log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log")
handler = logging.handlers.WatchedFileHandler(log_path)
# formatter = logging.Formatter(logging.BASIC_FORMAT)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
handler.setFormatter(formatter)
custom_logger.addHandler(handler)
set_pyftpsync_logger(custom_logger)
custom_logger.setLevel(logging.DEBUG)
print("print 1")
write("write info 1")
write_error("write error 1")
custom_logger.setLevel(logging.WARNING)
write("write info 2")
write_error("write error 2")
handler.flush()
log_data = read_test_file("pyftpsync.log")
assert "print 1" not in log_data
assert "write info 1" in log_data
assert "write error 1" in log_data
assert "write info 2" not in log_data, "Loglevel honored"
assert "write error 2" in log_data
# Cleanup properly (log file would be locked otherwise)
custom_logger.removeHandler(handler)
handler.close()
def _lock(self, break_existing=False):
"""Write a special file to the target root folder."""
# write("_lock")
data = {"lock_time": time.time(), "lock_holder": None}
try:
assert self.cur_dir == self.root_dir
self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data))
self.lock_data = data
self.lock_write_time = time.time()
except Exception as e:
errmsg = "{}".format(e)
write_error("Could not write lock file: {}".format(errmsg))
if errmsg.startswith("550") and self.ftp.passiveserver:
try:
self.ftp.makepasv()
except Exception:
write_error(
"The server probably requires FTP Active mode. "
"Try passing the --ftp-active option."
)
# Set to False, so we don't try to remove later
self.lock_data = False
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".format(
self.root_dir
)
)
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}".format(
self.cur_dir
)
)
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
try:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
except Exception as e:
# I have seen '226 Closing data connection' responses here,
# probably when a previous command threw another error.
# However here, 2xx response should be Ok(?):
# A 226 reply code is sent by the server before closing the
# data connection after successfully processing the previous client command
if e.args[0][:3] == "226":
write_error("Ignoring 226 response for ftp.delete() lockfile")
else:
raise
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _unlock(self, closing=False):
"""Remove lock file to the target root folder.
"""
# write("_unlock", closing)
try:
if self.cur_dir != self.root_dir:
if closing:
write(
"Changing to ftp root folder to remove lock file: {}".
format(self.root_dir))
self.cwd(self.root_dir)
else:
write_error(
"Could not remove lock file, because CWD != ftp root: {}"
.format(self.cur_dir))
return
if self.lock_data is False:
if self.get_option("verbose", 3) >= 4:
write("Skip remove lock file (was not written).")
else:
# direct delete, without updating metadata or checking for target access:
try:
self.ftp.delete(DirMetadata.LOCK_FILE_NAME)
# self.remove_file(DirMetadata.LOCK_FILE_NAME)
except Exception as e:
# I have seen '226 Closing data connection' responses here,
# probably when a previous command threw another error.
# However here, 2xx response should be Ok(?):
# A 226 reply code is sent by the server before closing the
# data connection after successfully processing the previous client command
if e.args[0][:3] == "226":
write_error(
"Ignoring 226 response for ftp.delete() lockfile")
else:
raise
self.lock_data = None
except Exception as e:
write_error("Could not remove lock file: {}".format(e))
raise
def _get_dir_impl(self):
entry_list = []
entry_map = {}
has_meta = False
attr_list = self.sftp.listdir_attr()
for de in attr_list:
is_dir = stat.S_ISDIR(de.st_mode)
name = de.filename
entry = None
if is_dir:
if name not in (".", ".."):
entry = DirectoryEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
elif name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
has_meta = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. Compare reported mtime with
# local upload time
self._probe_lock_file(de.st_mtime)
else:
entry = FileEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if has_meta:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error(
"Could not read meta info {}: {}".format(self.cur_dir_meta, e)
)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the SFTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {}).".format(
n, entry_map[n].size, meta.get("s")
)
)
missing.append(n)
elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (modified {} > {}).".format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
)
)
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by SFTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on SFTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
# print("entry_list", entry_list)
return entry_list
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
def get_dir(self):
entry_list = []
entry_map = {}
local_var = {
"has_meta": False
} # pass local variables outside func scope
encoding = self.encoding
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
# We use a custom wrapper here, so we can implement a codding fall back:
self._ftp_retrlines_native("MLSD", _addline, encoding)
# self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command.")
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_var["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error("Could not read meta info {}: {}".format(
self.cur_dir_meta, e))
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {})."
.format(n, entry_map[n].size, meta.get("s")))
missing.append(n)
elif (entry_map[n].mtime -
upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write("Removing meta entry {} (modified {} > {}).".
format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
))
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def open(self):
assert not self.ftp_socket_connected
super(FTPTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
self.ftp.connect(self.host, self.port, self.timeout)
# if self.timeout:
# self.ftp.connect(self.host, self.port, self.timeout)
# else:
# # Py2.7 uses -999 as default for `timeout`, Py3 uses None
# self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(self.host,
options,
force_user=self.username)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if verbose >= 4:
write("Login as '{}'.".format(
self.username if self.username else "anonymous"))
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error("Could not login to {}@{}: {}".format(
self.username, self.host, e))
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.syst_response = self.ftp.sendcmd("SYST")
if verbose >= 5:
write("SYST: '{}'.".format(
self.syst_response.replace("\n", " ")))
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
# TODO: case sensitivity?
except Exception as e:
write("SYST command failed: '{}'".format(e))
try:
self.feat_response = self.ftp.sendcmd("FEAT")
self.support_utf8 = "UTF8" in self.feat_response
if verbose >= 5:
write("FEAT: '{}'.".format(
self.feat_response.replace("\n", " ")))
except Exception as e:
write("FEAT command failed: '{}'".format(e))
if self.encoding == "utf-8":
if not self.support_utf8 and verbose >= 4:
write(
"Server does not list utf-8 as supported feature (using it anyway).",
warning=True,
)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00
# Note: this RFC is inactive, expired, and failed on Strato
self.ftp.sendcmd("OPTS UTF-8")
if verbose >= 4:
write("Sent 'OPTS UTF-8'.")
except Exception as e:
if verbose >= 4:
write("Could not send 'OPTS UTF-8': '{}'".format(e),
warning=True)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/rfc2389
# https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608
# Note: this was accepted on Strato
self.ftp.sendcmd("OPTS UTF8 ON")
if verbose >= 4:
write("Sent 'OPTS UTF8 ON'.")
except Exception as e:
write("Could not send 'OPTS UTF8 ON': '{}'".format(e),
warning=True)
if hasattr(self.ftp, "encoding"):
# Python 3 encodes using latin-1 by default(!)
# (In Python 2 ftp.encoding does not exist, but ascii is used)
if self.encoding != codecs.lookup(self.ftp.encoding).name:
write("Setting FTP encoding to {} (was {}).".format(
self.encoding, self.ftp.encoding))
self.ftp.encoding = self.encoding
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".
format(self.root_dir, e))
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".
format(self.root_dir, pwd))
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return
def open(self):
assert not self.ftp_socket_connected
super(FtpTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
if self.timeout:
self.ftp.connect(self.host, self.port, self.timeout)
else:
# Py2.7 uses -999 as default for `timeout`, Py3 uses None
self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(
self.host, options, force_user=self.username
)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if verbose >= 4:
write(
"Login as '{}'.".format(
self.username if self.username else "anonymous"
)
)
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error(
"Could not login to {}@{}: {}".format(self.username, self.host, e)
)
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.syst_response = self.ftp.sendcmd("SYST")
if verbose >= 5:
write("SYST: '{}'.".format(self.syst_response.replace("\n", " ")))
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
# TODO: case sensitivity?
except Exception as e:
write("SYST command failed: '{}'".format(e))
try:
self.feat_response = self.ftp.sendcmd("FEAT")
self.support_utf8 = "UTF8" in self.feat_response
if verbose >= 5:
write("FEAT: '{}'.".format(self.feat_response.replace("\n", " ")))
except Exception as e:
write("FEAT command failed: '{}'".format(e))
if self.encoding == "utf-8":
if not self.support_utf8 and verbose >= 4:
write(
"Server does not list utf-8 as supported feature (using it anyway).",
warning=True,
)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00
# Note: this RFC is inactive, expired, and failed on Strato
self.ftp.sendcmd("OPTS UTF-8")
if verbose >= 4:
write("Sent 'OPTS UTF-8'.")
except Exception as e:
if verbose >= 4:
write("Could not send 'OPTS UTF-8': '{}'".format(e), warning=True)
try:
# Announce our wish to use UTF-8 to the server as proposed here:
# See https://tools.ietf.org/html/rfc2389
# https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608
# Note: this was accepted on Strato
self.ftp.sendcmd("OPTS UTF8 ON")
if verbose >= 4:
write("Sent 'OPTS UTF8 ON'.")
except Exception as e:
write("Could not send 'OPTS UTF8 ON': '{}'".format(e), warning=True)
if hasattr(self.ftp, "encoding"):
# Python 3 encodes using latin-1 by default(!)
# (In Python 2 ftp.encoding does not exist, but ascii is used)
if self.encoding != codecs.lookup(self.ftp.encoding).name:
write(
"Setting FTP encoding to {} (was {}).".format(
self.encoding, self.ftp.encoding
)
)
self.ftp.encoding = self.encoding
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".format(
self.root_dir, e
)
)
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".format(
self.root_dir, pwd
)
)
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return
def get_dir(self):
entry_list = []
entry_map = {}
local_res = {
"has_meta": False
} # pass local variables outside func scope
def _addline(line):
data, _, name = line.partition("; ")
res_type = size = mtime = unique = None
fields = data.split(";")
# http://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_res["has_meta"] = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command.")
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_res["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error("Could not read meta info {}: {}".format(
self.cur_dir_meta, e))
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
if (entry_map[n].size == meta.get("s")
and FileEntry._eps_compare(entry_map[n].mtime,
upload_time) <= 0):
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
# entry_map[n].dt_modified = datetime.fromtimestamp(meta["m"])
else:
# Discard stored meta-data if
# 1. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
# or
# 2. the reported files size is different than the
# size we stored in the meta-data
if self.get_option("verbose", 3) >= 5:
write(
("META: Removing outdated meta entry {}\n" +
" modified after upload ({} > {}), or\n"
" cur. size ({}) != meta size ({})"
).format(n, time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
entry_map[n].size, meta.get("s")))
missing.append(n)
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:".format(
encoding
),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:", name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f")
)
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S")
)
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (
name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir
):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type)
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
def get_dir(self):
entry_list = []
entry_map = {}
local_var = {"has_meta": False} # pass local variables outside func scope
encoding = self.encoding
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:".format(
encoding
),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:", name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f")
)
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S")
)
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (
name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir
):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type)
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
# We use a custom wrapper here, so we can implement a codding fall back:
self._ftp_retrlines_native("MLSD", _addline, encoding)
# self.ftp.retrlines("MLSD", _addline)
except ftplib.error_perm as e:
# write_error("The FTP server responded with {}".format(e))
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError(
"The FTP server does not support the 'MLSD' command."
)
raise
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_var["has_meta"]:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error(
"Could not read meta info {}: {}".format(self.cur_dir_meta, e)
)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the FTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {}).".format(
n, entry_map[n].size, meta.get("s")
)
)
missing.append(n)
elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (modified {} > {}).".format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
)
)
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on FTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list
def open(self):
assert not self.ftp_socket_connected
super(FtpTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
# Optionally use FTP active mode (default: PASV) (issue #21)
force_active = self.get_option("ftp_active", False)
self.ftp.set_pasv(not force_active)
if self.timeout:
self.ftp.connect(self.host, self.port, self.timeout)
else:
# Py2.7 uses -999 as default for `timeout`, Py3 uses None
self.ftp.connect(self.host, self.port)
self.ftp_socket_connected = True
if self.username is None or self.password is None:
creds = get_credentials_for_url(self.host,
options,
force_user=self.username)
if creds:
self.username, self.password = creds
while True:
try:
# Login (as 'anonymous' if self.username is undefined):
self.ftp.login(self.username, self.password)
if self.get_option("verbose", 3) >= 4:
write("Login as '{}'.".format(
self.username if self.username else "anonymous"))
break
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("530"):
raise # error other then '530 Login incorrect'
write_error("Could not login to {}@{}: {}".format(
self.username, self.host, e))
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
if self.tls:
# Upgrade data connection to TLS.
self.ftp.prot_p()
try:
self.ftp.cwd(self.root_dir)
except ftplib.error_perm as e:
# If credentials were passed, but authentication fails, prompt
# for new password
if not e.args[0].startswith("550"):
raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".
format(self.root_dir, e))
pwd = self.ftp.pwd()
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".
format(self.root_dir, pwd))
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
# TODO: case sensitivity?
# resp = self.ftp.sendcmd("system")
# self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests
self._lock()
return
def open(self):
assert not self.ftp_socket_connected
super(SFTPTarget, self).open()
options = self.get_options_dict()
no_prompt = self.get_option("no_prompt", True)
store_password = self.get_option("store_password", False)
verbose = self.get_option("verbose", 3)
verify_host_keys = not self.get_option("no_verify_host_keys", False)
if self.get_option("ftp_active", False):
raise RuntimeError("SFTP does not have active/passive mode.")
if verbose <= 3:
logging.getLogger("paramiko.transport").setLevel(logging.WARNING)
write("Connecting {}:*** to sftp://{}".format(self.username, self.host))
cnopts = pysftp.CnOpts()
cnopts.log = self.get_option("ftp_debug", False)
if not verify_host_keys:
cnopts.hostkeys = None
if self.username is None or self.password is None:
creds = get_credentials_for_url(
self.host, options, force_user=self.username
)
if creds:
self.username, self.password = creds
assert self.sftp is None
while True:
try:
self.sftp = pysftp.Connection(
self.host,
username=self.username,
password=self.password,
port=self.port,
cnopts=cnopts,
)
break
except paramiko.ssh_exception.AuthenticationException as e:
write_error(
"Could not login to {}@{}: {}".format(self.username, self.host, e)
)
if no_prompt or not self.username:
raise
creds = prompt_for_password(self.host, self.username)
self.username, self.password = creds
# Continue while-loop
except paramiko.ssh_exception.SSHException as e:
write_error(
"{exc}: Try `ssh-keyscan HOST` to add it "
"(or pass `--no-verify-host-keys` if you don't care about security).".format(
exc=e
)
)
raise
if verbose >= 4:
write(
"Login as '{}'.".format(self.username if self.username else "anonymous")
)
if self.sftp.logfile:
write("Logging to {}".format(self.sftp.logfile))
self.sftp.timeout = self.timeout
self.ftp_socket_connected = True
try:
self.sftp.cwd(self.root_dir)
except IOError as e:
# if not e.args[0].startswith("550"):
# raise # error other then 550 No such directory'
write_error(
"Could not change directory to {} ({}): missing permissions?".format(
self.root_dir, e
)
)
pwd = self.pwd()
# pwd = self.to_unicode(pwd)
if pwd != self.root_dir:
raise RuntimeError(
"Unable to navigate to working directory {!r} (now at {!r})".format(
self.root_dir, pwd
)
)
self.cur_dir = pwd
# self.ftp_initialized = True
# Successfully authenticated: store password
if store_password:
save_password(self.host, self.username, self.password)
self._lock()
return
