def remove_file(self, name):
"""Remove cur_dir/name."""
assert compat.is_native(name)
self.check_write(name)
# self.cur_dir_meta.remove(name)
self.ftp.delete(name)
self.remove_sync_info(name)
def __init__(self, root_dir, extra_opts):
# All internal paths should use unicode.
# (We cannot convert here, since we don't know the target encoding.)
assert compat.is_native(root_dir)
if root_dir != "/":
root_dir = root_dir.rstrip("/")
# This target is not thread safe
self._rlock = threading.RLock()
#: The target's top-level folder
self.root_dir = root_dir
self.extra_opts = extra_opts or {}
self.readonly = False
self.dry_run = False
self.host = None
self.synchronizer = None # Set by BaseSynchronizer.__init__()
self.peer = None
self.cur_dir = None
self.connected = False
self.save_mode = True
self.case_sensitive = None # TODO: don't know yet
#: Time difference between <local upload time> and the mtime that the server reports afterwards.
#: The value is added to the 'u' time stored in meta data.
#: (This is only a rough estimation, derived from the lock-file.)
self.server_time_ofs = None
#: Maximum allowed difference between a reported mtime and the last known update time,
#: before we classify the entry as 'modified externally'
self.mtime_compare_eps = FileEntry.EPS_TIME
self.cur_dir_meta = DirMetadata(self)
self.meta_stack = []
# Optionally define an encoding for this target, but don't override
# derived class's setting
if not hasattr(self, "encoding"):
#: Assumed encoding for this target. Used to decode binary paths.
self.encoding = _get_encoding_opt(None, extra_opts, None)
return
def remove_file(self, name):
"""Remove cur_dir/name."""
assert compat.is_native(name)
self.check_write(name)
# self.cur_dir_meta.remove(name)
self.ftp.delete(name)
self.remove_sync_info(name)
def set_mtime(self, name, mtime, size):
assert compat.is_native(name)
self.check_write(name)
# write("META set_mtime(%s): %s" % (name, time.ctime(mtime)))
# We cannot set the mtime on FTP servers, so we store this as additional
# meta data in the same directory
# TODO: try "SITE UTIME", "MDTM (set version)", or "SRFT" command
self.cur_dir_meta.set_mtime(name, mtime, size)
def set_mtime(self, name, mtime, size):
assert compat.is_native(name)
self.check_write(name)
# write("META set_mtime(%s): %s" % (name, time.ctime(mtime)))
# We cannot set the mtime on FTP servers, so we store this as additional
# meta data in the same directory
# TODO: try "SITE UTIME", "MDTM (set version)", or "SRFT" command
self.cur_dir_meta.set_mtime(name, mtime, size)
def __init__(
self,
path,
host,
port=0,
username=None,
password=None,
tls=False,
timeout=None,
extra_opts=None,
):
"""Create FTP target with host, initial path, optional credentials and options.
Args:
path (str): root path on FTP server, relative to *host*
host (str): hostname of FTP server
port (int): FTP port (defaults to 21)
username (str):
password (str):
tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
timeout (int): the timeout to set against the ftp socket (seconds)
extra_opts (dict):
"""
self.encoding = _get_encoding_opt(None, extra_opts, "utf-8")
# path = self.to_unicode(path)
path = path or "/"
assert compat.is_native(path)
super(FtpTarget, self).__init__(path, extra_opts)
if tls:
try:
self.ftp = ftplib.FTP_TLS()
except AttributeError:
write("Python 2.7/3.2+ required for FTPS (TLS).")
raise
else:
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
self.host = host
self.port = port or 0
self.username = username
self.password = password
self.tls = tls
self.timeout = timeout
#: dict: written to ftp target root folder before synchronization starts.
#: set to False, if write failed. Default: None
self.lock_data = None
self.lock_write_time = None
self.feat_response = None
self.syst_response = None
self.is_unix = None
#: True if server reports FEAT UTF8
self.support_utf8 = None
#: Time difference between <local upload time> and the mtime that the server reports afterwards.
#: The value is added to the 'u' time stored in meta data.
#: (This is only a rough estimation, derived from the lock-file.)
self.server_time_ofs = None
self.ftp_socket_connected = False
self.support_set_time = False
def __init__(
self,
path,
host,
port=0,
username=None,
password=None,
tls=False,
timeout=None,
extra_opts=None,
):
"""Create FTP target with host, initial path, optional credentials and options.
Args:
path (str): root path on FTP server, relative to *host*
host (str): hostname of FTP server
port (int): FTP port (defaults to 21)
username (str):
password (str):
tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
timeout (int): the timeout to set against the ftp socket (seconds)
extra_opts (dict):
"""
self.encoding = _get_encoding_opt(None, extra_opts, "utf-8")
# path = self.to_unicode(path)
path = path or "/"
assert compat.is_native(path)
super(FtpTarget, self).__init__(path, extra_opts)
if tls:
try:
self.ftp = ftplib.FTP_TLS()
except AttributeError:
write("Python 2.7/3.2+ required for FTPS (TLS).")
raise
else:
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(self.get_option("ftp_debug", 0))
self.host = host
self.port = port or 0
self.username = username
self.password = password
self.tls = tls
self.timeout = timeout
#: dict: written to ftp target root folder before synchronization starts.
#: set to False, if write failed. Default: None
self.lock_data = None
self.lock_write_time = None
self.feat_response = None
self.syst_response = None
self.is_unix = None
#: True if server reports FEAT UTF8
self.support_utf8 = None
#: Time difference between <local upload time> and the mtime that the server reports afterwards.
#: The value is added to the 'u' time stored in meta data.
#: (This is only a rough estimation, derived from the lock-file.)
self.server_time_ofs = None
self.ftp_socket_connected = False
self.support_set_time = False
def check_write(self, name):
"""Raise exception if writing cur_dir/name is not allowed."""
assert compat.is_native(name)
if self.readonly and name not in (
DirMetadata.META_FILE_NAME,
DirMetadata.LOCK_FILE_NAME,
):
raise RuntimeError("Target is read-only: {} + {} / ".format(
self, name))
def cwd(self, dir_name):
assert compat.is_native(dir_name)
path = normpath_url(join_url(self.cur_dir, dir_name))
if not path.startswith(self.root_dir):
# paranoic check to prevent that our sync tool goes berserk
raise RuntimeError("Tried to navigate outside root %r: %r" %
(self.root_dir, path))
self.ftp.cwd(dir_name)
self.cur_dir = path
self.cur_dir_meta = None
return self.cur_dir
def cwd(self, dir_name):
assert compat.is_native(dir_name)
path = normpath_url(join_url(self.cur_dir, dir_name))
if not path.startswith(self.root_dir):
# paranoic check to prevent that our sync tool goes berserk
raise RuntimeError(
"Tried to navigate outside root %r: %r" % (self.root_dir, path)
)
self.ftp.cwd(dir_name)
self.cur_dir = path
self.cur_dir_meta = None
return self.cur_dir
def _ftp_nlst(self, dir_name):
"""Variant of `self.ftp.nlst()` that supports encoding-fallback."""
assert compat.is_native(dir_name)
lines = []
def _add_line(status, line):
lines.append(line)
cmd = "NLST " + dir_name
self._ftp_retrlines_native(cmd, _add_line, self.encoding)
# print(cmd, lines)
return lines
def _ftp_nlst(self, dir_name):
"""Variant of `self.ftp.nlst()` that supports encoding-fallback."""
assert compat.is_native(dir_name)
lines = []
def _add_line(status, line):
lines.append(line)
cmd = "NLST " + dir_name
self._ftp_retrlines_native(cmd, _add_line, self.encoding)
# print(cmd, lines)
return lines
def write_file(self, name, fp_src, blocksize=DEFAULT_BLOCKSIZE, callback=None):
"""Write file-like `fp_src` to cur_dir/name.
Args:
name (str): file name, located in self.curdir
fp_src (file-like): must support read() method
blocksize (int, optional):
callback (function, optional):
Called like `func(buf)` for every written chunk
"""
# print("FTP write_file({})".format(name), blocksize)
assert compat.is_native(name)
self.check_write(name)
self.ftp.storbinary("STOR {}".format(name), fp_src, blocksize, callback)
def open_readable(self, name):
"""Open cur_dir/name for reading.
Note: we read everything into a buffer that supports .read().
Args:
name (str): file name, located in self.curdir
Returns:
file-like (must support read() method)
"""
# print("FTP open_readable({})".format(name))
assert compat.is_native(name)
out = SpooledTemporaryFile(max_size=self.MAX_SPOOL_MEM, mode="w+b")
self.ftp.retrbinary("RETR {}".format(name), out.write,
FtpTarget.DEFAULT_BLOCKSIZE)
out.seek(0)
return out
def open_readable(self, name):
"""Open cur_dir/name for reading.
Note: we read everything into a buffer that supports .read().
Args:
name (str): file name, located in self.curdir
Returns:
file-like (must support read() method)
"""
# print("FTP open_readable({})".format(name))
assert compat.is_native(name)
out = SpooledTemporaryFile(max_size=self.MAX_SPOOL_MEM, mode="w+b")
self.ftp.retrbinary(
"RETR {}".format(name), out.write, FtpTarget.DEFAULT_BLOCKSIZE
)
out.seek(0)
return out
def copy_to_file(self, name, fp_dest, callback=None):
"""Write cur_dir/name to file-like `fp_dest`.
Args:
name (str): file name, located in self.curdir
fp_dest (file-like): must support write() method
callback (function, optional):
Called like `func(buf)` for every written chunk
"""
assert compat.is_native(name)
def _write_to_file(data):
# print("_write_to_file() {} bytes.".format(len(data)))
fp_dest.write(data)
if callback:
callback(data)
self.ftp.retrbinary("RETR {}".format(name), _write_to_file,
FtpTarget.DEFAULT_BLOCKSIZE)
def write_file(self,
name,
fp_src,
blocksize=DEFAULT_BLOCKSIZE,
callback=None):
"""Write file-like `fp_src` to cur_dir/name.
Args:
name (str): file name, located in self.curdir
fp_src (file-like): must support read() method
blocksize (int, optional):
callback (function, optional):
Called like `func(buf)` for every written chunk
"""
# print("FTP write_file({})".format(name), blocksize)
assert compat.is_native(name)
self.check_write(name)
self.ftp.storbinary("STOR {}".format(name), fp_src, blocksize,
callback)
def copy_to_file(self, name, fp_dest, callback=None):
"""Write cur_dir/name to file-like `fp_dest`.
Args:
name (str): file name, located in self.curdir
fp_dest (file-like): must support write() method
callback (function, optional):
Called like `func(buf)` for every written chunk
"""
assert compat.is_native(name)
def _write_to_file(data):
# print("_write_to_file() {} bytes.".format(len(data)))
fp_dest.write(data)
if callback:
callback(data)
self.ftp.retrbinary(
"RETR {}".format(name), _write_to_file, FtpTarget.DEFAULT_BLOCKSIZE
)
def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None):
# FTP does not support deletion of non-empty directories.
assert compat.is_native(dir_name)
self.check_write(dir_name)
names = []
nlst_res = self._ftp_nlst(dir_name)
# nlst_res = self.ftp.nlst(dir_name)
# write("rmdir(%s): %s" % (dir_name, nlst_res))
for name in nlst_res:
# name = self.re_encode_to_native(name)
if "/" in name:
name = os.path.basename(name)
if name in (".", ".."):
continue
if predicate and not predicate(name):
continue
names.append(name)
if len(names) > 0:
self.ftp.cwd(dir_name)
try:
for name in names:
try:
# try to delete this as a file
self.ftp.delete(name)
except ftplib.all_errors as _e:
write(
" ftp.delete({}) failed: {}, trying rmdir()...".format(
name, _e
)
)
# assume <name> is a folder
self.rmdir(name)
finally:
if dir_name != ".":
self.ftp.cwd("..")
# write("ftp.rmd(%s)..." % (dir_name, ))
if not keep_root_folder:
self.ftp.rmd(dir_name)
return
def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None):
# FTP does not support deletion of non-empty directories.
assert compat.is_native(dir_name)
self.check_write(dir_name)
names = []
nlst_res = self._ftp_nlst(dir_name)
# nlst_res = self.ftp.nlst(dir_name)
# write("rmdir(%s): %s" % (dir_name, nlst_res))
for name in nlst_res:
# name = self.re_encode_to_native(name)
if "/" in name:
name = os.path.basename(name)
if name in (".", ".."):
continue
if predicate and not predicate(name):
continue
names.append(name)
if len(names) > 0:
self.ftp.cwd(dir_name)
try:
for name in names:
try:
# try to delete this as a file
self.ftp.delete(name)
except ftplib.all_errors as _e:
write(
" ftp.delete({}) failed: {}, trying rmdir()...".
format(name, _e))
# assume <name> is a folder
self.rmdir(name)
finally:
if dir_name != ".":
self.ftp.cwd("..")
# write("ftp.rmd(%s)..." % (dir_name, ))
if not keep_root_folder:
self.ftp.rmd(dir_name)
return
def mkdir(self, dir_name):
assert compat.is_native(dir_name)
self.check_write(dir_name)
self.ftp.mkd(dir_name)
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:".format(
encoding
),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:", name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f")
)
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S")
)
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (
name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir
):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type)
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert compat.is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
def mkdir(self, dir_name):
assert compat.is_native(dir_name)
self.check_write(dir_name)
self.ftp.mkd(dir_name)