def test_logging(self): import logging import logging.handlers import os # Create and use a custom logger custom_logger = logging.getLogger("pyftpsync_test") log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log") handler = logging.handlers.WatchedFileHandler(log_path) # formatter = logging.Formatter(logging.BASIC_FORMAT) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) custom_logger.addHandler(handler) set_pyftpsync_logger(custom_logger) custom_logger.setLevel(logging.DEBUG) print("print 1") write("write info 1") write_error("write error 1") custom_logger.setLevel(logging.WARNING) write("write info 2") write_error("write error 2") handler.flush() log_data = read_test_file("pyftpsync.log") assert "print 1" not in log_data assert "write info 1" in log_data assert "write error 1" in log_data assert "write info 2" not in log_data, "Loglevel honored" assert "write error 2" in log_data # Cleanup properly (log file would be locked otherwise) custom_logger.removeHandler(handler) handler.close()
def _unlock(self, closing=False): """Remove lock file to the target root folder. """ # write("_unlock", closing) try: if self.cur_dir != self.root_dir: if closing: write( "Changing to ftp root folder to remove lock file: {}". format(self.root_dir)) self.cwd(self.root_dir) else: write_error( "Could not remove lock file, because CWD != ftp root: {}" .format(self.cur_dir)) return if self.lock_data is False: if self.get_option("verbose", 3) >= 4: write("Skip remove lock file (was not written).") else: # direct delete, without updating metadata or checking for target access: self.ftp.delete(DirMetadata.LOCK_FILE_NAME) # self.remove_file(DirMetadata.LOCK_FILE_NAME) self.lock_data = None except Exception as e: write_error("Could not remove lock file: {}".format(e)) raise
def _paramiko_py3compat_u_wrapper(s, encoding="utf8"): try: return SFTPTarget._paramiko_py3compat_u(s, encoding) except UnicodeDecodeError: write_error( "Failed to decode {} using {}. Trying cp1252...".format(s, encoding) ) s = s.decode("cp1252") return s
def close(self): if self.lock_data: self._unlock(closing=True) if self.ftp_socket_connected: try: self.ftp.quit() except (CompatConnectionError, EOFError) as e: write_error("ftp.quit() failed: {}".format(e)) self.ftp_socket_connected = False super(FtpTarget, self).close()
def _lock(self, break_existing=False): """Write a special file to the target root folder.""" # write("_lock") data = {"lock_time": time.time(), "lock_holder": None} try: assert self.cur_dir == self.root_dir self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data)) self.lock_data = data self.lock_write_time = time.time() except Exception as e: write_error("Could not write lock file: {}".format(e)) # Set to False, so we don't try to remove later self.lock_data = False
def read(self): """Initialize self from .pyftpsync-meta.json file.""" assert self.path == self.target.cur_dir try: self.modified_list = False self.modified_sync = False is_valid_file = False s = self.target.read_text(self.filename) # print("s", s) if self.target.synchronizer: self.target.synchronizer._inc_stat("meta_bytes_read", len(s)) self.was_read = True # True if a file exists (even invalid) self.dir = json.loads(s) # import pprint # print("dir") # print(pprint.pformat(self.dir)) self.dir = make_native_dict_keys(self.dir) # print(pprint.pformat(self.dir)) self.list = self.dir["mtimes"] self.peer_sync = self.dir["peer_sync"] is_valid_file = True # write"DirMetadata: read(%s)" % (self.filename, ), self.dir) # except IncompatibleMetadataVersion: # raise # We want version errors to terminate the app except Exception as e: write_error("Could not read meta info {}: {!r}".format(self, e)) # If the version is incompatible, we stop, unless: # if --migrate is set, we simply ignore this file (and probably replace it # with a current version) if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION: if not self.target or not self.target.get_option("migrate"): raise IncompatibleMetadataVersion( "Invalid meta data version: {} (expected {}).\n" "Consider passing --migrate to discard old data.".format( self.dir.get("_file_version"), self.VERSION ) ) # write( "Migrating meta data version from {} to {} (discarding old): {}".format( self.dir.get("_file_version"), self.VERSION, self.filename ) ) self.list = {} self.peer_sync = {} return
def read(self): """Initialize self from .pyftpsync-meta.json file.""" assert self.path == self.target.cur_dir try: self.modified_list = False self.modified_sync = False is_valid_file = False s = self.target.read_text(self.filename) # print("s", s) if self.target.synchronizer: self.target.synchronizer._inc_stat("meta_bytes_read", len(s)) self.was_read = True # True if a file exists (even invalid) self.dir = json.loads(s) # import pprint # print("dir") # print(pprint.pformat(self.dir)) self.dir = make_native_dict_keys(self.dir) # print(pprint.pformat(self.dir)) self.list = self.dir["mtimes"] self.peer_sync = self.dir["peer_sync"] is_valid_file = True # write"DirMetadata: read(%s)" % (self.filename, ), self.dir) # except IncompatibleMetadataVersion: # raise # We want version errors to terminate the app except Exception as e: write_error("Could not read meta info {}: {!r}".format(self, e)) # If the version is incompatible, we stop, unless: # if --migrate is set, we simply ignore this file (and probably replace it # with a current version) if is_valid_file and self.dir.get("_file_version", 0) != self.VERSION: if not self.target or not self.target.get_option("migrate"): raise IncompatibleMetadataVersion( "Invalid meta data version: {} (expected {}).\n" "Consider passing --migrate to discard old data.".format( self.dir.get("_file_version"), self.VERSION)) # write( "Migrating meta data version from {} to {} (i.e. discarding it): {}" .format(self.dir.get("_file_version"), self.VERSION, self.filename)) self.list = {} self.peer_sync = {} # This will remove .pyftpsync-meta.json: self.flush() return
def _lock(self, break_existing=False): """Write a special file to the target root folder.""" # write("_lock") data = {"lock_time": time.time(), "lock_holder": None} try: assert self.cur_dir == self.root_dir self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data)) self.lock_data = data except Exception as e: errmsg = "{}".format(e) write_error("Could not write lock file: {}".format(errmsg)) if errmsg.startswith("550") and self.ftp.passiveserver: try: self.ftp.makepasv() except Exception: write_error( "The server probably requires FTP Active mode. " "Try passing the --ftp-active option.") # Set to False, so we don't try to remove later self.lock_data = False
def test_logging(self): import logging import logging.handlers import os # Create and use a custom logger custom_logger = logging.getLogger("pyftpsync_test") log_path = os.path.join(PYFTPSYNC_TEST_FOLDER, "pyftpsync.log") handler = logging.handlers.WatchedFileHandler(log_path) # formatter = logging.Formatter(logging.BASIC_FORMAT) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s" ) handler.setFormatter(formatter) custom_logger.addHandler(handler) set_pyftpsync_logger(custom_logger) custom_logger.setLevel(logging.DEBUG) print("print 1") write("write info 1") write_error("write error 1") custom_logger.setLevel(logging.WARNING) write("write info 2") write_error("write error 2") handler.flush() log_data = read_test_file("pyftpsync.log") assert "print 1" not in log_data assert "write info 1" in log_data assert "write error 1" in log_data assert "write info 2" not in log_data, "Loglevel honored" assert "write error 2" in log_data # Cleanup properly (log file would be locked otherwise) custom_logger.removeHandler(handler) handler.close()
def _lock(self, break_existing=False): """Write a special file to the target root folder.""" # write("_lock") data = {"lock_time": time.time(), "lock_holder": None} try: assert self.cur_dir == self.root_dir self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data)) self.lock_data = data self.lock_write_time = time.time() except Exception as e: errmsg = "{}".format(e) write_error("Could not write lock file: {}".format(errmsg)) if errmsg.startswith("550") and self.ftp.passiveserver: try: self.ftp.makepasv() except Exception: write_error( "The server probably requires FTP Active mode. " "Try passing the --ftp-active option." ) # Set to False, so we don't try to remove later self.lock_data = False
def _unlock(self, closing=False): """Remove lock file to the target root folder. """ # write("_unlock", closing) try: if self.cur_dir != self.root_dir: if closing: write( "Changing to ftp root folder to remove lock file: {}".format( self.root_dir ) ) self.cwd(self.root_dir) else: write_error( "Could not remove lock file, because CWD != ftp root: {}".format( self.cur_dir ) ) return if self.lock_data is False: if self.get_option("verbose", 3) >= 4: write("Skip remove lock file (was not written).") else: # direct delete, without updating metadata or checking for target access: try: self.ftp.delete(DirMetadata.LOCK_FILE_NAME) # self.remove_file(DirMetadata.LOCK_FILE_NAME) except Exception as e: # I have seen '226 Closing data connection' responses here, # probably when a previous command threw another error. # However here, 2xx response should be Ok(?): # A 226 reply code is sent by the server before closing the # data connection after successfully processing the previous client command if e.args[0][:3] == "226": write_error("Ignoring 226 response for ftp.delete() lockfile") else: raise self.lock_data = None except Exception as e: write_error("Could not remove lock file: {}".format(e)) raise
def _unlock(self, closing=False): """Remove lock file to the target root folder. """ # write("_unlock", closing) try: if self.cur_dir != self.root_dir: if closing: write( "Changing to ftp root folder to remove lock file: {}". format(self.root_dir)) self.cwd(self.root_dir) else: write_error( "Could not remove lock file, because CWD != ftp root: {}" .format(self.cur_dir)) return if self.lock_data is False: if self.get_option("verbose", 3) >= 4: write("Skip remove lock file (was not written).") else: # direct delete, without updating metadata or checking for target access: try: self.ftp.delete(DirMetadata.LOCK_FILE_NAME) # self.remove_file(DirMetadata.LOCK_FILE_NAME) except Exception as e: # I have seen '226 Closing data connection' responses here, # probably when a previous command threw another error. # However here, 2xx response should be Ok(?): # A 226 reply code is sent by the server before closing the # data connection after successfully processing the previous client command if e.args[0][:3] == "226": write_error( "Ignoring 226 response for ftp.delete() lockfile") else: raise self.lock_data = None except Exception as e: write_error("Could not remove lock file: {}".format(e)) raise
def _get_dir_impl(self): entry_list = [] entry_map = {} has_meta = False attr_list = self.sftp.listdir_attr() for de in attr_list: is_dir = stat.S_ISDIR(de.st_mode) name = de.filename entry = None if is_dir: if name not in (".", ".."): entry = DirectoryEntry( self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None ) elif name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored has_meta = True elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir: # this is the root lock file. Compare reported mtime with # local upload time self._probe_lock_file(de.st_mtime) else: entry = FileEntry( self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None ) if entry: entry_map[name] = entry entry_list.append(entry) # load stored meta data if present self.cur_dir_meta = DirMetadata(self) if has_meta: try: self.cur_dir_meta.read() except IncompatibleMetadataVersion: raise # this should end the script (user should pass --migrate) except Exception as e: write_error( "Could not read meta info {}: {}".format(self.cur_dir_meta, e) ) meta_files = self.cur_dir_meta.list # Adjust file mtime from meta-data if present missing = [] for n in meta_files: meta = meta_files[n] if n in entry_map: # We have a meta-data entry for this resource upload_time = meta.get("u", 0) # Discard stored meta-data if # 1. the reported files size is different than the # size we stored in the meta-data # or # 2. the the mtime reported by the SFTP server is later # than the stored upload time (which indicates # that the file was modified directly on the server) if entry_map[n].size != meta.get("s"): if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (size changed from {} to {}).".format( n, entry_map[n].size, meta.get("s") ) ) missing.append(n) elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps: if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (modified {} > {}).".format( n, time.ctime(entry_map[n].mtime), time.ctime(upload_time), ) ) missing.append(n) else: # Use meta-data mtime instead of the one reported by SFTP server entry_map[n].meta = meta entry_map[n].mtime = meta["m"] else: # File is stored in meta-data, but no longer exists on SFTP server # write("META: Removing missing meta entry %s" % n) missing.append(n) # Remove missing or invalid files from cur_dir_meta for n in missing: self.cur_dir_meta.remove(n) # print("entry_list", entry_list) return entry_list
def _addline(status, line): # _ftp_retrlines_native() made sure that we always get `str` type lines assert status in (0, 1, 2) assert is_native(line) data, _, name = line.partition("; ") # print(status, name, u_name) if status == 1: write( "WARNING: File name seems not to be {}; re-encoded from CP-1252:" .format(encoding), name, ) elif status == 2: write_error("File name is neither UTF-8 nor CP-1252 encoded:", name) res_type = size = mtime = unique = None fields = data.split(";") # https://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f")) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S")) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_var["has_meta"] = True elif (name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir): # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: write_error("Could not parse '{}'".format(line)) raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type)) if entry: entry_map[name] = entry entry_list.append(entry)
def get_dir(self): entry_list = [] entry_map = {} local_var = { "has_meta": False } # pass local variables outside func scope encoding = self.encoding def _addline(status, line): # _ftp_retrlines_native() made sure that we always get `str` type lines assert status in (0, 1, 2) assert is_native(line) data, _, name = line.partition("; ") # print(status, name, u_name) if status == 1: write( "WARNING: File name seems not to be {}; re-encoded from CP-1252:" .format(encoding), name, ) elif status == 2: write_error("File name is neither UTF-8 nor CP-1252 encoded:", name) res_type = size = mtime = unique = None fields = data.split(";") # https://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f")) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S")) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_var["has_meta"] = True elif (name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir): # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: write_error("Could not parse '{}'".format(line)) raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type)) if entry: entry_map[name] = entry entry_list.append(entry) try: # We use a custom wrapper here, so we can implement a codding fall back: self._ftp_retrlines_native("MLSD", _addline, encoding) # self.ftp.retrlines("MLSD", _addline) except ftplib.error_perm as e: # write_error("The FTP server responded with {}".format(e)) # raises error_perm "500 Unknown command" if command is not supported if "500" in str(e.args): raise RuntimeError( "The FTP server does not support the 'MLSD' command.") raise # load stored meta data if present self.cur_dir_meta = DirMetadata(self) if local_var["has_meta"]: try: self.cur_dir_meta.read() except IncompatibleMetadataVersion: raise # this should end the script (user should pass --migrate) except Exception as e: write_error("Could not read meta info {}: {}".format( self.cur_dir_meta, e)) meta_files = self.cur_dir_meta.list # Adjust file mtime from meta-data if present missing = [] for n in meta_files: meta = meta_files[n] if n in entry_map: # We have a meta-data entry for this resource upload_time = meta.get("u", 0) # Discard stored meta-data if # 1. the reported files size is different than the # size we stored in the meta-data # or # 2. the the mtime reported by the FTP server is later # than the stored upload time (which indicates # that the file was modified directly on the server) if entry_map[n].size != meta.get("s"): if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (size changed from {} to {})." .format(n, entry_map[n].size, meta.get("s"))) missing.append(n) elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps: if self.get_option("verbose", 3) >= 5: write("Removing meta entry {} (modified {} > {}).". format( n, time.ctime(entry_map[n].mtime), time.ctime(upload_time), )) missing.append(n) else: # Use meta-data mtime instead of the one reported by FTP server entry_map[n].meta = meta entry_map[n].mtime = meta["m"] else: # File is stored in meta-data, but no longer exists on FTP server # write("META: Removing missing meta entry %s" % n) missing.append(n) # Remove missing or invalid files from cur_dir_meta for n in missing: self.cur_dir_meta.remove(n) return entry_list
def open(self): assert not self.ftp_socket_connected super(FTPTarget, self).open() options = self.get_options_dict() no_prompt = self.get_option("no_prompt", True) store_password = self.get_option("store_password", False) verbose = self.get_option("verbose", 3) self.ftp.set_debuglevel(self.get_option("ftp_debug", 0)) # Optionally use FTP active mode (default: PASV) (issue #21) force_active = self.get_option("ftp_active", False) self.ftp.set_pasv(not force_active) self.ftp.connect(self.host, self.port, self.timeout) # if self.timeout: # self.ftp.connect(self.host, self.port, self.timeout) # else: # # Py2.7 uses -999 as default for `timeout`, Py3 uses None # self.ftp.connect(self.host, self.port) self.ftp_socket_connected = True if self.username is None or self.password is None: creds = get_credentials_for_url(self.host, options, force_user=self.username) if creds: self.username, self.password = creds while True: try: # Login (as 'anonymous' if self.username is undefined): self.ftp.login(self.username, self.password) if verbose >= 4: write("Login as '{}'.".format( self.username if self.username else "anonymous")) break except ftplib.error_perm as e: # If credentials were passed, but authentication fails, prompt # for new password if not e.args[0].startswith("530"): raise # error other then '530 Login incorrect' write_error("Could not login to {}@{}: {}".format( self.username, self.host, e)) if no_prompt or not self.username: raise creds = prompt_for_password(self.host, self.username) self.username, self.password = creds # Continue while-loop if self.tls: # Upgrade data connection to TLS. self.ftp.prot_p() try: self.syst_response = self.ftp.sendcmd("SYST") if verbose >= 5: write("SYST: '{}'.".format( self.syst_response.replace("\n", " "))) # self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests # TODO: case sensitivity? except Exception as e: write("SYST command failed: '{}'".format(e)) try: self.feat_response = self.ftp.sendcmd("FEAT") self.support_utf8 = "UTF8" in self.feat_response if verbose >= 5: write("FEAT: '{}'.".format( self.feat_response.replace("\n", " "))) except Exception as e: write("FEAT command failed: '{}'".format(e)) if self.encoding == "utf-8": if not self.support_utf8 and verbose >= 4: write( "Server does not list utf-8 as supported feature (using it anyway).", warning=True, ) try: # Announce our wish to use UTF-8 to the server as proposed here: # See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00 # Note: this RFC is inactive, expired, and failed on Strato self.ftp.sendcmd("OPTS UTF-8") if verbose >= 4: write("Sent 'OPTS UTF-8'.") except Exception as e: if verbose >= 4: write("Could not send 'OPTS UTF-8': '{}'".format(e), warning=True) try: # Announce our wish to use UTF-8 to the server as proposed here: # See https://tools.ietf.org/html/rfc2389 # https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608 # Note: this was accepted on Strato self.ftp.sendcmd("OPTS UTF8 ON") if verbose >= 4: write("Sent 'OPTS UTF8 ON'.") except Exception as e: write("Could not send 'OPTS UTF8 ON': '{}'".format(e), warning=True) if hasattr(self.ftp, "encoding"): # Python 3 encodes using latin-1 by default(!) # (In Python 2 ftp.encoding does not exist, but ascii is used) if self.encoding != codecs.lookup(self.ftp.encoding).name: write("Setting FTP encoding to {} (was {}).".format( self.encoding, self.ftp.encoding)) self.ftp.encoding = self.encoding try: self.ftp.cwd(self.root_dir) except ftplib.error_perm as e: if not e.args[0].startswith("550"): raise # error other then 550 No such directory' write_error( "Could not change directory to {} ({}): missing permissions?". format(self.root_dir, e)) pwd = self.pwd() # pwd = self.to_unicode(pwd) if pwd != self.root_dir: raise RuntimeError( "Unable to navigate to working directory {!r} (now at {!r})". format(self.root_dir, pwd)) self.cur_dir = pwd # self.ftp_initialized = True # Successfully authenticated: store password if store_password: save_password(self.host, self.username, self.password) self._lock() return
def open(self): assert not self.ftp_socket_connected super(FtpTarget, self).open() options = self.get_options_dict() no_prompt = self.get_option("no_prompt", True) store_password = self.get_option("store_password", False) verbose = self.get_option("verbose", 3) self.ftp.set_debuglevel(self.get_option("ftp_debug", 0)) # Optionally use FTP active mode (default: PASV) (issue #21) force_active = self.get_option("ftp_active", False) self.ftp.set_pasv(not force_active) if self.timeout: self.ftp.connect(self.host, self.port, self.timeout) else: # Py2.7 uses -999 as default for `timeout`, Py3 uses None self.ftp.connect(self.host, self.port) self.ftp_socket_connected = True if self.username is None or self.password is None: creds = get_credentials_for_url( self.host, options, force_user=self.username ) if creds: self.username, self.password = creds while True: try: # Login (as 'anonymous' if self.username is undefined): self.ftp.login(self.username, self.password) if verbose >= 4: write( "Login as '{}'.".format( self.username if self.username else "anonymous" ) ) break except ftplib.error_perm as e: # If credentials were passed, but authentication fails, prompt # for new password if not e.args[0].startswith("530"): raise # error other then '530 Login incorrect' write_error( "Could not login to {}@{}: {}".format(self.username, self.host, e) ) if no_prompt or not self.username: raise creds = prompt_for_password(self.host, self.username) self.username, self.password = creds # Continue while-loop if self.tls: # Upgrade data connection to TLS. self.ftp.prot_p() try: self.syst_response = self.ftp.sendcmd("SYST") if verbose >= 5: write("SYST: '{}'.".format(self.syst_response.replace("\n", " "))) # self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests # TODO: case sensitivity? except Exception as e: write("SYST command failed: '{}'".format(e)) try: self.feat_response = self.ftp.sendcmd("FEAT") self.support_utf8 = "UTF8" in self.feat_response if verbose >= 5: write("FEAT: '{}'.".format(self.feat_response.replace("\n", " "))) except Exception as e: write("FEAT command failed: '{}'".format(e)) if self.encoding == "utf-8": if not self.support_utf8 and verbose >= 4: write( "Server does not list utf-8 as supported feature (using it anyway).", warning=True, ) try: # Announce our wish to use UTF-8 to the server as proposed here: # See https://tools.ietf.org/html/draft-ietf-ftpext-utf-8-option-00 # Note: this RFC is inactive, expired, and failed on Strato self.ftp.sendcmd("OPTS UTF-8") if verbose >= 4: write("Sent 'OPTS UTF-8'.") except Exception as e: if verbose >= 4: write("Could not send 'OPTS UTF-8': '{}'".format(e), warning=True) try: # Announce our wish to use UTF-8 to the server as proposed here: # See https://tools.ietf.org/html/rfc2389 # https://www.cerberusftp.com/phpBB3/viewtopic.php?t=2608 # Note: this was accepted on Strato self.ftp.sendcmd("OPTS UTF8 ON") if verbose >= 4: write("Sent 'OPTS UTF8 ON'.") except Exception as e: write("Could not send 'OPTS UTF8 ON': '{}'".format(e), warning=True) if hasattr(self.ftp, "encoding"): # Python 3 encodes using latin-1 by default(!) # (In Python 2 ftp.encoding does not exist, but ascii is used) if self.encoding != codecs.lookup(self.ftp.encoding).name: write( "Setting FTP encoding to {} (was {}).".format( self.encoding, self.ftp.encoding ) ) self.ftp.encoding = self.encoding try: self.ftp.cwd(self.root_dir) except ftplib.error_perm as e: if not e.args[0].startswith("550"): raise # error other then 550 No such directory' write_error( "Could not change directory to {} ({}): missing permissions?".format( self.root_dir, e ) ) pwd = self.pwd() # pwd = self.to_unicode(pwd) if pwd != self.root_dir: raise RuntimeError( "Unable to navigate to working directory {!r} (now at {!r})".format( self.root_dir, pwd ) ) self.cur_dir = pwd # self.ftp_initialized = True # Successfully authenticated: store password if store_password: save_password(self.host, self.username, self.password) self._lock() return
def get_dir(self): entry_list = [] entry_map = {} local_res = { "has_meta": False } # pass local variables outside func scope def _addline(line): data, _, name = line.partition("; ") res_type = size = mtime = unique = None fields = data.split(";") # http://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f")) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S")) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_res["has_meta"] = True elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir: # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type)) if entry: entry_map[name] = entry entry_list.append(entry) try: self.ftp.retrlines("MLSD", _addline) except ftplib.error_perm as e: # write_error("The FTP server responded with {}".format(e)) # raises error_perm "500 Unknown command" if command is not supported if "500" in str(e.args): raise RuntimeError( "The FTP server does not support the 'MLSD' command.") raise # load stored meta data if present self.cur_dir_meta = DirMetadata(self) if local_res["has_meta"]: try: self.cur_dir_meta.read() except IncompatibleMetadataVersion: raise # this should end the script (user should pass --migrate) except Exception as e: write_error("Could not read meta info {}: {}".format( self.cur_dir_meta, e)) meta_files = self.cur_dir_meta.list # Adjust file mtime from meta-data if present missing = [] for n in meta_files: meta = meta_files[n] if n in entry_map: # We have a meta-data entry for this resource upload_time = meta.get("u", 0) if (entry_map[n].size == meta.get("s") and FileEntry._eps_compare(entry_map[n].mtime, upload_time) <= 0): # Use meta-data mtime instead of the one reported by FTP server entry_map[n].meta = meta entry_map[n].mtime = meta["m"] # entry_map[n].dt_modified = datetime.fromtimestamp(meta["m"]) else: # Discard stored meta-data if # 1. the the mtime reported by the FTP server is later # than the stored upload time (which indicates # that the file was modified directly on the server) # or # 2. the reported files size is different than the # size we stored in the meta-data if self.get_option("verbose", 3) >= 5: write( ("META: Removing outdated meta entry {}\n" + " modified after upload ({} > {}), or\n" " cur. size ({}) != meta size ({})" ).format(n, time.ctime(entry_map[n].mtime), time.ctime(upload_time), entry_map[n].size, meta.get("s"))) missing.append(n) else: # File is stored in meta-data, but no longer exists on FTP server # write("META: Removing missing meta entry %s" % n) missing.append(n) # Remove missing or invalid files from cur_dir_meta for n in missing: self.cur_dir_meta.remove(n) return entry_list
def _addline(status, line): # _ftp_retrlines_native() made sure that we always get `str` type lines assert status in (0, 1, 2) assert compat.is_native(line) data, _, name = line.partition("; ") # print(status, name, u_name) if status == 1: write( "WARNING: File name seems not to be {}; re-encoded from CP-1252:".format( encoding ), name, ) elif status == 2: write_error("File name is neither UTF-8 nor CP-1252 encoded:", name) res_type = size = mtime = unique = None fields = data.split(";") # https://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f") ) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S") ) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_var["has_meta"] = True elif ( name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir ): # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: write_error("Could not parse '{}'".format(line)) raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type) ) if entry: entry_map[name] = entry entry_list.append(entry)
def get_dir(self): entry_list = [] entry_map = {} local_var = {"has_meta": False} # pass local variables outside func scope encoding = self.encoding def _addline(status, line): # _ftp_retrlines_native() made sure that we always get `str` type lines assert status in (0, 1, 2) assert compat.is_native(line) data, _, name = line.partition("; ") # print(status, name, u_name) if status == 1: write( "WARNING: File name seems not to be {}; re-encoded from CP-1252:".format( encoding ), name, ) elif status == 2: write_error("File name is neither UTF-8 nor CP-1252 encoded:", name) res_type = size = mtime = unique = None fields = data.split(";") # https://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f") ) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S") ) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_var["has_meta"] = True elif ( name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir ): # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: write_error("Could not parse '{}'".format(line)) raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type) ) if entry: entry_map[name] = entry entry_list.append(entry) try: # We use a custom wrapper here, so we can implement a codding fall back: self._ftp_retrlines_native("MLSD", _addline, encoding) # self.ftp.retrlines("MLSD", _addline) except ftplib.error_perm as e: # write_error("The FTP server responded with {}".format(e)) # raises error_perm "500 Unknown command" if command is not supported if "500" in str(e.args): raise RuntimeError( "The FTP server does not support the 'MLSD' command." ) raise # load stored meta data if present self.cur_dir_meta = DirMetadata(self) if local_var["has_meta"]: try: self.cur_dir_meta.read() except IncompatibleMetadataVersion: raise # this should end the script (user should pass --migrate) except Exception as e: write_error( "Could not read meta info {}: {}".format(self.cur_dir_meta, e) ) meta_files = self.cur_dir_meta.list # Adjust file mtime from meta-data if present missing = [] for n in meta_files: meta = meta_files[n] if n in entry_map: # We have a meta-data entry for this resource upload_time = meta.get("u", 0) # Discard stored meta-data if # 1. the reported files size is different than the # size we stored in the meta-data # or # 2. the the mtime reported by the FTP server is later # than the stored upload time (which indicates # that the file was modified directly on the server) if entry_map[n].size != meta.get("s"): if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (size changed from {} to {}).".format( n, entry_map[n].size, meta.get("s") ) ) missing.append(n) elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps: if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (modified {} > {}).".format( n, time.ctime(entry_map[n].mtime), time.ctime(upload_time), ) ) missing.append(n) else: # Use meta-data mtime instead of the one reported by FTP server entry_map[n].meta = meta entry_map[n].mtime = meta["m"] else: # File is stored in meta-data, but no longer exists on FTP server # write("META: Removing missing meta entry %s" % n) missing.append(n) # Remove missing or invalid files from cur_dir_meta for n in missing: self.cur_dir_meta.remove(n) return entry_list
def open(self): assert not self.ftp_socket_connected super(FtpTarget, self).open() options = self.get_options_dict() no_prompt = self.get_option("no_prompt", True) store_password = self.get_option("store_password", False) self.ftp.set_debuglevel(self.get_option("ftp_debug", 0)) # Optionally use FTP active mode (default: PASV) (issue #21) force_active = self.get_option("ftp_active", False) self.ftp.set_pasv(not force_active) if self.timeout: self.ftp.connect(self.host, self.port, self.timeout) else: # Py2.7 uses -999 as default for `timeout`, Py3 uses None self.ftp.connect(self.host, self.port) self.ftp_socket_connected = True if self.username is None or self.password is None: creds = get_credentials_for_url(self.host, options, force_user=self.username) if creds: self.username, self.password = creds while True: try: # Login (as 'anonymous' if self.username is undefined): self.ftp.login(self.username, self.password) if self.get_option("verbose", 3) >= 4: write("Login as '{}'.".format( self.username if self.username else "anonymous")) break except ftplib.error_perm as e: # If credentials were passed, but authentication fails, prompt # for new password if not e.args[0].startswith("530"): raise # error other then '530 Login incorrect' write_error("Could not login to {}@{}: {}".format( self.username, self.host, e)) if no_prompt or not self.username: raise creds = prompt_for_password(self.host, self.username) self.username, self.password = creds # Continue while-loop if self.tls: # Upgrade data connection to TLS. self.ftp.prot_p() try: self.ftp.cwd(self.root_dir) except ftplib.error_perm as e: # If credentials were passed, but authentication fails, prompt # for new password if not e.args[0].startswith("550"): raise # error other then 550 No such directory' write_error( "Could not change directory to {} ({}): missing permissions?". format(self.root_dir, e)) pwd = self.ftp.pwd() if pwd != self.root_dir: raise RuntimeError( "Unable to navigate to working directory {!r} (now at {!r})". format(self.root_dir, pwd)) self.cur_dir = pwd # self.ftp_initialized = True # Successfully authenticated: store password if store_password: save_password(self.host, self.username, self.password) # TODO: case sensitivity? # resp = self.ftp.sendcmd("system") # self.is_unix = "unix" in resp.lower() # not necessarily true, better check with r/w tests self._lock() return
def open(self): assert not self.ftp_socket_connected super(SFTPTarget, self).open() options = self.get_options_dict() no_prompt = self.get_option("no_prompt", True) store_password = self.get_option("store_password", False) verbose = self.get_option("verbose", 3) verify_host_keys = not self.get_option("no_verify_host_keys", False) if self.get_option("ftp_active", False): raise RuntimeError("SFTP does not have active/passive mode.") if verbose <= 3: logging.getLogger("paramiko.transport").setLevel(logging.WARNING) write("Connecting {}:*** to sftp://{}".format(self.username, self.host)) cnopts = pysftp.CnOpts() cnopts.log = self.get_option("ftp_debug", False) if not verify_host_keys: cnopts.hostkeys = None if self.username is None or self.password is None: creds = get_credentials_for_url( self.host, options, force_user=self.username ) if creds: self.username, self.password = creds assert self.sftp is None while True: try: self.sftp = pysftp.Connection( self.host, username=self.username, password=self.password, port=self.port, cnopts=cnopts, ) break except paramiko.ssh_exception.AuthenticationException as e: write_error( "Could not login to {}@{}: {}".format(self.username, self.host, e) ) if no_prompt or not self.username: raise creds = prompt_for_password(self.host, self.username) self.username, self.password = creds # Continue while-loop except paramiko.ssh_exception.SSHException as e: write_error( "{exc}: Try `ssh-keyscan HOST` to add it " "(or pass `--no-verify-host-keys` if you don't care about security).".format( exc=e ) ) raise if verbose >= 4: write( "Login as '{}'.".format(self.username if self.username else "anonymous") ) if self.sftp.logfile: write("Logging to {}".format(self.sftp.logfile)) self.sftp.timeout = self.timeout self.ftp_socket_connected = True try: self.sftp.cwd(self.root_dir) except IOError as e: # if not e.args[0].startswith("550"): # raise # error other then 550 No such directory' write_error( "Could not change directory to {} ({}): missing permissions?".format( self.root_dir, e ) ) pwd = self.pwd() # pwd = self.to_unicode(pwd) if pwd != self.root_dir: raise RuntimeError( "Unable to navigate to working directory {!r} (now at {!r})".format( self.root_dir, pwd ) ) self.cur_dir = pwd # self.ftp_initialized = True # Successfully authenticated: store password if store_password: save_password(self.host, self.username, self.password) self._lock() return