class TransferFTP: """ This class uploads files to a website, if the remote does not exists, it creates it first. .. exref:: :title: Transfer files to webste through FTP Simple sketch to transfer a list of ``files`` to a website through FTP :: ftp = TransferFTP('ftp.<website>', alias, password, fLOG=print) issues = [ ] done = [ ] notdone = [ ] for file in files : try : r = ftp.transfer (file, path) if r : done.append( (file, path) ) else : notdone.append ( (file, path) ) except Exception as e : issues.append( (file, e) ) try : ftp.close() except Exception as e : print ("unable to close FTP connection using ftp.close") The class may access to a server using :epkg:`SFTP` protocol but it relies on :epkg:`pysftp` and :epkg:`paramiko`. """ errorNoDirectory = "Can't change directory" blockSize = 2 ** 20 def __init__(self, site, login, password, ftps='FTP', fLOG=noLOG): """ @param site website @param login login @param password password @param ftps if ``'TLS'``, use class :epkg:`*py:ftplib:FTP_TLS`, if ``'FTP'``, use :epkg:`*py:ftplib:TLS`, if ``'SFTP'``, use :epkg:`pysftp` @param fLOG logging function """ self._ftps_ = ftps if site is not None: if ftps == 'TLS': cls = FTP_TLS self.is_sftp = False elif ftps == 'SFTP': import pysftp import paramiko import socket sock = socket.socket() sock.connect((site, 22)) trans = paramiko.transport.Transport(sock) trans.start_client() k = trans.get_remote_server_key() hk = paramiko.hostkeys.HostKeys() hk.add(site, 'ssh-rsa', k) cnopts = pysftp.CnOpts() cnopts.hostkeys = hk def cls(si, lo, pw, cnopts=cnopts): return pysftp.Connection( si, username=lo, password=pw, cnopts=cnopts) self._login_ = lambda si=site, lo=login, pw=password: cls( si, lo, pw) self.is_sftp = True elif ftps == 'FTP': cls = FTP self.is_sftp = False else: raise RuntimeError( # pragma: no cover "No implementation for '{}'.".format(ftps)) if not self.is_sftp: self._ftp = cls(site, login, password) self._logins = [(datetime.datetime.now(), site)] else: # mocking if ftps != 'FTP': raise NotImplementedError( # pragma: no cover "Option ftps is not implemented for mocking.") self._logins = [] self._ftp = FTP(site) self.is_sftp = False self.LOG = fLOG self._atts = dict(site=site, login=login, password=password) def _check_can_logged(self): if self.is_sftp and not hasattr(self, '_ftp'): self._ftp = self._login_() @property def Site(self): """ return the website """ return self._atts["site"] def _private_login(self): # pragma: no cover """ logs in """ self.LOG("reconnecting to ", self.Site, " - ", len(self._logins)) try: if self.is_sftp: self._ftp = self._login_() else: self._ftp.login() self._logins.append((datetime.datetime.now(), self.Site)) except Exception as e: se = str(e) if "You're already logged in" in se: return elif (not self.is_sftp and ( "An existing connection was forcibly closed by the remote host" in se or "An established connection was aborted by the software in your host machine" in se)): # it starts a new connection self.LOG("reconnecting failed, starting a new connection", self.Site, " - ", len(self._logins)) self._ftp = FTP(self.Site, self._atts[ "login"], self._atts["password"]) self._logins.append((datetime.datetime.now(), self.Site)) else: raise e def run_command(self, command, *args, **kwargs): """ Runs a FTP command. @param command command @param args list of argument @return output of the command or True for success, False for failure """ try: t = command(*args, **kwargs) if (command == self._ftp.pwd or command == getattr(self._ftp, 'dir', None) or command == getattr(self._ftp, 'mlsd', 'listdir')): return t elif command != self._ftp.cwd: pass return True except Exception as e: # pragma: no cover if self.is_sftp and 'No such file' in str(e): raise FileNotFoundError( "Unable to find {}.".format(args)) from e if TransferFTP.errorNoDirectory in str(e): raise e self.LOG(e) self.LOG(" ** run exc ", str(command), str(args)) self._private_login() if command == self._ftp.pwd or command is self._ftp.pwd: t = command(self) else: t = command(self, *args, **kwargs) self.LOG(" ** run ", str(command), str(args)) return t def print_list(self): # pragma: no cover """ Returns the list of files in the current directory the function sends everything to the logging function. @return output of the command or True for success, False for failure """ self._check_can_logged() if hasattr(self._ftp, 'retrlines'): return self.run_command(self._ftp.retrlines, 'LIST') raise NotImplementedError( "Not implemented for ftps='{}'.".format(self._ftps_)) def mkd(self, path): # pragma: no cover """ Creates a directory. @param path path to the directory @return True or False """ self._check_can_logged() self.LOG("[mkd]", path) cmd = self._ftp.mkd if hasattr(self._ftp, 'mkd') else self._ftp.mkdir return self.run_command(cmd, path) def cwd(self, path, create=False): """ Goes to a directory, if it does not exist, creates it (if *create* is True). @param path path to the directory @param create True to create it @return True or False """ self._check_can_logged() try: self.run_command(self._ftp.cwd, path) except EOFError as e: # pragma: no cover raise EOFError("unable to go to: {0}".format(path)) from e except FileNotFoundError as e: # pragma: no cover if create: self.mkd(path) self.cwd(path, create) else: raise e except Exception as e: # pragma: no cover if create and TransferFTP.errorNoDirectory in str(e): self.mkd(path) self.cwd(path, create) else: raise e def pwd(self): """ Returns the pathname of the current directory on the server. @return pathname """ self._check_can_logged() if hasattr(self._ftp, 'getcwd'): r = self._ftp.getcwd() return self._ftp.pwd else: return self.run_command(self._ftp.pwd) def dir(self, path='.'): """ Lists the content of a path. @param path path @return list of path See :meth:`enumerate_ls <pyquickhelper.filehelper.ftp_transfer.TransferFTP.enumerate_ls>` """ return list(self.enumerate_ls(path)) def ls(self, path='.'): """ Lists the content of a path. @param path path @return list of path see :meth:`enumerate_ls <pyquickhelper.filehelper.ftp_transfer.TransferFTP.enumerate_ls>` .. exref:: :title: List files from FTP site :: from pyquickhelper.filehelper import TransferFTP ftp = TransferFTP("ftp....", "login", "password") res = ftp.ls("path") for v in res: print(v["name"]) ftp.close() """ return list(self.enumerate_ls(path)) def enumerate_ls(self, path='.'): """ Enumerates the content of a path. @param path path @return list of dictionaries One dictionary:: {'name': 'www', 'type': 'dir', 'unique': 'aaaa', 'unix.uid': '1111', 'unix.mode': '111', 'sizd': '5', 'unix.gid': '000', 'modify': '111111'} """ self._check_can_logged() if not self.is_sftp: for a in self.run_command(self._ftp.mlsd, path): r = dict(name=a[0]) r.update(a[1]) yield r else: with self._ftp.cd(path): for name in self._ftp.listdir(): yield dict(name=name) def transfer(self, file, to, name, debug=False, blocksize=None, callback=None): """ Transfers a file. @param file file name or stream (binary, BytesIO) @param to destination (a folder) @param name name of the stream on the website @param debug if True, displays more information @param blocksize see :tpl:`py,m='ftplib',o='FTP.storbinary'` @param callback see :tpl:`py,m='ftplib',o='FTP.storbinary'` @return status When an error happens, the original current directory is restored. """ self._check_can_logged() path = to.split("/") path = [_ for _ in path if len(_) > 0] nb_logins = len(self._logins) cpwd = self.pwd() done = [] exc = None for i, p in enumerate(path): p_ = ('/' + '/'.join(path[:i + 1])) if self.is_sftp else p try: self.cwd(p_, True) except Exception as e: # pragma: no cover exc = e break done.append(p) if nb_logins != len(self._logins): raise CannotCompleteWithoutNewLoginException( # pragma: no cover "Cannot reach folder '{0}' without new login".format(to)) bs = blocksize if blocksize else TransferFTP.blockSize if not self.is_sftp: def runc(name, f, bs, callback): return self.run_command( self._ftp.storbinary, 'STOR ' + name, f, bs, callback) else: def runc(name, f, bs, callback): return self.run_command( self._ftp.putfo, remotepath=name, flo=f, file_size=bs, callback=None) if exc is None: try: if isinstance(file, str): if not os.path.exists(file): raise FileNotFoundError(file) # pragma: no cover with open(file, "rb") as f: r = runc(name, f, bs, callback) elif isinstance(file, BytesIO): r = runc(name, file, bs, callback) elif isinstance(file, bytes): st = BytesIO(file) r = runc(name, st, bs, callback) else: r = runc(name, file, bs, callback) except Exception as ee: # pragma: no cover exc = ee if nb_logins != len(self._logins): # pragma: no cover try: self.cwd(cpwd) done = [] except Exception as e: raise CannotCompleteWithoutNewLoginException( "Cannot transfer '{0}' without new login".format(to)) # It may fail here, it hopes not. nbtry = 0 nbth = len(done) * 2 + 1 while len(done) > 0: if nb_logins != len(self._logins): # pragma: no cover try: self.cwd(cpwd) break except Exception as e: raise CannotCompleteWithoutNewLoginException( "Cannot return to original folder'{0}' without new login".format(to)) from e nbtry += 1 try: self.cwd("..") done.pop() except Exception as e: # pragma: no cover time.sleep(0.5) self.LOG( " issue with command .. len(done) == {0}".format(len(done))) if nbtry > nbth: raise CannotReturnToFolderException( "len(path)={0} nbtry={1} exc={2} nbl={3} act={4}".format( len(done), nbtry, exc, nb_logins, len(self._logins))) from e if exc is not None: raise exc # pragma: no cover return r def retrieve(self, fold, name, file=None, debug=False): """ Downloads a file. @param file file name or stream (binary, BytesIO) @param fold full remote path @param name name of the stream on the website @param debug if True, displays more information @return status """ self._check_can_logged() if self.is_sftp: self.cwd(fold, False) else: path = fold.split("/") path = [_ for _ in path if len(_) > 0] for p in path: self.cwd(p, True) raise_exc = None if not self.is_sftp: def _retrbinary_(name, callback, size, f): r = self.run_command(self._ftp.retrbinary, 'RETR ' + name, callback, size) if isinstance(r, (bytes, str)): f.write(r) return r runc = _retrbinary_ else: def runc(name, callback, size, f): return self.run_command( self._ftp.getfo, remotepath=name, flo=f, callback=None) if isinstance(file, str): with open(file, "wb") as f: def callback(block): f.write(block) try: runc(name, callback, TransferFTP.blockSize, f) r = True except error_perm as e: # pragma: no cover raise_exc = e r = False elif isinstance(file, BytesIO): def callback(block): file.write(block) try: r = runc(name, callback, TransferFTP.blockSize, file) except error_perm as e: # pragma: no cover raise_exc = e r = False else: b = BytesIO() def callback(block): b.write(block) try: runc(name, callback, TransferFTP.blockSize, b) except error_perm as e: # pragma: no cover raise_exc = e r = b.getvalue() if not self.is_sftp: for p in path: self.cwd("..") if raise_exc: raise raise_exc # pragma: no cover return r def close(self): """ Closes the connection. """ self._check_can_logged() self._ftp.close() if self.is_sftp: self._ftp = None
ll = [] ll = ftp.dir() ftp.dir('CHANGEI.CNDWPROD.JCLMASTR') ftp.cwd('''CHANGEI.CNDWPROD.JCLMASTR.''') ftp.cwd('''CHANGEI.CNDWPROD.JCLMASTR''') ftp.cd('''CHANGEI.CNDWPROD.JCLMASTR''') ftp.dir() ftp.retrlines('LIST') ftp.retrlines('NLST') ftp.retrlines('dir') ftp.cwd("'CHANGEI.CNDWPROD.JCLMASTR'") import ftplib import os