class OdooFTP(): def __init__(self, host='', port=21, username='', password='', path='', path_done='', security=False): self.host = host self.port = port self.username = username self.password = password self.path = path self.path_done = path_done self.security = security self.ftp = None self.file_ftp = '' def get_connection(self): logging.info("CRON _import - Inicia ") if self.security: cnopts = pysftp.CnOpts() cnopts.hostkeys = None # ignore knownhosts self.ftp = pysftp.Connection(host=self.host, port=self.port, username=self.username, password=self.password, cnopts=cnopts) return self.ftp else: self.ftp = FTP() self.ftp.connect(self.host, self.port) self.ftp.login(self.username, self.password) self.ftp.cwd(self.path) return self.ftp def _getFTPData(self, imprt): if self.security: return self._getSFTPData(imprt) else: print("--0000000") return '' def _getSFTPData(self, imprt): self.ftp = self.get_connection() self.ftp.chdir(self.path) try: # Archivo de Control exists = self.ftp.exists(imprt.source_ftp_write_control) if exists: logging.info("CRON Import - Existe archivo Control %s " % (imprt.source_ftp_write_control)) return { 'error': "Existe archivo Control %s" % imprt.source_ftp_write_control } listdir = self.ftp.listdir() listdir_files = [ line for line in listdir if line.lower().endswith(".dat") ] if not listdir_files: logging.info("CRON Import - No hay archivos para procesar ") return { 'error': "CRON Import - No hay archivos para procesar " } output = StringIO() self.ftp.putfo(output, imprt.source_ftp_write_control) logging.info("CRON Import.Crear Control %s " % (imprt.source_ftp_write_control)) directory = "/tmp/tmpsftp%s" % (imprt.id) for line in listdir_files: self.ftp.get(line, directory + '/' + line) logging.info("CRON Import - File Transer %s " % (line)) self.ftp.close() except: if self.ftp.exists(imprt.source_ftp_write_control): self.ftp.remove(imprt.source_ftp_write_control) self.ftp.close() return {} def _get_dirfile(self, file_re): self.ftp = self.get_connection() regex = re.compile(file_re) file_ftp = '' listdir = [] if self.security: self.ftp.chdir(self.path) listdir = self.ftp.listdir() self.ftp.close() for line in listdir: validate = regex.match(line) and True or False if validate: file_ftp = line break else: file_ftp = '' else: self.ftp.dir(listdir.append) self.ftp.quit() for line in listdir: ll = line[29:].strip().split(' ') file_ftp = ll[-1] validate = regex.match(file_ftp) and True or False if validate: break else: file_ftp = '' logging.info("CRON _import Security %s - Files %s " % (self.security, file_ftp)) return file_ftp def get_file_exist(self, filename): self.ftp = self.get_connection() res = False if self.security: self.ftp.chdir(self.path) res = self.ftp.exists(filename) self.ftp.close() else: if filename in self.ftp.nlst(): res = True self.ftp.quit() logging.info("CRON _import Security %s - File Exists %s " % (self.security, res)) return res def get_file_push(self, filename): self.ftp = self.get_connection() output = StringIO() if self.security: self.ftp.chdir(self.path) self.ftp.putfo(output, filename) self.ftp.close() else: self.ftp.storbinary('STOR ' + filename, output) self.ftp.quit() output.close() logging.info("CRON _import Security %s - File Push %s " % (self.security, filename)) return True def get_file_read(self, filename): self.ftp = self.get_connection() info = "" flo = io.BytesIO() if self.security: self.ftp.chdir(self.path) self.ftp.getfo(filename, flo) self.ftp.close() else: self.ftp.retrbinary('RETR %s' % (filename), flo.write) self.ftp.quit() info = flo.getvalue() flo.close() logging.info("CRON _import Security %s - File Read %s " % (self.security, '')) return info def get_file_delete(self, filename): self.ftp = self.get_connection() if self.security: self.ftp.chdir(self.path) res = self.ftp.exists(filename) if res: self.ftp.remove(filename) self.ftp.close() else: if filename in self.ftp.nlst(): self.ftp.delete(filename) self.ftp.quit() logging.info("CRON _import Security %s - File Delete %s " % (self.security, filename)) return True def get_file_move(self, filename): self.ftp = self.get_connection() if self.security: self.ftp.chdir(self.path_done) if self.ftp.exists(filename): self.ftp.remove(filename) self.ftp.chdir(self.path) self.ftp.rename(filename, self.path_done + '/' + filename) self.ftp.close() else: self.ftp.rename(filename, self.path_done + '/' + filename) self.ftp.quit() logging.info("CRON _import Security %s - File Delete %s " % (self.security, filename)) return True def get_file_move_noprocesados(self, filename): self.ftp = self.get_connection() if self.security: self.ftp.chdir(self.path_done) if self.ftp.exists(filename): self.ftp.remove(filename) self.ftp.chdir(self.path) self.ftp.rename(filename, self.path + '/noprocesados/' + filename) self.ftp.close() else: self.ftp.rename(filename, self.path + '/noprocesados/' + filename) self.ftp.quit() logging.info("CRON _import Security %s - File Delete %s " % (self.security, filename)) return True
class FTPStore(StoreBase): def __init__( self, name="__dummy__", limit="", auto_manage=False, ip="127.0.0.1", root="store", login="******", password="******", sftp=True, ): StoreBase.__init__(self, name, limit, auto_manage) if ip == "": raise Exception("Server address cannot be blank") if login == "" or password == "": raise Exception("Login name and password cannot be blank") if type(sftp) != bool: raise Exception("SFTP must be True or False") self.ip = ip self.root = root self.login = login self.password = password self.sftp = sftp self.folder_stack = [] for attr in ["ip", "root", "login", "password", "sftp"]: self._persistent.append(attr) def __str__(self): return "FTP: %s@%s/%s %s" % (self.login, self.ip, self.root, str(self.sftp)) def copy(self): log.trace("FTPStore Copy Constructor") return FTPStore( self.name, self.limit, self.auto_manage, self.ip, self.root, self.login, self.password, self.sftp ) ######################################################### # # FTP implementation of a streaming interface. # ######################################################### def open(self, path, mode): if not self.connected: self.connect() if "w" in mode: self.io_state = ioWriting # Make sure the dest folder exists dir, dummy = os.path.split(path) self.make_dir(dir) else: self.io_state = ioReading self.io_path = os.path.join(self.root, path) if self.sftp and use_paramiko: # We dont have FTP_TLS AND we need the encryption. We will be using paramiko self.io_fd = ParamikoFTPStreamer(self.ftp, self.io_path, self.io_state) else: self.io_fd = FTPStreamer(self.ftp, self.io_path, self.io_state) self.io_fd.start() # Now we wait for some data, or an error if self.io_state == ioReading: log.debug("In open - waiting for data or error") while self.io_fd.bufsize == 0 and not self.io_fd.error: time.sleep(0.01) if self.io_fd.error: log.debug("Got exception", str(self.io_fd.error)) raise self.io_fd.error log.debug("Got data") return self def seek(self, offset, whence): pass def tell(self): return 0 def close(self): self.io_fd.close() self.io_fd.join() self.io_state = ioClosed if self.io_fd.error: log.error("Raising exception from FTPWorker", self.io_fd.error) raise Exception(str(self.io_fd.error)) ################################################################################### # # Implementation Of The Store API # ################################################################################### def _connect(self): """ Creates a connected and authenticated FTP object. Raises an exception if that ftp object cannot be connected. """ try: self.ftp = None self.transport = None if self.sftp: if use_paramiko: log.debug( "Connect SFTP paramiko version=", paramiko.__version__, "id", self.login, "addr", self.ip, "python version", sys.version, ) # paramiko.common.logging.basicConfig(level=paramiko.common.DEBUG) # self.transport = paramiko.SSHClient() # log.debug("Have transport") # self.transport.connect(str(self.ip), # username=str(self.login), # password=str(self.password)) # log.debug("Connected") # self.ftp = self.transport.open_sftp() self.transport = paramiko.Transport((self.ip, 22)) log.debug("Have transport") self.transport.connect(username=self.login, password=self.password) log.debug("Connected") self.ftp = paramiko.SFTPClient.from_transport(self.transport) log.debug("Connected and ready to go") else: log.debug("Connect SFTP") self.ftp = FTP_TLS() self.ftp.connect(self.ip, timeout=const.FTPTimeout) self.ftp.login(self.login, self.password, secure=True) self.ftp.prot_p() else: log.debug("Connect FTP") self.ftp = FTP() self.ftp.connect(self.ip, timeout=const.FTPTimeout) self.ftp.login(self.login, self.password) except Exception as e: # We failed connection and/or login, so FORCE close log.debug("Exception during connect and login: "******"Failed to connect or log in: %s" % str(e)) def _disconnect(self): if self.ftp: self.ftp.close() self.ftp = None if self.transport: self.transport.close() self.transport = None def _send(self, src, dest): dest = utils.join_paths(self.root, dest) with open(src, "rb") as ifd: if self.sftp and use_paramiko: log.info("Starting sftp.paramiko put call src", src, "dest", dest) ofd = self.ftp.open(dest, "w+") try: data = ifd.read(const.BufferSize) while data: log.debug("FTP read %d bytes", len(data)) ofd.write(data) data = ifd.read(const.BufferSize) finally: ofd.close() else: self.ftp.storbinary("STOR " + dest, ifd) def _get(self, src, dest): src = utils.join_paths(self.root, src) with open(dest, "wb") as ofd: if self.sftp and use_paramiko: ifd = self.ftp.open(src, "r") try: data = ifd.read(const.BufferSize) while data: log.debug("FTP read %d bytes", len(data)) ofd.write(data) data = ifd.read(const.BufferSize) finally: ifd.close() else: self.ftp.retrbinary("RETR " + src, ofd.write) def _make_dir(self, folder): """ Create a folder on thel FTP service. If its relative, the folder is made relative to cwd. Otherwise its absolute. We do it by attempting to create every folder in the chain. """ folder = utils.join_paths(self.root, folder) self._pushd(".") try: # Get all the anscestor paths. paths = utils.ancestor_paths(folder) for path in paths: if not path in ["", ".", "..", "/"]: try: # If we can't chdir, then we try to build self._pushd(path) self._popd() except: # If we fail to build - we fail if self.sftp and use_paramiko: self.ftp.mkdir(path) else: self.ftp.mkd(path) finally: self._popd() def _remove_file(self, path): path = utils.join_paths(self.root, path) if self.sftp and use_paramiko: self.ftp.remove(path) else: self.ftp.delete(path) def _remove_dir(self, path): if path in ["", ".", "/"]: path = self.root else: path = utils.join_paths(self.root, path) self._recurse_delete(path) def _list(self, path="."): path = utils.join_paths(self.root, path) # Save the folder self._pushd(path) try: if self.sftp and use_paramiko: contents = self.ftp.listdir() else: contents = self.ftp.nlst() finally: self._popd() # Restore the working folder return contents def _size(self, path): path = utils.join_paths(self.root, path) if self.sftp and use_paramiko: stat = self.ftp.stat(path) size = stat.st_size else: self.ftp.sendcmd("TYPE i") size = self.ftp.size(path) return size ################################################################################### # # Internal Routines # ################################################################################### def _pushd(self, folder): if self.sftp and use_paramiko: wd = self.ftp.getcwd() else: wd = self.ftp.pwd() self.folder_stack.append(wd) if self.sftp and use_paramiko: self.ftp.chdir(folder) else: self.ftp.cwd(folder) def _popd(self): if len(self.folder_stack) == 0: raise Exception("Folder stack is empty in popd") wd = self.folder_stack.pop() if self.sftp and use_paramiko: self.ftp.chdir(wd) else: self.ftp.cwd(wd) def _recurse_delete(self, folder): """ Internal recursive delete. Requires a full path @param folder: @type folder: """ # Get a list of files self._pushd(folder) try: if self.sftp and use_paramiko: files = self.ftp.listdir() else: files = self.ftp.nlst() for d in files: try: if self.sftp and use_paramiko: self.ftp.remove(d) else: self.ftp.delete(d) except: # Probably a folder, so lets recurse # If the failure had another cause - the exception will bubble up. self._recurse_delete(d) # The folder should now be empty. # Exceptions will bubble this up. except Exception as e: log.debug("NList exception", str(e)) raise e finally: self._popd() # Attempt to delete the folder itself if self.sftp and use_paramiko: self.ftp.rmdir(folder) else: self.ftp.rmd(folder)
class PyFileTransfer(object): ''' def __init__(self, transfName): transfConf = ConfigUtils.read_trasnfer_config(transfName) #hostname self.__hostname = transfConf['host'] #username self.__username = transfConf['user'] #password self.__password = transfConf['password'] #protocol self.__typeProtocol = transfConf['type'] #so if transfConf['so'] == 'unix': self.__SEP = '/' elif transfConf['so'] == 'win': self.__SEP = chr(92) #port if 'port' in transfConf: self.__port = transfConf['port'] else: self.__port = None #open transfering if self.__typeProtocol == 'ftp': if self.__port is None: self.__port = 21 #open self.__t = FTP() self.__t.connect(self.__hostname, self.__port, self.__timeout) elif self.__typeProtocol == 'sftp': if self.__port is None: self.__port = 22 #open self.__ssh = paramiko.Transport((self.__hostname, self.__port)) def connection(self): if self.__typeProtocol == 'ftp': self.__t.login(self.__username, self.__password) #default directory self.__defaultDirectory = self.__t.pwd() elif self.__typeProtocol == 'sftp': self.__ssh.connect(username = self.__username, password = self.__password) self.__t = paramiko.SFTPClient.from_transport(self.__ssh) #default directory self.__defaultDirectory = None ''' def __init__(self, typeProtocol='ftp', hostname='localhost', so='unix', port=None, timeout=None): #Protocol self.__typeProtocol = typeProtocol #host self.__hostname = hostname #so if so == 'unix': self.__SEP = '/' elif so == 'win': self.__SEP = chr(92) #timeout self.__timeout = timeout #port if port: self.__port = port #open transfering if self.__typeProtocol == 'ftp': if not port: self.__port = 21 #open self.__t = FTP() self.__t.connect(self.__hostname, self.__port, self.__timeout) elif self.__typeProtocol == 'sftp': if not port: self.__port = 22 #open self.__ssh = paramiko.Transport((self.__hostname, self.__port)) def connection(self, username, password): if self.__typeProtocol == 'ftp': self.__t.login(username, password) #default directory self.__defaultDirectory = self.__t.pwd() elif self.__typeProtocol == 'sftp': self.__ssh.connect(username=username, password=password) self.__t = paramiko.SFTPClient.from_transport(self.__ssh) self.__t.sock.settimeout(self.__timeout) #default directory self.__defaultDirectory = None def get(self, filename, remoteDirectory=None, localDirectory=None): if localDirectory is None: localDirectory = os.path.dirname(os.path.realpath(__file__)) if self.__typeProtocol == 'ftp': pwdAux = self.__t.pwd() if remoteDirectory is not None: self.__t.cwd(remoteDirectory) remoteFile = open(os.path.join(localDirectory, filename), 'wb').write self.__t.retrbinary("RETR " + filename, remoteFile) self.__t.cwd(pwdAux) remoteFile.close() elif self.__typeProtocol == 'sftp': if remoteDirectory is not None: self.__t.chdir(remoteDirectory) self.__t.get(filename, os.path.join(localDirectory, filename)) self.__t.chdir(None) def put(self, filename, remoteDirectory=None, localDirectory=None): if localDirectory is None: localDirectory = os.path.dirname(os.path.realpath(__file__)) if self.__typeProtocol == 'ftp': pwdAux = self.__t.pwd() if remoteDirectory is not None: self.__t.cwd(remoteDirectory) localFile = open(filename, 'r') self.__t.storbinary('RETR %s' % filename, localFile.write) self.__t.cwd(pwdAux) localFile.close() elif self.__typeProtocol == 'sftp': if remoteDirectory is not None: self.__t.chdir(remoteDirectory) self.__t.put(os.path.join(localDirectory, filename), filename) self.__t.chdir(None) def disconnect(self): if self.__typeProtocol == 'ftp': self.__t.quit() elif self.__typeProtocol == 'sftp': self.__t.close() self.__ssh.close() def pwd(self): if self.__typeProtocol == 'ftp': return self.__t.pwd() elif self.__typeProtocol == 'sftp': return self.__t.getcwd() def cwd(self, remoteDirectory=None): if self.__typeProtocol == 'ftp': self.__t.cwd(remoteDirectory) elif self.__typeProtocol == 'sftp': self.__t.chdir(remoteDirectory) def setDefaultDirectory(self): if self.__typeProtocol == 'ftp': self.__t.cwd(self.__defaultDirectory) elif self.__typeProtocol == 'sftp': self.__t.chdir(None) def remotePathJoin(self, *paths): """Returns separate paths to string""" if len(paths) == 0: return None if len(paths) == 1: return paths[0] else: path = paths[0] for i in paths[1:]: path += self.__SEP + i return path
class PyFileTransfer(object): ''' def __init__(self, transfName): transfConf = ConfigUtils.read_trasnfer_config(transfName) #hostname self.__hostname = transfConf['host'] #username self.__username = transfConf['user'] #password self.__password = transfConf['password'] #protocol self.__typeProtocol = transfConf['type'] #so if transfConf['so'] == 'unix': self.__SEP = '/' elif transfConf['so'] == 'win': self.__SEP = chr(92) #port if 'port' in transfConf: self.__port = transfConf['port'] else: self.__port = None #open transfering if self.__typeProtocol == 'ftp': if self.__port is None: self.__port = 21 #open self.__t = FTP() self.__t.connect(self.__hostname, self.__port, self.__timeout) elif self.__typeProtocol == 'sftp': if self.__port is None: self.__port = 22 #open self.__ssh = paramiko.Transport((self.__hostname, self.__port)) def connection(self): if self.__typeProtocol == 'ftp': self.__t.login(self.__username, self.__password) #default directory self.__defaultDirectory = self.__t.pwd() elif self.__typeProtocol == 'sftp': self.__ssh.connect(username = self.__username, password = self.__password) self.__t = paramiko.SFTPClient.from_transport(self.__ssh) #default directory self.__defaultDirectory = None ''' def __init__(self, typeProtocol='ftp', hostname = 'localhost', so='unix', port = None, timeout = None): #Protocol self.__typeProtocol = typeProtocol #host self.__hostname = hostname #so if so == 'unix': self.__SEP = '/' elif so == 'win': self.__SEP = chr(92) #timeout self.__timeout = timeout #port if port: self.__port = port #open transfering if self.__typeProtocol == 'ftp': if not port: self.__port = 21 #open self.__t = FTP() self.__t.connect(self.__hostname, self.__port, self.__timeout) elif self.__typeProtocol == 'sftp': if not port: self.__port = 22 #open self.__ssh = paramiko.Transport((self.__hostname, self.__port)) def connection(self, username, password): if self.__typeProtocol == 'ftp': self.__t.login(username, password) #default directory self.__defaultDirectory = self.__t.pwd() elif self.__typeProtocol == 'sftp': self.__ssh.connect(username = username, password = password) self.__t = paramiko.SFTPClient.from_transport(self.__ssh) self.__t.sock.settimeout(self.__timeout) #default directory self.__defaultDirectory = None def get(self, filename, remoteDirectory=None, localDirectory=None): if localDirectory is None: localDirectory = os.path.dirname(os.path.realpath(__file__)) if self.__typeProtocol == 'ftp': pwdAux = self.__t.pwd() if remoteDirectory is not None: self.__t.cwd(remoteDirectory) remoteFile = open(os.path.join(localDirectory, filename), 'wb').write self.__t.retrbinary("RETR " + filename, remoteFile) self.__t.cwd(pwdAux) remoteFile.close() elif self.__typeProtocol == 'sftp': if remoteDirectory is not None: self.__t.chdir(remoteDirectory) self.__t.get(filename, os.path.join(localDirectory, filename)) self.__t.chdir(None) def put(self, filename, remoteDirectory=None, localDirectory=None): if localDirectory is None: localDirectory = os.path.dirname(os.path.realpath(__file__)) if self.__typeProtocol == 'ftp': pwdAux = self.__t.pwd() if remoteDirectory is not None: self.__t.cwd(remoteDirectory) localFile = open(filename, 'r') self.__t.storbinary('RETR %s' % filename, localFile.write) self.__t.cwd(pwdAux) localFile.close() elif self.__typeProtocol == 'sftp': if remoteDirectory is not None: self.__t.chdir(remoteDirectory) self.__t.put(os.path.join(localDirectory, filename), filename) self.__t.chdir(None) def disconnect(self): if self.__typeProtocol == 'ftp': self.__t.quit() elif self.__typeProtocol == 'sftp': self.__t.close() self.__ssh.close() def pwd(self): if self.__typeProtocol == 'ftp': return self.__t.pwd() elif self.__typeProtocol == 'sftp': return self.__t.getcwd() def cwd(self, remoteDirectory = None): if self.__typeProtocol == 'ftp': self.__t.cwd(remoteDirectory) elif self.__typeProtocol == 'sftp': self.__t.chdir(remoteDirectory) def setDefaultDirectory(self): if self.__typeProtocol == 'ftp': self.__t.cwd(self.__defaultDirectory) elif self.__typeProtocol == 'sftp': self.__t.chdir(None) def remotePathJoin (self, *paths): """Returns separate paths to string""" if len(paths)== 0: return None if len(paths)== 1: return paths[0] else: path = paths[0] for i in paths[1:]: path += self.__SEP + i return path
class FTPClient: def __init__(self, host, user, password, port=21, protocol='ftp'): assert protocol.strip().lower() in [ 'ftp', 'sftp' ], 'Invalid protocol, options are: SFTP, FTP' self.host = host self.user = user self.password = password self.port = port self.protocol = protocol.strip().lower() self._transport = None self._conn = None def __enter__(self): self._start_connection() return self def __exit__(self, exc_type, exc_val, exc_tb): self._close_connection() def __repr__(self): return f'<FTPClient({self.host}, {self.user}, xxxx, port = {self.port}, protocol = {self.protocol})>' def _start_connection(self): func = { 'ftp': self._start_ftp_connection, 'sftp': self._start_sftp_connection }[self.protocol] try: func() except: raise InvalidConnection( f'Error connecting to {self.protocol.upper()} server') def _close_connection(self): if self._transport is not None: self._transport.close() if self._conn is not None: self._conn.close() def _start_ftp_connection(self): self._conn = FTP() self._conn.connect(self.host, self.port) self._conn.login(self.user, self.password) def _start_sftp_connection(self): self._transport = paramiko.Transport((self.host, self.port)) self._transport.connect(None, self.user, self.password) self._conn = paramiko.SFTPClient.from_transport(self._transport) def listdir(self, path): return self._conn.listdir( path) if self.protocol == 'sftp' else self._conn.nlst(path) def chdir(self, path): self._conn.chdir(path) if self.protocol == 'sftp' else self._conn.cwd( path) return True def mkdir(self, path): self._conn.mkdir(path) if self.protocol == 'sftp' else self._conn.mkd( path) return True def rmdir(self, path): self._conn.rmdir(path) if self.protocol == 'sftp' else self._conn.rmd( path) return True def pwd(self): return self._conn.getcwd( ) if self.protocol == 'sftp' else self._conn.pwd() def rename(self, oldpath, newpath): self._conn.rename(oldpath, newpath) return True def delete(self, path): self._conn.remove( path) if self.protocol == 'sftp' else self._conn.delete(path) return True def get(self, remotepath, localpath, callback=None): if callback is not None: assert callable(callback), "Callback must to be a function" path = os.path.split(localpath)[0] if not os.path.exists(path): os.mkdir(path) if self.protocol == 'sftp': self._conn.get(remotepath, localpath, callback=callback) else: with open(localpath, 'wb') as f: self._conn.retrbinary(f'RETR {remotepath}', f.write) if callback is not None: callback() def put(self, localpath, remotepath, callback=None): if callback is not None: assert callable(callback), "Callback must to be a function" if not os.path.exists(localpath): raise Exception(f'{localpath} does not exists') if self.protocol == 'sftp': self._conn.put(localpath, remotepath, callback=callback) else: with open(localpath, 'rb') as f: self._conn.storbinary(f'STOR {remotepath}', f) if callback is not None: callback()