def get_ftp_files(**kwargs):
conn = FTPHook('accela_ftp')
d = conn.describe_directory('/opentext/TEST')
for t in tables:
r = get_recent_file(t, d)
conn.retrieve_file(r['name'], f"/tmp/{t}.csv")
class FTPFSHook(FileSystemHook):
conn_type = 'ftp_filesystem'
conn_type_long = 'FTP FileSystem'
def __init__(self, conn_params: Connection):
from airflow.contrib.hooks.ftp_hook import FTPHook
self.conn_id = conn_params.conn_id
self.af_ftp_hook = FTPHook(ftp_conn_id=self.conn_id)
self.base_path = Path(conn_params.extra_dejson('base_path', '/'))
def list_path(self, path: str, recursive: bool = False) -> List[str]:
if recursive:
raise NotImplementedError('Recursive list not implemented for FTP')
else:
return self.af_ftp_hook.list_directory(str(self.base_path / path))
def write_data(self, path: str, data: Union[str, bytes, BytesIO]):
if isinstance(data, str):
data = data.encode()
if isinstance(data, bytes):
data = BytesIO(data)
self.af_ftp_hook.store_file(str(self.base_path / path), data)
def read_data(self, path: str) -> BytesIO:
result = BytesIO()
self.af_ftp_hook.retrieve_file(str(self.base_path / path), result)
return result