def execute(self, context):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket)
with NamedTemporaryFile() as local_tmp_file:
s3_obj.download_fileobj(local_tmp_file)
ftp_hook.store_file(self.ftp_path, local_tmp_file.name)
def execute(self, context: 'Context'):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket)
with NamedTemporaryFile() as local_tmp_file:
self.log.info('Downloading file from %s', self.s3_key)
s3_obj.download_fileobj(local_tmp_file)
local_tmp_file.seek(0)
ftp_hook.store_file(self.ftp_path, local_tmp_file.name)
self.log.info('File stored in %s', {self.ftp_path})
def execute(self, context):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
with NamedTemporaryFile() as local_tmp_file:
ftp_hook.retrieve_file(
remote_full_path=self.ftp_path,
local_full_path_or_buffer=local_tmp_file.name)
s3_hook.load_file(
filename=local_tmp_file.name,
key=self.s3_key,
bucket_name=self.s3_bucket,
replace=self.replace,
encrypt=self.encrypt,
gzip=self.gzip,
acl_policy=self.acl_policy,
)
def execute(self, context: 'Context'):
self.ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
self.s3_hook = S3Hook(self.aws_conn_id)
if self.ftp_filenames:
if isinstance(self.ftp_filenames, str):
self.log.info(f'Getting files in {self.ftp_path}')
list_dir = self.ftp_hook.list_directory(path=self.ftp_path, )
if self.ftp_filenames == '*':
files = list_dir
else:
ftp_filename: str = self.ftp_filenames
files = list(filter(lambda f: ftp_filename in f, list_dir))
for file in files:
self.log.info(f'Moving file {file}')
if self.s3_filenames and isinstance(
self.s3_filenames, str):
filename = file.replace(self.ftp_filenames,
self.s3_filenames)
else:
filename = file
s3_file_key = f'{self.s3_key}{filename}'
self.__upload_to_s3_from_ftp(file, s3_file_key)
else:
if self.s3_filenames:
for ftp_file, s3_file in zip(self.ftp_filenames,
self.s3_filenames):
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file,
self.s3_key + s3_file)
else:
for ftp_file in self.ftp_filenames:
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file,
self.s3_key + ftp_file)
else:
self.__upload_to_s3_from_ftp(self.ftp_path, self.s3_key)
class FTPToS3Operator(BaseOperator):
"""
This operator enables the transfer of files from a FTP server to S3. It can be used to
transfer one or multiple files.
:param ftp_path: The ftp remote path. For one file it is mandatory to include the file as well.
For multiple files, it is the route where the files will be found.
:param s3_bucket: The targeted s3 bucket in which to upload the file(s).
:param s3_key: The targeted s3 key. For one file it must include the file path. For several,
it must end with "/".
:param ftp_filenames: Only used if you want to move multiple files. You can pass a list
with exact filenames present in the ftp path, or a prefix that all files must meet. It
can also be the string '*' for moving all the files within the ftp path.
:param s3_filenames: Only used if you want to move multiple files and name them different from
the originals from the ftp. It can be a list of filenames or file prefix (that will replace
the ftp prefix).
:param ftp_conn_id: The ftp connection id. The name or identifier for
establishing a connection to the FTP server.
:param aws_conn_id: The s3 connection id. The name or identifier for
establishing a connection to S3.
:param replace: A flag to decide whether or not to overwrite the key
if it already exists. If replace is False and the key exists, an
error will be raised.
:param encrypt: If True, the file will be encrypted on the server-side
by S3 and will be stored in an encrypted form while at rest in S3.
:param gzip: If True, the file will be compressed locally
:param acl_policy: String specifying the canned ACL policy for the file being
uploaded to the S3 bucket.
"""
template_fields: Sequence[str] = ('ftp_path', 's3_bucket', 's3_key', 'ftp_filenames', 's3_filenames')
def __init__(
self,
*,
ftp_path: str,
s3_bucket: str,
s3_key: str,
ftp_filenames: Optional[Union[str, List[str]]] = None,
s3_filenames: Optional[Union[str, List[str]]] = None,
ftp_conn_id: str = 'ftp_default',
aws_conn_id: str = 'aws_default',
replace: bool = False,
encrypt: bool = False,
gzip: bool = False,
acl_policy: Optional[str] = None,
**kwargs,
):
super().__init__(**kwargs)
self.ftp_path = ftp_path
self.s3_bucket = s3_bucket
self.s3_key = s3_key
self.ftp_filenames = ftp_filenames
self.s3_filenames = s3_filenames
self.aws_conn_id = aws_conn_id
self.ftp_conn_id = ftp_conn_id
self.replace = replace
self.encrypt = encrypt
self.gzip = gzip
self.acl_policy = acl_policy
self.s3_hook: Optional[S3Hook] = None
self.ftp_hook: Optional[FTPHook] = None
def __upload_to_s3_from_ftp(self, remote_filename, s3_file_key):
with NamedTemporaryFile() as local_tmp_file:
self.ftp_hook.retrieve_file(
remote_full_path=remote_filename, local_full_path_or_buffer=local_tmp_file.name
)
self.s3_hook.load_file(
filename=local_tmp_file.name,
key=s3_file_key,
bucket_name=self.s3_bucket,
replace=self.replace,
encrypt=self.encrypt,
gzip=self.gzip,
acl_policy=self.acl_policy,
)
self.log.info(f'File upload to {s3_file_key}')
def execute(self, context: 'Context'):
self.ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
self.s3_hook = S3Hook(self.aws_conn_id)
if self.ftp_filenames:
if isinstance(self.ftp_filenames, str):
self.log.info(f'Getting files in {self.ftp_path}')
list_dir = self.ftp_hook.list_directory(
path=self.ftp_path,
)
if self.ftp_filenames == '*':
files = list_dir
else:
ftp_filename: str = self.ftp_filenames
files = list(filter(lambda f: ftp_filename in f, list_dir))
for file in files:
self.log.info(f'Moving file {file}')
if self.s3_filenames and isinstance(self.s3_filenames, str):
filename = file.replace(self.ftp_filenames, self.s3_filenames)
else:
filename = file
s3_file_key = f'{self.s3_key}{filename}'
self.__upload_to_s3_from_ftp(file, s3_file_key)
else:
if self.s3_filenames:
for ftp_file, s3_file in zip(self.ftp_filenames, self.s3_filenames):
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + s3_file)
else:
for ftp_file in self.ftp_filenames:
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + ftp_file)
else:
self.__upload_to_s3_from_ftp(self.ftp_path, self.s3_key)
def _create_hook(self) -> FTPHook:
"""Return connection hook."""
return FTPHook(ftp_conn_id=self.ftp_conn_id)
