def execute(self, context):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket)
with NamedTemporaryFile() as local_tmp_file:
s3_obj.download_fileobj(local_tmp_file)
ftp_hook.store_file(self.ftp_path, local_tmp_file.name)
def execute(self, context: 'Context'):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket)
with NamedTemporaryFile() as local_tmp_file:
self.log.info('Downloading file from %s', self.s3_key)
s3_obj.download_fileobj(local_tmp_file)
local_tmp_file.seek(0)
ftp_hook.store_file(self.ftp_path, local_tmp_file.name)
self.log.info('File stored in %s', {self.ftp_path})
def execute(self, context):
s3_hook = S3Hook(self.aws_conn_id)
ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
with NamedTemporaryFile() as local_tmp_file:
ftp_hook.retrieve_file(
remote_full_path=self.ftp_path,
local_full_path_or_buffer=local_tmp_file.name)
s3_hook.load_file(
filename=local_tmp_file.name,
key=self.s3_key,
bucket_name=self.s3_bucket,
replace=self.replace,
encrypt=self.encrypt,
gzip=self.gzip,
acl_policy=self.acl_policy,
)
def execute(self, context: 'Context'):
self.ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id)
self.s3_hook = S3Hook(self.aws_conn_id)
if self.ftp_filenames:
if isinstance(self.ftp_filenames, str):
self.log.info(f'Getting files in {self.ftp_path}')
list_dir = self.ftp_hook.list_directory(path=self.ftp_path, )
if self.ftp_filenames == '*':
files = list_dir
else:
ftp_filename: str = self.ftp_filenames
files = list(filter(lambda f: ftp_filename in f, list_dir))
for file in files:
self.log.info(f'Moving file {file}')
if self.s3_filenames and isinstance(
self.s3_filenames, str):
filename = file.replace(self.ftp_filenames,
self.s3_filenames)
else:
filename = file
s3_file_key = f'{self.s3_key}{filename}'
self.__upload_to_s3_from_ftp(file, s3_file_key)
else:
if self.s3_filenames:
for ftp_file, s3_file in zip(self.ftp_filenames,
self.s3_filenames):
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file,
self.s3_key + s3_file)
else:
for ftp_file in self.ftp_filenames:
self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file,
self.s3_key + ftp_file)
else:
self.__upload_to_s3_from_ftp(self.ftp_path, self.s3_key)
def _create_hook(self) -> FTPHook:
"""Return connection hook."""
return FTPHook(ftp_conn_id=self.ftp_conn_id)