def execute(self, context): s3_hook = S3Hook(self.aws_conn_id) ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket) with NamedTemporaryFile() as local_tmp_file: s3_obj.download_fileobj(local_tmp_file) ftp_hook.store_file(self.ftp_path, local_tmp_file.name)
def execute(self, context: 'Context'): s3_hook = S3Hook(self.aws_conn_id) ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) s3_obj = s3_hook.get_key(self.s3_key, self.s3_bucket) with NamedTemporaryFile() as local_tmp_file: self.log.info('Downloading file from %s', self.s3_key) s3_obj.download_fileobj(local_tmp_file) local_tmp_file.seek(0) ftp_hook.store_file(self.ftp_path, local_tmp_file.name) self.log.info('File stored in %s', {self.ftp_path})
def execute(self, context): s3_hook = S3Hook(self.aws_conn_id) ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) with NamedTemporaryFile() as local_tmp_file: ftp_hook.retrieve_file( remote_full_path=self.ftp_path, local_full_path_or_buffer=local_tmp_file.name) s3_hook.load_file( filename=local_tmp_file.name, key=self.s3_key, bucket_name=self.s3_bucket, replace=self.replace, encrypt=self.encrypt, gzip=self.gzip, acl_policy=self.acl_policy, )
def execute(self, context: 'Context'): self.ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) self.s3_hook = S3Hook(self.aws_conn_id) if self.ftp_filenames: if isinstance(self.ftp_filenames, str): self.log.info(f'Getting files in {self.ftp_path}') list_dir = self.ftp_hook.list_directory(path=self.ftp_path, ) if self.ftp_filenames == '*': files = list_dir else: ftp_filename: str = self.ftp_filenames files = list(filter(lambda f: ftp_filename in f, list_dir)) for file in files: self.log.info(f'Moving file {file}') if self.s3_filenames and isinstance( self.s3_filenames, str): filename = file.replace(self.ftp_filenames, self.s3_filenames) else: filename = file s3_file_key = f'{self.s3_key}{filename}' self.__upload_to_s3_from_ftp(file, s3_file_key) else: if self.s3_filenames: for ftp_file, s3_file in zip(self.ftp_filenames, self.s3_filenames): self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + s3_file) else: for ftp_file in self.ftp_filenames: self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + ftp_file) else: self.__upload_to_s3_from_ftp(self.ftp_path, self.s3_key)
class FTPToS3Operator(BaseOperator): """ This operator enables the transfer of files from a FTP server to S3. It can be used to transfer one or multiple files. :param ftp_path: The ftp remote path. For one file it is mandatory to include the file as well. For multiple files, it is the route where the files will be found. :param s3_bucket: The targeted s3 bucket in which to upload the file(s). :param s3_key: The targeted s3 key. For one file it must include the file path. For several, it must end with "/". :param ftp_filenames: Only used if you want to move multiple files. You can pass a list with exact filenames present in the ftp path, or a prefix that all files must meet. It can also be the string '*' for moving all the files within the ftp path. :param s3_filenames: Only used if you want to move multiple files and name them different from the originals from the ftp. It can be a list of filenames or file prefix (that will replace the ftp prefix). :param ftp_conn_id: The ftp connection id. The name or identifier for establishing a connection to the FTP server. :param aws_conn_id: The s3 connection id. The name or identifier for establishing a connection to S3. :param replace: A flag to decide whether or not to overwrite the key if it already exists. If replace is False and the key exists, an error will be raised. :param encrypt: If True, the file will be encrypted on the server-side by S3 and will be stored in an encrypted form while at rest in S3. :param gzip: If True, the file will be compressed locally :param acl_policy: String specifying the canned ACL policy for the file being uploaded to the S3 bucket. """ template_fields: Sequence[str] = ('ftp_path', 's3_bucket', 's3_key', 'ftp_filenames', 's3_filenames') def __init__( self, *, ftp_path: str, s3_bucket: str, s3_key: str, ftp_filenames: Optional[Union[str, List[str]]] = None, s3_filenames: Optional[Union[str, List[str]]] = None, ftp_conn_id: str = 'ftp_default', aws_conn_id: str = 'aws_default', replace: bool = False, encrypt: bool = False, gzip: bool = False, acl_policy: Optional[str] = None, **kwargs, ): super().__init__(**kwargs) self.ftp_path = ftp_path self.s3_bucket = s3_bucket self.s3_key = s3_key self.ftp_filenames = ftp_filenames self.s3_filenames = s3_filenames self.aws_conn_id = aws_conn_id self.ftp_conn_id = ftp_conn_id self.replace = replace self.encrypt = encrypt self.gzip = gzip self.acl_policy = acl_policy self.s3_hook: Optional[S3Hook] = None self.ftp_hook: Optional[FTPHook] = None def __upload_to_s3_from_ftp(self, remote_filename, s3_file_key): with NamedTemporaryFile() as local_tmp_file: self.ftp_hook.retrieve_file( remote_full_path=remote_filename, local_full_path_or_buffer=local_tmp_file.name ) self.s3_hook.load_file( filename=local_tmp_file.name, key=s3_file_key, bucket_name=self.s3_bucket, replace=self.replace, encrypt=self.encrypt, gzip=self.gzip, acl_policy=self.acl_policy, ) self.log.info(f'File upload to {s3_file_key}') def execute(self, context: 'Context'): self.ftp_hook = FTPHook(ftp_conn_id=self.ftp_conn_id) self.s3_hook = S3Hook(self.aws_conn_id) if self.ftp_filenames: if isinstance(self.ftp_filenames, str): self.log.info(f'Getting files in {self.ftp_path}') list_dir = self.ftp_hook.list_directory( path=self.ftp_path, ) if self.ftp_filenames == '*': files = list_dir else: ftp_filename: str = self.ftp_filenames files = list(filter(lambda f: ftp_filename in f, list_dir)) for file in files: self.log.info(f'Moving file {file}') if self.s3_filenames and isinstance(self.s3_filenames, str): filename = file.replace(self.ftp_filenames, self.s3_filenames) else: filename = file s3_file_key = f'{self.s3_key}{filename}' self.__upload_to_s3_from_ftp(file, s3_file_key) else: if self.s3_filenames: for ftp_file, s3_file in zip(self.ftp_filenames, self.s3_filenames): self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + s3_file) else: for ftp_file in self.ftp_filenames: self.__upload_to_s3_from_ftp(self.ftp_path + ftp_file, self.s3_key + ftp_file) else: self.__upload_to_s3_from_ftp(self.ftp_path, self.s3_key)
def _create_hook(self) -> FTPHook: """Return connection hook.""" return FTPHook(ftp_conn_id=self.ftp_conn_id)