def test_execute_with_key(self):
try:
os.makedirs(self._data_dir)
dummy_pass = os.path.join(self._data_dir, "id_rsa")
with open(dummy_pass, "w") as f:
f.write("test")
instance = SftpDownload()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(__name__))
Helper.set_property(instance, "host", "dummy.host")
Helper.set_property(instance, "user", "dummy_user")
Helper.set_property(instance, "key", dummy_pass)
Helper.set_property(instance, "src_dir", "/")
Helper.set_property(instance, "src_pattern", ".*.txt")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "step", "sftp_class")
with ExitStack() as stack:
mock_sftp = stack.enter_context(
patch("cliboa.util.sftp.Sftp.list_files"))
mock_sftp.return_value = ["test.txt"]
instance.execute()
assert mock_sftp.called
assert ObjectStore.get("sftp_class") == ["test.txt"]
finally:
shutil.rmtree(self._data_dir)
def execute(self, *args):
dl_files = ObjectStore.get(self._symbol)
if len(dl_files) > 0:
self._logger.info("Delete files %s" % dl_files)
client = self._gcs_client()
bucket = client.get_bucket(super().get_step_argument("bucket"))
for blob in bucket.list_blobs(
prefix=super().get_step_argument("prefix"),
delimiter=super().get_step_argument("delimiter"),
):
for dl_f in dl_files:
if dl_f == blob.name:
blob.delete()
break
else:
self._logger.info("No files to delete.")
def execute(self, *args):
files = ObjectStore.get(self._symbol)
if files is not None and len(files) > 0:
self._logger.info("Delete files %s" % files)
sftp = Sftp(
super().get_step_argument("host"),
super().get_step_argument("user"),
super().get_step_argument("password"),
super().get_step_argument("key"),
super().get_step_argument("timeout"),
super().get_step_argument("retry_count"),
super().get_step_argument("port"),
)
for file in files:
sftp.remove_specific_file(super().get_step_argument("src_dir"), file)
self._logger.info("%s is successfully deleted." % file)
else:
self._logger.info("No files to delete.")
def execute(self, *args):
files = ObjectStore.get(self._symbol)
if files is not None and len(files) > 0:
self._logger.info("Delete files %s" % files)
if isinstance(super().get_step_argument("key"), str):
self._logger.warning((
"DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types. "
"Please see more information "
"https://github.com/BrainPad/cliboa/blob/master/docs/modules/sftp_download_file_delete.md" # noqa
))
key_filepath = super().get_step_argument("key")
else:
key_filepath = self._source_path_reader(
super().get_step_argument("key"))
sftp = Sftp(
super().get_step_argument("host"),
super().get_step_argument("user"),
super().get_step_argument("password"),
key_filepath,
super().get_step_argument("timeout"),
super().get_step_argument("retry_count"),
super().get_step_argument("port"),
)
endfile_suffix = super().get_step_argument("endfile_suffix")
for file in files:
sftp.remove_specific_file(super().get_step_argument("src_dir"),
file)
self._logger.info("%s is successfully deleted." % file)
if endfile_suffix:
sftp.remove_specific_file(
super().get_step_argument("src_dir"),
file + endfile_suffix)
self._logger.info("%s is successfully deleted." %
(file + endfile_suffix))
else:
self._logger.info("No files to delete.")
def execute(self, *args):
files = ObjectStore.get(self._symbol)
if files is not None and len(files) > 0:
self._logger.info("Delete files %s" % files)
ftp_util = FtpUtil(
super().get_step_argument("host"),
super().get_step_argument("user"),
super().get_step_argument("password"),
super().get_step_argument("timeout"),
super().get_step_argument("retry_count"),
super().get_step_argument("port"),
super().get_step_argument("tls"),
)
for file in files:
ftp_util.remove_specific_file(
super().get_step_argument("src_dir"), file)
else:
self._logger.info("No files to delete.")
def test_execute_with_files(self):
instance = SftpDownload()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "host", "dummy.host")
Helper.set_property(instance, "user", "dummy_user")
Helper.set_property(instance, "password", "dummy_pass")
Helper.set_property(instance, "src_dir", "/")
Helper.set_property(instance, "src_pattern", ".*.txt")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "step", "sftp_class")
with ExitStack() as stack:
mock_sftp = stack.enter_context(
patch("cliboa.util.sftp.Sftp.list_files"))
mock_sftp.return_value = ["test.txt"]
instance.execute()
assert ObjectStore.get("sftp_class") == ["test.txt"]
shutil.rmtree(self._data_dir)
def execute(self, *args):
for k, v in self.__dict__.items():
self._logger.info("%s : %s" % (k, v))
dl_files = ObjectStore.get(self._symbol)
if len(dl_files) > 0:
self._logger.info("Delete files %s" % dl_files)
c = storage.Client(
super().get_step_argument("project_id"),
credentials=ServiceAccount.auth(
super().get_step_argument("credentials")),
)
bucket = c.get_bucket(super().get_step_argument("bucket"))
for blob in bucket.list_blobs(
prefix=super().get_step_argument("prefix"),
delimiter=super().get_step_argument("delimiter"),
):
for dl_f in dl_files:
if dl_f == blob.name:
blob.delete()
break
else:
self._logger.info("No files to delete.")