def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[
self._host, self._user, self._src_dir, self._src_pattern,
self._dest_dir
],
)
valid()
if isinstance(self._key, str):
self._logger.warning(
("DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types."))
key_filepath = self._key
else:
key_filepath = self._source_path_reader(self._key)
sftp = Sftp(
self._host,
self._user,
self._password,
key_filepath,
self._passphrase,
self._timeout,
self._retry_count,
self._port,
)
files = super().get_target_files(self._src_dir, self._src_pattern)
for file in files:
sftp.put_file(file,
os.path.join(self._dest_dir, os.path.basename(file)))
def execute(self, *args):
for k, v in self.__dict__.items():
self._logger.info("%s : %s" % (k, v))
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[
self._host, self._user, self._src_dir, self._src_pattern,
self._dest_dir
],
)
valid()
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
files = super().get_target_files(self._src_dir, self._src_pattern)
if len(files) > 0:
for file in files:
sftp.put_file(
file, os.path.join(self._dest_dir, os.path.basename(file)))
else:
self._logger.info(
"Files to upload do not exist. File pattern: {}".format(
os.path.join(self._src_dir, self._src_pattern)))
return 0
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
if isinstance(self._key, str):
self._logger.warning((
"DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types. "
"Please see more information "
"https://github.com/BrainPad/cliboa/blob/master/docs/modules/sftp_delete.md"
))
key_filepath = self._key
else:
key_filepath = self._source_path_reader(self._key)
# remove src
sftp = Sftp(
self._host,
self._user,
self._password,
key_filepath,
self._passphrase,
self._timeout,
self._retry_count,
self._port,
)
sftp.clear_files(self._src_dir, re.compile(self._src_pattern))
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
os.makedirs(self._dest_dir, exist_ok=True)
# fetch src
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
files = sftp.list_files(
self._src_dir, self._dest_dir, re.compile(self._src_pattern)
)
if self._quit is True and len(files) == 0:
self._logger.info("No file was found. After process will not be processed")
return StepStatus.SUCCESSFUL_TERMINATION
self._logger.info("Files downloaded %s" % files)
# cache downloaded file names
ObjectStore.put(self._step, files)
def execute(self, *args):
for k, v in self.__dict__.items():
self._logger.info("%s : %s" % (k, v))
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
os.makedirs(self._dest_dir, exist_ok=True)
# fetch src
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
files = sftp.list_files(self._src_dir, self._dest_dir,
re.compile(self._src_pattern))
if self.__quit is True and len(files) == 0:
self._logger.info(
"No file was found. After process will not be processed")
return 0
# cache downloaded file names
ObjectStore.put(self._step, files)
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[
self._host, self._user, self._src_dir, self._src_pattern,
self._dest_dir
],
)
valid()
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
files = super().get_target_files(self._src_dir, self._src_pattern)
for file in files:
sftp.put_file(file,
os.path.join(self._dest_dir, os.path.basename(file)))
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[
self._host, self._user, self._src_dir, self._src_pattern,
self._dest_dir
],
)
valid()
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
files = super().get_target_files(self._src_dir, self._src_pattern)
if len(files) > 0:
for file in files:
sftp.put_file(
file, os.path.join(self._dest_dir, os.path.basename(file)))
self._logger.info("%s is successfully uploaded." % file)
else:
self._logger.info(
"Files to upload do not exist. File pattern: {}".format(
os.path.join(self._src_dir, self._src_pattern)))
if self._quit is True:
return StepStatus.SUCCESSFUL_TERMINATION
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[
self._host, self._user, self._src_dir, self._src_pattern,
self._dest_dir
],
)
valid()
if isinstance(self._key, str):
self._logger.warning((
"DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types. "
"Please see more information "
"https://github.com/BrainPad/cliboa/blob/master/docs/modules/sftp_upload.md"
))
key_filepath = self._key
else:
key_filepath = self._source_path_reader(self._key)
sftp = Sftp(
self._host,
self._user,
self._password,
key_filepath,
self._passphrase,
self._timeout,
self._retry_count,
self._port,
)
files = super().get_target_files(self._src_dir, self._src_pattern)
if len(files) > 0:
for file in files:
if self._ignore_empty_file and os.path.getsize(file) == 0:
self._logger.info("0 byte file will no be uploaded %s." %
file)
continue
sftp.put_file(
file,
os.path.join(self._dest_dir, os.path.basename(file)),
self._endfile_suffix,
)
self._logger.info("%s is successfully uploaded." % file)
else:
self._logger.info(
"Files to upload do not exist. File pattern: {}".format(
os.path.join(self._src_dir, self._src_pattern)))
if self._quit is True:
return StepStatus.SUCCESSFUL_TERMINATION
def test_put_file(self):
if sys.version_info.minor < 6:
# ignore test if python version is less 3.6(assert_called is not supported)
return
with ExitStack() as stack:
mock_func = stack.enter_context(
patch("cliboa.util.sftp.put_file_func"))
stack.enter_context(patch("paramiko.SSHClient.connect"))
stack.enter_context(patch("paramiko.SSHClient.open_sftp"))
sftp = Sftp(host="test", user="******", password="******")
sftp.put_file(None, None)
mock_func.assert_called_once()
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
os.makedirs(self._dest_dir, exist_ok=True)
if isinstance(self._key, str):
self._logger.warning((
"DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types. "
"Please see more information "
"https://github.com/BrainPad/cliboa/blob/master/docs/modules/sftp_download.md"
))
key_filepath = self._key
else:
key_filepath = self._source_path_reader(self._key)
# fetch src
sftp = Sftp(
self._host,
self._user,
self._password,
key_filepath,
self._passphrase,
self._timeout,
self._retry_count,
self._port,
)
files = sftp.list_files(
self._src_dir,
self._dest_dir,
re.compile(self._src_pattern),
self._endfile_suffix,
self._ignore_empty_file,
)
if self._quit is True and len(files) == 0:
self._logger.info(
"No file was found. After process will not be processed")
return StepStatus.SUCCESSFUL_TERMINATION
self._logger.info("Files downloaded %s" % files)
# cache downloaded file names
ObjectStore.put(self._step, files)
def execute(self, *args):
files = ObjectStore.get(self._symbol)
if files is not None and len(files) > 0:
self._logger.info("Delete files %s" % files)
sftp = Sftp(
super().get_step_argument("host"),
super().get_step_argument("user"),
super().get_step_argument("password"),
super().get_step_argument("key"),
super().get_step_argument("timeout"),
super().get_step_argument("retry_count"),
super().get_step_argument("port"),
)
for file in files:
sftp.remove_specific_file(super().get_step_argument("src_dir"), file)
self._logger.info("%s is successfully deleted." % file)
else:
self._logger.info("No files to delete.")
def execute(self, *args):
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
# remove src
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
sftp.clear_files(self._src_dir, re.compile(self._src_pattern))
def execute(self, *args):
for k, v in self.__dict__.items():
self._logger.info("%s : %s" % (k, v))
# essential parameters check
valid = EssentialParameters(
self.__class__.__name__,
[self._host, self._user, self._src_dir, self._src_pattern],
)
valid()
# remove src
sftp = Sftp(
self._host,
self._user,
self._password,
self._key,
self._timeout,
self._retry_count,
self._port,
)
sftp.clear_files(self._src_dir, re.compile(self._src_pattern))
def execute(self, *args):
files = ObjectStore.get(self._symbol)
if files is not None and len(files) > 0:
self._logger.info("Delete files %s" % files)
if isinstance(super().get_step_argument("key"), str):
self._logger.warning((
"DeprecationWarning: "
"In the near future, "
"the `key` will be changed to accept only dictionary types. "
"Please see more information "
"https://github.com/BrainPad/cliboa/blob/master/docs/modules/sftp_download_file_delete.md" # noqa
))
key_filepath = super().get_step_argument("key")
else:
key_filepath = self._source_path_reader(
super().get_step_argument("key"))
sftp = Sftp(
super().get_step_argument("host"),
super().get_step_argument("user"),
super().get_step_argument("password"),
key_filepath,
super().get_step_argument("timeout"),
super().get_step_argument("retry_count"),
super().get_step_argument("port"),
)
endfile_suffix = super().get_step_argument("endfile_suffix")
for file in files:
sftp.remove_specific_file(super().get_step_argument("src_dir"),
file)
self._logger.info("%s is successfully deleted." % file)
if endfile_suffix:
sftp.remove_specific_file(
super().get_step_argument("src_dir"),
file + endfile_suffix)
self._logger.info("%s is successfully deleted." %
(file + endfile_suffix))
else:
self._logger.info("No files to delete.")