def notify(notifier: Notifier):
"""
This module allows to send a notification based on the custom implementation of the
``Notifier`` abstract class. A developer needs to provide implementation for the notifier.
The notifier dependency will be injected by the caller as per the use case. This behaviour
allows to plug in either existing notifier or custom notifiers.
This method also captures general audit log.
Parameters
----------
notifier : Notifier
an instance of custom implementation of ``Notifier``
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_NOTIFICATION,
Sc.STATUS_PROCESSING,
'Sending notification')
notifier.notify()
audit_params(Sc.OPERATION_NOTIFICATION,
Sc.STATUS_COMPLETE,
'Notification sent' + time_taken(_start))
def download(downloader: Downloader) -> list:
"""
The caller should inject the appropriate implementation to perform
download operation as per use case
Parameters
----------
downloader: Downloader
instance of custom downloader implementation which has overridden
download() function
Returns
----------
list
list of downloaded files ordered by last modified time,
earlier will be the first element
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_PROCESSING, 'Files are downloading from source to destination')
files = downloader.download()
audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_COMPLETE, 'Files are downloaded to destination' + time_taken(_start))
return files
def upload(uploader: Uploader) -> list:
"""
The caller should inject the appropriate implementation to perform
upload operation as per use case
Parameters
----------
uploader: Uploader
instance of custom uploader implementation which has overridden
upload() function
Returns
--------
list
a list of uploaded files
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_PROCESSING, 'Files are uploading from source to destination')
files = uploader.upload()
audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_COMPLETE, 'Files are uploaded to destination' + time_taken(_start))
return files
def upload(self) -> list:
"""
Connects to SFTP host and transfers files from a remote source to destination directory
Returns
-------
list
a list of files transferred
"""
_sftp = None
try:
if self._sftpSecret.find(os.sep) == -1:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
password=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
else:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
private_key=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
_sftp.chdir(self._srcDir)
for file in self._files:
_src = os.path.join(self._srcDir, os.path.basename(file))
_dest = os.path.join(self._destDir, os.path.basename(file))
_start = current_time_in_millis()
file_comments = '({}) ==> ({}) transfer'.format(_src, _dest)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
_sftp.rename(_src, _dest)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
finally:
if _sftp is not None:
_sftp.close()
return self._files
def download(self) -> list:
"""
Connects to SFTP host using configurations
and scans for files as per the extensions mentioned in the configurations.
If there are any matching files, they will be downloaded to destination
with total seconds taken to download files.
Returns
--------
list
a list of downloaded files ordered by modified time in ascending.
Earliest will be the first.
"""
_sftp = None
try:
if self._sftpSecret.find(os.sep) == -1:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
password=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
else:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
private_key=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
_sftp.chdir(self._srcDir)
_start = current_time_in_millis()
# audit_params(operation=Sc.OPERATION_DOWNLOAD,
# status=Sc.STATUS_PROCESSING,
# comments=Sc.MSG_FILES_DOWNLOADING)
self._download_them(_sftp)
# audit_params(operation=Sc.OPERATION_DOWNLOAD,
# status=Sc.STATUS_COMPLETE,
# comments=Sc.MSG_FILES_DOWNLOADED + time_taken(_start))
finally:
if _sftp is not None:
_sftp.close()
return self._downloaded_files()
def upload(self) -> list:
"""
Connects to SFTP host and transfers files passed as an input
Returns
--------
list
a list of uploaded files
"""
_sftp = None
try:
if self._sftpSecret.find(os.sep) == -1:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
password=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
else:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
private_key=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
_sftp.chdir(self._destDir)
_start = current_time_in_millis()
# audit_params(operation=Sc.OPERATION_UPLOAD,
# status=Sc.STATUS_PROCESSING,
# comments=Sc.MSG_FILES_UPLOADING)
for file in self._files:
_sftp.put(localpath=file,
callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_COMPLETE,
comments="{} {} ({})% ".format('<--<<', file, str("%.2f" % (100 * (int(transfered) / int(size)))))))
# audit_params(operation=Sc.OPERATION_UPLOAD,
# status=Sc.STATUS_COMPLETE,
# comments=Sc.MSG_FILES_UPLOADED + time_taken(_start))
finally:
if _sftp is not None:
_sftp.close()
return self._files
def cleanup(cleaner: Cleaner):
"""
The caller should inject the appropriate implementation to perform
cleanup operation as per use case
Parameters
----------
cleaner: Cleaner
instance of custom cleaner implementation which has overridden
clean() function
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_PROCESSING, 'Cleaning proecssed files')
cleaner.clean()
audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_COMPLETE, 'Processed files are cleaned' + time_taken(_start))
def download(self) -> list:
"""
Copy files from source directory to destination.
Only files will be considered which are matches with the extension mentioned in the configurations.
Returns
--------
list
a list of copied files
"""
files = []
# scans source directory for files and filters out with extension and then copy
with os.scandir(self._srcDir) as it:
for entry in it:
if self._isfile(entry) and self._is_file_asked(entry.name):
files.append(entry)
dest_path = os.path.join(self._destDir, entry.name)
file_comments = '({}) ==> ({}) transfer'.format(
entry.path, dest_path)
_start = current_time_in_millis()
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
shutil.copy2(entry.path, dest_path)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
# sorts file entries by modified time. I
# t is sorted by earliest modified time first and latest at the last
if len(files) > 0:
files.sort(key=lambda f: f.stat().st_mtime)
print(files)
return list(
map(lambda f: os.path.join(self._destDir, f.name), files))
else:
return []
def upload(self) -> list:
"""
Copy or move files to destination
Returns
--------
list
a list of files transferred to destination
"""
_done = []
for entry in self._files:
if self._isfile(entry) and self._is_file_asked(entry):
dest_path = os.path.join(self._destDir,
os.path.basename(entry))
src_path = os.path.join(self._srcDir, os.path.basename(entry))
file_comments = '({}) ==> ({}) transfer'.format(
src_path, dest_path)
_start = current_time_in_millis()
audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
if self._move:
shutil.move(src=src_path, dst=dest_path)
else:
shutil.copy2(src_path, dest_path)
audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
_done.append(dest_path)
return _done