Пример #1
0
def notify(notifier: Notifier):
    """
    This module allows to send a notification based on the custom implementation of the
    ``Notifier`` abstract class. A developer needs to provide implementation for the notifier.

    The notifier dependency will be injected by the caller as per the use case. This behaviour
    allows to plug in either existing notifier or custom notifiers.

    This method also captures general audit log.

    Parameters
    ----------
    notifier : Notifier
                an instance of custom implementation of ``Notifier``

    """
    _start = current_time_in_millis()
    audit_params(Sc.OPERATION_NOTIFICATION,
                 Sc.STATUS_PROCESSING,
                 'Sending notification')

    notifier.notify()

    audit_params(Sc.OPERATION_NOTIFICATION,
                 Sc.STATUS_COMPLETE,
                 'Notification sent' + time_taken(_start))
Пример #2
0
def download(downloader: Downloader) -> list:
    """
    The caller should inject the appropriate implementation to perform
    download operation as per use case

    Parameters
    ----------
    downloader: Downloader
                instance of custom downloader implementation which has overridden
                download() function

    Returns
    ----------
    list
        list of downloaded files ordered by last modified time,
        earlier will be the first element

    """
    _start = current_time_in_millis()
    audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_PROCESSING, 'Files are downloading from source to destination')

    files = downloader.download()

    audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_COMPLETE, 'Files are downloaded to destination' + time_taken(_start))

    return files
Пример #3
0
def upload(uploader: Uploader) -> list:
    """
    The caller should inject the appropriate implementation to perform
    upload operation as per use case

    Parameters
    ----------
    uploader: Uploader
              instance of custom uploader implementation which has overridden
              upload() function

    Returns
    --------
    list
        a list of uploaded files

    """
    _start = current_time_in_millis()
    audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_PROCESSING, 'Files are uploading from source to destination')

    files = uploader.upload()

    audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_COMPLETE, 'Files are uploaded to destination' + time_taken(_start))

    return files
Пример #4
0
    def upload(self) -> list:
        """
        Connects to SFTP host and transfers files from a remote source to destination directory

        Returns
        -------
        list
            a list of files transferred

        """

        _sftp = None
        try:
            if self._sftpSecret.find(os.sep) == -1:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          password=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            else:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          private_key=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            _sftp.chdir(self._srcDir)

            for file in self._files:
                _src = os.path.join(self._srcDir, os.path.basename(file))
                _dest = os.path.join(self._destDir, os.path.basename(file))
                _start = current_time_in_millis()
                file_comments = '({}) ==> ({}) transfer'.format(_src, _dest)

                audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
                             status=Sc.STATUS_PROCESSING,
                             comments=file_comments + 'ing...')

                _sftp.rename(_src, _dest)

                audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
                             status=Sc.STATUS_COMPLETE,
                             comments=file_comments + 'ed' +
                             time_taken(_start))

        finally:
            if _sftp is not None:
                _sftp.close()

        return self._files
Пример #5
0
    def download(self) -> list:
        """
        Connects to SFTP host using configurations
        and scans for files as per the extensions mentioned in the configurations.
        If there are any matching files, they will be downloaded to destination
        with total seconds taken to download files.

        Returns
        --------
        list
            a list of downloaded files ordered by modified time in ascending.
            Earliest will be the first.

        """
        _sftp = None
        try:
            if self._sftpSecret.find(os.sep) == -1:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          password=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            else:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          private_key=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            _sftp.chdir(self._srcDir)

            _start = current_time_in_millis()

            # audit_params(operation=Sc.OPERATION_DOWNLOAD,
            #              status=Sc.STATUS_PROCESSING,
            #              comments=Sc.MSG_FILES_DOWNLOADING)

            self._download_them(_sftp)

            # audit_params(operation=Sc.OPERATION_DOWNLOAD,
            #              status=Sc.STATUS_COMPLETE,
            #              comments=Sc.MSG_FILES_DOWNLOADED + time_taken(_start))
        finally:
            if _sftp is not None:
                _sftp.close()

        return self._downloaded_files()
Пример #6
0
    def upload(self) -> list:
        """
        Connects to SFTP host and transfers files passed as an input

        Returns
        --------
        list
            a list of uploaded files

        """
        _sftp = None
        try:
            if self._sftpSecret.find(os.sep) == -1:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          password=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            else:
                _sftp = pysftp.Connection(self._sftpHost,
                                          username=self._sftpUsername,
                                          private_key=self._sftpSecret,
                                          port=self._sftpPort,
                                          cnopts=cnopts)
            _sftp.chdir(self._destDir)

            _start = current_time_in_millis()
            # audit_params(operation=Sc.OPERATION_UPLOAD,
            #              status=Sc.STATUS_PROCESSING,
            #              comments=Sc.MSG_FILES_UPLOADING)

            for file in self._files:
                _sftp.put(localpath=file,
                          callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_UPLOAD,
                                                                         status=Sc.STATUS_COMPLETE,
                                                                         comments="{} {} ({})% ".format('<--<<', file, str("%.2f" % (100 * (int(transfered) / int(size)))))))

            # audit_params(operation=Sc.OPERATION_UPLOAD,
            #              status=Sc.STATUS_COMPLETE,
            #              comments=Sc.MSG_FILES_UPLOADED + time_taken(_start))
        finally:
            if _sftp is not None:
                _sftp.close()

        return self._files
Пример #7
0
def cleanup(cleaner: Cleaner):
    """
    The caller should inject the appropriate implementation to perform
    cleanup operation as per use case

    Parameters
    ----------
    cleaner: Cleaner
             instance of custom cleaner implementation which has overridden
             clean() function

    """
    _start = current_time_in_millis()
    audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_PROCESSING, 'Cleaning proecssed files')

    cleaner.clean()

    audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_COMPLETE, 'Processed files are cleaned' + time_taken(_start))
Пример #8
0
    def download(self) -> list:
        """
        Copy files from source directory to destination.
        Only files will be considered which are matches with the extension mentioned in the configurations.

        Returns
        --------
        list
            a list of copied files

        """
        files = []

        # scans source directory for files and filters out with extension and then copy
        with os.scandir(self._srcDir) as it:
            for entry in it:
                if self._isfile(entry) and self._is_file_asked(entry.name):
                    files.append(entry)
                    dest_path = os.path.join(self._destDir, entry.name)
                    file_comments = '({}) ==> ({}) transfer'.format(
                        entry.path, dest_path)
                    _start = current_time_in_millis()
                    audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
                                 status=Sc.STATUS_PROCESSING,
                                 comments=file_comments + 'ing...')

                    shutil.copy2(entry.path, dest_path)

                    audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
                                 status=Sc.STATUS_COMPLETE,
                                 comments=file_comments + 'ed' +
                                 time_taken(_start))

        # sorts file entries by modified time. I
        # t is sorted by earliest modified time first and latest at the last
        if len(files) > 0:
            files.sort(key=lambda f: f.stat().st_mtime)
            print(files)
            return list(
                map(lambda f: os.path.join(self._destDir, f.name), files))
        else:
            return []
Пример #9
0
    def upload(self) -> list:
        """
        Copy or move files to destination

        Returns
        --------
        list
            a list of files transferred to destination

        """
        _done = []
        for entry in self._files:
            if self._isfile(entry) and self._is_file_asked(entry):
                dest_path = os.path.join(self._destDir,
                                         os.path.basename(entry))
                src_path = os.path.join(self._srcDir, os.path.basename(entry))
                file_comments = '({}) ==> ({}) transfer'.format(
                    src_path, dest_path)
                _start = current_time_in_millis()

                audit_params(operation=Sc.OPERATION_UPLOAD,
                             status=Sc.STATUS_PROCESSING,
                             comments=file_comments + 'ing...')

                if self._move:
                    shutil.move(src=src_path, dst=dest_path)
                else:
                    shutil.copy2(src_path, dest_path)

                audit_params(operation=Sc.OPERATION_UPLOAD,
                             status=Sc.STATUS_COMPLETE,
                             comments=file_comments + 'ed' +
                             time_taken(_start))

                _done.append(dest_path)

        return _done