def start(self, print_next_run=True):
"""
Starts the scheduler as per the configurations and parameters. Calling this method more than once has
no effect if ``shutdown`` requested or already started.
If ``separate_thread`` is true, then scheduler will run in separate thread, ``main`` thread otherwise.
If ``run_continuous`` is true, then job will be run continuously, once otherwise
Parameters
----------
print_next_run : bool
if ``True`` it will keep printing next rnu schedule at every ``pulse_seconds``.
Default every 5 seconds
"""
if self._shutdown_requested or self._started:
return
print('Starting scheduler > ')
self._print_etr = print_next_run
audit_params(Sc.OPERATION_START_JOBS, Sc.STATUS_STARTING,
'Starting scheduled jobs')
if self._separate_thread:
self._stop_event = self._schedule_in_separate_thread()
else:
self._schedule_in_main_thread()
def notify(notifier: Notifier):
"""
This module allows to send a notification based on the custom implementation of the
``Notifier`` abstract class. A developer needs to provide implementation for the notifier.
The notifier dependency will be injected by the caller as per the use case. This behaviour
allows to plug in either existing notifier or custom notifiers.
This method also captures general audit log.
Parameters
----------
notifier : Notifier
an instance of custom implementation of ``Notifier``
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_NOTIFICATION,
Sc.STATUS_PROCESSING,
'Sending notification')
notifier.notify()
audit_params(Sc.OPERATION_NOTIFICATION,
Sc.STATUS_COMPLETE,
'Notification sent' + time_taken(_start))
def shutdown(self, force: bool = False):
"""
Shuts down the scheduler. When requested, calling cancel any job has no effect.
Parameters
----------
force : bool
if ``True`` it will not wait until the jobs complete, otherwise will wait until
all jobs complete and then safely shutdown
"""
if inspect.stack()[1].function != '_shut_it_down':
return
print(Sc.MSG_SHUTTING_DOWN_SCHEDULER)
audit_params(Sc.OPERATION_SHUTDOWN, Sc.STATUS_STARTING,
Sc.MSG_SHUTTING_DOWN_SCHEDULER + ' Force=' + str(force))
if self._run_continuous:
self.__wait = True
self._shutdown_requested = True
if self._separate_thread:
self._stop_event.set()
else:
self._run_continuous = False
if not force:
self._wait_until_safely_shutdown()
schedule.clear()
audit_params(Sc.OPERATION_SHUTDOWN, Sc.STATUS_COMPLETE,
Sc.MSG_SCHEDULER_SHUTDOWN_COMPLETE)
print(Sc.MSG_SCHEDULER_SHUTDOWN_COMPLETE)
def download(downloader: Downloader) -> list:
"""
The caller should inject the appropriate implementation to perform
download operation as per use case
Parameters
----------
downloader: Downloader
instance of custom downloader implementation which has overridden
download() function
Returns
----------
list
list of downloaded files ordered by last modified time,
earlier will be the first element
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_PROCESSING, 'Files are downloading from source to destination')
files = downloader.download()
audit_params(Sc.OPERATION_DOWNLOAD, Sc.STATUS_COMPLETE, 'Files are downloaded to destination' + time_taken(_start))
return files
def upload(uploader: Uploader) -> list:
"""
The caller should inject the appropriate implementation to perform
upload operation as per use case
Parameters
----------
uploader: Uploader
instance of custom uploader implementation which has overridden
upload() function
Returns
--------
list
a list of uploaded files
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_PROCESSING, 'Files are uploading from source to destination')
files = uploader.upload()
audit_params(Sc.OPERATION_UPLOAD, Sc.STATUS_COMPLETE, 'Files are uploaded to destination' + time_taken(_start))
return files
def _wait_until_safely_shutdown(self):
"""
If not force shutdown, then this method will ensure all jobs will complete before shutdown safely.
"""
audit_params(Sc.OPERATION_SHUTDOWN, Sc.STATUS_WAITING,
Sc.MSG_WAIT_UNTIL_SAFE_SHUTDOWN)
while self.__wait:
print("Please wait...")
time.sleep(2)
def upload(self) -> list:
"""
Connects to SFTP host and transfers files from a remote source to destination directory
Returns
-------
list
a list of files transferred
"""
_sftp = None
try:
if self._sftpSecret.find(os.sep) == -1:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
password=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
else:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
private_key=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
_sftp.chdir(self._srcDir)
for file in self._files:
_src = os.path.join(self._srcDir, os.path.basename(file))
_dest = os.path.join(self._destDir, os.path.basename(file))
_start = current_time_in_millis()
file_comments = '({}) ==> ({}) transfer'.format(_src, _dest)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
_sftp.rename(_src, _dest)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
finally:
if _sftp is not None:
_sftp.close()
return self._files
def cleanup(cleaner: Cleaner):
"""
The caller should inject the appropriate implementation to perform
cleanup operation as per use case
Parameters
----------
cleaner: Cleaner
instance of custom cleaner implementation which has overridden
clean() function
"""
_start = current_time_in_millis()
audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_PROCESSING, 'Cleaning proecssed files')
cleaner.clean()
audit_params(Sc.OPERATION_FILE_DELETE, Sc.STATUS_COMPLETE, 'Processed files are cleaned' + time_taken(_start))
def download(self) -> list:
"""
Copy files from source directory to destination.
Only files will be considered which are matches with the extension mentioned in the configurations.
Returns
--------
list
a list of copied files
"""
files = []
# scans source directory for files and filters out with extension and then copy
with os.scandir(self._srcDir) as it:
for entry in it:
if self._isfile(entry) and self._is_file_asked(entry.name):
files.append(entry)
dest_path = os.path.join(self._destDir, entry.name)
file_comments = '({}) ==> ({}) transfer'.format(
entry.path, dest_path)
_start = current_time_in_millis()
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
shutil.copy2(entry.path, dest_path)
audit_params(operation=Sc.OPERATION_FILE_TRANSFER,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
# sorts file entries by modified time. I
# t is sorted by earliest modified time first and latest at the last
if len(files) > 0:
files.sort(key=lambda f: f.stat().st_mtime)
print(files)
return list(
map(lambda f: os.path.join(self._destDir, f.name), files))
else:
return []
def _init():
"""
Initialise and load all configurations from file. This method also validates configurations for errors.
"""
print(Sc.MSG_CONFIG_LOADING)
audit_params(Sc.OPERATION_CONFIGURATION, Sc.STATUS_LOADING,
'Loading configurations from file')
_working_dir = os.path.dirname(__file__)
_scheduler_config_file = _working_dir + Sc.CONFIG_FILE
# config_file_path = scheduler_config_file
with open(_scheduler_config_file) as jf:
__box.all_configs = yaml.load(jf)
audit_params(Sc.OPERATION_CONFIGURATION, Sc.STATUS_LOADED,
'Loaded configurations from file')
errors = _validate()
__box.valid = _empty_arr(errors)
if not __box.valid:
audit_params(Sc.OPERATION_CONFIGURATION, Sc.STATUS_INVALID,
'Invalid configurations found')
print(Sc.MSG_CONFIG_LOADED)
def upload(self) -> list:
"""
Copy or move files to destination
Returns
--------
list
a list of files transferred to destination
"""
_done = []
for entry in self._files:
if self._isfile(entry) and self._is_file_asked(entry):
dest_path = os.path.join(self._destDir,
os.path.basename(entry))
src_path = os.path.join(self._srcDir, os.path.basename(entry))
file_comments = '({}) ==> ({}) transfer'.format(
src_path, dest_path)
_start = current_time_in_millis()
audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_PROCESSING,
comments=file_comments + 'ing...')
if self._move:
shutil.move(src=src_path, dst=dest_path)
else:
shutil.copy2(src_path, dest_path)
audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_COMPLETE,
comments=file_comments + 'ed' +
time_taken(_start))
_done.append(dest_path)
return _done
def upload(self) -> list:
"""
Connects to SFTP host and transfers files passed as an input
Returns
--------
list
a list of uploaded files
"""
_sftp = None
try:
if self._sftpSecret.find(os.sep) == -1:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
password=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
else:
_sftp = pysftp.Connection(self._sftpHost,
username=self._sftpUsername,
private_key=self._sftpSecret,
port=self._sftpPort,
cnopts=cnopts)
_sftp.chdir(self._destDir)
_start = current_time_in_millis()
# audit_params(operation=Sc.OPERATION_UPLOAD,
# status=Sc.STATUS_PROCESSING,
# comments=Sc.MSG_FILES_UPLOADING)
for file in self._files:
_sftp.put(localpath=file,
callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_UPLOAD,
status=Sc.STATUS_COMPLETE,
comments="{} {} ({})% ".format('<--<<', file, str("%.2f" % (100 * (int(transfered) / int(size)))))))
# audit_params(operation=Sc.OPERATION_UPLOAD,
# status=Sc.STATUS_COMPLETE,
# comments=Sc.MSG_FILES_UPLOADED + time_taken(_start))
finally:
if _sftp is not None:
_sftp.close()
return self._files
def _download_them(self, _sftp: pysftp.Connection):
"""
This method does the hard work downloading files. It also preserves the modified time
after downloaded from the SFTP host.
The call back lambda function prints the % byes downloaded,
which is based on the size of the file.
Parameters
----------
_sftp : pysftp.Connection
a sftp connection object
"""
entries = _sftp.listdir(self._srcDir)
for entry in entries:
if _sftp.isfile(entry) and self._is_file_asked(entry):
_sftp.get(os.path.join(self._srcDir, entry),
os.path.join(self._destDir, entry),
callback=lambda transfered, size: audit_params(operation=Sc.OPERATION_DOWNLOAD,
status=Sc.STATUS_COMPLETE,
comments="{} {} ({})%".format('>>-->', entry, str("%.2f" % (100*(int(transfered)/int(size)))))),
preserve_mtime=True)
def _schedule_in_main_thread(self):
"""
Run scheduler in the main thread.
If ``run_continuous`` is true then it will continuously run the jobs on schedule.
Otherwise run all jobs at once.
"""
if self._run_continuous:
audit_params(Sc.OPERATION_START_JOBS, Sc.STATUS_STARTED,
Sc.MSG_JOB_STARTED.format(str(self._print_etr)))
self._started = True
while True:
try:
if not self._run_continuous:
print(Sc.MSG_SHUTDOWN_SCHEDULER_RUNNING_ALL)
schedule.run_all()
self.__wait = False
self._shutdown_requested = False
self._started = False
break
self._next_run = schedule.next_run()
self._idle_seconds = schedule.idle_seconds()
self._log_etr()
schedule.run_pending()
time.sleep(self._pulse)
except KeyboardInterrupt:
audit_params(operation=Sc.OPERATION_SHUTDOWN,
status=Sc.STATUS_INTERRUPTED,
comments=Sc.MSG_SCHEDULER_INTERRUPTED)
schedule.run_all()
self.__wait = False
self._shutdown_requested = False
self._started = False
break
else:
audit_params(Sc.OPERATION_START_JOBS, Sc.STATUS_STARTED,
Sc.MSG_JOB_STARTED.format(str(self._print_etr)))
schedule.run_all()
def run(cls):
if self._run_continuous:
audit_params(
Sc.OPERATION_START_JOBS, Sc.STATUS_STARTED,
Sc.MSG_JOB_STARTED.format(str(self._print_etr)))
self._started = True
while True:
try:
if stop_continuous_run.is_set():
print(Sc.MSG_SHUTDOWN_SCHEDULER_RUNNING_ALL)
schedule.run_all()
self.__wait = False
self._shutdown_requested = False
self._started = False
break
self._next_run = schedule.next_run()
self._idle_seconds = schedule.idle_seconds()
self._log_etr()
schedule.run_pending()
time.sleep(self._pulse)
except KeyboardInterrupt:
audit_params(operation=Sc.OPERATION_SHUTDOWN,
status=Sc.STATUS_INTERRUPTED,
comments=Sc.MSG_SCHEDULER_INTERRUPTED)
self._stop_event.set()
schedule.run_all()
self.__wait = False
self._shutdown_requested = False
self._started = False
break
else:
audit_params(
Sc.OPERATION_START_JOBS, Sc.STATUS_STARTED,
Sc.MSG_JOB_STARTED.format(str(self._print_etr)))
schedule.run_all()
def schedule_job(self,
job: Job,
schedule_config: ScheduleConfig = None,
run_continuous: bool = False,
execute_parallel: bool = False,
pulse_seconds: int = Sc.DEFAULT_PULSE):
"""
Schedules a job with ``name`` and user can specify whether the job should run once or should run continuously.
Parameters
----------
job : Job
an instance of custom implementation of the ``Job`` module. Default ``None``
schedule_config : ScheduleConfig
if user wants to change the ``every`` or ``at`` and ``time_unit`` before job schedule,
then user can create an instance on ScheduleConfig and specify new values
for schedule
run_continuous : bool
if ``True`` the scheduler will continuously run jobs as per schedule,
run jobs once otherwise
execute_parallel : bool
if ``True`` the job will be scheduled to execute parallel by spawning a thread,
sequential otherwise
pulse_seconds : int
seconds to keep checking next schedule. Default 5 seconds.
Examples
--------
1. ``pulse_seconds=10`` will check next scheduler every 10 seconds
2. ``pulse_seconds=30`` will check next scheduler every 30 seconds
"""
if not is_valid_implementation(job, Job):
raise Exception(Sc.MSG_EX_ILLEGAL_JOB)
if not isnone(schedule_config):
self._override_schedule(schedule_config)
self._run_continuous = run_continuous
self._pulse = pulse_seconds
audit_params(Sc.OPERATION_SCHEDULE, Sc.STATUS_SCHEDULING,
'Scheduling a job')
if not isnone(self._every):
evaluation_str = 'schedule.every(int(self._every)).' + self._unit
else:
evaluation_str = 'schedule.every().' + self._unit
job_name = job.name()
evaluation_str += self._at()
if execute_parallel:
evaluation_str += '.do(self._spawn_thread, job.goal).tag(job_name)'
else:
evaluation_str += '.do(job.goal).tag(job_name)'
eval(evaluation_str)
comments = StringBuilder(', ')
comments.append('A job scheduled. Summary (Every=' + str(self._every)) \
.append('TimeUnit=' + str(self._unit)) \
.append('At=' + str(self._at_time)) \
.append('SeparateThread=' + str(self._separate_thread)) \
.append('RunningContinuously=' + str(self._run_continuous)) \
.append('Pulse=' + str(self._pulse) + ')')
audit_params(Sc.OPERATION_SCHEDULE, Sc.STATUS_SCHEDULED,
comments.to_string())
print(Sc.MSG_JOB_SCHEDULED)