def test_threadpool():
pool = ThreadPool(core_threads=2, keepalive=0)
event1 = Event()
event2 = Event()
event3 = Event()
pool.submit(event1.set)
pool.submit(event2.set)
pool.submit(event3.set)
event1.wait(1)
event2.wait(1)
event3.wait(1)
assert event1.isSet()
assert event2.isSet()
assert event3.isSet()
sleep(0.3)
eq_(repr(pool), '<ThreadPool at %x; threads=2/20>' % id(pool))
pool.shutdown()
eq_(repr(pool), '<ThreadPool at %x; threads=0/20>' % id(pool))
# Make sure double shutdown is ok
pool.shutdown()
# Make sure one can't submit tasks to a thread pool that has been shut down
assert_raises(RuntimeError, pool.submit, event1.set)
def configure(self, gconfig={}, **options):
"""
Reconfigures the scheduler with the given options. Can only be done when the scheduler isn't running.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Set general options
config = combine_opts(gconfig, 'apscheduler.', options)
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
# Configure the thread pool
if 'threadpool' in config:
self._threadpool = maybe_ref(config['threadpool'])
else:
threadpool_opts = combine_opts(config, 'threadpool.')
self._threadpool = ThreadPool(**threadpool_opts)
# Configure job stores
jobstore_opts = combine_opts(config, 'jobstore.')
jobstores = {}
for key, value in jobstore_opts.items():
store_name, option = key.split('.', 1)
opts_dict = jobstores.setdefault(store_name, {})
opts_dict[option] = value
for alias, opts in jobstores.items():
classname = opts.pop('class')
cls = maybe_ref(classname)
jobstore = cls(**opts)
self.add_jobstore(jobstore, alias, True)
def test_threadpool_nocore():
pool = ThreadPool(keepalive=0)
event = Event()
pool.submit(event.set)
event.wait(1)
assert event.isSet()
sleep(1)
eq_(repr(pool), '<ThreadPool at %x; threads=0/20>' % id(pool))
def test_threadpool_maxthreads():
pool = ThreadPool(core_threads=2, max_threads=1)
eq_(pool.max_threads, 2)
pool = ThreadPool(core_threads=2, max_threads=3)
eq_(pool.max_threads, 3)
pool = ThreadPool(core_threads=0, max_threads=0)
eq_(pool.max_threads, 1)
def configure(self, gconfig={}, **options):
"""
Reconfigures the scheduler with the given options. Can only be done
when the scheduler isn't running.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Set general options
config = combine_opts(gconfig, 'apscheduler.', options)
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
# Configure the thread pool
if 'threadpool' in config:
self._threadpool = maybe_ref(config['threadpool'])
else:
threadpool_opts = combine_opts(config, 'threadpool.')
self._threadpool = ThreadPool(**threadpool_opts)
# Configure job stores
jobstore_opts = combine_opts(config, 'jobstore.')
jobstores = {}
for key, value in jobstore_opts.items():
store_name, option = key.split('.', 1)
opts_dict = jobstores.setdefault(store_name, {})
opts_dict[option] = value
for alias, opts in jobstores.items():
classname = opts.pop('class')
cls = maybe_ref(classname)
jobstore = cls(**opts)
self.add_jobstore(jobstore, alias, True)
def configure(self, **options):
try:
redis = StrictRedis(**settings.REDISES["default"])
redis2 = StrictRedis(**settings.REDISES["default"])
redis3 = StrictRedis(**settings.REDISES["default"])
except:
logging.exception("cannot connect to redis")
raise
self._timezone = gettz("Asia/Chongqing")
self._threadpool = ThreadPool()
self._jobstore = JobStore(redis, self._timezone) # todo Jobstore
self._jobstats = JobStats(redis2)
self._jobsync = JobSync(redis3)
self.misfire_grace_time = 1
self.coalesce = True
self.daemonic = True
def configure(self, gconfig={}, **options):
if self.running:
raise SchedulerAlreadyRunningError
config = combine_opts(gconfig, 'main.', options)
self._config = config
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
timezone = config.pop('timezone', None)
self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()
# config threadpool
threadpool_opts = combine_opts(config, 'threadpool.')
self._worker_threadpool = ThreadPool(**threadpool_opts)
# config jobstore
jobstore_opts = combine_opts(config, 'jobstore.')
self._job_store = SQLAlchemyJobStore(**jobstore_opts)
# config syncqueue
syncqueue_opts = combine_opts(config, 'syncqueue.')
self._changes_queue = HotQueue(**syncqueue_opts)
# config statstore
statstore_opts = combine_opts(config, 'statstore.')
self._stat_store = JobReporter(**statstore_opts)
# config statqueue
statqueue_opts = combine_opts(config, 'statqueue.')
self._stats_queue = HotQueue(**statqueue_opts)
# configure logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
class Scheduler(object):
"""
This class is responsible for scheduling jobs and triggering their execution.
"""
_stopped = False
_thread = None
def __init__(self, gconfig={}, **options):
self._wakeup = Event()
self._jobstores = {}
self._jobstores_lock = Lock()
self._listeners = []
self._listeners_lock = Lock()
self._pending_jobs = []
self.configure(gconfig, **options)
def configure(self, gconfig={}, **options):
"""
Reconfigures the scheduler with the given options. Can only be done when the scheduler isn't running.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Set general options
config = combine_opts(gconfig, 'apscheduler.', options)
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
# Configure the thread pool
if 'threadpool' in config:
self._threadpool = maybe_ref(config['threadpool'])
else:
threadpool_opts = combine_opts(config, 'threadpool.')
self._threadpool = ThreadPool(**threadpool_opts)
# Configure job stores
jobstore_opts = combine_opts(config, 'jobstore.')
jobstores = {}
for key, value in jobstore_opts.items():
store_name, option = key.split('.', 1)
opts_dict = jobstores.setdefault(store_name, {})
opts_dict[option] = value
for alias, opts in jobstores.items():
classname = opts.pop('class')
cls = maybe_ref(classname)
jobstore = cls(**opts)
self.add_jobstore(jobstore, alias, True)
def start(self):
"""
Starts the scheduler in a new thread.
In threaded mode (the default), this method will return immediately after starting the scheduler thread.
In standalone mode, this method will block until there are no more scheduled jobs.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Create a RAMJobStore as the default if there is no default job store
if not 'default' in self._jobstores:
self.add_jobstore(RAMJobStore(), 'default', True)
# Schedule all pending jobs
for job, jobstore in self._pending_jobs:
self._real_add_job(job, jobstore, False)
del self._pending_jobs[:]
self._stopped = False
if self.standalone:
self._main_loop()
else:
self._thread = Thread(target=self._main_loop, name='APScheduler')
self._thread.setDaemon(self.daemonic)
self._thread.start()
def shutdown(self,
wait=True,
shutdown_threadpool=True,
close_jobstores=True):
"""
Shuts down the scheduler and terminates the thread. Does not interrupt any currently running jobs.
:param wait: ``True`` to wait until all currently executing jobs have finished (if ``shutdown_threadpool`` is
also ``True``)
:param shutdown_threadpool: ``True`` to shut down the thread pool
:param close_jobstores: ``True`` to close all job stores after shutdown
"""
if not self.running:
return
self._stopped = True
self._wakeup.set()
# Shut down the thread pool
if shutdown_threadpool:
self._threadpool.shutdown(wait)
# Wait until the scheduler thread terminates
if self._thread:
self._thread.join()
# Close all job stores
if close_jobstores:
for jobstore in itervalues(self._jobstores):
jobstore.close()
@property
def running(self):
return not self._stopped and self._thread and self._thread.isAlive()
def add_jobstore(self, jobstore, alias, quiet=False):
"""
Adds a job store to this scheduler.
:param jobstore: job store to be added
:param alias: alias for the job store
:param quiet: True to suppress scheduler thread wakeup
:type jobstore: instance of :class:`~apscheduler.jobstores.base.JobStore`
:type alias: str
"""
with self._jobstores_lock:
if alias in self._jobstores:
raise KeyError('Alias "%s" is already in use' % alias)
self._jobstores[alias] = jobstore
jobstore.load_jobs()
# Notify listeners that a new job store has been added
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_ADDED, alias))
# Notify the scheduler so it can scan the new job store for jobs
if not quiet:
self._wakeup.set()
def remove_jobstore(self, alias, close=True):
"""
Removes the job store by the given alias from this scheduler.
:param close: ``True`` to close the job store after removing it
:type alias: str
"""
with self._jobstores_lock:
jobstore = self._jobstores.pop(alias)
if not jobstore:
raise KeyError('No such job store: %s' % alias)
# Close the job store if requested
if close:
jobstore.close()
# Notify listeners that a job store has been removed
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_REMOVED, alias))
def add_listener(self, callback, mask=EVENT_ALL):
"""
Adds a listener for scheduler events. When a matching event occurs, ``callback`` is executed with the event
object as its sole argument. If the ``mask`` parameter is not provided, the callback will receive events of all
types.
:param callback: any callable that takes one argument
:param mask: bitmask that indicates which events should be listened to
"""
with self._listeners_lock:
self._listeners.append((callback, mask))
def remove_listener(self, callback):
"""
Removes a previously added event listener.
"""
with self._listeners_lock:
for i, (cb, _) in enumerate(self._listeners):
if callback == cb:
del self._listeners[i]
def _notify_listeners(self, event):
with self._listeners_lock:
listeners = tuple(self._listeners)
for cb, mask in listeners:
if event.code & mask:
try:
cb(event)
except:
logger.exception('Error notifying listener')
def _real_add_job(self, job, jobstore, wakeup):
job.compute_next_run_time(datetime.now())
if not job.next_run_time:
raise ValueError('Not adding job since it would never be run')
with self._jobstores_lock:
try:
store = self._jobstores[jobstore]
except KeyError:
raise KeyError('No such job store: %s' % jobstore)
store.add_job(job)
# Notify listeners that a new job has been added
event = JobStoreEvent(EVENT_JOBSTORE_JOB_ADDED, jobstore, job)
self._notify_listeners(event)
logger.info('Added job "%s" to job store "%s"', job, jobstore)
# Notify the scheduler about the new job
if wakeup:
self._wakeup.set()
def add_job(self,
trigger,
func,
args,
kwargs,
jobstore='default',
**options):
"""
Adds the given job to the job list and notifies the scheduler thread.
The ``func`` argument can be given either as a callable object or a textual reference in the
``package.module:some.object`` format, where the first half (separated by ``:``) is an importable module and the
second half is a reference to the callable object, relative to the module.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
:param trigger: trigger that determines when ``func`` is called
:param func: callable (or a textual reference to one) to run at the given time
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param jobstore: alias of the job store to store the job in
:rtype: :class:`~apscheduler.job.Job`
"""
job = Job(trigger, func, args or [], kwargs or {},
options.pop('misfire_grace_time', self.misfire_grace_time),
options.pop('coalesce', self.coalesce), **options)
if not self.running:
self._pending_jobs.append((job, jobstore))
logger.info(
'Adding job tentatively -- it will be properly scheduled when the scheduler starts'
)
else:
self._real_add_job(job, jobstore, True)
return job
def _remove_job(self, job, alias, jobstore):
jobstore.remove_job(job)
# Notify listeners that a job has been removed
event = JobStoreEvent(EVENT_JOBSTORE_JOB_REMOVED, alias, job)
self._notify_listeners(event)
logger.info('Removed job "%s"', job)
def add_date_job(self, func, date, args=None, kwargs=None, **options):
"""
Schedules a job to be completed on a specific date and time.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
:param func: callable to run at the given time
:param date: the date/time to run the job at
:param name: name of the job
:param jobstore: stored the job in the named (or given) job store
:param misfire_grace_time: seconds after the designated run time that the job is still allowed to be run
:type date: :class:`datetime.date`
:rtype: :class:`~apscheduler.job.Job`
"""
trigger = SimpleTrigger(date)
return self.add_job(trigger, func, args, kwargs, **options)
def add_interval_job(self,
func,
weeks=0,
days=0,
hours=0,
minutes=0,
seconds=0,
start_date=None,
args=None,
kwargs=None,
**options):
"""
Schedules a job to be completed on specified intervals.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
:param func: callable to run
:param weeks: number of weeks to wait
:param days: number of days to wait
:param hours: number of hours to wait
:param minutes: number of minutes to wait
:param seconds: number of seconds to wait
:param start_date: when to first execute the job and start the counter (default is after the given interval)
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param name: name of the job
:param jobstore: alias of the job store to add the job to
:param misfire_grace_time: seconds after the designated run time that the job is still allowed to be run
:rtype: :class:`~apscheduler.job.Job`
"""
interval = timedelta(weeks=weeks,
days=days,
hours=hours,
minutes=minutes,
seconds=seconds)
trigger = IntervalTrigger(interval, start_date)
return self.add_job(trigger, func, args, kwargs, **options)
def add_cron_job(self,
func,
year=None,
month=None,
day=None,
week=None,
day_of_week=None,
hour=None,
minute=None,
second=None,
start_date=None,
args=None,
kwargs=None,
**options):
"""
Schedules a job to be completed on times that match the given expressions.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
:param func: callable to run
:param year: year to run on
:param month: month to run on
:param day: day of month to run on
:param week: week of the year to run on
:param day_of_week: weekday to run on (0 = Monday)
:param hour: hour to run on
:param second: second to run on
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param name: name of the job
:param jobstore: alias of the job store to add the job to
:param misfire_grace_time: seconds after the designated run time that the job is still allowed to be run
:return: the scheduled job
:rtype: :class:`~apscheduler.job.Job`
"""
trigger = CronTrigger(year=year,
month=month,
day=day,
week=week,
day_of_week=day_of_week,
hour=hour,
minute=minute,
second=second,
start_date=start_date)
return self.add_job(trigger, func, args, kwargs, **options)
def cron_schedule(self, **options):
"""
Decorator version of :meth:`add_cron_job`.
This decorator does not wrap its host function.
Unscheduling decorated functions is possible by passing the ``job`` attribute of the scheduled function to
:meth:`unschedule_job`.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
"""
def inner(func):
func.job = self.add_cron_job(func, **options)
return func
return inner
def interval_schedule(self, **options):
"""
Decorator version of :meth:`add_interval_job`.
This decorator does not wrap its host function.
Unscheduling decorated functions is possible by passing the ``job`` attribute of the scheduled function to
:meth:`unschedule_job`.
Any extra keyword arguments are passed along to the constructor of the :class:`~apscheduler.job.Job` class
(see :ref:`job_options`).
"""
def inner(func):
func.job = self.add_interval_job(func, **options)
return func
return inner
def get_jobs(self):
"""
Returns a list of all scheduled jobs.
:return: list of :class:`~apscheduler.job.Job` objects
"""
with self._jobstores_lock:
jobs = []
for jobstore in itervalues(self._jobstores):
jobs.extend(jobstore.jobs)
return jobs
def unschedule_job(self, job):
"""
Removes a job, preventing it from being run any more.
"""
with self._jobstores_lock:
for alias, jobstore in iteritems(self._jobstores):
if job in list(jobstore.jobs):
self._remove_job(job, alias, jobstore)
return
raise KeyError('Job "%s" is not scheduled in any job store' % job)
def unschedule_func(self, func):
"""
Removes all jobs that would execute the given function.
"""
found = False
with self._jobstores_lock:
for alias, jobstore in iteritems(self._jobstores):
for job in list(jobstore.jobs):
if job.func == func:
self._remove_job(job, alias, jobstore)
found = True
if not found:
raise KeyError(
'The given function is not scheduled in this scheduler')
def print_jobs(self, out=None):
"""
Prints out a textual listing of all jobs currently scheduled on this
scheduler.
:param out: a file-like object to print to (defaults to **sys.stdout** if nothing is given)
"""
out = out or sys.stdout
job_strs = []
with self._jobstores_lock:
for alias, jobstore in iteritems(self._jobstores):
job_strs.append('Jobstore %s:' % alias)
if jobstore.jobs:
for job in jobstore.jobs:
job_strs.append(' %s' % job)
else:
job_strs.append(' No scheduled jobs')
out.write(os.linesep.join(job_strs) + os.linesep)
def _run_job(self, job, run_times):
"""
Acts as a harness that runs the actual job code in a thread.
"""
for run_time in run_times:
# See if the job missed its run time window, and handle possible
# misfires accordingly
difference = datetime.now() - run_time
grace_time = timedelta(seconds=job.misfire_grace_time)
if difference > grace_time:
# Notify listeners about a missed run
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._notify_listeners(event)
logger.warning('Run time of job "%s" was missed by %s', job,
difference)
else:
try:
job.add_instance()
except MaxInstancesReachedError:
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._notify_listeners(event)
logger.warning(
'Execution of job "%s" skipped: maximum number of running instances reached (%d)',
job, job.max_instances)
break
logger.info('Running job "%s" (scheduled at %s)', job,
run_time)
try:
retval = job.func(*job.args, **job.kwargs)
except:
# Notify listeners about the exception
exc, tb = sys.exc_info()[1:]
event = JobEvent(EVENT_JOB_ERROR,
job,
run_time,
exception=exc,
traceback=tb)
self._notify_listeners(event)
logger.exception('Job "%s" raised an exception', job)
else:
# Notify listeners about successful execution
event = JobEvent(EVENT_JOB_EXECUTED,
job,
run_time,
retval=retval)
self._notify_listeners(event)
logger.info('Job "%s" executed successfully', job)
job.remove_instance()
# If coalescing is enabled, don't attempt any further runs
if job.coalesce:
break
def _process_jobs(self, now):
"""
Iterates through jobs in every jobstore, starts pending jobs and figures out the next wakeup time.
"""
next_wakeup_time = None
with self._jobstores_lock:
for alias, jobstore in iteritems(self._jobstores):
for job in tuple(jobstore.jobs):
run_times = job.get_run_times(now)
if run_times:
self._threadpool.submit(self._run_job, job, run_times)
# Increase the job's run count
if job.coalesce:
job.runs += 1
else:
job.runs += len(run_times)
# Update the job, but don't keep finished jobs around
if job.compute_next_run_time(now + timedelta(
microseconds=1)):
jobstore.update_job(job)
else:
self._remove_job(job, alias, jobstore)
if not next_wakeup_time:
next_wakeup_time = job.next_run_time
elif job.next_run_time:
next_wakeup_time = min(next_wakeup_time,
job.next_run_time)
return next_wakeup_time
def _main_loop(self):
"""Executes jobs on schedule."""
logger.info('Scheduler started')
self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_START))
self._wakeup.clear()
while not self._stopped:
logger.debug('Looking for jobs to run')
now = datetime.now()
next_wakeup_time = self._process_jobs(now)
# Sleep until the next job is scheduled to be run,
# a new job is added or the scheduler is stopped
if next_wakeup_time is not None:
wait_seconds = time_difference(next_wakeup_time, now)
logger.debug('Next wakeup is due at %s (in %f seconds)',
next_wakeup_time, wait_seconds)
self._wakeup.wait(wait_seconds)
self._wakeup.clear()
elif self.standalone:
logger.debug('No jobs left; shutting down scheduler')
self.shutdown()
break
else:
logger.debug('No jobs; waiting until a job is added')
self._wakeup.wait()
self._wakeup.clear()
logger.info('Scheduler has been shut down')
self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN))
def prepare_scheduler(self):
self.scheduler = LogshipperScheduler(
threadpool=ThreadPool(max_threads=len(self.logfile_configs), ))
jirasync.sync_issues(accounts, au.index)
except (OperationalError, InvalidRequestError, StatementError) as e:
app.logger.exception("Database error processing accounts %s, cleaning up session.", accounts)
db.session.remove()
store_exception("scheduler-audit-changes", None, e)
def _clear_old_exceptions():
print("Clearing out exceptions that have an expired TTL...")
clear_old_exceptions()
print("Completed clearing out exceptions that have an expired TTL.")
pool = ThreadPool(
core_threads=app.config.get('CORE_THREADS', 25),
max_threads=app.config.get('MAX_THREADS', 30),
keepalive=0
)
scheduler = Scheduler(
standalone=True,
threadpool=pool,
coalesce=True,
misfire_grace_time=30
)
def setup_scheduler():
"""Sets up the APScheduler"""
log = logging.getLogger('apscheduler')
try:
find_iamgroup_changes(account)
app.logger.info("Account {} is done with IAMGROUP".format(account))
find_iamrole_changes(account)
app.logger.info("Account {} is done with IAMROLE".format(account))
find_keypair_changes(account)
app.logger.info("Account {} is done with KEYPAIR".format(account))
find_sns_changes(account)
app.logger.info("Account {} is done with SNS".format(account))
time2 = time.time()
app.logger.info('Run Account %s took %0.1f s' % (account, (time2-time1)))
from apscheduler.threadpool import ThreadPool
from apscheduler.scheduler import Scheduler
import traceback
import time
pool = ThreadPool(core_threads=25, max_threads=30, keepalive=0)
scheduler = Scheduler(standalone=True, threadpool=pool, coalesce=True, misfire_grace_time=30)
interval = 15
def setup_scheduler():
"""Sets up the APScheduler"""
log = logging.getLogger('apscheduler')
log.setLevel(app.config.get('LOG_LEVEL'))
log.addHandler(handler)
try:
accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all()
accounts = [account.name for account in accounts]
for account in accounts:
print "Scheduler adding account {}".format(account)
def _run_jobs(self, core):
logger.debug("Starting patched threadpool worker!")
with app.app_context():
ThreadPool._old_run_jobs(self, core)
class LocalScheduler(object):
_stopped = False
_main_thread = None
#init worker thread pool，reporter thread，updater thread
def __init__(self, gconfig={}, **options):
self._wakeup = Event()
self._job_store = None
self._stat_store = None
self._jobs = {}
self.logger = None
self._stats_queue = None
self._changes_queue = None
self._jobs_locks = {}
self._jobs_lock = Lock()
self._log_queue_lock = Lock()
self._worker_threadpool = None
self._reporter_thread = None
self._main_thread = None
self._updater_thread = None
self._monitor_thread = None
self.configure(gconfig, **options)
def configure(self, gconfig={}, **options):
if self.running:
raise SchedulerAlreadyRunningError
config = combine_opts(gconfig, 'main.', options)
self._config = config
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
self.standalone = asbool(config.pop('standalone', False))
timezone = config.pop('timezone', None)
self.timezone = gettz(timezone) if isinstance(timezone, basestring) else timezone or tzlocal()
# config threadpool
threadpool_opts = combine_opts(config, 'threadpool.')
self._worker_threadpool = ThreadPool(**threadpool_opts)
# config jobstore
jobstore_opts = combine_opts(config, 'jobstore.')
self._job_store = SQLAlchemyJobStore(**jobstore_opts)
# config syncqueue
syncqueue_opts = combine_opts(config, 'syncqueue.')
self._changes_queue = HotQueue(**syncqueue_opts)
# config statstore
statstore_opts = combine_opts(config, 'statstore.')
self._stat_store = JobReporter(**statstore_opts)
# config statqueue
statqueue_opts = combine_opts(config, 'statqueue.')
self._stats_queue = HotQueue(**statqueue_opts)
# configure logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
def start(self):
if self.running:
raise SchedulerAlreadyRunningError
self.load_jobs()
self._stopped = False
if self.standalone:
self._main_loop()
else:
self._main_thread = Thread(target = self._main_loop, name = 'main')
self._main_thread.setDaemon(self.daemonic)
self._main_thread.start()
print 'main thread is startted'
self._updater_thread = Thread(target = self._sync_changes, name = 'update')
self._updater_thread.setDaemon(self.daemonic)
self._updater_thread.start()
print 'update thread is started'
self._stater_thread = Thread(target = self._stat_runs, name = 'stat')
self._stater_thread.setDaemon(self.daemonic)
self._stater_thread.start()
print 'stat thread is started'
def shutdown(self, shutdown_threadpool=True, close_jobstore=True):
if not self.running:
return
self._stopped = True
self._wakeup.set()
if shutdown_threadpool:
self._worker_threadpool.shutdown()
if self._main_thread:
self._main_thread.join()
if close_jobstore:
self._job_store.close()
@property
def running(self):
return not self._stopped and self._main_thread and self._main_thread.isAlive()
def now(self):
return datetime.now(self.timezone)
def set_jobs(self, jobs):
now = self.now()
with self._jobs_lock:
for job in jobs:
job.compute_next_run_time(now)
self._jobs[job.id] = job
self._jobs_locks[job.id] = Lock()
# loads jobs pool from db
def load_jobs(self):
jobs = self._job_store.load_jobs()
now = self.now()
with self._jobs_lock:
for job in jobs:
self._add_job(job)
def _add_job(self, job):
try:
now = self.now()
job.compute_next_run_time(now)
if job.next_run_time:
self._jobs[job.id] = job
self._jobs_locks[job.id] = Lock()
except:
logger.exception("add job(id=%d, name=%s) failed" % (job.id, job.name))
return False
return True
def _remove_job(self, job_id):
try:
with self._jobs_locks[job_id]:
del self._jobs[job_id]
del self._jobs_locks[job_id]
except:
logger.exception("remove job(id=%d) failed" % (job_id))
return False
return True
def _main_loop(self):
print "get into the main loop"
self._wakeup.clear()
while not self._stopped:
print 'check again'
now = self.now()
next_wakeup_time = self._process_jobs(now)
print "next_wakeup_time:", next_wakeup_time
if next_wakeup_time is not None:
wait_seconds = time_difference(next_wakeup_time, now)
self._wakeup.wait(wait_seconds)
self._wakeup.clear()
else:
self._wakeup.wait()
self._wakeup.clear()
print "get out the main loop"
def _process_jobs(self, now):
next_wakeup_time = None
print self._jobs
for job in self._jobs.values():
run_time_list = job.get_run_times(now)
if run_time_list:
self._worker_threadpool.submit(self._run_job, job, run_time_list)
with self._jobs_locks[job.id]:
next_run_time = job.compute_next_run_time(now + timedelta(microseconds=1))
if not next_run_time:
self._remove_job(job.id)
print 'job.next_run_time:', job.id, job.next_run_time
if not next_wakeup_time:
next_wakeup_time = job.next_run_time
elif job.next_run_time:
next_wakeup_time = min(next_wakeup_time, job.next_run_time)
return next_wakeup_time
def _run_job(self, job, run_time_list):
for run_time in run_time_list:
now = self.now()
difference = now - run_time
grace_time = timedelta(seconds=self.misfire_grace_time)
if difference > grace_time:
self.logger.warning('Run time of job "%s" was missed by %s', job, difference)
self._put_stat(job.id, 'missed', next_run_time=job.next_run_time)
else:
try:
# maybe add a timeout handle by join thread.
# t = Thread(job.run); t.start(); t.join(timeout)
# refer: http://augustwu.iteye.com/blog/554827
self._put_stat(job.id, 'running', next_run_time=job.next_run_time)
result = job.run()
print 'job runned success'
cost = self.now() - now
self._put_stat(job.id, 'succed', cost=cost)
except:
self.logger.exception('Job "%s" raised an exception', job)
cost = self.now() - now
self._put_stat(job.id, 'failed', cost=cost)
if self.coalesce:
break
def _put_stat(self, job_id, status, next_run_time=None, cost=timedelta(seconds=0)):
msg = {
'time': pickle.dumps(self.now()),
'job_id': job_id,
'status': status,
'next_run_time':pickle.dumps(next_run_time),
'cost': cost.total_seconds() + cost.microseconds / 1000000
}
try:
self._stats_queue.put(msg)
except:
logger.exception('failed to put stat item ' + json.dumps(msg))
def _stat_runs(self):
while not self._stopped:
try:
msg = self._stats_queue.get(block=True, timeout=1)
except:
logger.exception('get stat item failed')
msg = None
if not msg:
continue
try:
msg["time"] = pickle.loads(msg['time'])
msg["next_run_time"] = pickle.loads(msg['next_run_time'])
self._stat_store.report(**msg)
except:
traceback.print_exc()
logger.exception('report job status failed ' + pickle.dumps(msg))
def _sync_changes(self):
count = 0
max_items_once = int(self._config.pop('max_items_once', 0))
while not self._stopped:
try:
msg = self._changes_queue.get(block=True, timeout=1)
except:
logger.exception('get sync item failed')
msg = None
if msg:
opt_type = msg['opt_type']
job_id = msg['job_id']
if job_id > 0 and isinstance(opt_type, basestring):
try:
self._apply_change(opt_type, job_id)
except:
pass
self.logger.info('apply change "%s" for job(%d)', opt_type, job_id)
count += 1
if not msg or (max_items_once > 0 and count > max_items_once):
if count > 0:
self.logger.info('wakeup main thread by sync thread with %d updates' % count)
self._wakeup.set()
count = 0
def _apply_change(self, opt_type, job_id):
if opt_type == 'add' or opt_type == 'update':
try:
job = self._job_store.get_job(job_id)
except Exception as e:
self.logger.exception(e)
if job:
if opt_type == 'add':
if not self._jobs.has_key(job_id):
self._add_job(job)
else:
logger.exception("apply channge '%s job(id=%d, name=%s)' failed" % (opt_type, job.id, job.name))
else:
#!todo check if compute next_run_time again is necessary
now = self.now()
job.compute_next_run_time(now)
with self._jobs_locks[job_id]:
self._jobs[job_id] = job
elif opt_type == 'delete' or opt_type == 'pause':
self._remove_job(job_id)
else:
self.logger.exception('opt %s job(%d) to jobs pool is not supported' % (opt_type, job_id))
class Scheduler(object):
"""
This class is responsible for scheduling jobs and triggering
their execution.
"""
_stopped = False
_thread = None
def __init__(self, gconfig={}, **options):
self._wakeup = Event()
self._jobstores = {}
self._jobstores_lock = Lock()
self._listeners = []
self._listeners_lock = Lock()
self._pending_jobs = []
self._jobs_num = 0
self._jobs_num_lock = Lock()
self.configure(gconfig, **options)
def configure(self, gconfig={}, **options):
"""
Reconfigures the scheduler with the given options. Can only be done
when the scheduler isn't running.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Set general options
config = combine_opts(gconfig, 'apscheduler.', options)
self.misfire_grace_time = int(config.pop('misfire_grace_time', 30))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
# Configure the thread pool
if 'threadpool' in config:
self._threadpool = maybe_ref(config['threadpool'])
else:
threadpool_opts = combine_opts(config, 'threadpool.')
self._threadpool = ThreadPool(**threadpool_opts)
# Configure job stores
jobstore_opts = combine_opts(config, 'jobstore.')
jobstores = {}
for key, value in jobstore_opts.items():
store_name, option = key.split('.', 1)
opts_dict = jobstores.setdefault(store_name, {})
opts_dict[option] = value
for alias, opts in jobstores.items():
classname = opts.pop('class')
cls = maybe_ref(classname)
jobstore = cls(**opts)
self.add_jobstore(jobstore, alias, True)
def start(self):
"""
Starts the scheduler in a new thread.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Create a RAMJobStore as the default if there is no default job store
if not 'default' in self._jobstores:
self.add_jobstore(RAMJobStore(), 'default', True)
# Schedule all pending jobs
for job, jobstore in self._pending_jobs:
self._real_add_job(job, jobstore, False)
del self._pending_jobs[:]
self._stopped = False
self._thread = Thread(target=self._main_loop, name='APScheduler')
self._thread.setDaemon(self.daemonic)
self._thread.start()
def shutdown(self, wait=True, shutdown_threadpool=True, close_jobstores=True):
"""
Shuts down the scheduler and terminates the thread.
Does not interrupt any currently running jobs.
:param wait: ``True`` to wait until all currently executing jobs have
finished (if ``shutdown_threadpool`` is also ``True``)
:param shutdown_threadpool: ``True`` to shut down the thread pool
:param close_jobstores: ``True`` to close all job stores after shutdown
"""
if not self.running:
return
self._stopped = True
self._wakeup.set()
# Shut down the thread pool
if shutdown_threadpool:
self._threadpool.shutdown(wait)
# Wait until the scheduler thread terminates
self._thread.join()
# Close all job stores
if close_jobstores:
for jobstore in itervalues(self._jobstores):
jobstore.close()
@property
def running(self):
return not self._stopped and self._thread and self._thread.isAlive()
def _add_jobs_num(self):
"""add number of current jobs"""
self._jobs_num_lock.acquire()
try:
self._jobs_num += 1
finally:
self._jobs_num_lock.release()
def _sub_jobs_num(self):
"""remove number of current jobs"""
self._jobs_num_lock.acquire()
try:
self._jobs_num -= 1
finally:
self._jobs_num_lock.release()
def add_jobstore(self, jobstore, alias, quiet=False):
"""
Adds a job store to this scheduler.
:param jobstore: job store to be added
:param alias: alias for the job store
:param quiet: True to suppress scheduler thread wakeup
:type jobstore: instance of
:class:`~apscheduler.jobstores.base.JobStore`
:type alias: str
"""
self._jobstores_lock.acquire()
try:
if alias in self._jobstores:
raise KeyError('Alias "%s" is already in use' % alias)
self._jobstores[alias] = jobstore
jobstore.load_jobs()
finally:
self._jobstores_lock.release()
# Notify listeners that a new job store has been added
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_ADDED, alias))
# Notify the scheduler so it can scan the new job store for jobs
if not quiet:
self._wakeup.set()
def remove_jobstore(self, alias, close=True):
"""
Removes the job store by the given alias from this scheduler.
:param close: ``True`` to close the job store after removing it
:type alias: str
"""
self._jobstores_lock.acquire()
try:
jobstore = self._jobstores.pop(alias)
if not jobstore:
raise KeyError('No such job store: %s' % alias)
finally:
self._jobstores_lock.release()
# Close the job store if requested
if close:
jobstore.close()
# Notify listeners that a job store has been removed
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_REMOVED, alias))
def add_listener(self, callback, mask=EVENT_ALL):
"""
Adds a listener for scheduler events. When a matching event occurs,
``callback`` is executed with the event object as its sole argument.
If the ``mask`` parameter is not provided, the callback will receive
events of all types.
:param callback: any callable that takes one argument
:param mask: bitmask that indicates which events should be listened to
"""
self._listeners_lock.acquire()
try:
self._listeners.append((callback, mask))
finally:
self._listeners_lock.release()
def remove_listener(self, callback):
"""
Removes a previously added event listener.
"""
self._listeners_lock.acquire()
try:
for i, (cb, _) in enumerate(self._listeners):
if callback == cb:
del self._listeners[i]
finally:
self._listeners_lock.release()
def _notify_listeners(self, event):
self._listeners_lock.acquire()
try:
listeners = tuple(self._listeners)
finally:
self._listeners_lock.release()
for cb, mask in listeners:
if event.code & mask:
try:
cb(event)
except Exception, e:
logger.exception('Error notifying listener')
class Scheduler(object):
"""
This class is responsible for scheduling jobs and triggering
their execution.
"""
_stopped = False
_thread = None
def __init__(self, gconfig={}, **options):
self._wakeup = Event()
self._jobstores = {}
self._jobstores_lock = Lock()
self._listeners = []
self._listeners_lock = Lock()
self._pending_jobs = []
self.configure(gconfig, **options)
def configure(self, gconfig={}, **options):
"""
Reconfigures the scheduler with the given options. Can only be done
when the scheduler isn't running.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Set general options
config = combine_opts(gconfig, 'apscheduler.', options)
self.misfire_grace_time = int(config.pop('misfire_grace_time', 1))
self.coalesce = asbool(config.pop('coalesce', True))
self.daemonic = asbool(config.pop('daemonic', True))
# Configure the thread pool
if 'threadpool' in config:
self._threadpool = maybe_ref(config['threadpool'])
else:
threadpool_opts = combine_opts(config, 'threadpool.')
self._threadpool = ThreadPool(**threadpool_opts)
# Configure job stores
jobstore_opts = combine_opts(config, 'jobstore.')
jobstores = {}
for key, value in jobstore_opts.items():
store_name, option = key.split('.', 1)
opts_dict = jobstores.setdefault(store_name, {})
opts_dict[option] = value
for alias, opts in jobstores.items():
classname = opts.pop('class')
cls = maybe_ref(classname)
jobstore = cls(**opts)
self.add_jobstore(jobstore, alias, True)
def start(self):
"""
Starts the scheduler in a new thread.
"""
if self.running:
raise SchedulerAlreadyRunningError
# Create a RAMJobStore as the default if there is no default job store
if not 'default' in self._jobstores:
self.add_jobstore(RAMJobStore(), 'default', True)
# Schedule all pending jobs
for job, jobstore in self._pending_jobs:
self._real_add_job(job, jobstore, False)
del self._pending_jobs[:]
self._stopped = False
self._thread = Thread(target=self._main_loop, name='APScheduler')
self._thread.setDaemon(self.daemonic)
self._thread.start()
def shutdown(self, wait=True, shutdown_threadpool=True):
"""
Shuts down the scheduler and terminates the thread.
Does not interrupt any currently running jobs.
:param wait: ``True`` to wait until all currently executing jobs have
finished (if ``shutdown_threadpool`` is also ``True``)
:param shutdown_threadpool: ``True`` to shut down the thread pool
"""
if not self.running:
return
self._stopped = True
self._wakeup.set()
# Shut down the thread pool
if shutdown_threadpool:
self._threadpool.shutdown(wait)
# Wait until the scheduler thread terminates
self._thread.join()
@property
def running(self):
return not self._stopped and self._thread and self._thread.isAlive()
def add_jobstore(self, jobstore, alias, quiet=False):
"""
Adds a job store to this scheduler.
:param jobstore: job store to be added
:param alias: alias for the job store
:param quiet: True to suppress scheduler thread wakeup
:type jobstore: instance of
:class:`~apscheduler.jobstores.base.JobStore`
:type alias: str
"""
self._jobstores_lock.acquire()
try:
if alias in self._jobstores:
raise KeyError('Alias "%s" is already in use' % alias)
self._jobstores[alias] = jobstore
jobstore.load_jobs()
finally:
self._jobstores_lock.release()
# Notify listeners that a new job store has been added
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_ADDED, alias))
# Notify the scheduler so it can scan the new job store for jobs
if not quiet:
self._wakeup.set()
def remove_jobstore(self, alias):
"""
Removes the job store by the given alias from this scheduler.
:type alias: str
"""
self._jobstores_lock.acquire()
try:
try:
del self._jobstores[alias]
except KeyError:
raise KeyError('No such job store: %s' % alias)
finally:
self._jobstores_lock.release()
# Notify listeners that a job store has been removed
self._notify_listeners(JobStoreEvent(EVENT_JOBSTORE_REMOVED, alias))
def add_listener(self, callback, mask=EVENT_ALL):
"""
Adds a listener for scheduler events. When a matching event occurs,
``callback`` is executed with the event object as its sole argument.
If the ``mask`` parameter is not provided, the callback will receive
events of all types.
:param callback: any callable that takes one argument
:param mask: bitmask that indicates which events should be listened to
"""
self._listeners_lock.acquire()
try:
self._listeners.append((callback, mask))
finally:
self._listeners_lock.release()
def remove_listener(self, callback):
"""
Removes a previously added event listener.
"""
self._listeners_lock.acquire()
try:
for i, (cb, _) in enumerate(self._listeners):
if callback == cb:
del self._listeners[i]
finally:
self._listeners_lock.release()
def _notify_listeners(self, event):
self._listeners_lock.acquire()
try:
listeners = tuple(self._listeners)
finally:
self._listeners_lock.release()
for cb, mask in listeners:
if event.code & mask:
try:
cb(event)
except:
logger.exception('Error notifying listener')
def _real_add_job(self, job, jobstore, wakeup):
job.compute_next_run_time(datetime.now())
if not job.next_run_time:
raise ValueError('Not adding job since it would never be run')
self._jobstores_lock.acquire()
try:
try:
store = self._jobstores[jobstore]
except KeyError:
raise KeyError('No such job store: %s' % jobstore)
store.add_job(job)
finally:
self._jobstores_lock.release()
# Notify listeners that a new job has been added
event = JobStoreEvent(EVENT_JOBSTORE_JOB_ADDED, jobstore, job)
self._notify_listeners(event)
logger.info('Added job "%s" to job store "%s"', job, jobstore)
# Notify the scheduler about the new job
if wakeup:
self._wakeup.set()
def add_job(self, trigger, func, args, kwargs, jobstore='default',
**options):
"""
Adds the given job to the job list and notifies the scheduler thread.
:param trigger: alias of the job store to store the job in
:param func: callable to run at the given time
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param jobstore: alias of the job store to store the job in
:rtype: :class:`~apscheduler.job.Job`
"""
job = Job(trigger, func, args or [], kwargs or {},
options.pop('misfire_grace_time', self.misfire_grace_time),
options.pop('coalesce', self.coalesce), **options)
if not self.running:
self._pending_jobs.append((job, jobstore))
logger.info('Adding job tentatively -- it will be properly '
'scheduled when the scheduler starts')
else:
self._real_add_job(job, jobstore, True)
return job
def _remove_job(self, job, alias, jobstore):
jobstore.remove_job(job)
# Notify listeners that a job has been removed
event = JobStoreEvent(EVENT_JOBSTORE_JOB_REMOVED, alias, job)
self._notify_listeners(event)
logger.info('Removed job "%s"', job)
def add_date_job(self, func, date, args=None, kwargs=None, **options):
"""
Schedules a job to be completed on a specific date and time.
:param func: callable to run at the given time
:param date: the date/time to run the job at
:param name: name of the job
:param jobstore: stored the job in the named (or given) job store
:param misfire_grace_time: seconds after the designated run time that
the job is still allowed to be run
:type date: :class:`datetime.date`
:rtype: :class:`~apscheduler.job.Job`
"""
trigger = SimpleTrigger(date)
return self.add_job(trigger, func, args, kwargs, **options)
def add_interval_job(self, func, weeks=0, days=0, hours=0, minutes=0,
seconds=0, start_date=None, args=None, kwargs=None,
**options):
"""
Schedules a job to be completed on specified intervals.
:param func: callable to run
:param weeks: number of weeks to wait
:param days: number of days to wait
:param hours: number of hours to wait
:param minutes: number of minutes to wait
:param seconds: number of seconds to wait
:param start_date: when to first execute the job and start the
counter (default is after the given interval)
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param name: name of the job
:param jobstore: alias of the job store to add the job to
:param misfire_grace_time: seconds after the designated run time that
the job is still allowed to be run
:rtype: :class:`~apscheduler.job.Job`
"""
interval = timedelta(weeks=weeks, days=days, hours=hours,
minutes=minutes, seconds=seconds)
trigger = IntervalTrigger(interval, start_date)
return self.add_job(trigger, func, args, kwargs, **options)
def add_cron_job(self, func, year='*', month='*', day='*', week='*',
day_of_week='*', hour='*', minute='*', second='*',
start_date=None, args=None, kwargs=None, **options):
"""
Schedules a job to be completed on times that match the given
expressions.
:param func: callable to run
:param year: year to run on
:param month: month to run on (0 = January)
:param day: day of month to run on
:param week: week of the year to run on
:param day_of_week: weekday to run on (0 = Monday)
:param hour: hour to run on
:param second: second to run on
:param args: list of positional arguments to call func with
:param kwargs: dict of keyword arguments to call func with
:param name: name of the job
:param jobstore: alias of the job store to add the job to
:param misfire_grace_time: seconds after the designated run time that
the job is still allowed to be run
:return: the scheduled job
:rtype: :class:`~apscheduler.job.Job`
"""
trigger = CronTrigger(year=year, month=month, day=day, week=week,
day_of_week=day_of_week, hour=hour,
minute=minute, second=second,
start_date=start_date)
return self.add_job(trigger, func, args, kwargs, **options)
def cron_schedule(self, **options):
"""
Decorator version of :meth:`add_cron_job`.
This decorator does not wrap its host function.
Unscheduling decorated functions is possible by passing the ``job``
attribute of the scheduled function to :meth:`unschedule_job`.
"""
def inner(func):
func.job = self.add_cron_job(func, **options)
return func
return inner
def interval_schedule(self, **options):
"""
Decorator version of :meth:`add_interval_job`.
This decorator does not wrap its host function.
Unscheduling decorated functions is possible by passing the ``job``
attribute of the scheduled function to :meth:`unschedule_job`.
"""
def inner(func):
func.job = self.add_interval_job(func, **options)
return func
return inner
def get_jobs(self):
"""
Returns a list of all scheduled jobs.
:return: list of :class:`~apscheduler.job.Job` objects
"""
self._jobstores_lock.acquire()
try:
jobs = []
for jobstore in itervalues(self._jobstores):
jobs.extend(jobstore.jobs)
return jobs
finally:
self._jobstores_lock.release()
def unschedule_job(self, job):
"""
Removes a job, preventing it from being run any more.
"""
self._jobstores_lock.acquire()
try:
for alias, jobstore in iteritems(self._jobstores):
if job in list(jobstore.jobs):
self._remove_job(job, alias, jobstore)
return
finally:
self._jobstores_lock.release()
raise KeyError('Job "%s" is not scheduled in any job store' % job)
def unschedule_func(self, func):
"""
Removes all jobs that would execute the given function.
"""
found = False
self._jobstores_lock.acquire()
try:
for alias, jobstore in iteritems(self._jobstores):
for job in list(jobstore.jobs):
if job.func == func:
self._remove_job(job, alias, jobstore)
found = True
finally:
self._jobstores_lock.release()
if not found:
raise KeyError('The given function is not scheduled in this '
'scheduler')
def print_jobs(self, out=None):
"""
Prints out a textual listing of all jobs currently scheduled on this
scheduler.
:param out: a file-like object to print to (defaults to **sys.stdout**
if nothing is given)
"""
out = out or sys.stdout
job_strs = []
self._jobstores_lock.acquire()
try:
for alias, jobstore in iteritems(self._jobstores):
job_strs.append('Jobstore %s:' % alias)
if jobstore.jobs:
for job in jobstore.jobs:
job_strs.append(' %s' % job)
else:
job_strs.append(' No scheduled jobs')
finally:
self._jobstores_lock.release()
out.write(os.linesep.join(job_strs))
def _run_job(self, job, run_times):
"""
Acts as a harness that runs the actual job code in a thread.
"""
for run_time in run_times:
# See if the job missed its run time window, and handle possible
# misfires accordingly
difference = datetime.now() - run_time
grace_time = timedelta(seconds=job.misfire_grace_time)
if difference > grace_time:
# Notify listeners about a missed run
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._notify_listeners(event)
logger.warning('Run time of job "%s" was missed by %s',
job, difference)
else:
try:
job.add_instance()
except MaxInstancesReachedError:
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._notify_listeners(event)
logger.warning('Execution of job "%s" skipped: '
'maximum number of running instances '
'reached (%d)', job, job.max_instances)
break
logger.info('Running job "%s" (scheduled at %s)', job,
run_time)
try:
retval = job.func(*job.args, **job.kwargs)
except:
# Notify listeners about the exception
exc, tb = sys.exc_info()[1:]
event = JobEvent(EVENT_JOB_ERROR, job, run_time,
exception=exc, traceback=tb)
self._notify_listeners(event)
logger.exception('Job "%s" raised an exception', job)
else:
# Notify listeners about successful execution
event = JobEvent(EVENT_JOB_EXECUTED, job, run_time,
retval=retval)
self._notify_listeners(event)
logger.info('Job "%s" executed successfully', job)
job.remove_instance()
# If coalescing is enabled, don't attempt any further runs
if job.coalesce:
break
def _process_jobs(self, now):
"""
Iterates through jobs in every jobstore, starts pending jobs
and figures out the next wakeup time.
"""
next_wakeup_time = None
self._jobstores_lock.acquire()
try:
for alias, jobstore in iteritems(self._jobstores):
for job in tuple(jobstore.jobs):
run_times = job.get_run_times(now)
if run_times:
self._threadpool.submit(self._run_job, job, run_times)
# Increase the job's run count
if job.coalesce:
job.runs += 1
else:
job.runs += len(run_times)
# Update the job, but don't keep finished jobs around
if job.compute_next_run_time(now + timedelta(microseconds=1)):
jobstore.update_job(job)
else:
self._remove_job(job, alias, jobstore)
if not next_wakeup_time:
next_wakeup_time = job.next_run_time
elif job.next_run_time:
next_wakeup_time = min(next_wakeup_time,
job.next_run_time)
return next_wakeup_time
finally:
self._jobstores_lock.release()
def _main_loop(self):
"""Executes jobs on schedule."""
logger.info('Scheduler started')
self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_START))
self._wakeup.clear()
while not self._stopped:
logger.debug('Looking for jobs to run')
now = datetime.now()
next_wakeup_time = self._process_jobs(now)
# Sleep until the next job is scheduled to be run,
# a new job is added or the scheduler is stopped
if next_wakeup_time is not None:
wait_seconds = time_difference(next_wakeup_time, now)
logger.debug('Next wakeup is due at %s (in %f seconds)',
next_wakeup_time, wait_seconds)
self._wakeup.wait(wait_seconds)
else:
logger.debug('No jobs; waiting until a job is added')
self._wakeup.wait()
self._wakeup.clear()
logger.info('Scheduler has been shut down')
self._notify_listeners(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN))
def _run_jobs(self, core):
logger.debug("Starting patched threadpool worker!")
with app.app_context():
ThreadPool._old_run_jobs(self, core)
class Scheduler(object):
def __init__(self, **options):
self._wakeup = Event()
self._jobstore = None
self._threadpool = None
self._jobstats = None
self._jobsync = None
self._jobsync_lock = Lock()
self._jobstats_lock = Lock()
self._jobstore_lock = Lock()
self._listeners = []
self._listeners_lock = Lock()
self.configure(**options)
def configure(self, **options):
try:
redis = StrictRedis(**settings.REDISES["default"])
redis2 = StrictRedis(**settings.REDISES["default"])
redis3 = StrictRedis(**settings.REDISES["default"])
except:
logging.exception("cannot connect to redis")
raise
self._timezone = gettz("Asia/Chongqing")
self._threadpool = ThreadPool()
self._jobstore = JobStore(redis, self._timezone) # todo Jobstore
self._jobstats = JobStats(redis2)
self._jobsync = JobSync(redis3)
self.misfire_grace_time = 1
self.coalesce = True
self.daemonic = True
def start(self):
logger.info("Scheduler is starting...")
with self._jobstore_lock:
self._jobstore.load_jobs(datetime.now(self._timezone))
self._stopped = False
self._thread = Thread(target=self._main_loop, name="APScheduler")
self._thread.setDaemon(True)
logger.info("start main loop thread")
self._thread.start()
self._sync_thread = Thread(target=self._sync_jobs, name="JobsSync")
self._sync_thread.setDaemon(True)
logger.info("start job sync thread")
self._sync_thread.start()
def shutdown(self, wait=True, shutdown_threadpool=True, close_jobstore=True):
if not self.running:
return
logger.info("Scheduler is stopping")
self._stopped = True
self._wakeup.set()
self._threadpool.shutdown(wait)
if self._sync_thread:
self._sync_thread.join()
if self._thread:
self._thread.join()
if close_jobstore:
self._jobstore.close()
logger.info("Scheduler is stoped")
@property
def running(self):
return not self._stopped and self._thread and self._thread.isAlive()
def _update_job(self, job, change_type):
with self._jobstore_lock:
if change_type == "add":
ret = self._jobstore.add_job(job)
elif change_type == "remove":
ret = self._jobstore.remove_job(job)
elif change_type == "update":
ret = self._jobstore.update_job(job)
else:
ret = "invalid change_type %s" % change_type
def _sync_jobs(self):
while not self._stopped:
if self._jobsync.has_notifier():
while True:
now = datetime.now(self._timezone)
job_id, change_type, job = self._jobsync.pop(now)
if job_id == 0:
break
logger.debug("pop %d|%s" % (job_id, change_type))
self._update_job(job, change_type)
self._wakeup.set()
time.sleep(3)
def _main_loop(self):
logger.info("Scheduler is started")
self._fire(SchedulerEvent(EVENT_SCHEDULER_START))
self._wakeup.clear()
while not self._stopped:
now = datetime.now(self._timezone)
next_wakeup_time = self._process_jobs(now)
if next_wakeup_time is not None:
wait_seconds = time_difference(next_wakeup_time, now)
logger.debug("Next wakeup is due at %s (in %f seconds)", next_wakeup_time, wait_seconds)
self._wakeup.wait(wait_seconds)
self._wakeup.clear()
else:
logger.debug("No jobs; waiting until a job is added")
self._wakeup.wait()
self._wakeup.clear()
logger.info("Scheduler has been shut down")
self._fire(SchedulerEvent(EVENT_SCHEDULER_SHUTDOWN))
def _run_job(self, job, run_times):
for run_time in run_times:
difference = datetime.now(self._timezone) - run_time
grace_time = timedelta(seconds=job.misfire_grace_time)
if difference > grace_time:
# Notify listeners about a missed run
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._fire(event)
with self._jobstats_lock:
self._jobstats.miss(job.id)
logger.warning('Run time of job "%s" was missed by %s', job, difference)
else:
try:
job.add_instance()
except MaxInstancesReachedError:
event = JobEvent(EVENT_JOB_MISSED, job, run_time)
self._fire(event)
with self._jobstats_lock:
self._jobstats.miss(job.id)
logger.warning(
'Execution of job "%s" skipped: maximum number of running instances reached (%d)',
job,
job.max_instances,
)
break
logger.info('1Running job "%s" (scheduled at %s)', job, run_time)
try:
retval = job.run()
except:
exc, tb = sys.exec_info()[1:]
with self._jobstats_lock:
self._jobstats.fail(job.id)
event = JobEvent(EVENT_JOB_ERROR, job, run_time, exception=exc, traceback=tb)
self._fire(event)
logger.exception('Job "%s" raised an exception', job)
else:
with self._jobstats_lock:
self._jobstats.succ(job.id)
event = JobEvent(EVENT_JOB_EXECUTED, job, run_time, retval=retval)
self._fire(event)
logger.info('Job "%s" executed successfully', job)
logger.info('2runned job "%s" (scheduled at %s)', job, run_time)
job.remove_instance()
if job.coalesce:
break
def _process_jobs(self, now):
logger.debug("processing jobs")
next_wakeup_time = None
with self._jobstore_lock:
jobs = tuple(self._jobstore.jobs)
for job in jobs:
run_times = job.get_run_times(now)
if run_times:
self._threadpool.submit(self._run_job, job, run_times)
if job.compute_next_run_time(now + timedelta(microseconds=1)):
with self._jobstore_lock:
self._jobstore.update_job(job)
else:
with self._jobstore_lock:
self._jobstore.remove_job(job)
if not next_wakeup_time:
next_wakeup_time = job.next_run_time
elif job.next_run_time:
next_wakeup_time = min(next_wakeup_time, job.next_run_time)
logger.debug("processing jobs end")
return next_wakeup_time
def add_listener(self, callback, mask=EVENT_ALL):
"""
添加事件监听器
"""
with self._listeners_lock:
self._listeners.append((callback, mask))
def remove_listener(self, callback):
"""
移除事件监听器
"""
with self._listeners_lock:
for i, (cb, _) in enumerate(self._listeners):
if callback == cb:
del self._listeners[i]
def _fire(self, event):
"""
事件分发
"""
with self._listeners_lock:
listeners = tuple(self._listeners)
for cb, mask in listeners:
if event.code & mask:
try:
cb(event)
except:
logger.exception("Error notifying listener")
