示例#1
0
def setup_schedule_config(self, endpoint_explorer, node_address):
    scheduler = GeventScheduler()
    scheduler.add_job(
        lambda: notice_explorer_to_be_alive(endpoint_explorer, node_address),
        'interval',
        minutes=30)
    scheduler.start()
示例#2
0
def delayed_finish():
    final_sched = Scheduler()
    final_sched.start()
    now = datetime.today()
    deltaFinal = timedelta( seconds = final_wait )
    starttime = now + deltaFinal
    final_sched.add_job( final_finish, 'date', run_date = starttime, args= [ ] )
示例#3
0
文件: setup.py 项目: virrius/lumino
def setup_schedule_config(self, endpoint_explorer, discoverable, node_address,
                          raiden_instance):
    scheduler = GeventScheduler()
    scheduler.add_job(lambda: notice_explorer_to_be_alive(
        endpoint_explorer, discoverable, node_address, raiden_instance),
                      'interval',
                      minutes=30)
    scheduler.start()
示例#4
0
class ScheduledEventProducer(EventProducer):
    '''
    Desciption:
        Continuously generates an event based on a defined interval, or if an event is consumed

    Parameters:
        producers (Optional[int]):
            | The number of greenthreads to spawn that each spawn events at the provided interval
            | Default: 1
        interval (Optional[float] OR dict):
            | The interval (in seconds) between each generated event.
            | Should have a value > 0.
            | Can also be a dict, supporting values of weeks, days, hours, minutes, and seconds
            | default: 5
        delay (Optional[float]):
            | The time (in seconds) to wait before initial event generation.
            | Default: 0
        interval_grace_time (Optional[int]):
            | Sometimes the scheduler can fail to wakeup and execute a job right at the set interval. This is how much
            | time (in seconds) the scheduler can miss the interval time by and the job will still be run.
            | Default: None (will use apscheduler's default miss_grace_time of 1 second)
    '''
    '''
    MIXIN Attributes:
        DEFAULT_INTERVAL = {}
        def _parse_interval(self, interval):
        def _initialize_jobs(self):
    '''
    def __init__(self, name, *args, **kwargs):
        super(ScheduledEventProducer, self).__init__(name, *args, **kwargs)
        self._init_scheduler(*args, **kwargs)

    def _init_scheduler(self,
                        producers=1,
                        interval=5,
                        delay=0,
                        scheduler=None,
                        interval_grace_time=None,
                        *args,
                        **kwargs):
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.producers = producers
        self.scheduler = scheduler
        self.interval_grace_time = interval_grace_time
        if not self.scheduler:
            self.scheduler = GeventScheduler(
                misfire_grace_time=self.interval_grace_time)

    def pre_hook(self):
        #super(ScheduledEventProducer, self).pre_hook()
        self._initialize_jobs()
        gevent.sleep(self.delay)
        self.scheduler.start()

    def post_hook(self):
        #super(ScheduledEventProducer, self).post_hook()
        self.scheduler.shutdown()
示例#5
0
def scheduler_loop(arg):
    global sched
    sched = Scheduler()
    sched.start()
    logging.basicConfig()
    scheduler_main("")
    while True:
        sleep( 1 )
        sys.stdout.write( '.' )
        sys.stdout.flush()
示例#6
0
def scheduler_loop(arg):
    global sched
    ready_for_scheduler.wait()
    print("STARTING SCHEDULER.")
    sched = Scheduler()
    sched.start()
    logging.basicConfig()
    scheduler_main("")
    ready_for_queue.set()
    while True:
        sys.stdout.write( '.' )
        sys.stdout.flush()
        gevent.sleep(1)
示例#7
0
class ScheduledEventProducer(EventProducer):
    '''
    Desciption:
        Continuously generates an event based on a defined interval, or if an event is consumed

    Parameters:
        producers (Optional[int]):
            | The number of greenthreads to spawn that each spawn events at the provided interval
            | Default: 1
        interval (Optional[float] OR dict):
            | The interval (in seconds) between each generated event.
            | Should have a value > 0.
            | Can also be a dict, supporting values of weeks, days, hours, minutes, and seconds
            | default: 5
        delay (Optional[float]):
            | The time (in seconds) to wait before initial event generation.
            | Default: 0
        interval_grace_time (Optional[int]):
            | Sometimes the scheduler can fail to wakeup and execute a job right at the set interval. This is how much
            | time (in seconds) the scheduler can miss the interval time by and the job will still be run.
            | Default: None (will use apscheduler's default miss_grace_time of 1 second)
    '''

    '''
    MIXIN Attributes:
        DEFAULT_INTERVAL = {}
        def _parse_interval(self, interval):
        def _initialize_jobs(self):
    '''
    def __init__(self, name, *args, **kwargs):
        super(ScheduledEventProducer, self).__init__(name, *args, **kwargs)
        self._init_scheduler(*args, **kwargs)

    def _init_scheduler(self, producers=1, interval=5, delay=0, scheduler=None, interval_grace_time=None, *args, **kwargs):
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.producers = producers
        self.scheduler = scheduler
        self.interval_grace_time = interval_grace_time
        if not self.scheduler:
            self.scheduler = GeventScheduler(misfire_grace_time=self.interval_grace_time)

    def pre_hook(self):
        #super(ScheduledEventProducer, self).pre_hook()
        self._initialize_jobs()
        gevent.sleep(self.delay)
        self.scheduler.start()

    def post_hook(self):
        #super(ScheduledEventProducer, self).post_hook()
        self.scheduler.shutdown()
class MockedSchedulingActor(PrepClazz):
    def __init__(self, name, producers=1, interval=5, delay=0, scheduler=None, *args, **kwargs):
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.producers = producers
        self.scheduler = scheduler
        if not self.scheduler:
            self.scheduler = GeventScheduler()

    def _do_produce(self):
        self.mock_do_produce()

    def pre_hook(self):
        self.scheduler.start()
示例#9
0
class TaskManager:
    """Simple wrapper for Advanced Python Scheduler"""
    def __init__(self):
        print "init Task Manager"
        self.logger = logging.getLogger('dls')
        executors = {
            'default': ThreadPoolExecutor(config.EXECUTOR_THREADS_NUMBER),
            'monitor': GeventExecutor(),
        }

        self.scheduler = GeventScheduler(executors=executors)
        self.scheduler.start()

        # Map of tasks for tracking them on UI
        self.tasks = {}
        self.scheduler.add_job(self.report_progress,
                               'interval',
                               seconds=config.JOB_MONITOR_INTERVAL,
                               executor='monitor')

    # Starts new task
    def start_task(self, task):

        self.scheduler.add_job(func=task.execute,
                               misfire_grace_time=config.MISFIRE_GRACE_TIME)
        self.tasks[task.id] = task

    # Kills task by it's ID
    def term_task(self, index):
        task = self.tasks[index]
        task.kill()

    def shutdown(self):
        self.scheduler.shutdown()

    def report_progress(self):
        """Gathers information from task and sends to clients"""
        # self.logger.info("sending tasks progress")
        task_data = []
        for t in self.tasks.values():
            task_data.append(t.status())
        # from pprint import pprint
        # pprint (task_data)
        socketio.emit('task_monitor', json.dumps(task_data))
        return task_data

    def task_info(self, id):
        t = self.tasks[int(id)]
        return t.detailed_status()
示例#10
0
def register_schedule(minutes=0):
    minutes = minutes or 60
    scheduler = GeventScheduler()
    func = ansible_inventory_patrol
    name = func.__name__
    job_id = '5db150f3e3f7e0677091329f'
    if scheduler.state != 1:
        scheduler.start()
    job = scheduler.get_job(job_id=job_id)
    if not job:
        scheduler.add_job(func=func,
                          trigger='interval',
                          minutes=minutes,
                          name=name,
                          id=job_id)
    def run(self):
        """Start apscheduler tasks"""
        jobstores = {'mongo': MongoDBJobStore()}

        executors = {
            'default': ThreadPoolExecutor(self.poolsize),
            'processpool': ProcessPoolExecutor(self.procsize)
        }

        job_defaults = {'coalesce': False, 'max_instances': 3}

        scheduler = GeventScheduler()
        scheduler.configure(jobstores=jobstores,
                            executors=executors,
                            job_defaults=job_defaults,
                            timezone=utc)
        scheduler.add_job(self.job_worker, 'interval', seconds=0.001)

        green_let = scheduler.start()
        print('Ctrl+{0} to exit.'.format('Break' if os.name == 'nt' else 'C'))

        # Execution will block here util Ctrl+C (Ctrl+Break on Windows).
        try:
            green_let.join()
        except (KeyboardInterrupt, SystemExit):
            pass
示例#12
0
class ScheduledEventProducer(EventProducer):
    '''
    Desciption:
        Continuously generates an event based on a defined interval, or if an event is consumed

    Parameters:
        producers (Optional[int]):
            | The number of greenthreads to spawn that each spawn events at the provided interval
            | Default: 1
        interval (Optional[float] OR dict):
            | The interval (in seconds) between each generated event.
            | Should have a value > 0.
            | Can also be a dict, supporting values of weeks, days, hours, minutes, and seconds
            | default: 5
        delay (Optional[float]):
            | The time (in seconds) to wait before initial event generation.
            | Default: 0
    '''

    '''
    MIXIN Attributes:
        DEFAULT_INTERVAL = {}
        def _parse_interval(self, interval):
        def _initialize_jobs(self):
    '''
    def __init__(self, name, *args, **kwargs):
        super(ScheduledEventProducer, self).__init__(name, *args, **kwargs)
        self._init_scheduler(*args, **kwargs)

    def _init_scheduler(self, producers=1, interval=5, delay=0, scheduler=None, *args, **kwargs):
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.producers = producers
        self.scheduler = scheduler
        if not self.scheduler:
            self.scheduler = GeventScheduler()

    def pre_hook(self):
        #super(ScheduledEventProducer, self).pre_hook()
        self._initialize_jobs()
        gevent.sleep(self.delay)
        self.scheduler.start()

    def post_hook(self):
        #super(ScheduledEventProducer, self).post_hook()
        self.scheduler.shutdown()
示例#13
0
 def deactivate_player(self, player):
     player.active = False
     scheduler = GeventScheduler()
     timeout_date = datetime.now() + timedelta(seconds=PLAYER_TIMEOUT)
     scheduler.add_job(self.delete_player,
                       'date',
                       run_date=timeout_date,
                       args=[player])
     g = scheduler.start()
     g.join()
示例#14
0
def GeventScheduler_test():
    sched = GeventScheduler()
    sched.add_job(tick, 'interval', seconds=3)
    g = sched.start()
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
def startScheduledExecution(interval):
    scheduler = GeventScheduler()
    fetchAndSaveData()
    scheduler.add_job(fetchAndSaveData, 'interval', minutes=interval)
    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#16
0
class MockedSchedulingActor(PrepClazz):
    def __init__(self,
                 name,
                 producers=1,
                 interval=5,
                 delay=0,
                 scheduler=None,
                 *args,
                 **kwargs):
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.producers = producers
        self.scheduler = scheduler
        if not self.scheduler:
            self.scheduler = GeventScheduler()

    def _do_produce(self):
        self.mock_do_produce()

    def pre_hook(self):
        self.scheduler.start()
示例#17
0
class Scheduler:

    def __init__(self, app=None):
        self._scheduler = GeventScheduler()
        self.jobs_list = [monitor_ac_usage, monitor_temperatures]
        if app:
            self.init_app(app)

    def init_app(self, app):
        for job in self.jobs_list:
            self.add_jobs(app, job)

    def add_jobs(self, app, job):
        def call_func(*args, **kwargs):
            with app.app_context():
                func(*args, **kwargs)
        func = job

        self._scheduler.add_job(call_func, 'interval', minutes=func.minutes, coalesce=True)

    def start(self):
        self._scheduler.start()
示例#18
0
def watch():
    scheduler = GeventScheduler()

    for web in watchweb.get_watch_webs():
        s = int(web[watchweb.INTERVAL_SECONDS])
        scheduler.add_job(check_web, "interval", seconds=s, kwargs=web)

    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print ("Press Ctrl+{0} to exit".format("Break" if os.name == "nt" else "C"))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#19
0
class Schedule:
    def __init__(self, irc_c, config):
        self.config = config
        self.scheduler = GeventScheduler()

        self._scheduler_greenlet = self.scheduler.start()

        log_propagation_message = partial(
            irc_c.PRIVMSG,
            CONFIG.external['propagation']['logging']['channel'],
        )

        # Scheduled full wiki update
        self.scheduler.add_job(
            Propagate.get_wiki_data,
            'cron',
            kwargs={'reply': log_propagation_message},
            **self.cron_to_kwargs(
                CONFIG.external['propagation']['all_articles']['often']),
        )

        # Scheduled recent pages update
        self.scheduler.add_job(
            Propagate.get_wiki_data,
            'cron',
            kwargs={
                'reply': log_propagation_message,
                'seconds': 259200
            },
            **self.cron_to_kwargs(
                CONFIG.external['propagation']['new_articles']['often']),
        )

    @staticmethod
    def cron_to_kwargs(cronstring):
        """Converts a cron string to cron kwargs"""
        crons = cronstring.split(" ")
        if len(crons) != 5:
            raise ValueError("Invalid cron {}".format(cronstring))
        crons = [cron.replace("_", " ") for cron in crons]
        kwargs = {
            'minute': crons[0],
            'hour': crons[1],
            'day': crons[2],
            'month': crons[3],
            'day_of_week': crons[4],
        }
        return kwargs
示例#20
0
def gevent_schedule():
    from apscheduler.schedulers.gevent import GeventScheduler

    def tick():
        print('Tick! The time is: %s' % datetime.now())

    scheduler = GeventScheduler()
    scheduler.add_job(tick, 'interval', seconds=3)
    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#21
0
def gevent_schedule():
    from apscheduler.schedulers.gevent import GeventScheduler

    def tick():
        print('Tick! The time is: %s' % datetime.now())

    scheduler = GeventScheduler()
    scheduler.add_job(tick, 'interval', seconds=3)
    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#22
0
def watch():
	scheduler = GeventScheduler()
	
	for web in watchweb.get_watch_webs():
		s = int(web[watchweb.INTERVAL_SECONDS])
		scheduler.add_job(check_web, 'interval', seconds=s,kwargs=web)


	g = scheduler.start()  # g is the greenlet that runs the scheduler loop
	print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

	# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
	try:
		g.join()
	except (KeyboardInterrupt, SystemExit):
		pass
示例#23
0
def production(*_, **settings):
    """Hooks exceptions and returns the Flask app."""

    hook_exceptions()

    app.shiptoasts = ShipToasts()
    app.shiptoasts.initial_fill()

    scheduler = GeventScheduler()
    scheduler.add_job(app.shiptoasts.periodic_call, "interval", seconds=30)
    cleaner = scheduler.start()
    listener = gevent.Greenlet.spawn(app.shiptoasts.listen_for_updates)

    atexit.register(cleaner.join, timeout=2)
    atexit.register(listener.join, timeout=2)
    atexit.register(scheduler.shutdown)

    return app
示例#24
0
def main():

    scheduler = GeventScheduler()

    url = os.environ.get('SQLALCHEMY_DATABASE_URI', 'sqlite:///database.db')

    scheduler.add_jobstore('sqlalchemy', url=url)

    scheduler.add_job(tick, 'interval', seconds=3, id='example_job', replace_existing=True)

    # g is the greenlet that runs the scheduler loop.
    g = scheduler.start()

    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#25
0
def main():

    scheduler = GeventScheduler()

    url = os.environ.get('SQLALCHEMY_DATABASE_URI', 'sqlite:///database.db')

    scheduler.add_jobstore('sqlalchemy', url=url)

    scheduler.add_job(tick,
                      'interval',
                      seconds=3,
                      id='example_job',
                      replace_existing=True)

    # g is the greenlet that runs the scheduler loop.
    g = scheduler.start()

    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#26
0
class EventGenerator(Actor):

    '''**Generates a test event at the chosen interval.**

    Parameters:

        name (str):
            | The instance name
        event_class (Optional[compysition.event.Event]):
            | The class that the generated event should be created as
            | Default: Event
        event_kwargs (Optional[int]):
            | Any additional kwargs to add to the event, including data
        producers (Optional[int]):
            | The number of greenthreads to spawn that each spawn events at the provided interval
            | Default: 1
        interval (Optional[float] OR dict):
            | The interval (in seconds) between each generated event.
            | Should have a value > 0.
            | Can also be a dict, supporting values of weeks, days, hours, minutes, and seconds
            | default: 5
        delay (Optional[float]):
            | The time (in seconds) to wait before initial event generation.
            | Default: 0
        generate_error (Optional[bool]):
            | Whether or not to also send the event via Actor.send_error
            | Default: False

    '''

    DEFAULT_INTERVAL = {'weeks': 0,
                         'days': 0,
                         'hours': 0,
                         'minutes': 0,
                         'seconds': 5}

    def __init__(self, name, event_class=Event, event_kwargs=None, producers=1, interval=5, delay=0, generate_error=False, *args, **kwargs):
        super(EventGenerator, self).__init__(name, *args, **kwargs)
        self.blockdiag_config["shape"] = "flowchart.input"
        self.generate_error = generate_error
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.event_kwargs = event_kwargs or {}
        self.output = event_class
        self.producers = producers
        self.scheduler = GeventScheduler()

    def _parse_interval(self, interval):
        _interval = self.DEFAULT_INTERVAL

        if isinstance(interval, int):
            _interval['seconds'] = interval
        elif isinstance(interval, dict):
            _interval.update(interval)

        return _interval

    def _initialize_jobs(self):
        for i in xrange(self.producers):
            self.scheduler.add_job(self._do_produce, 'interval', **self.interval)

    def pre_hook(self):
        self._initialize_jobs()
        gevent.sleep(self.delay)
        self.scheduler.start()

    def post_hook(self):
        self.scheduler.shutdown()

    def _do_produce(self):
        event = self.output[0](**self.event_kwargs)
        self.logger.debug("Generated new event {event_id}".format(event_id=event.event_id))
        self.send_event(event)
        if self.generate_error:
            event = self.output(**self.event_kwargs)
            self.send_error(event)

    def consume(self, event, *args, **kwargs):
        self._do_produce()
示例#27
0
class W5Timer(object):
    _instance_lock = threading.Lock()

    def __init__(self):
        self.scheduler = None

    def __new__(cls, *args, **kwargs):
        if not hasattr(W5Timer, "_instance"):
            with W5Timer._instance_lock:
                if not hasattr(W5Timer, "_instance"):
                    W5Timer._instance = object.__new__(cls)
        return W5Timer._instance

    def create_scheduler(self):
        self.scheduler = GeventScheduler()

    def start(self):
        self.scheduler.start()

    def shutdown(self):
        self.scheduler.shutdown()

    def pause(self, uuid):
        self.scheduler.pause_job(uuid)

    def pause_all(self):
        self.scheduler.pause()

    def resume(self, uuid):
        self.scheduler.resume_job(uuid)

    def resume_all(self):
        self.scheduler.resume()

    def remove_job(self, uuid):
        self.scheduler.remove_job(uuid)

    def get_jobs(self):
        return self.scheduler.get_jobs()

    def add_date(self, run_date=None, uuid=None, timer_uuid=None):
        self.scheduler.add_job(
            auto_execute,
            'date',
            run_date=run_date,
            id=timer_uuid,
            args=(uuid,)
        )

    def update_date(self, uuid, run_date=None):
        self.scheduler.reschedule_job(
            uuid,
            trigger='date',
            run_date=run_date
        )

    def add_interval(self, t, interval, uuid=None, timer_uuid=None, start_date=None, end_date=None, jitter=0):
        if t == "seconds":
            self.scheduler.add_job(
                auto_execute,
                'interval',
                seconds=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter,
                id=timer_uuid,
                args=(uuid,)
            )
        elif t == "minutes":
            self.scheduler.add_job(
                auto_execute,
                'interval',
                minutes=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter,
                id=timer_uuid,
                args=(uuid,)
            )
        elif t == "hours":
            self.scheduler.add_job(
                auto_execute,
                'interval',
                hours=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter,
                id=timer_uuid,
                args=(uuid,)
            )
        elif t == "days":
            self.scheduler.add_job(
                auto_execute,
                'interval',
                days=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter,
                id=timer_uuid,
                args=(uuid,)
            )
        elif t == "weeks":
            self.scheduler.add_job(
                auto_execute,
                'interval',
                weeks=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter,
                id=timer_uuid,
                args=(uuid,)
            )

    def update_interval(self, uuid, t, interval, start_date=None, end_date=None, jitter=0):
        if t == "seconds":
            self.scheduler.reschedule_job(
                uuid,
                trigger="interval",
                seconds=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter
            )
        elif t == "minutes":
            self.scheduler.reschedule_job(
                uuid,
                trigger="interval",
                minutes=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter
            )
        elif t == "hours":
            self.scheduler.reschedule_job(
                uuid,
                trigger="interval",
                hours=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter
            )
        elif t == "days":
            self.scheduler.reschedule_job(
                uuid,
                trigger="interval",
                days=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter
            )
        elif t == "weeks":
            self.scheduler.reschedule_job(
                uuid,
                trigger="interval",
                weeks=interval,
                start_date=start_date,
                end_date=end_date,
                jitter=jitter
            )

    def add_cron(self, cron, uuid=None, timer_uuid=None, start_date=None, end_date=None, jitter=0):
        self.scheduler.add_job(
            auto_execute,
            CronTrigger.from_crontab(cron),
            start_date=start_date,
            end_date=end_date,
            jitter=jitter,
            id=timer_uuid,
            args=(uuid,)
        )

    def update_cron(self, uuid, cron, start_date=None, end_date=None, jitter=0):
        values = cron.split()

        if len(values) != 5:
            raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))

        self.scheduler.reschedule_job(
            uuid,
            None,
            "cron",
            minute=values[0],
            hour=values[1],
            day=values[2],
            month=values[3],
            day_of_week=values[4],
            start_date=start_date,
            end_date=end_date,
            jitter=jitter
        )
示例#28
0
class Application(object):

    def __init__(self, config_file, data_dir, verbose=0):
        """
        Construct a new application instance.
        """

        self.config_file = config_file
        self.data_dir = data_dir
        self.verbose = verbose

        self.server = None
        self.provider = None
        self.connections = {}

        # Setup all parts of the application
        self.setup_config()
        self.setup_open_files()
        self.setup_database()
        self.setup_state()
        self.setup_connections()
        self.setup_cache()
        self.setup_provider()
        self.setup_server()
        self.setup_tasks()

    def setup_config(self):
        """
        Load the application config from file.
        """

        logger.debug("Loading config from %s", self.config_file)
        self.config = config.get_config(self.config_file)

    def setup_open_files(self):
        """
        Get and set open files limit.
        """

        open_files_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        new_open_files_limit = self.config["Advanced"]["open files limit"]

        logger.info(
            "System reports open files limit is %d.", open_files_limit)

        if new_open_files_limit != -1:
            logger.info(
                "Changing open files limit to %d.", new_open_files_limit)

            try:
                resource.setrlimit(resource.RLIMIT_NOFILE, (
                    new_open_files_limit, resource.RLIM_INFINITY))
            except resource.error as e:
                logger.warning(
                    "Failed to increase the number of open files: %s", e)

    def setup_database(self):
        """
        Initialize database.
        """

        self.db = Database(self.config["Provider"]["database"])
        self.db.create_database(drop_all=False)

    def setup_state(self):
        """
        Setup state.
        """

        self.state = State(os.path.join(
            self.get_cache_dir(), "provider.state"))

    def setup_cache(self):
        """
        Setup the caches for items and artwork.
        """

        # Initialize caches for items and artwork.
        item_cache = cache.ItemCache(
            path=self.get_cache_dir(
                self.config["Provider"]["item cache dir"]),
            max_size=self.config["Provider"]["item cache size"],
            prune_threshold=self.config[
                "Provider"]["item cache prune threshold"])
        artwork_cache = cache.ArtworkCache(
            path=self.get_cache_dir(self.config[
                "Provider"]["artwork cache dir"]),
            max_size=self.config["Provider"]["artwork cache size"],
            prune_threshold=self.config[
                "Provider"]["artwork cache prune threshold"])

        # Create a cache manager
        self.cache_manager = cache.CacheManager(
            db=self.db,
            item_cache=item_cache,
            artwork_cache=artwork_cache,
            connections=self.connections)

    def setup_connections(self):
        """
        Initialize the connections.
        """

        for name, section in self.config["Connections"].iteritems():
            index = len(self.connections) + 1

            self.connections[index] = Connection(
                db=self.db,
                state=self.state,
                index=index,
                name=name,
                url=section["url"],
                username=section["username"],
                password=section["password"],
                synchronization=section["synchronization"],
                synchronization_interval=section["synchronization interval"],
                transcode=section["transcode"],
                transcode_unsupported=section["transcode unsupported"])

    def setup_provider(self):
        """
        Setup the provider.
        """

        # Create provider.
        logger.debug(
            "Setting up provider for %d connection(s).", len(self.connections))

        self.provider = Provider(
            server_name=self.config["Provider"]["name"],
            db=self.db,
            state=self.state,
            connections=self.connections,
            cache_manager=self.cache_manager)

        # Do an initial synchronization if required.
        for connection in self.connections.itervalues():
            connection.synchronizer.provider = self.provider
            connection.synchronizer.synchronize(initial=True)

    def setup_server(self):
        """
        Create the DAAP server.
        """

        logger.debug(
            "Setting up DAAP server at %s:%d",
            self.config["Daap"]["interface"], self.config["Daap"]["port"])

        self.server = DaapServer(
            provider=self.provider,
            password=self.config["Daap"]["password"],
            ip=self.config["Daap"]["interface"],
            port=self.config["Daap"]["port"],
            cache=self.config["Daap"]["cache"],
            cache_timeout=self.config["Daap"]["cache timeout"] * 60,
            bonjour=self.config["Daap"]["zeroconf"],
            debug=self.verbose > 1)

        # Extend server with a web interface
        if self.config["Daap"]["web interface"]:
            webserver.extend_server_app(self, self.server.app)

    def setup_tasks(self):
        """
        Setup all tasks that run periodically.
        """

        self.scheduler = GeventScheduler()

        # Add an initial job
        def _job():
            job.remove()
            self.synchronize(synchronization="startup")
        job = self.scheduler.add_job(
            _job, max_instances=1, trigger="interval", seconds=1)

        # Scheduler task to clean and expire the cache.
        cache_interval = self.config['Provider']['item cache prune interval']

        self.scheduler.add_job(
            self.cache_manager.expire,
            max_instances=1, trigger="interval", minutes=cache_interval)
        self.scheduler.add_job(
            self.cache_manager.clean,
            max_instances=1, trigger="interval", minutes=cache_interval)

        # Schedule tasks to synchronize each connection.
        for connection in self.connections.itervalues():
            self.scheduler.add_job(
                self.synchronize, args=([connection, "interval"]),
                max_instances=1, trigger="interval",
                minutes=connection.synchronization_interval)

    def synchronize(self, connections=None, synchronization="manual"):
        """
        Synchronize selected connections (or all) given a synchronization
        event.
        """

        count = 0
        connections = connections or self.connections.values()

        logger.debug("Synchronization triggered via '%s'.", synchronization)

        for connection in connections:
            if synchronization == "interval":
                if connection.synchronization == "interval":
                    connection.synchronizer.synchronize()
                    count += 1
            elif synchronization == "startup":
                if connection.synchronization == "startup":
                    if not connection.synchronizer.is_initial_synced:
                        connection.synchronizer.synchronize()
                        count += 1
            elif synchronization == "manual":
                connection.synchronizer.synchronize()
                count += 1

        logger.debug("Synchronized %d connections.", count)

        # Update the cache.
        self.cache_manager.cache()

    def start(self):
        """
        Start the server.
        """

        logger.debug("Starting task scheduler.")
        self.scheduler.start()

        logger.debug("Starting DAAP server.")
        self.server.serve_forever()

    def stop(self):
        """
        Stop the server.
        """

        logger.debug("Stopping DAAP server.")
        self.server.stop()

        logger.debug("Stopping task scheduler.")
        self.scheduler.shutdown()

    def get_cache_dir(self, *path):
        """
        Resolve the path to a cache directory. The path is relative to the data
        directory. The directory will be created if it does not exists, and
        will be tested for writing.
        """

        full_path = os.path.abspath(os.path.normpath(
            os.path.join(self.data_dir, *path)))
        logger.debug("Resolved %s to %s", path, full_path)

        # Create path if required.
        try:
            os.makedirs(full_path, 0755)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(full_path):
                pass
            else:
                raise Exception("Could not create folder: %s" % full_path)

        # Test for writing.
        ok = True
        test_file = os.path.join(full_path, ".write-test")

        while os.path.exists(test_file):
            test_file = test_file + str(random.randint(0, 9))

        try:
            with open(test_file, "w") as fp:
                fp.write("test")
        except IOError:
            ok = False
        finally:
            try:
                os.remove(test_file)
            except OSError:
                ok = False

        if not ok:
            raise Exception("Could not write to cache folder: %s" % full_path)

        # Cache directory created and tested for writing.
        return full_path
示例#29
0
class Controller(object):
    def __init__(self, name="defaultController"):
        self.name = name
        self.workflows = {}
        self.load_all_workflows_from_directory()
        self.instances = {}
        self.tree = None
        self.eventlog = []
        self.schedulerStatusListener = SchedulerStatusListener(self.eventlog)
        self.jobStatusListener = JobStatusListener(self.eventlog)
        self.jobExecutionListener = JobExecutionListener(self.eventlog)
        self.scheduler = GeventScheduler()
        self.scheduler.add_listener(
            self.schedulerStatusListener.callback(self),
            EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
            | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED)
        self.scheduler.add_listener(self.jobStatusListener.callback(self),
                                    EVENT_JOB_ADDED | EVENT_JOB_REMOVED)
        self.scheduler.add_listener(self.jobExecutionListener.callback(self),
                                    EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
        self.ancestry = [self.name]

        # MULTIPROCESSING
        # self.pool = multiprocessing.Pool(processes=5)
        # self.manager = multiprocessing.Manager()
        # self.queue = self.manager.SimpleQueue()

    def load_workflow_from_file(self,
                                path,
                                workflow_name,
                                name_override=None,
                                playbook_override=None):
        self.tree = et.ElementTree(file=path)
        playbook_name = playbook_override if playbook_override else os.path.splitext(
            os.path.basename(path))[0]
        for workflow in self.tree.iter(tag="workflow"):
            current_workflow_name = workflow.get('name')
            if current_workflow_name == workflow_name:
                if name_override:
                    workflow_name = name_override
                name = construct_workflow_name_key(playbook_name,
                                                   workflow_name)
                key = _WorkflowKey(playbook_name, workflow_name)
                self.workflows[key] = wf.Workflow(name=name,
                                                  workflowConfig=workflow,
                                                  parent_name=self.name,
                                                  filename=playbook_name)
                break
        else:
            return False

        self.addChildWorkflows()
        self.addWorkflowScheduledJobs()
        return True

    def loadWorkflowsFromFile(self,
                              path,
                              name_override=None,
                              playbook_override=None):
        self.tree = et.ElementTree(file=path)
        playbook_name = playbook_override if playbook_override else os.path.splitext(
            os.path.basename(path))[0]
        for workflow in self.tree.iter(tag='workflow'):
            workflow_name = name_override if name_override else workflow.get(
                'name')
            name = construct_workflow_name_key(playbook_name, workflow_name)
            key = _WorkflowKey(playbook_name, workflow_name)
            self.workflows[key] = wf.Workflow(name=name,
                                              workflowConfig=workflow,
                                              parent_name=self.name,
                                              filename=playbook_name)
        self.addChildWorkflows()
        self.addWorkflowScheduledJobs()

    def load_all_workflows_from_directory(self, path=config.workflowsPath):
        for workflow in locate_workflows_in_directory(path):
            self.loadWorkflowsFromFile(
                os.path.join(config.workflowsPath, workflow))

    def addChildWorkflows(self):
        for workflow in self.workflows:
            playbook_name = workflow.playbook
            children = self.workflows[workflow].options.children
            for child in children:
                workflow_key = _WorkflowKey(
                    playbook_name,
                    extract_workflow_name(child, playbook_name=playbook_name))
                if workflow_key in self.workflows:
                    children[child] = self.workflows[workflow_key]

    def addWorkflowScheduledJobs(self):
        for workflow in self.workflows:
            if (self.workflows[workflow].options.enabled
                    and self.workflows[workflow].options.scheduler["autorun"]
                    == "true"):
                schedule_type = self.workflows[workflow].options.scheduler[
                    "type"]
                schedule = self.workflows[workflow].options.scheduler["args"]
                self.scheduler.add_job(self.workflows[workflow].execute,
                                       trigger=schedule_type,
                                       replace_existing=True,
                                       **schedule)

    def create_workflow_from_template(self,
                                      playbook_name,
                                      workflow_name,
                                      template_playbook='emptyWorkflow',
                                      template_name='emptyWorkflow'):
        path = '{0}{1}{2}.workflow'.format(config.templatesPath, sep,
                                           template_playbook)
        return self.load_workflow_from_file(path=path,
                                            workflow_name=template_name,
                                            name_override=workflow_name,
                                            playbook_override=playbook_name)

    def create_playbook_from_template(self,
                                      playbook_name,
                                      template_playbook='emptyWorkflow'):
        #TODO: Need a handler for returning workflow key and status
        path = '{0}{1}{2}.workflow'.format(config.templatesPath, sep,
                                           template_playbook)
        self.loadWorkflowsFromFile(path=path, playbook_override=playbook_name)

    def removeWorkflow(self, playbook_name, workflow_name):
        name = _WorkflowKey(playbook_name, workflow_name)
        if name in self.workflows:
            del self.workflows[name]
            return True
        return False

    def remove_playbook(self, playbook_name):
        for name in [
                workflow for workflow in self.workflows
                if workflow.playbook == playbook_name
        ]:
            del self.workflows[name]
            return True
        return False

    def get_all_workflows(self):
        result = {}
        for key in self.workflows.keys():
            if key.playbook not in result:
                result[key.playbook] = []
            result[key.playbook].append(key.workflow)
        return result

    def is_workflow_registered(self, playbook_name, workflow_name):
        return _WorkflowKey(playbook_name, workflow_name) in self.workflows

    def is_playbook_registerd(self, playbook_name):
        return any(workflow_key.playbook == playbook_name
                   for workflow_key in self.workflows)

    def update_workflow_name(self, old_playbook, old_workflow, new_playbook,
                             new_workflow):
        old_key = _WorkflowKey(old_playbook, old_workflow)
        new_key = _WorkflowKey(new_playbook, new_workflow)
        self.workflows[new_key] = self.workflows.pop(old_key)
        self.workflows[new_key].name = construct_workflow_name_key(
            new_playbook, new_workflow)

    def update_playbook_name(self, old_playbook, new_playbook):
        for key in [
                name for name in self.workflows.keys()
                if name.playbook == old_playbook
        ]:
            self.update_workflow_name(old_playbook, key.workflow, new_playbook,
                                      key.workflow)

    # def executeWorkflowWorker(self):
    #
    #     print("Thread " + str(os.getpid()) + " starting up...")
    #
    #     while (True):
    #         while (self.queue.empty()):
    #             continue
    #         name,start,data = self.queue.get()
    #         print("Thread " + str(os.getpid()) + " received and executing workflow "+name)
    #         steps, instances = self.workflows[name].execute(start=start, data=data)

    def executeWorkflow(self,
                        playbook_name,
                        workflow_name,
                        start="start",
                        data=None):
        self.workflows[_WorkflowKey(playbook_name,
                                    workflow_name)].execute(start=start,
                                                            data=data)
        # print("Boss thread putting "+name+" workflow on queue...:")
        # self.queue.put((name, start, data))
        self.jobExecutionListener.execute_event_code(self, 'JobExecuted')

    def get_workflow(self, playbook_name, workflow_name):
        key = _WorkflowKey(playbook_name, workflow_name)
        if key in self.workflows:
            return self.workflows[key]
        return None

    def playbook_to_xml(self, playbook_name):
        workflows = [
            workflow for key, workflow in self.workflows.items()
            if key.playbook == playbook_name
        ]
        if workflows:
            xml = et.Element("workflows")
            for workflow in workflows:
                xml.append(workflow.to_xml())
            return xml
        else:
            return None

    # Starts active execution
    def start(self):
        self.scheduler.start()

    # Stops active execution
    def stop(self, wait=True):
        self.scheduler.shutdown(wait=wait)

    # Pauses active execution
    def pause(self):
        self.scheduler.pause()

    # Resumes active execution
    def resume(self):
        self.scheduler.resume()

    # Pauses active execution of specific job
    def pauseJob(self, job_id):
        self.scheduler.pause_job(job_id=job_id)

    # Resumes active execution of specific job
    def resumeJob(self, job_id):
        self.scheduler.resume_job(job_id=job_id)

    # Returns jobs scheduled for active execution
    def getScheduledJobs(self):
        self.scheduler.get_jobs()
示例#30
0
def setup():
    print('setting up Scheduler')
    scheduler = GeventScheduler()
    scheduler.add_job(func=schedule, trigger='cron', **data.update_cron)
    scheduler.start()
示例#31
0
class EventGenerator(Actor):
    '''**Generates a test event at the chosen interval.**

    Parameters:

        name (str):
            | The instance name
        event_class (Optional[compysition.event.Event]):
            | The class that the generated event should be created as
            | Default: Event
        event_kwargs (Optional[int]):
            | Any additional kwargs to add to the event, including data
        producers (Optional[int]):
            | The number of greenthreads to spawn that each spawn events at the provided interval
            | Default: 1
        interval (Optional[float] OR dict):
            | The interval (in seconds) between each generated event.
            | Should have a value > 0.
            | Can also be a dict, supporting values of weeks, days, hours, minutes, and seconds
            | default: 5
        delay (Optional[float]):
            | The time (in seconds) to wait before initial event generation.
            | Default: 0
        generate_error (Optional[bool]):
            | Whether or not to also send the event via Actor.send_error
            | Default: False

    '''

    DEFAULT_INTERVAL = {
        'weeks': 0,
        'days': 0,
        'hours': 0,
        'minutes': 0,
        'seconds': 5
    }

    def __init__(self,
                 name,
                 event_class=Event,
                 event_kwargs=None,
                 producers=1,
                 interval=5,
                 delay=0,
                 generate_error=False,
                 *args,
                 **kwargs):
        super(EventGenerator, self).__init__(name, *args, **kwargs)
        self.blockdiag_config["shape"] = "flowchart.input"
        self.generate_error = generate_error
        self.interval = self._parse_interval(interval)
        self.delay = delay
        self.event_kwargs = event_kwargs or {}
        self.output = event_class
        self.producers = producers
        self.scheduler = GeventScheduler()

    def _parse_interval(self, interval):
        _interval = self.DEFAULT_INTERVAL

        if isinstance(interval, int):
            _interval['seconds'] = interval
        elif isinstance(interval, dict):
            _interval.update(interval)

        return _interval

    def _initialize_jobs(self):
        for i in xrange(self.producers):
            self.scheduler.add_job(self._do_produce, 'interval',
                                   **self.interval)

    def pre_hook(self):
        self._initialize_jobs()
        gevent.sleep(self.delay)
        self.scheduler.start()

    def post_hook(self):
        self.scheduler.shutdown()

    def _do_produce(self):
        event = self.output[0](**self.event_kwargs)
        self.logger.debug(
            "Generated new event {event_id}".format(event_id=event.event_id))
        self.send_event(event)
        if self.generate_error:
            event = self.output(**self.event_kwargs)
            self.send_error(event)

    def consume(self, event, *args, **kwargs):
        self._do_produce()
class TileScheduler:
    def __init__(self, socketio):
        self.__socketio = socketio
        self.__jobs = {}
        self.__tiles = {}
        self.__cache = {}
        executors = {
            'default': {'type': 'threadpool', 'max_workers': 20}
        }

        self.__scheduler = GeventScheduler(executors=executors)

    @staticmethod
    def get_full_name(pageName: str, tileName: str) -> str:
        return f'{pageName.replace(" ", "_")}_{tileName}'

    def register_tile(self, pageName: str, tile: Tile):
        fullName = self.get_full_name(pageName, tile.get_uniqueName())
        if fullName in self.__jobs:
            LOGGER.warning(f'Tile "{fullName}" already registered')
            return

        seconds = tile.get_intervalInSeconds()
        nextRunTime = datetime.now()
        if seconds == -1:  # disable automatic refresh
            seconds = 9999999999  # 317 years
            nextRunTime = None  # job is paused

        job = self.__scheduler.add_job(tile.update, 'interval',
                                       [pageName],
                                       seconds=seconds,
                                       next_run_time=nextRunTime)

        self.__jobs[fullName] = job
        self.__cache[fullName] = None
        self.__tiles[fullName] = tile
        LOGGER.debug(f'Registered "{fullName}" (scheduled every {tile.get_intervalInSeconds()} seconds)')

    def unregister_tile(self, pageName: str, tile: Tile):
        fullName = self.get_full_name(pageName, tile.get_uniqueName())
        if fullName not in self.__jobs:
            LOGGER.warning(f'Tile "{fullName}" is not registered')

        self.__jobs[fullName].remove()
        del self.__jobs[fullName]
        del self.__cache[fullName]
        del self.__tiles[fullName]
        LOGGER.debug(f'Unregistered "{fullName}"')

    def emit_from_cache(self):
        for fullName, value in self.__cache.items():
            self.__emit_update(fullName, value)

    def run(self):
        def JobListener(event):
            if event.exception:
                LOGGER.error(event.exception)
            else:
                name, value = event.retval
                self.__cache[name] = value
                self.__emit_update(name, value)

        self.__scheduler.add_listener(JobListener, EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
        self.__scheduler.start()

    def __emit_update(self, fullName: str, content: str):
        data = {'fullName': fullName, 'content': content}
        self.__socketio.emit('tileUpdate', json.dumps(data), namespace='/update')

    def get_tiles(self) -> Dict[str, Tile]:
        return self.__tiles

    def get_jobs(self) -> Dict[str, Job]:
        return self.__jobs

    def force_refresh(self, fullName: str):
        job = self.__get_job_by_name(fullName)
        if job is not None:
            LOGGER.debug(f'Manual refresh for tile "{fullName}"')
            job.modify(next_run_time=datetime.now())

    def __get_job_by_name(self, fullName: str) -> Job or None:
        if fullName not in self.__jobs:
            LOGGER.warning(f'Ignoring request to refresh non-existing tile "{fullName}"')
            return None
        return self.__jobs[fullName]
示例#33
0
class AuctionsDataBridge(object):
    """Auctions Data Bridge"""
    def __init__(self, config, activate=False):
        super(AuctionsDataBridge, self).__init__()
        self.config = config
        self.tenders_ids_list = []
        self.activate = activate
        self.client = ApiClient(
            '',
            host_url=self.config_get('tenders_api_server'),
            api_version=self.config_get('tenders_api_version'))
        params = {'opt_fields': 'status,auctionPeriod', 'mode': '_all_'}
        if parse_version(
                self.config_get('tenders_api_version')) > parse_version('0.9'):
            params['opt_fields'] += ',lots'
        self.client.params.update(params)
        self.tz = tzlocal()

        self.couch_url = urljoin(self.config_get('couch_url'),
                                 self.config_get('auctions_db'))
        self.db = Database(self.couch_url,
                           session=Session(retry_delays=range(10)))

        if self.activate:
            self.queue = Queue()
            self.scheduler = GeventScheduler()
            self.scheduler.add_job(self.run_systemd_cmds,
                                   'interval',
                                   max_instances=1,
                                   minutes=2,
                                   id='run_systemd_cmds')
            self.scheduler.start()

    def config_get(self, name):
        return self.config.get('main').get(name)

    def run_systemd_cmds(self):
        auctions = []
        logger.info('Start systemd units activator')
        while True:
            try:
                auctions.append(self.queue.get_nowait())
            except Empty, e:
                break
        if auctions:
            logger.info('Handle systemctl daemon-reload')
            do_until_success(
                check_call,
                (['/usr/bin/systemctl', '--user', 'daemon-reload'], ))
            for planning_data in auctions:
                if len(planning_data) == 1:
                    logger.info('Tender {0} selected for activate'.format(
                        *planning_data))
                    self.start_auction_worker_cmd('activate', planning_data[0])
                elif len(planning_data) == 2:
                    logger.info(
                        'Lot {1} of tender {0} selected for activate'.format(
                            *planning_data))
                    self.start_auction_worker_cmd('activate',
                                                  planning_data[0],
                                                  lot_id=planning_data[1])
        else:
            logger.info('No auctions to activate')
示例#34
0
class Scheduler(object):
    def __init__(self):
        self.scheduler = GeventScheduler()
        self.scheduler.add_listener(self.__scheduler_listener(),
                                    EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
                                    | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
                                    | EVENT_JOB_ADDED | EVENT_JOB_REMOVED
                                    | EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
        self.id = 'controller'
        self.app = None

    def schedule_workflows(self, task_id, executable, workflow_ids, trigger):
        """
        Schedules a workflow for execution

        Args:
            task_id (int): Id of the scheduled task
            executable (func): A callable to execute must take in one argument -- a workflow id
            workflow_ids (iterable(str)): An iterable of workflow ids
            trigger (Trigger): The trigger to use for this scheduled task
        """

        def execute(id_):
            with self.app.app_context():
                executable(id_)

        for workflow_id in workflow_ids:
            self.scheduler.add_job(execute, args=(workflow_id,),
                                   id=construct_task_id(task_id, workflow_id),
                                   trigger=trigger, replace_existing=True)

    def get_all_scheduled_workflows(self):
        """
        Gets all the scheduled workflows

        Returns:
             (dict{str: list[str]}) A dict of task_id to workflow execution ids
        """
        tasks = {}
        for job in self.scheduler.get_jobs():
            task, workflow_execution_id = split_task_id(job.id)
            if task not in tasks:
                tasks[task] = [workflow_execution_id]
            else:
                tasks[task].append(workflow_execution_id)
        return tasks

    def get_scheduled_workflows(self, task_id):
        """
        Gets all the scheduled worfklows for a given task id

        Args:
            task_id (str): The task id

        Returns:
            (list[str]) A list fo workflow execution id associated with this task id
        """
        tasks = []
        for job in self.scheduler.get_jobs():
            task, workflow_execution_id = split_task_id(job.id)
            if task == task_id:
                tasks.append(workflow_execution_id)
        return tasks

    def update_workflows(self, task_id, trigger):
        """
        Updates the workflows for a given task id to use a different trigger

        Args:
            task_id (str|int): The task id to update
            trigger (Trigger): The new trigger to use
        """
        existing_tasks = {construct_task_id(task_id, workflow_execution_id) for workflow_execution_id in
                          self.get_scheduled_workflows(task_id)}
        for job_id in existing_tasks:
            self.scheduler.reschedule_job(job_id=job_id, trigger=trigger)

    def unschedule_workflows(self, task_id, workflow_execution_ids):
        """
        Unschedules a workflow

        Args:
            task_id (str|int): The task ID to unschedule
            workflow_execution_ids (list[str]): The list of workflow execution IDs to update
        """
        for workflow_execution_id in workflow_execution_ids:
            try:
                self.scheduler.remove_job(construct_task_id(task_id, workflow_execution_id))
            except JobLookupError:
                logger.warning('Cannot delete task {}. '
                               'No task found in scheduler'.format(construct_task_id(task_id, workflow_execution_id)))

    def start(self):
        """Starts the scheduler for active execution. This function must be called before any workflows are executed.

        Returns:
            The state of the scheduler if successful, error message if scheduler is in "stopped" state.
        """
        if self.scheduler.state == STATE_STOPPED:
            logger.info('Starting scheduler')
            self.scheduler.start()
        else:
            logger.warning('Cannot start scheduler. Scheduler is already running or is paused')
            return "Scheduler already running."
        return self.scheduler.state

    def stop(self, wait=True):
        """Stops active execution.

        Args:
            wait (bool, optional): Boolean to synchronously or asynchronously wait for the scheduler to shutdown.
                Default is True.

        Returns:
            The state of the scheduler if successful, error message if scheduler is already in "stopped" state.
        """
        if self.scheduler.state != STATE_STOPPED:
            logger.info('Stopping scheduler')
            self.scheduler.shutdown(wait=wait)
        else:
            logger.warning('Cannot stop scheduler. Scheduler is already stopped')
            return "Scheduler already stopped."
        return self.scheduler.state

    def pause(self):
        """Pauses active execution.

        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "running" state.
        """
        if self.scheduler.state == STATE_RUNNING:
            logger.info('Pausing scheduler')
            self.scheduler.pause()
        elif self.scheduler.state == STATE_PAUSED:
            logger.warning('Cannot pause scheduler. Scheduler is already paused')
            return "Scheduler already paused."
        elif self.scheduler.state == STATE_STOPPED:
            logger.warning('Cannot pause scheduler. Scheduler is stopped')
            return "Scheduler is in STOPPED state and cannot be paused."
        return self.scheduler.state

    def resume(self):
        """Resumes active execution.

        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "paused" state.
        """
        if self.scheduler.state == STATE_PAUSED:
            logger.info('Resuming scheduler')
            self.scheduler.resume()
        else:
            logger.warning("Scheduler is not in PAUSED state and cannot be resumed.")
            return "Scheduler is not in PAUSED state and cannot be resumed."
        return self.scheduler.state

    def pause_workflows(self, task_id, workflow_execution_ids):
        """
        Pauses some workflows associated with a task

        Args:
            task_id (int|str): The id of the task to pause
            workflow_execution_ids (list[str]): The list of workflow execution IDs to pause
        """
        for workflow_execution_id in workflow_execution_ids:
            job_id = construct_task_id(task_id, workflow_execution_id)
            try:
                self.scheduler.pause_job(job_id=job_id)
                logger.info('Paused job {0}'.format(job_id))
            except JobLookupError:
                logger.warning('Cannot pause scheduled workflow {}. Workflow ID not found'.format(job_id))

    def resume_workflows(self, task_id, workflow_execution_ids):
        """
        Resumes some workflows associated with a task

        Args:
            task_id (int|str): The id of the task to pause
            workflow_execution_ids (list[str]): The list of workflow execution IDs to resume
        """
        for workflow_execution_id in workflow_execution_ids:
            job_id = construct_task_id(task_id, workflow_execution_id)
            try:
                self.scheduler.resume_job(job_id=job_id)
                logger.info('Resumed job {0}'.format(job_id))
            except JobLookupError:
                logger.warning('Cannot resume scheduled workflow {}. Workflow ID not found'.format(job_id))

    def __scheduler_listener(self):
        event_selector_map = {EVENT_SCHEDULER_START: WalkoffEvent.SchedulerStart,
                              EVENT_SCHEDULER_SHUTDOWN: WalkoffEvent.SchedulerShutdown,
                              EVENT_SCHEDULER_PAUSED: WalkoffEvent.SchedulerPaused,
                              EVENT_SCHEDULER_RESUMED: WalkoffEvent.SchedulerResumed,
                              EVENT_JOB_ADDED: WalkoffEvent.SchedulerJobAdded,
                              EVENT_JOB_REMOVED: WalkoffEvent.SchedulerJobRemoved,
                              EVENT_JOB_EXECUTED: WalkoffEvent.SchedulerJobExecuted,
                              EVENT_JOB_ERROR: WalkoffEvent.SchedulerJobError}

        def event_selector(event):
            try:
                event = event_selector_map[event.code]
                event.send(self)
            except KeyError:  # pragma: no cover
                logger.error('Unknown event sent triggered in scheduler {}'.format(event))

        return event_selector
示例#35
0
state = HomerHelper.getSettingValue('State')
timezone = HomerHelper.calcTimeZone(street, city, state)

jobstores = {
    'default': MemoryJobStore
}
executors = {
    'default': ThreadPoolExecutor(40)
}
job_defaults = {
    'misfire_grace_time': None,
    'coalesce': True,
    'max_instances': 3
}
scheduler = GeventScheduler(executors=executors, job_defaults=job_defaults, timezone=timezone)
scheduler.start()

def schedule(*args, **kwargs):
    job = scheduler.add_job(*args, **kwargs)
    return job

def KillJob(*args, **kwargs):
    scheduler.remove_job(*args, **kwargs)

def KillScheduler():
    scheduler.shutdown()

def GetJob(*args, **kwargs):
    job = scheduler.get_job(*args, **kwargs)
    return job
示例#36
0
class Scheduler(Service):
    """
    Service running scheduled and periodic commands
    """

    _schedulers = weakref.WeakSet()

    def __init__(self, name):
        super().__init__(name)
        self._commands = {}  # cmd_id : command
        # self._jobstores = {'default': SQLAlchemyJobStore(url=constants.scheduler_database_url,
        #                                                 tablename="jobs")}
        self._jobstores = {}
        self._executors = {}
        self._job_defaults = {
            'max_instances': 10,
            'coalesce': True,
        }
        self._triggers = {}  # cmd_id : trigger
        self._scheduler = GeventScheduler(
            jobstores=self._jobstores,
            executors=self._executors,
            job_defaults=self._job_defaults,
            timezone=pytz.utc,  # TODO: make user configurable
        )
        self._schedulers.add(self)
        s_cmds = constants.internaldb.scheduler_commands.get({})
        v = s_cmds.setdefault(self.identifier(), {})
        if not v:
            constants.internaldb.scheduler_commands.set(s_cmds)

    def _get_job_id(self, cmd_id):
        s_cmds = constants.internaldb.scheduler_commands.get()
        return s_cmds[self.identifier()].get(cmd_id)

    def _set_job_id(self, cmd_id, job_id):
        if not isinstance(job_id, int):
            job_id = job_id.id
        s_cmds = constants.internaldb.scheduler_commands.get()
        s_cmds[self.identifier()][cmd_id] = job_id
        constants.internaldb.scheduler_commands.set(s_cmds)

    def start(self):
        "Start the scheduler"
        self._scheduler.start()
        log.d("Started scheduler service:", self.name)
        return self

    def add_command(self, cmd, trigger=None):
        "Add a command to this scheduler and return a command id"
        assert isinstance(cmd, command.CoreCommand)
        command_id = super().add_command(cmd)
        self._commands[command_id] = cmd
        if trigger:
            self.set_trigger(command_id, trigger)
        else:
            raise NotImplementedError
        return command_id

    def set_trigger(self, cmd_id, trigger):
        "Change the trigger for a command in this scheduler"
        self._triggers[cmd_id] = trigger

    def remove_command(self, cmd_id):
        "Remove a command from this scheduler"
        j_id = self._get_job_id(cmd_id)
        self._scheduler.remove_job(j_id)

    def pause_command(self, cmd_id):
        "Pause a command in this scheduler, returns command state"
        raise NotImplementedError

    def resume_command(self, cmd_id):
        "Resume a command in this scheduler, returns command state"
        raise NotImplementedError

    def start_command(self, cmd_id, *args, **kwargs):
        "Start running a command in this scheduler, returns command state"
        cmd = self.get_command(cmd_id)
        j_id = self._scheduler.add_job(cmd._run,
                                       self._triggers[cmd_id],
                                       args,
                                       kwargs,
                                       name=cmd.__class__.__name__)
        self._set_job_id(cmd_id, j_id)

    def stop_command(self, cmd_id):
        "alias for remove_command"
        self.remove_command(cmd_id)

    def shutdown(self, wait=True):
        "Shutdown scheduler"
        self._scheduler.shutdown(wait=wait)
        return self

    @classmethod
    def shutdown_all(cls, wait=True):
        "Shutdown all schedulers"
        for s in cls._schedulers:
            s.shutdown(wait=wait)

    def pause(self):
        "Pause scheduler"
        raise NotImplementedError
        return self

    def resume(self):
        "Resume scheduler"
        raise NotImplementedError
        return self
示例#37
0
class Controller(object):
    def __init__(self,
                 name='defaultController',
                 workflows_path=core.config.paths.workflows_path):
        """Initializes a Controller object.
        
        Args:
            name (str, optional): Name for the controller.
            workflows_path (str, optional): Path to the workflows.
        """
        self.name = name
        self.workflows = {}
        self.load_all_workflows_from_directory(path=workflows_path)
        self.instances = {}
        self.tree = None

        self.scheduler = GeventScheduler()
        self.scheduler.add_listener(
            self.__scheduler_listener(),
            EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
            | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
            | EVENT_JOB_ADDED | EVENT_JOB_REMOVED
            | EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
        self.ancestry = [self.name]
        self.paused_workflows = {}

    def reconstruct_ancestry(self):
        """Reconstructs the ancestry list field of a workflow in case it changes.
        """
        for key in self.workflows:
            self.workflows[key].reconstruct_ancestry(self.ancestry)

    def load_workflow_from_file(self,
                                path,
                                workflow_name,
                                name_override=None,
                                playbook_override=None):
        """Loads a workflow from a file.
        
        Args:
            path (str): Path to the workflow.
            workflow_name (str): Name of the workflow to load.
            name_override (str, optional): Name that the workflow should be changed to.
            playbook_override (str, optional): Name that the playbook should be changed to.
            
        Returns:
            True on success, False otherwise.
        """
        self.tree = cElementTree.ElementTree(file=path)
        playbook_name = playbook_override if playbook_override else os.path.splitext(
            os.path.basename(path))[0]
        for workflow in self.tree.iter(tag='workflow'):
            current_workflow_name = workflow.get('name')
            if current_workflow_name == workflow_name:
                if name_override:
                    workflow_name = name_override
                name = construct_workflow_name_key(playbook_name,
                                                   workflow_name)
                key = _WorkflowKey(playbook_name, workflow_name)
                self.workflows[key] = wf.Workflow(name=name,
                                                  xml=workflow,
                                                  parent_name=self.name,
                                                  playbook_name=playbook_name)
                logger.info('Adding workflow {0} to controller'.format(name))
                break
        else:
            logger.warning(
                'Workflow {0} not found in playbook {0}. Cannot load.'.format(
                    workflow_name, playbook_name))
            return False

        self.add_child_workflows()
        self.add_workflow_scheduled_jobs()
        return True

    def load_workflows_from_file(self,
                                 path,
                                 name_override=None,
                                 playbook_override=None):
        """Loads multiple workloads from a file.
        
        Args:
            path (str): Path to the workflow.
            name_override (str, optional): Name that the workflow should be changed to. 
            playbook_override (str, optional): Name that the playbook should be changed to.
        """
        self.tree = cElementTree.ElementTree(file=path)
        playbook_name = playbook_override if playbook_override else os.path.splitext(
            os.path.basename(path))[0]
        for workflow in self.tree.iter(tag='workflow'):
            workflow_name = name_override if name_override else workflow.get(
                'name')
            name = construct_workflow_name_key(playbook_name, workflow_name)
            key = _WorkflowKey(playbook_name, workflow_name)
            self.workflows[key] = wf.Workflow(name=name,
                                              xml=workflow,
                                              parent_name=self.name,
                                              playbook_name=playbook_name)
            logger.info('Adding workflow {0} to controller'.format(name))
        self.add_child_workflows()
        self.add_workflow_scheduled_jobs()

    def load_all_workflows_from_directory(self, path=None):
        """Loads all workflows from a directory.
        
        Args:
            path (str, optional): Path to the directory to load from. Defaults to the configuration workflows_path. 
        """
        if path is None:
            path = core.config.paths.workflows_path
        for workflow in locate_workflows_in_directory(path):
            self.load_workflows_from_file(os.path.join(path, workflow))

    def add_child_workflows(self):
        for workflow in self.workflows:
            playbook_name = workflow.playbook
            children = self.workflows[workflow].options.children
            for child in children:
                workflow_key = _WorkflowKey(
                    playbook_name,
                    extract_workflow_name(child, playbook_name=playbook_name))
                if workflow_key in self.workflows:
                    logger.info(
                        'Adding child workflow {0} to workflow {1}'.format(
                            child, self.workflows[workflow_key].name))
                    children[child] = self.workflows[workflow_key]

    def add_workflow_scheduled_jobs(self):
        """Schedules the workflow to run based on workflow options.
        """
        for workflow in self.workflows:
            if (self.workflows[workflow].options.enabled
                    and self.workflows[workflow].options.scheduler['autorun']
                    == 'true'):
                schedule_type = self.workflows[workflow].options.scheduler[
                    'type']
                schedule = self.workflows[workflow].options.scheduler['args']
                self.scheduler.add_job(self.workflows[workflow].execute,
                                       trigger=schedule_type,
                                       replace_existing=True,
                                       **schedule)
                logger.info('Added scheduled job for workflow {0}'.format(
                    self.workflows[workflow].name))

    def create_workflow_from_template(self,
                                      playbook_name,
                                      workflow_name,
                                      template_playbook='emptyWorkflow',
                                      template_name='emptyWorkflow'):
        """Creates a workflow from a workflow template.
        
        Args:
            playbook_name (str): The name of the new playbook. 
            workflow_name (str): The name of the new workflow.
            template_playbook (str): The name of the playbook template to load. Default is "emptyWorkflow".
            template_name (str): The name of the workflow template to load. Default is "emptyWorkflow".
            
        Returns:
            True on success, False if otherwise.
        """
        path = '{0}{1}{2}.workflow'.format(core.config.paths.templates_path,
                                           sep, template_playbook)
        return self.load_workflow_from_file(path=path,
                                            workflow_name=template_name,
                                            name_override=workflow_name,
                                            playbook_override=playbook_name)

    def create_playbook_from_template(self,
                                      playbook_name,
                                      template_playbook='emptyWorkflow'):
        """Creates a playbook from a playbook template.
        
        Args:
            playbook_name (str): The name of the new playbook.
            template_playbook (str): The name of the playbook template to load. Default is "emptyWorkflow".
        """
        # TODO: Need a handler for returning workflow key and status
        path = '{0}{1}{2}.workflow'.format(core.config.paths.templates_path,
                                           sep, template_playbook)
        self.load_workflows_from_file(path=path,
                                      playbook_override=playbook_name)

    def remove_workflow(self, playbook_name, workflow_name):
        """Removes a workflow.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow to remove.
            
        Returns:
            True on success, False otherwise.
        """

        name = _WorkflowKey(playbook_name, workflow_name)
        if name in self.workflows:
            del self.workflows[name]

            logger.debug('Removed workflow {0}'.format(name))
            return True
        logger.warning(
            'Cannot remove workflow {0}. Does not exist in controller'.format(
                name))
        return False

    def remove_playbook(self, playbook_name):
        """Removes a playbook and all workflows within it.
        
        Args:
            playbook_name (str): The name of the playbook to remove.
            
        Returns:
            True on success, False otherwise.
        """
        for name in [
                workflow for workflow in self.workflows
                if workflow.playbook == playbook_name
        ]:
            del self.workflows[name]
            logger.debug('Removed workflow {0}'.format(name))
        logger.debug('Removed playbook {0}'.format(playbook_name))
        return True

    def get_all_workflows(self):
        """Gets all of the currently loaded workflows.
        
        Returns:
            A dict with key being the playbook, mapping to a list of workflow names for each playbook.
        """
        result = {}
        for key in self.workflows.keys():
            if key.playbook not in result:
                result[key.playbook] = []
            result[key.playbook].append(key.workflow)
        return result

    def get_all_playbooks(self):
        """Gets a list of all playbooks.
        
        Returns:
            A list containing all currently loaded playbook names.
        """
        return list(set(key.playbook for key in self.workflows.keys()))

    def is_workflow_registered(self, playbook_name, workflow_name):
        """Checks whether or not a workflow is currently registered in the system.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow.
            
        Returns:
            True if the workflow is registered, false otherwise.
        """
        return _WorkflowKey(playbook_name, workflow_name) in self.workflows

    def is_playbook_registered(self, playbook_name):
        """Checks whether or not a playbook is currently registered in the system.
        
        Args:
            playbook_name (str): The name of the playbook.
            
        Returns:
            True if the playbook is registered, false otherwise.
        """
        return any(workflow_key.playbook == playbook_name
                   for workflow_key in self.workflows)

    def update_workflow_name(self, old_playbook, old_workflow, new_playbook,
                             new_workflow):
        """Update the name of a workflow.
        
        Args:
            old_playbook (str): Name of the current playbook.
            old_workflow (str): Name of the current workflow.
            new_playbook (str): The new name of the playbook.
            new_workflow (str): The new name of the workflow.
        """
        old_key = _WorkflowKey(old_playbook, old_workflow)
        new_key = _WorkflowKey(new_playbook, new_workflow)
        self.workflows[new_key] = self.workflows.pop(old_key)
        self.workflows[new_key].name = construct_workflow_name_key(
            new_playbook, new_workflow)
        self.workflows[new_key].reconstruct_ancestry([self.name])
        logger.debug('updated workflow name {0} to {1}'.format(
            old_key, new_key))

    def update_playbook_name(self, old_playbook, new_playbook):
        """Update the name of a playbook.
        
        Args:
            old_playbook (str): Name of the current playbook.
            new_playbook (str): The new name of the playbook.
        """
        for key in [
                name for name in self.workflows.keys()
                if name.playbook == old_playbook
        ]:
            self.update_workflow_name(old_playbook, key.workflow, new_playbook,
                                      key.workflow)

    def add_workflow_breakpoint_steps(self, playbook_name, workflow_name,
                                      steps):
        """Adds a breakpoint (for debugging purposes) in the specified steps.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow under which the steps are located.
            steps (list[str]): The list of step names for which the user would like to pause execution.
        """
        workflow = self.get_workflow(playbook_name, workflow_name)
        if workflow:
            workflow.breakpoint_steps.extend(steps)

    def execute_workflow(self, playbook_name, workflow_name, start=None):
        """Executes a workflow.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): Workflow to execute.
            start (str, optional): The name of the first step. Defaults to "start".
        """
        global pool
        global workflows
        global threading_is_initialized

        key = _WorkflowKey(playbook_name, workflow_name)
        if key in self.workflows:
            workflow = self.workflows[key]
            subs = deepcopy(subscription.subscriptions)

            # If threading has not been initialized, initialize it.
            if not threading_is_initialized:
                initialize_threading()
            if start is not None:
                logger.info('Executing workflow {0} for step {1}'.format(
                    key, start))
                workflows.append(
                    pool.submit(execute_workflow_worker, workflow, subs,
                                start))
            else:
                logger.info(
                    'Executing workflow {0} with default starting step'.format(
                        key, start))
                workflows.append(
                    pool.submit(execute_workflow_worker, workflow, subs))
            callbacks.SchedulerJobExecuted.send(self)
        else:
            logger.error(
                'Attempted to execute playbook which does not exist in controller'
            )

    def get_workflow(self, playbook_name, workflow_name):
        """Get a workflow object.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow.
            
        Returns:
            The workflow object if found, else None.
        """
        key = _WorkflowKey(playbook_name, workflow_name)
        if key in self.workflows:
            return self.workflows[key]
        return None

    def get_all_workflows_by_playbook(self, playbook_name):
        """Get a list of all workflow objects in a playbook.
        
        Args:
            playbook_name: The name of the playbook.
            
        Returns:
            A list of all workflow objects in a playbook.
        """
        _workflows = []
        for key in self.workflows.keys():
            if key.playbook == playbook_name:
                _workflows.append(self.workflows[key].name)
        return _workflows

    def playbook_to_xml(self, playbook_name):
        """Returns the XML representation of a playbook.
        
        Args:
            playbook_name: The name of the playbook.
            
        Returns:
            The XML representation of the playbook if the playbook has any workflows under it, else None.
        """
        all_workflows = [
            workflow for key, workflow in self.workflows.items()
            if key.playbook == playbook_name
        ]
        if all_workflows:
            xml = cElementTree.Element('workflows')
            for workflow in all_workflows:
                xml.append(workflow.to_xml())
            return xml
        else:
            logger.debug(
                'No workflows are registered in controller to convert to XML')
            return None

    def copy_workflow(self, old_playbook_name, new_playbook_name,
                      old_workflow_name, new_workflow_name):
        """Duplicates a workflow into its current playbook, or a different playbook.
        
        Args:
            old_playbook_name (str): Playbook name under which the workflow is located.
            new_playbook_name (str): The new playbook name for the duplicated workflow.
            old_workflow_name (str): The name of the workflow to be copied.
            new_workflow_name (str): The new name of the duplicated workflow.
        """
        workflow = self.get_workflow(old_playbook_name, old_workflow_name)
        workflow_copy = deepcopy(workflow)
        workflow_copy.playbook_name = new_playbook_name
        workflow_copy.name = construct_workflow_name_key(
            new_playbook_name, new_workflow_name)

        key = _WorkflowKey(new_playbook_name, new_workflow_name)
        self.workflows[key] = workflow_copy
        self.workflows[key].reconstruct_ancestry([self.name])
        logger.info('Workflow copied from {0}-{1} to {2}-{3}'.format(
            old_playbook_name, old_workflow_name, new_playbook_name,
            new_workflow_name))

    def copy_playbook(self, old_playbook_name, new_playbook_name):
        """Copies a playbook
        
        Args:
            old_playbook_name (str): The name of the playbook to be copied.
            new_playbook_name (str): The new name of the duplicated playbook.
        """
        for workflow in [
                workflow.workflow for workflow in self.workflows
                if workflow.playbook == old_playbook_name
        ]:
            self.copy_workflow(old_playbook_name, new_playbook_name, workflow,
                               workflow)

    def pause_workflow(self, playbook_name, workflow_name):
        """Pauses a workflow that is currently executing.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow.
            
        Returns:
            A randomly-generated key that needs to be used in order to resume the workflow. This feature is added for
            security purposes.
        """
        workflow = self.get_workflow(playbook_name, workflow_name)
        wf_key = _WorkflowKey(playbook_name, workflow_name)
        self.paused_workflows[wf_key] = uuid.uuid4()
        if workflow:
            logger.info('Pausing workflow {0}'.format(workflow.name))
            workflow.pause()
        return self.paused_workflows[wf_key].hex

    def resume_workflow(self, playbook_name, workflow_name, validate_uuid):
        """Resumes a workflow that has been paused.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow.
            validate_uuid (str): The randomly-generated hexadecimal key that was returned from pause_workflow(). This
            is needed to resume a workflow for security purposes.
            
        Returns:
            "Success" if it is successful, or other error messages.
        """
        workflow = self.get_workflow(playbook_name, workflow_name)
        wf_key = _WorkflowKey(playbook_name, workflow_name)
        if workflow:
            if validate_uuid == self.paused_workflows[wf_key].hex:
                logger.info('Resuming workflow {0}'.format(workflow.name))
                workflow.resume()
                return True
            else:
                logger.warning(
                    'Cannot resume workflow {0}. Invalid key'.format(
                        workflow.name))
                return False

    def resume_breakpoint_step(self, playbook_name, workflow_name):
        """Resumes a step that has been specified as a breakpoint.
        
        Args:
            playbook_name (str): Playbook name under which the workflow is located.
            workflow_name (str): The name of the workflow.
        """
        workflow = self.get_workflow(playbook_name, workflow_name)
        if workflow:
            logger.debug('Resuming workflow {0} from breakpoint'.format(
                workflow.name))
            workflow.resume_breakpoint_step()

    # Starts active execution
    def start(self):
        """Starts the scheduler for active execution. This function must be called before any workflows are executed.
        
        Returns:
            The state of the scheduler if successful, error message if scheduler is in "stopped" state.
        """
        if self.scheduler.state != STATE_RUNNING and self.scheduler.state != STATE_PAUSED:
            logger.info('Starting scheduler')
            self.scheduler.start()
        else:
            logger.warning(
                'Cannot start scheduler. Scheduler is already running or is paused'
            )
            return "Scheduler already running."
        return self.scheduler.state

    # Stops active execution
    def stop(self, wait=True):
        """Stops active execution. 
        
        Args:
            wait (bool, optional): Boolean to synchronously or asynchronously wait for the scheduler to shutdown.
                Default is True.
                
        Returns:
            The state of the scheduler if successful, error message if scheduler is already in "stopped" state.
        """
        if self.scheduler.state != STATE_STOPPED:
            logger.info('Stopping scheduler')
            self.scheduler.shutdown(wait=wait)
        else:
            logger.warning(
                'Cannot stop scheduler. Scheduler is already stopped')
            return "Scheduler already stopped."
        return self.scheduler.state

    # Pauses active execution
    def pause(self):
        """Pauses active execution.
        
        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "running" state.
        """
        if self.scheduler.state == STATE_RUNNING:
            logger.info('Pausing scheduler')
            self.scheduler.pause()
        elif self.scheduler.state == STATE_PAUSED:
            logger.warning(
                'Cannot pause scheduler. Scheduler is already paused')
            return "Scheduler already paused."
        elif self.scheduler.state == STATE_STOPPED:
            logger.warning('Cannot pause scheduler. Scheduler is stopped')
            return "Scheduler is in STOPPED state and cannot be paused."
        return self.scheduler.state

    # Resumes active execution
    def resume(self):
        """Resumes active execution.
        
        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "paused" state.
        """
        if self.scheduler.state == STATE_PAUSED:
            logger.info('Resuming scheduler')
            self.scheduler.resume()
        else:
            logger.warning(
                "Scheduler is not in PAUSED state and cannot be resumed.")
            return "Scheduler is not in PAUSED state and cannot be resumed."
        return self.scheduler.state

    # Pauses active execution of specific job
    def pause_job(self, job_id):
        """Pauses active execution of a specific job.
        
        Args:
            job_id (str): ID of the job to pause.
        """
        logger.info('Pausing job {0}'.format(job_id))
        self.scheduler.pause_job(job_id=job_id)

    # Resumes active execution of specific job
    def resume_job(self, job_id):
        """Resumes active execution of a specific job.
        
        Args:
            job_id (str): ID of the job to resume.
        """
        logger.info('Resuming job {0}'.format(job_id))
        self.scheduler.resume_job(job_id=job_id)

    # Returns jobs scheduled for active execution
    def get_scheduled_jobs(self):
        """Get all actively scheduled jobs.
        
        Returns:
             A list of all actively scheduled jobs.
        """
        self.scheduler.get_jobs()

    def __scheduler_listener(self):
        event_selector_map = {
            EVENT_SCHEDULER_START:
            (lambda: callbacks.SchedulerStart.send(self)),
            EVENT_SCHEDULER_SHUTDOWN:
            (lambda: callbacks.SchedulerShutdown.send(self)),
            EVENT_SCHEDULER_PAUSED:
            (lambda: callbacks.SchedulerPaused.send(self)),
            EVENT_SCHEDULER_RESUMED:
            (lambda: callbacks.SchedulerResumed.send(self)),
            EVENT_JOB_ADDED: (lambda: callbacks.SchedulerJobAdded.send(self)),
            EVENT_JOB_REMOVED:
            (lambda: callbacks.SchedulerJobRemoved.send(self)),
            EVENT_JOB_EXECUTED:
            (lambda: callbacks.SchedulerJobExecuted.send(self)),
            EVENT_JOB_ERROR: (lambda: callbacks.SchedulerJobError.send(self))
        }

        def event_selector(event):
            try:
                event_selector_map[event.code]()
            except KeyError:
                print('Error: Unknown event sent!')

        return event_selector
示例#38
0
            client.connect(broker, port=9001)

            client.subscribe(topic=channel_common_000, qos=0)
            client.subscribe(topic=channel_liwei_000, qos=0)
            client.subscribe(topic=channel_liwei_111, qos=0)
            client.subscribe(topic=channel_lot_message, qos=0)

            client.loop_forever()

        except (KeyboardInterrupt, SystemExit):
            client.disconnect()

    elif s == 'p':
        #         sched = BackgroundScheduler()
        #         sched.add_job(transmitMQTT, 'interval', seconds=1, args=(m,))  #interval 闂撮殧璋冨害锛堟瘡闅斿涔呮墽琛岋級
        #         sched.start()
        #         try:
        #             while True:
        #                 time.sleep(10)
        #         except (KeyboardInterrupt, SystemExit):
        #             sched.shutdown()

        sched = GeventScheduler()
        sched.add_job(transmitMQTT, 'interval', seconds=1, args=(m, ))
        g = sched.start()

        try:
            g.join()
        except (KeyboardInterrupt, SystemExit):
            pass
示例#39
0

def doPdf():
    print('doPdf start')
    try:
        info = pdfkit.from_url('http://localhost:7070/table', 'tableOut.pdf')
        if (info):
            print('merging pdfs...')
            main2.merge4()
        print('Info: ', info)
    except (Exception):
        print('Cannot open url or sth.')


def tick():
    print('Tick! The time is: %s' % datetime.now())


scheduler = GeventScheduler()

if __name__ == '__main__':
    scheduler.add_job(webapp_run)
    # scheduler.add_job(doPdf, 'interval', seconds=15)
    # scheduler.add_job(tick, 'interval', seconds=4)

    g = scheduler.start()
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        scheduler.shutdown()
示例#40
0
"""
Demonstrates how to use the gevent compatible scheduler to schedule a job that executes on 3 second intervals.
"""

from datetime import datetime
import os

from apscheduler.schedulers.gevent import GeventScheduler


def tick():
    print('Tick! The time is: %s' % datetime.now())


if __name__ == '__main__':
    scheduler = GeventScheduler()
    scheduler.add_job(tick, 'interval', seconds=3)
    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#41
0
# -*- coding: utf-8 -*-

import pickle

from handler.config import database_config

from apscheduler.schedulers.gevent import GeventScheduler
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.redis import RedisJobStore

jobstores = {
    'redis':
    RedisJobStore(db=10,
                  jobs_key='apscheduler.jobs',
                  run_times_key='apscheduler.run_times',
                  pickle_protocol=pickle.HIGHEST_PROTOCOL,
                  host=database_config.get("redis", "host"),
                  port=int(database_config.get("redis", "port")),
                  password=database_config.get("redis", "password"),
                  max_connections=int(
                      database_config.get("redis", "max_connections")))
}
test_task_scheduler = GeventScheduler(jobstores=jobstores)
# test_task_scheduler = BackgroundScheduler(jobstores=jobstores)
test_task_scheduler.start()
示例#42
0
    if int(app.conf.automation_status) == 1 or forced is True:
        if forced is False:
            global l_t_check
            l_t_check = app.datetime.now().strftime("%d/%m/%Y, %H:%M:%S")
        logging.info("Checking for releases in torrents")
        todays_date = app.datetime.now()
        schd_albums = app.QueueAlbum.query.all()
        for query in schd_albums:
            date = app.datetime.strptime(query.date, "%d %B %Y")
            if date <= todays_date:
                if int(query.status) == 0:
                    app.download(query.album_name)
        data = ({"album": "C_T", "date": "C_T"})
        app.pushtoListener(data)


def reschedule():
    sched.reschedule_job(job_id="auto_A", trigger='interval', minutes=int(app.conf.automation_interval) * 60)
    sched.reschedule_job(job_id="auto_T", trigger='interval', minutes=int(app.conf.automation_interval) * 60)


# ugly
l_t_check = "Never"
l_a_check = "Never"

sched = GeventScheduler()
sched.add_job(look_for_artist, 'interval', id="auto_A", minutes=int(app.conf.automation_interval) * 60)
sched.add_job(look_for_torrents, 'interval', id="auto_T", minutes=int(app.conf.automation_interval) * 60)
sched.add_job(generateSuggestions, 'interval', id="auto_S", seconds=6200)
sched.start()
示例#43
0
class Application(object):
    def __init__(self, config_file, data_dir, verbose=0):
        """
        Construct a new application instance.
        """

        self.config_file = config_file
        self.data_dir = data_dir
        self.verbose = verbose

        self.server = None
        self.provider = None
        self.connections = {}

        # Setup all parts of the application
        self.setup_config()
        self.setup_open_files()
        self.setup_database()
        self.setup_state()
        self.setup_connections()
        self.setup_cache()
        self.setup_provider()
        self.setup_server()
        self.setup_tasks()

    def setup_config(self):
        """
        Load the application config from file.
        """

        logger.debug("Loading config from %s", self.config_file)
        self.config = config.get_config(self.config_file)

    def setup_open_files(self):
        """
        Get and set open files limit.
        """

        open_files_limit = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        new_open_files_limit = self.config["Advanced"]["open files limit"]

        logger.info("System reports open files limit is %d.", open_files_limit)

        if new_open_files_limit != -1:
            logger.info("Changing open files limit to %d.",
                        new_open_files_limit)

            try:
                resource.setrlimit(
                    resource.RLIMIT_NOFILE,
                    (new_open_files_limit, resource.RLIM_INFINITY))
            except resource.error as e:
                logger.warning(
                    "Failed to increase the number of open files: %s", e)

    def setup_database(self):
        """
        Initialize database.
        """

        self.db = Database(self.config["Provider"]["database"])
        self.db.create_database(drop_all=False)

    def setup_state(self):
        """
        Setup state.
        """

        self.state = State(os.path.join(self.get_cache_dir(),
                                        "provider.state"))

    def setup_cache(self):
        """
        Setup the caches for items and artwork.
        """

        # Initialize caches for items and artwork.
        item_cache = cache.ItemCache(
            path=self.get_cache_dir(self.config["Provider"]["item cache dir"]),
            max_size=self.config["Provider"]["item cache size"],
            prune_threshold=self.config["Provider"]
            ["item cache prune threshold"])
        artwork_cache = cache.ArtworkCache(
            path=self.get_cache_dir(
                self.config["Provider"]["artwork cache dir"]),
            max_size=self.config["Provider"]["artwork cache size"],
            prune_threshold=self.config["Provider"]
            ["artwork cache prune threshold"])

        # Create a cache manager
        self.cache_manager = cache.CacheManager(db=self.db,
                                                item_cache=item_cache,
                                                artwork_cache=artwork_cache,
                                                connections=self.connections)

    def setup_connections(self):
        """
        Initialize the connections.
        """

        for name, section in self.config["Connections"].iteritems():
            index = len(self.connections) + 1

            self.connections[index] = Connection(
                db=self.db,
                state=self.state,
                index=index,
                name=name,
                url=section["url"],
                username=section["username"],
                password=section["password"],
                synchronization=section["synchronization"],
                synchronization_interval=section["synchronization interval"],
                transcode=section["transcode"],
                transcode_unsupported=section["transcode unsupported"])

    def setup_provider(self):
        """
        Setup the provider.
        """

        # Create provider.
        logger.debug("Setting up provider for %d connection(s).",
                     len(self.connections))

        self.provider = Provider(server_name=self.config["Provider"]["name"],
                                 db=self.db,
                                 state=self.state,
                                 connections=self.connections,
                                 cache_manager=self.cache_manager)

        # Do an initial synchronization if required.
        for connection in self.connections.itervalues():
            connection.synchronizer.provider = self.provider
            connection.synchronizer.synchronize(initial=True)

    def setup_server(self):
        """
        Create the DAAP server.
        """

        logger.debug("Setting up DAAP server at %s:%d",
                     self.config["Daap"]["interface"],
                     self.config["Daap"]["port"])

        self.server = DaapServer(
            provider=self.provider,
            password=self.config["Daap"]["password"],
            ip=self.config["Daap"]["interface"],
            port=self.config["Daap"]["port"],
            cache=self.config["Daap"]["cache"],
            cache_timeout=self.config["Daap"]["cache timeout"] * 60,
            bonjour=self.config["Daap"]["zeroconf"],
            debug=self.verbose > 1)

        # Extend server with a web interface
        if self.config["Daap"]["web interface"]:
            webserver.extend_server_app(self, self.server.app)

    def setup_tasks(self):
        """
        Setup all tasks that run periodically.
        """

        self.scheduler = GeventScheduler()

        # Add an initial job
        def _job():
            job.remove()
            self.synchronize(synchronization="startup")

        job = self.scheduler.add_job(_job,
                                     max_instances=1,
                                     trigger="interval",
                                     seconds=1)

        # Scheduler task to clean and expire the cache.
        cache_interval = self.config['Provider']['item cache prune interval']

        self.scheduler.add_job(self.cache_manager.expire,
                               max_instances=1,
                               trigger="interval",
                               minutes=cache_interval)
        self.scheduler.add_job(self.cache_manager.clean,
                               max_instances=1,
                               trigger="interval",
                               minutes=cache_interval)

        # Schedule tasks to synchronize each connection.
        for connection in self.connections.itervalues():
            self.scheduler.add_job(self.synchronize,
                                   args=([connection, "interval"]),
                                   max_instances=1,
                                   trigger="interval",
                                   minutes=connection.synchronization_interval)

    def synchronize(self, connections=None, synchronization="manual"):
        """
        Synchronize selected connections (or all) given a synchronization
        event.
        """

        count = 0
        connections = connections or self.connections.values()

        logger.debug("Synchronization triggered via '%s'.", synchronization)

        for connection in connections:
            if synchronization == "interval":
                if connection.synchronization == "interval":
                    connection.synchronizer.synchronize()
                    count += 1
            elif synchronization == "startup":
                if connection.synchronization == "startup":
                    if not connection.synchronizer.is_initial_synced:
                        connection.synchronizer.synchronize()
                        count += 1
            elif synchronization == "manual":
                connection.synchronizer.synchronize()
                count += 1

        logger.debug("Synchronized %d connections.", count)

        # Update the cache.
        self.cache_manager.cache()

    def start(self):
        """
        Start the server.
        """

        logger.debug("Starting task scheduler.")
        self.scheduler.start()

        logger.debug("Starting DAAP server.")
        self.server.serve_forever()

    def stop(self):
        """
        Stop the server.
        """

        logger.debug("Stopping DAAP server.")
        self.server.stop()

        logger.debug("Stopping task scheduler.")
        self.scheduler.shutdown()

    def get_cache_dir(self, *path):
        """
        Resolve the path to a cache directory. The path is relative to the data
        directory. The directory will be created if it does not exists, and
        will be tested for writing.
        """

        full_path = os.path.abspath(
            os.path.normpath(os.path.join(self.data_dir, *path)))
        logger.debug("Resolved %s to %s", path, full_path)

        # Create path if required.
        try:
            os.makedirs(full_path, 0755)
        except OSError as e:
            if e.errno == errno.EEXIST and os.path.isdir(full_path):
                pass
            else:
                raise Exception("Could not create folder: %s" % full_path)

        # Test for writing.
        ok = True
        test_file = os.path.join(full_path, ".write-test")

        while os.path.exists(test_file):
            test_file = test_file + str(random.randint(0, 9))

        try:
            with open(test_file, "w") as fp:
                fp.write("test")
        except IOError:
            ok = False
        finally:
            try:
                os.remove(test_file)
            except OSError:
                ok = False

        if not ok:
            raise Exception("Could not write to cache folder: %s" % full_path)

        # Cache directory created and tested for writing.
        return full_path
示例#44
0
from pytz import utc

from .db import engine
jobstores = {
    'default': SQLAlchemyJobStore(engine=engine)
}
if 'gunicorn' in os.environ.get('SERVER_SOFTWARE',''):
    from apscheduler.schedulers.gevent import GeventScheduler as Scheduler
else:
    from apscheduler.schedulers.background import BackgroundScheduler as Scheduler
logging.getLogger('apscheduler').setLevel(logging.DEBUG)
scheduler = Scheduler(jobstores=jobstores, timezone=utc)
app.apscheduler = scheduler
scheduler.app = app
if (os.getenv('PSITURK_DO_SCHEDULER', False)):
    scheduler.start()


#
# Dashboard
#
if (os.getenv('PSITURK_ENABLE_DASHBOARD', False)):
    from .dashboard import dashboard, init_app as dashboard_init_app # management dashboard
    app.register_blueprint(dashboard)
    dashboard_init_app(app)

    from .api import api_blueprint
    app.register_blueprint(api_blueprint)

init_db()
示例#45
0
class Scheduler(object):
    def __init__(self):
        self.scheduler = GeventScheduler()
        self.scheduler.add_listener(self.__scheduler_listener(),
                                    EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN
                                    | EVENT_SCHEDULER_PAUSED | EVENT_SCHEDULER_RESUMED
                                    | EVENT_JOB_ADDED | EVENT_JOB_REMOVED
                                    | EVENT_JOB_EXECUTED | EVENT_JOB_ERROR)
        self.id = 'controller'
        self.app = None

    def schedule_workflows(self, task_id, executable, workflow_ids, trigger):
        """
        Schedules a workflow for execution

        Args:
            task_id (int): Id of the scheduled task
            executable (func): A callable to execute must take in one argument -- a workflow id
            workflow_ids (iterable(str)): An iterable of workflow ids
            trigger (Trigger): The trigger to use for this scheduled task
        """

        def execute(id_):
            with self.app.app_context():
                executable(id_)

        for workflow_id in workflow_ids:
            self.scheduler.add_job(execute, args=(workflow_id,),
                                   id=construct_task_id(task_id, workflow_id),
                                   trigger=trigger, replace_existing=True)

    def get_all_scheduled_workflows(self):
        """
        Gets all the scheduled workflows

        Returns:
             (dict{str: list[str]}) A dict of task_id to workflow execution ids
        """
        tasks = {}
        for job in self.scheduler.get_jobs():
            task, workflow_execution_id = split_task_id(job.id)
            if task not in tasks:
                tasks[task] = [workflow_execution_id]
            else:
                tasks[task].append(workflow_execution_id)
        return tasks

    def get_scheduled_workflows(self, task_id):
        """
        Gets all the scheduled worfklows for a given task id

        Args:
            task_id (str): The task id

        Returns:
            (list[str]) A list fo workflow execution id associated with this task id
        """
        tasks = []
        for job in self.scheduler.get_jobs():
            task, workflow_execution_id = split_task_id(job.id)
            if task == task_id:
                tasks.append(workflow_execution_id)
        return tasks

    def update_workflows(self, task_id, trigger):
        """
        Updates the workflows for a given task id to use a different trigger

        Args:
            task_id (str|int): The task id to update
            trigger (Trigger): The new trigger to use
        """
        existing_tasks = {construct_task_id(task_id, workflow_execution_id) for workflow_execution_id in
                          self.get_scheduled_workflows(task_id)}
        for job_id in existing_tasks:
            self.scheduler.reschedule_job(job_id=job_id, trigger=trigger)

    def unschedule_workflows(self, task_id, workflow_execution_ids):
        """
        Unschedules a workflow

        Args:
            task_id (str|int): The task ID to unschedule
            workflow_execution_ids (list[str]): The list of workflow execution IDs to update
        """
        for workflow_execution_id in workflow_execution_ids:
            try:
                self.scheduler.remove_job(construct_task_id(task_id, workflow_execution_id))
            except JobLookupError:
                logger.warning('Cannot delete task {}. '
                               'No task found in scheduler'.format(construct_task_id(task_id, workflow_execution_id)))

    def start(self):
        """Starts the scheduler for active execution. This function must be called before any workflows are executed.

        Returns:
            The state of the scheduler if successful, error message if scheduler is in "stopped" state.
        """
        if self.scheduler.state == STATE_STOPPED:
            logger.info('Starting scheduler')
            self.scheduler.start()
        else:
            logger.warning('Cannot start scheduler. Scheduler is already running or is paused')
            return "Scheduler already running."
        return self.scheduler.state

    def stop(self, wait=True):
        """Stops active execution.

        Args:
            wait (bool, optional): Boolean to synchronously or asynchronously wait for the scheduler to shutdown.
                Default is True.

        Returns:
            The state of the scheduler if successful, error message if scheduler is already in "stopped" state.
        """
        if self.scheduler.state != STATE_STOPPED:
            logger.info('Stopping scheduler')
            self.scheduler.shutdown(wait=wait)
        else:
            logger.warning('Cannot stop scheduler. Scheduler is already stopped')
            return "Scheduler already stopped."
        return self.scheduler.state

    def pause(self):
        """Pauses active execution.

        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "running" state.
        """
        if self.scheduler.state == STATE_RUNNING:
            logger.info('Pausing scheduler')
            self.scheduler.pause()
        elif self.scheduler.state == STATE_PAUSED:
            logger.warning('Cannot pause scheduler. Scheduler is already paused')
            return "Scheduler already paused."
        elif self.scheduler.state == STATE_STOPPED:
            logger.warning('Cannot pause scheduler. Scheduler is stopped')
            return "Scheduler is in STOPPED state and cannot be paused."
        return self.scheduler.state

    def resume(self):
        """Resumes active execution.

        Returns:
            The state of the scheduler if successful, error message if scheduler is not in the "paused" state.
        """
        if self.scheduler.state == STATE_PAUSED:
            logger.info('Resuming scheduler')
            self.scheduler.resume()
        else:
            logger.warning("Scheduler is not in PAUSED state and cannot be resumed.")
            return "Scheduler is not in PAUSED state and cannot be resumed."
        return self.scheduler.state

    def pause_workflows(self, task_id, workflow_execution_ids):
        """
        Pauses some workflows associated with a task

        Args:
            task_id (int|str): The id of the task to pause
            workflow_execution_ids (list[str]): The list of workflow execution IDs to pause
        """
        for workflow_execution_id in workflow_execution_ids:
            job_id = construct_task_id(task_id, workflow_execution_id)
            try:
                self.scheduler.pause_job(job_id=job_id)
                logger.info('Paused job {0}'.format(job_id))
            except JobLookupError:
                logger.warning('Cannot pause scheduled workflow {}. Workflow ID not found'.format(job_id))

    def resume_workflows(self, task_id, workflow_execution_ids):
        """
        Resumes some workflows associated with a task

        Args:
            task_id (int|str): The id of the task to pause
            workflow_execution_ids (list[str]): The list of workflow execution IDs to resume
        """
        for workflow_execution_id in workflow_execution_ids:
            job_id = construct_task_id(task_id, workflow_execution_id)
            try:
                self.scheduler.resume_job(job_id=job_id)
                logger.info('Resumed job {0}'.format(job_id))
            except JobLookupError:
                logger.warning('Cannot resume scheduled workflow {}. Workflow ID not found'.format(job_id))

    def __scheduler_listener(self):
        event_selector_map = {EVENT_SCHEDULER_START: WalkoffEvent.SchedulerStart,
                              EVENT_SCHEDULER_SHUTDOWN: WalkoffEvent.SchedulerShutdown,
                              EVENT_SCHEDULER_PAUSED: WalkoffEvent.SchedulerPaused,
                              EVENT_SCHEDULER_RESUMED: WalkoffEvent.SchedulerResumed,
                              EVENT_JOB_ADDED: WalkoffEvent.SchedulerJobAdded,
                              EVENT_JOB_REMOVED: WalkoffEvent.SchedulerJobRemoved,
                              EVENT_JOB_EXECUTED: WalkoffEvent.SchedulerJobExecuted,
                              EVENT_JOB_ERROR: WalkoffEvent.SchedulerJobError}

        def event_selector(event):
            try:
                event = event_selector_map[event.code]
                event.send(self)
            except KeyError:  # pragma: no cover
                logger.error('Unknown event sent triggered in scheduler {}'.format(event))

        return event_selector
示例#46
0
"""
Demonstrates how to use the gevent compatible scheduler to schedule a job that executes on 3 second
intervals.
"""

from datetime import datetime
import os

from apscheduler.schedulers.gevent import GeventScheduler


def tick():
    print('Tick! The time is: %s' % datetime.now())


if __name__ == '__main__':
    scheduler = GeventScheduler()
    scheduler.add_job(tick, 'interval', seconds=3)
    g = scheduler.start()  # g is the greenlet that runs the scheduler loop
    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))

    # Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
    try:
        g.join()
    except (KeyboardInterrupt, SystemExit):
        pass
示例#47
0
from pytz import utc

from .db import engine
jobstores = {'default': SQLAlchemyJobStore(engine=engine)}
if 'gunicorn' in os.environ.get('SERVER_SOFTWARE', ''):
    from apscheduler.schedulers.gevent import GeventScheduler as Scheduler
else:
    from apscheduler.schedulers.background import BackgroundScheduler as Scheduler
logging.getLogger('apscheduler').setLevel(logging.DEBUG)
scheduler = Scheduler(jobstores=jobstores, timezone=utc)
app.apscheduler = scheduler
scheduler.app = app

if CONFIG.getboolean('Server Parameters', 'do_scheduler'):
    app.logger.info("Scheduler starting!")
    scheduler.start()
else:
    app.logger.info(
        "Starting scheduler in 'paused' mode -- it will not run any tasks, but it can be used to create, modify, or delete tasks."
    )
    scheduler.start(paused=True)

#
# Dashboard
#
if CONFIG.getboolean('Server Parameters', 'enable_dashboard'):
    from .dashboard import dashboard, init_app as dashboard_init_app  # management dashboard
    app.register_blueprint(dashboard)
    dashboard_init_app(app)

    from .api import api_blueprint