Example #1
0
    def test_interleaved_workers(self):
        class A(DummyTask):
            pass

        a = A()

        class B(DummyTask):

            def requires(self):
                return a

        class ExternalB(ExternalTask):
            task_family = "B"

            def complete(self):
                return False

        b = B()
        eb = ExternalB()
        self.assertEqual(str(eb), "B()")

        sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10)
        with Worker(scheduler=sch, worker_id='X') as w, Worker(scheduler=sch, worker_id='Y') as w2:
            self.assertTrue(w.add(b))
            self.assertTrue(w2.add(eb))
            logging.debug("RUNNING BROKEN WORKER")
            self.assertTrue(w2.run())
            self.assertFalse(a.complete())
            self.assertFalse(b.complete())
            logging.debug("RUNNING FUNCTIONAL WORKER")
            self.assertTrue(w.run())
            self.assertTrue(a.complete())
            self.assertTrue(b.complete())
Example #2
0
    def __init__(self, scheduler=None, worker_id=None, worker_processes=1, assistant=False, **kwargs):
        if scheduler is None:
            scheduler = CentralPlannerScheduler()

        self.worker_processes = int(worker_processes)
        self._worker_info = self._generate_worker_info()

        if not worker_id:
            worker_id = 'Worker(%s)' % ', '.join(['%s=%s' % (k, v) for k, v in self._worker_info])

        self._config = worker(**kwargs)

        self._id = worker_id
        self._scheduler = scheduler
        self._assistant = assistant

        self.host = socket.gethostname()
        self._scheduled_tasks = {}
        self._suspended_tasks = {}

        self._first_task = None

        self.add_succeeded = True
        self.run_succeeded = True
        self.unfulfilled_counts = collections.defaultdict(int)

        self._keep_alive_thread = KeepAliveThread(self._scheduler, self._id, self._config.ping_interval)
        self._keep_alive_thread.daemon = True
        self._keep_alive_thread.start()

        # Keep info about what tasks are running (could be in other processes)
        self._task_result_queue = multiprocessing.Queue()
        self._running_tasks = {}
Example #3
0
    def test_ping_retry(self):
        """ Worker ping fails once. Ping continues to try to connect to scheduler

        Kind of ugly since it uses actual timing with sleep to test the thread
        """
        sch = CentralPlannerScheduler(
            retry_delay=100,
            remove_delay=1000,
            worker_disconnect_delay=10,
        )

        self._total_pings = 0  # class var so it can be accessed from fail_ping

        def fail_ping(worker):
            # this will be called from within keep-alive thread...
            self._total_pings += 1
            raise Exception("Some random exception")

        sch.ping = fail_ping

        with Worker(
                scheduler=sch,
                worker_id="foo",
                ping_interval=0.01  # very short between pings to make test fast
        ):
            # let the keep-alive thread run for a bit...
            time.sleep(
                0.1)  # yes, this is ugly but it's exactly what we need to test
        self.assertTrue(self._total_pings > 1,
                        msg="Didn't retry pings (%d pings performed)" %
                        (self._total_pings, ))
Example #4
0
 def run(self, result=None):
     super(WorkerEmailTest, self).setUp()
     sch = CentralPlannerScheduler(retry_delay=100,
                                   remove_delay=1000,
                                   worker_disconnect_delay=10)
     with Worker(scheduler=sch, worker_id="foo") as self.worker:
         super(WorkerEmailTest, self).run(result)
 def _build(self, tasks):
     self.scheduler = CentralPlannerScheduler(prune_on_get_work=True)
     w = luigi.worker.Worker(scheduler=self.scheduler, worker_processes=1)
     for t in tasks:
         w.add(t)
     w.run()
     w.stop()
Example #6
0
 def test_task_list_beyond_limit(self):
     sch = CentralPlannerScheduler(max_shown_tasks=3)
     for c in 'ABCD':
         sch.add_task(worker=WORKER, task_id=c)
     self.assertEqual(set('ABCD'),
                      set(sch.task_list('PENDING', '', False).keys()))
     self.assertEqual({'num_tasks': 4}, sch.task_list('PENDING', ''))
Example #7
0
    def test_interleaved_workers3(self):
        class A(DummyTask):

            def run(self):
                logging.debug('running A')
                time.sleep(0.1)
                super(A, self).run()

        a = A()

        class B(DummyTask):

            def requires(self):
                return a

            def run(self):
                logging.debug('running B')
                super(B, self).run()

        b = B()

        sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10)

        with Worker(scheduler=sch, worker_id='X', keep_alive=True, count_uniques=True) as w:
            with Worker(scheduler=sch, worker_id='Y', keep_alive=True, count_uniques=True, wait_interval=0.1) as w2:
                self.assertTrue(w.add(a))
                self.assertTrue(w2.add(b))

                threading.Thread(target=w.run).start()
                self.assertTrue(w2.run())

                self.assertTrue(a.complete())
                self.assertTrue(b.complete())
Example #8
0
    def test_die_for_non_unique_pending(self):
        class A(DummyTask):

            def run(self):
                logging.debug('running A')
                time.sleep(0.1)
                super(A, self).run()

        a = A()

        class B(DummyTask):

            def requires(self):
                return a

            def run(self):
                logging.debug('running B')
                super(B, self).run()

        b = B()

        sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10)

        with Worker(scheduler=sch, worker_id='X', keep_alive=True, count_uniques=True) as w:
            with Worker(scheduler=sch, worker_id='Y', keep_alive=True, count_uniques=True, wait_interval=0.1) as w2:
                self.assertTrue(w.add(b))
                self.assertTrue(w2.add(b))

                self.assertEqual(w._get_work()[0], a.task_id)
                self.assertTrue(w2.run())

                self.assertFalse(a.complete())
                self.assertFalse(b.complete())
Example #9
0
def run(api_port=8082, address=None, scheduler=None, responder=None):
    """
    Runs one instance of the API server.
    """
    if scheduler is None:
        scheduler = CentralPlannerScheduler()

    # load scheduler state
    scheduler.load()

    _init_api(scheduler, responder, api_port, address)

    # prune work DAG every 60 seconds
    pruner = tornado.ioloop.PeriodicCallback(scheduler.prune, 60000)
    pruner.start()

    def shutdown_handler(foo=None, bar=None):
        logger.info("Scheduler instance shutting down")
        scheduler.dump()
        os._exit(0)

    signal.signal(signal.SIGINT, shutdown_handler)
    signal.signal(signal.SIGTERM, shutdown_handler)
    if os.name == 'nt':
        signal.signal(signal.SIGBREAK, shutdown_handler)
    else:
        signal.signal(signal.SIGQUIT, shutdown_handler)
    atexit.register(shutdown_handler)

    logger.info("Scheduler starting up")

    tornado.ioloop.IOLoop.instance().start()
Example #10
0
    def test_complete_exception(self):
        "Tests that a task is still scheduled if its sister task crashes in the complete() method"

        class A(DummyTask):
            def complete(self):
                raise Exception("doh")

        a = A()

        class C(DummyTask):
            pass

        c = C()

        class B(DummyTask):
            def requires(self):
                return a, c

        b = B()
        sch = CentralPlannerScheduler(retry_delay=100,
                                      remove_delay=1000,
                                      worker_disconnect_delay=10)
        with Worker(scheduler=sch, worker_id="foo") as w:
            self.assertFalse(w.add(b))
            self.assertTrue(w.run())
            self.assertFalse(b.has_run)
            self.assertTrue(c.has_run)
            self.assertFalse(a.has_run)
Example #11
0
    def test_interleaved_workers2(self):
        # two tasks without dependencies, one external, one not
        class B(DummyTask):
            pass

        class ExternalB(ExternalTask):
            task_family = "B"

            def complete(self):
                return False

        b = B()
        eb = ExternalB()

        self.assertEqual(str(eb), "B()")

        sch = CentralPlannerScheduler(retry_delay=100,
                                      remove_delay=1000,
                                      worker_disconnect_delay=10)
        with Worker(scheduler=sch,
                    worker_id='X') as w, Worker(scheduler=sch,
                                                worker_id='Y') as w2:
            self.assertTrue(w2.add(eb))
            self.assertTrue(w.add(b))

            self.assertTrue(w2.run())
            self.assertFalse(b.complete())
            self.assertTrue(w.run())
            self.assertTrue(b.complete())
Example #12
0
    def test_requires_exception(self):
        class A(DummyTask):
            def requires(self):
                raise Exception("doh")

        a = A()

        class D(DummyTask):
            pass

        d = D()

        class C(DummyTask):
            def requires(self):
                return d

        c = C()

        class B(DummyTask):
            def requires(self):
                return c, a

        b = B()
        sch = CentralPlannerScheduler(retry_delay=100,
                                      remove_delay=1000,
                                      worker_disconnect_delay=10)
        with Worker(scheduler=sch, worker_id="foo") as w:
            self.assertFalse(w.add(b))
            self.assertTrue(w.run())
            self.assertFalse(b.has_run)
            self.assertTrue(c.has_run)
            self.assertTrue(d.has_run)
            self.assertFalse(a.has_run)
 def setUp(self):
     self.scheduler = CentralPlannerScheduler(retry_delay=0.01,
                                              remove_delay=3,
                                              worker_disconnect_delay=3,
                                              disable_persist=3,
                                              disable_window=5,
                                              disable_failures=2)
Example #14
0
 def setUp(self):
     # InstanceCache.disable()
     self.sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10)
     self.w = Worker(scheduler=self.sch, worker_id='X')
     self.w_raise = Worker(scheduler=self.sch, worker_id='X_raise', raise_on_error=True)
     self.w2 = Worker(scheduler=self.sch, worker_id='Y')
     self.time = time.time
Example #15
0
 def setUp(self):
     # InstanceCache.disable()
     self.sch = CentralPlannerScheduler(retry_delay=100,
                                        remove_delay=1000,
                                        worker_disconnect_delay=10)
     self.w = Worker(scheduler=self.sch, worker_id='X').__enter__()
     self.w2 = Worker(scheduler=self.sch, worker_id='Y').__enter__()
     self.time = time.time
Example #16
0
 def setUp(self):
     self.sch = CentralPlannerScheduler(retry_delay=100,
                                        remove_delay=1000,
                                        worker_disconnect_delay=10,
                                        disable_persist=10,
                                        disable_window=10,
                                        disable_failures=3)
     self.time = time.time
Example #17
0
 def test_search_results_beyond_limit(self):
     sch = CentralPlannerScheduler(max_shown_tasks=3)
     sch.add_task(worker=WORKER, task_id='task_a')
     sch.add_task(worker=WORKER, task_id='task_b')
     sch.add_task(worker=WORKER, task_id='task_c')
     sch.add_task(worker=WORKER, task_id='task_d')
     self.assertEqual({'num_tasks': 4}, sch.task_list('PENDING', '', search='a'))
     self.assertEqual(['task_a'], list(sch.task_list('PENDING', '', search='_a').keys()))
Example #18
0
 def setUp(self):
     self.sch = CentralPlannerScheduler(retry_delay=100,
                                        remove_delay=1000,
                                        worker_disconnect_delay=10)
     self.w = Worker(scheduler=self.sch, worker_id='X')
     self.assistant = Worker(scheduler=self.sch,
                             worker_id='Y',
                             assistant=True)
Example #19
0
 def test_task_lists_some_beyond_limit(self):
     sch = CentralPlannerScheduler(max_shown_tasks=3)
     for c in 'ABCD':
         sch.add_task(worker=WORKER, task_id=c, status=DONE)
     for c in 'EFG':
         sch.add_task(worker=WORKER, task_id=c)
     self.assertEqual(set('EFG'), set(sch.task_list('PENDING', '').keys()))
     self.assertEqual({'num_tasks': 4}, sch.task_list('DONE', ''))
Example #20
0
    def __init__(self,
                 scheduler=None,
                 worker_id=None,
                 worker_processes=1,
                 assistant=False,
                 **kwargs):
        if scheduler is None:
            scheduler = CentralPlannerScheduler()

        self.worker_processes = int(worker_processes)
        self._worker_info = self._generate_worker_info()

        if not worker_id:
            worker_id = 'Worker(%s)' % ', '.join(
                ['%s=%s' % (k, v) for k, v in self._worker_info])

        self._config = worker(**kwargs)

        assert self._config.wait_interval >= _WAIT_INTERVAL_EPS, "[worker] wait_interval must be positive"
        assert self._config.wait_jitter >= 0.0, "[worker] wait_jitter must be equal or greater than zero"

        self._id = worker_id
        self._scheduler = scheduler
        self._assistant = assistant
        self._stop_requesting_work = False

        self.host = socket.gethostname()
        self._scheduled_tasks = {}
        self._suspended_tasks = {}

        self._first_task = None

        self.add_succeeded = True
        self.run_succeeded = True
        self.unfulfilled_counts = collections.defaultdict(int)

        try:
            signal.signal(signal.SIGUSR1, self.handle_interrupt)
        except AttributeError:
            pass

        self._keep_alive_thread = KeepAliveThread(self._scheduler, self._id,
                                                  self._config.ping_interval)
        self._keep_alive_thread.daemon = True
        self._keep_alive_thread.start()

        # Keep info about what tasks are running (could be in other processes)
        if worker_processes == 1:
            self._task_result_queue = DequeQueue()
        else:
            self._task_result_queue = multiprocessing.Queue()

        self._running_tasks = {}

        # Stuff for execution_summary
        self._add_task_history = []
        self._get_work_response_history = []
Example #21
0
    def run(self, result=None):
        self.sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10)
        self.time = time.time
        with Worker(scheduler=self.sch, worker_id='X') as w, Worker(scheduler=self.sch, worker_id='Y') as w2:
            self.w = w
            self.w2 = w2
            super(WorkerTest, self).run(result)

        if time.time != self.time:
            time.time = self.time
Example #22
0
 def run(self, result=None):
     self.sch = CentralPlannerScheduler(retry_delay=100,
                                        remove_delay=1000,
                                        worker_disconnect_delay=10)
     self.assistant = Worker(scheduler=self.sch,
                             worker_id='Y',
                             assistant=True)
     with Worker(scheduler=self.sch, worker_id='X') as w:
         self.w = w
         super(AssistantTest, self).run(result)
Example #23
0
    def __init__(self,
                 scheduler=None,
                 worker_id=None,
                 worker_processes=1,
                 assistant=False,
                 raise_on_error=False,
                 **kwargs):
        if scheduler is None:
            scheduler = CentralPlannerScheduler()

        self.worker_processes = int(worker_processes)
        self._worker_info = self._generate_worker_info()

        if not worker_id:
            worker_id = 'Worker(%s)' % ', '.join(
                ['%s=%s' % (k, v) for k, v in self._worker_info])

        self._config = worker(**kwargs)

        # multiprocessing.Queue.get() is undefined for timeout=0
        assert self._config.wait_interval >= 0.00001, "[worker] wait_interval must be positive"

        self._id = worker_id
        self._scheduler = scheduler
        self._assistant = assistant
        self._stop_requesting_work = False

        self.host = socket.gethostname()
        self._scheduled_tasks = {}
        self._suspended_tasks = {}

        self._first_task = None

        self.add_succeeded = True
        self.run_succeeded = True
        self.unfulfilled_counts = collections.defaultdict(int)

        signal.signal(signal.SIGUSR1, self.handle_interrupt)

        self._keep_alive_thread = KeepAliveThread(self._scheduler, self._id,
                                                  self._config.ping_interval)
        self._keep_alive_thread.daemon = True
        self._keep_alive_thread.start()

        # Keep info about what tasks are running (could be in other processes)
        self._task_result_queue = multiprocessing.Queue()
        self._running_tasks = {}

        # Stuff for execution_summary
        self._add_task_history = []
        self._get_work_response_history = []

        # whether to raise when a task throws an exception
        self._raise_on_error = raise_on_error
 def test_search_results_beyond_limit(self):
     sch = CentralPlannerScheduler(max_shown_tasks=3)
     for i in range(4):
         sch.add_task(worker=WORKER,
                      family='Test',
                      params={'p': str(i)},
                      task_id='Test_%i' % i)
     self.assertEqual({'num_tasks': 4},
                      sch.task_list('PENDING', '', search='Test'))
     self.assertEqual(['Test_0'],
                      list(sch.task_list('PENDING', '', search='0').keys()))
Example #25
0
    def test_automatic_re_enable_with_one_failure_allowed(self):
        self.sch = CentralPlannerScheduler(disable_failures=1, disable_persist=100)
        self.setTime(0)
        self.sch.add_task(worker=WORKER, task_id='A', status=FAILED)

        # should be disabled now
        self.assertEqual(DISABLED, self.sch.task_list('', '')['A']['status'])

        # re-enables after 100 seconds
        self.setTime(101)
        self.assertEqual(FAILED, self.sch.task_list('', '')['A']['status'])
Example #26
0
    def test_no_automatic_re_enable_after_manual_disable(self):
        self.sch = CentralPlannerScheduler(disable_persist=100)
        self.setTime(0)
        self.sch.add_task(worker=WORKER, task_id='A', status=DISABLED)

        # should be disabled now
        self.assertEqual(DISABLED, self.sch.task_list('', '')['A']['status'])

        # should not re-enable after 100 seconds
        self.setTime(101)
        self.assertEqual(DISABLED, self.sch.task_list('', '')['A']['status'])
Example #27
0
    def __init__(self, scheduler=None, worker_id=None, worker_processes=1, assistant=False, **kwargs):
        if scheduler is None:
            scheduler = CentralPlannerScheduler()

        self.worker_processes = int(worker_processes)

        self._worker_info = self._generate_worker_info()
        config = configuration.get_config()

        if not worker_id:
            default_worker_id = 'Worker(%s)' % ', '.join(['%s=%s' % (k, v) for k, v in self._worker_info])
            worker_id = config.get('worker_metadata', 'worker_id', default_worker_id)

        if config.getboolean('worker_history', 'record_worker_history_sqs', False):
            import sqs_history  # Needs boto, thus imported here
            self._worker_history_impl = sqs_history.SqsWorkerHistory()
        else:
            self._worker_history_impl = NopWorkerHistory()

        self._config = worker(**kwargs)

        self._id = worker_id
        self._scheduler = scheduler
        self._assistant = assistant

        self.host = socket.gethostname()
        self._scheduled_tasks = {}
        self._suspended_tasks = {}

        self._first_task = None

        self.add_succeeded = True
        self.run_succeeded = True
        self.unfulfilled_counts = collections.defaultdict(int)

        self._worker_history_impl.worker_started(worker_id)

        self._keep_alive_thread = KeepAliveThread(self._scheduler, self._id, self._config.ping_interval)
        self._keep_alive_thread.daemon = True
        self._keep_alive_thread.start()

        # Keep info about what tasks are running (could be in other processes)
        self._task_result_queue = multiprocessing.Queue()
        self._running_tasks = {}
Example #28
0
    def setUp(self):
        try:
            from luigi.sqs_history import SqsHistory, SqsTaskHistory, SqsWorkerHistory
        except ImportError as e:
            raise unittest.SkipTest(
                'Could not test WorkerTaskGlobalEventHandlerTests: %s' % e)

        # Replace _config method with one that uses our dummy queue.
        def fake_config(s, *args):
            s._queue = DummyQueue()

        SqsHistory._config = fake_config

        # InstanceCache.disable()
        self.sch = CentralPlannerScheduler(retry_delay=100,
                                           remove_delay=1000,
                                           worker_disconnect_delay=10)
        self.w = Worker(scheduler=self.sch, worker_id='X')
        self.w2 = Worker(scheduler=self.sch, worker_id='Y')
        self.time = time.time
Example #29
0
 def test_task_list_within_limit(self):
     sch = CentralPlannerScheduler(max_shown_tasks=4)
     for c in 'ABCD':
         sch.add_task(worker=WORKER, task_id=c)
     self.assertEqual(set('ABCD'), set(sch.task_list('PENDING', '').keys()))
Example #30
0
 def setUp(self):
     super(CentralPlannerTest, self).setUp()
     conf = self.get_scheduler_config()
     self.sch = CentralPlannerScheduler(**conf)
     self.time = time.time