def test_ping_retry(self):
""" Worker ping fails once. Ping continues to try to connect to scheduler
Kind of ugly since it uses actual timing with sleep to test the thread
"""
sch = Scheduler(
retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10,
)
self._total_pings = 0 # class var so it can be accessed from fail_ping
def fail_ping(worker):
# this will be called from within keep-alive thread...
self._total_pings += 1
raise Exception("Some random exception")
sch.ping = fail_ping
with Worker(
scheduler=sch,
worker_id="foo",
ping_interval=0.01 # very short between pings to make test fast
):
# let the keep-alive thread run for a bit...
time.sleep(
0.1) # yes, this is ugly but it's exactly what we need to test
self.assertTrue(self._total_pings > 1,
msg="Didn't retry pings (%d pings performed)" %
(self._total_pings, ))
def test_complete_exception(self):
"Tests that a task is still scheduled if its sister task crashes in the complete() method"
class A(DummyTask):
def complete(self):
raise Exception("doh")
a = A()
class C(DummyTask):
pass
c = C()
class B(DummyTask):
def requires(self):
return a, c
b = B()
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch, worker_id="foo") as w:
self.assertFalse(w.add(b))
self.assertTrue(w.run())
self.assertFalse(b.has_run)
self.assertTrue(c.has_run)
self.assertFalse(a.has_run)
def run(self, result=None):
super(WorkerEmailTest, self).setUp()
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch, worker_id="foo") as self.worker:
super(WorkerEmailTest, self).run(result)
def test_interleaved_workers2(self):
# two tasks without dependencies, one external, one not
class B(DummyTask):
pass
class ExternalB(ExternalTask):
task_family = "B"
def complete(self):
return False
b = B()
eb = ExternalB()
self.assertEqual(str(eb), "B()")
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch,
worker_id='X') as w, Worker(scheduler=sch,
worker_id='Y') as w2:
self.assertTrue(w2.add(eb))
self.assertTrue(w.add(b))
self.assertTrue(w2.run())
self.assertFalse(b.complete())
self.assertTrue(w.run())
self.assertTrue(b.complete())
def test_requires_exception(self):
class A(DummyTask):
def requires(self):
raise Exception("doh")
a = A()
class D(DummyTask):
pass
d = D()
class C(DummyTask):
def requires(self):
return d
c = C()
class B(DummyTask):
def requires(self):
return c, a
b = B()
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch, worker_id="foo") as w:
self.assertFalse(w.add(b))
self.assertTrue(w.run())
self.assertFalse(b.has_run)
self.assertTrue(c.has_run)
self.assertTrue(d.has_run)
self.assertFalse(a.has_run)
def __init__(self,
scheduler=None,
worker_id=None,
worker_processes=1,
assistant=False,
**kwargs):
if scheduler is None:
scheduler = Scheduler()
self.worker_processes = int(worker_processes)
self._worker_info = self._generate_worker_info()
if not worker_id:
worker_id = 'Worker(%s)' % ', '.join(
['%s=%s' % (k, v) for k, v in self._worker_info])
self._config = worker(**kwargs)
assert self._config.wait_interval >= _WAIT_INTERVAL_EPS, "[worker] wait_interval must be positive"
assert self._config.wait_jitter >= 0.0, "[worker] wait_jitter must be equal or greater than zero"
self._id = worker_id
self._scheduler = scheduler
self._assistant = assistant
self._stop_requesting_work = False
self.host = socket.gethostname()
self._scheduled_tasks = {}
self._suspended_tasks = {}
self._batch_running_tasks = {}
self._batch_families_sent = set()
self._first_task = None
self.add_succeeded = True
self.run_succeeded = True
self.unfulfilled_counts = collections.defaultdict(int)
# note that ``signal.signal(signal.SIGUSR1, fn)`` only works inside the main execution thread, which is why we
# provide the ability to conditionally install the hook.
if not self._config.no_install_shutdown_handler:
try:
signal.signal(signal.SIGUSR1, self.handle_interrupt)
signal.siginterrupt(signal.SIGUSR1, False)
except AttributeError:
pass
# Keep info about what tasks are running (could be in other processes)
if worker_processes == 1:
self._task_result_queue = DequeQueue()
else:
self._task_result_queue = multiprocessing.Queue()
self._running_tasks = {}
# Stuff for execution_summary
self._add_task_history = []
self._get_work_response_history = []
def run(self, result=None):
self.sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
self.assistant = Worker(scheduler=self.sch,
worker_id='Y',
assistant=True)
with Worker(scheduler=self.sch, worker_id='X') as w:
self.w = w
super(AssistantTest, self).run(result)
def run(self, result=None):
"""
Common setup code. Due to the contextmanager cant use normal setup
"""
self.sch = Scheduler(retry_delay=0.00000001, retry_count=2)
with Worker(scheduler=self.sch,
worker_id='X',
keep_alive=True,
wait_interval=0.1,
wait_jitter=0) as w:
self.w = w
super(WorkerKeepAliveUpstreamTest, self).run(result)
def run(self, result=None):
self.sch = Scheduler()
with Worker(scheduler=self.sch,
worker_id='X',
ping_interval=1,
max_reschedules=0) as w:
self.w = w
# also save scheduler's worker struct
self.sw = self.sch._state.get_worker(self.w._id)
super(WorkerSchedulerCommunicationTest, self).run(result)
def run(self, result=None):
self.sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
self.time = time.time
with Worker(scheduler=self.sch,
worker_id='X') as w, Worker(scheduler=self.sch,
worker_id='Y') as w2:
self.w = w
self.w2 = w2
super(WorkerTest, self).run(result)
if time.time != self.time:
time.time = self.time
def test_send_event_on_task_disabled(self):
s = Scheduler(metrics_collector=MetricsCollectors.datadog,
disable_persist=10,
retry_count=2,
disable_window=2)
task = self.startTask(scheduler=s)
self.collector.handle_task_disabled(task, s._config)
self.mock_create.assert_called_once_with(
alert_type='error',
priority='normal',
tags=[
'task_name:DDTaskName', 'task_state:DISABLED',
'environment:development', 'application:luigi'
],
text='A task has been disabled in the pipeline named: DDTaskName. '
+ 'The task has failed 2 times in the last 2 seconds' +
', so it is being disabled for 10 seconds.',
title='Luigi: A task has been disabled!')
def run(api_port=8082, address=None, unix_socket=None, scheduler=None, responder=None):
"""
Runs one instance of the API server.
"""
if scheduler is None:
scheduler = Scheduler()
# load scheduler state
scheduler.load()
_init_api(
scheduler=scheduler,
responder=responder,
api_port=api_port,
address=address,
unix_socket=unix_socket,
)
# prune work DAG every 60 seconds
pruner = tornado.ioloop.PeriodicCallback(scheduler.prune, 60000)
pruner.start()
def shutdown_handler(signum, frame):
exit_handler()
sys.exit(0)
@atexit.register
def exit_handler():
logger.info("Scheduler instance shutting down")
scheduler.dump()
stop()
signal.signal(signal.SIGINT, shutdown_handler)
signal.signal(signal.SIGTERM, shutdown_handler)
if os.name == 'nt':
signal.signal(signal.SIGBREAK, shutdown_handler)
else:
signal.signal(signal.SIGQUIT, shutdown_handler)
logger.info("Scheduler starting up")
tornado.ioloop.IOLoop.instance().start()
def test_interleaved_workers3(self):
class A(DummyTask):
def run(self):
logging.debug('running A')
time.sleep(0.1)
super(A, self).run()
a = A()
class B(DummyTask):
def requires(self):
return a
def run(self):
logging.debug('running B')
super(B, self).run()
b = B()
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch,
worker_id='X',
keep_alive=True,
count_uniques=True) as w:
with Worker(scheduler=sch,
worker_id='Y',
keep_alive=True,
count_uniques=True,
wait_interval=0.1) as w2:
self.assertTrue(w.add(a))
self.assertTrue(w2.add(b))
threading.Thread(target=w.run).start()
self.assertTrue(w2.run())
self.assertTrue(a.complete())
self.assertTrue(b.complete())
def test_die_for_non_unique_pending(self):
class A(DummyTask):
def run(self):
logging.debug('running A')
time.sleep(0.1)
super(A, self).run()
a = A()
class B(DummyTask):
def requires(self):
return a
def run(self):
logging.debug('running B')
super(B, self).run()
b = B()
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch,
worker_id='X',
keep_alive=True,
count_uniques=True) as w:
with Worker(scheduler=sch,
worker_id='Y',
keep_alive=True,
count_uniques=True,
wait_interval=0.1) as w2:
self.assertTrue(w.add(b))
self.assertTrue(w2.add(b))
self.assertEqual(w._get_work()[0], a.task_id)
self.assertTrue(w2.run())
self.assertFalse(a.complete())
self.assertFalse(b.complete())
def test_interleaved_workers(self):
class A(DummyTask):
pass
a = A()
class B(DummyTask):
def requires(self):
return a
class ExternalB(ExternalTask):
task_family = "B"
def complete(self):
return False
b = B()
eb = ExternalB()
self.assertEqual(str(eb), "B()")
sch = Scheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
with Worker(scheduler=sch,
worker_id='X') as w, Worker(scheduler=sch,
worker_id='Y') as w2:
self.assertTrue(w.add(b))
self.assertTrue(w2.add(eb))
logging.debug("RUNNING BROKEN WORKER")
self.assertTrue(w2.run())
self.assertFalse(a.complete())
self.assertFalse(b.complete())
logging.debug("RUNNING FUNCTIONAL WORKER")
self.assertTrue(w.run())
self.assertTrue(a.complete())
self.assertTrue(b.complete())
def setUp(self):
self.sch = Scheduler()
def _make_worker(self):
self.scheduler = Scheduler(prune_on_get_work=True)
return luigi.worker.Worker(scheduler=self.scheduler,
worker_processes=1)
def get_app(self):
return luigi.server.app(Scheduler())
def get_app(self):
conf = self.get_scheduler_config()
sch = Scheduler(**conf)
return luigi.server.app(sch)
def setUp(self):
self.mockDatadog()
self.time = time.time
self.collector = DatadogMetricsCollector()
self.s = Scheduler(metrics_collector=MetricsCollectors.datadog)
def setUp(self):
self.collector = PrometheusMetricsCollector()
self.s = Scheduler(metrics_collector=MetricsCollectors.prometheus)
self.gauge_name = 'luigi_task_execution_time_seconds'
self.labels = {'family': TASK_FAMILY}
