def __init__(self,
retry_delay=900.0,
remove_delay=600.0,
worker_disconnect_delay=60.0,
state_path='/var/lib/luigi-server/state.pickle',
task_history=None,
resources=None,
disable_persist=0,
disable_window=0,
disable_failures=None):
'''
(all arguments are in seconds)
Keyword Arguments:
retry_delay -- How long after a Task fails to try it again, or -1 to never retry
remove_delay -- How long after a Task finishes to remove it from the scheduler
state_path -- Path to state file (tasks and active workers)
worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers
'''
self._retry_delay = retry_delay
self._remove_delay = remove_delay
self._worker_disconnect_delay = worker_disconnect_delay
self._task_history = task_history or history.NopHistory()
self._state = SimpleTaskState(state_path)
self._task_history = task_history or history.NopHistory()
self._resources = resources
self._disable_failures = disable_failures
self._disable_window = disable_window
self._make_task = functools.partial(
Task,
disable_failures=disable_failures,
disable_window=datetime.timedelta(seconds=disable_window))
self._disable_persist = disable_persist
self._disable_time = datetime.timedelta(seconds=disable_persist)
def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0,
state_path='/var/lib/luigi-server/state.pickle', task_history=None,
resources=None, disable_persist=0, disable_window=0, disable_failures=None,
max_shown_tasks=100000):
"""
(all arguments are in seconds)
Keyword Arguments:
:param retry_delay: how long after a Task fails to try it again, or -1 to never retry.
:param remove_delay: how long after a Task finishes to remove it from the scheduler.
:param state_path: path to state file (tasks and active workers).
:param worker_disconnect_delay: if a worker hasn't communicated for this long, remove it from active workers.
"""
self._config = SchedulerConfig(
retry_delay=retry_delay,
remove_delay=remove_delay,
worker_disconnect_delay=worker_disconnect_delay,
disable_failures=disable_failures,
disable_window=disable_window,
disable_persist=disable_persist,
disable_time=disable_persist,
max_shown_tasks=max_shown_tasks,
)
self._state = SimpleTaskState(state_path)
self._task_history = task_history or history.NopHistory()
self._resources = resources
self._make_task = functools.partial(
Task, disable_failures=disable_failures,
disable_window=disable_window)
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler',
'worker-disconnect-delay', 60.0)
state_path = config.get('scheduler', 'state-path',
'/var/lib/luigi-server/state.pickle')
# Jobs are disabled if we see more than disable_failures failures in disable_window seconds.
# These disables last for disable_persist seconds.
disable_window = config.getint('scheduler', 'disable-window-seconds', 3600)
disable_failures = config.getint('scheduler', 'disable-num-failures', None)
disable_persist = config.getint('scheduler', 'disable-persist-seconds',
86400)
max_shown_tasks = config.getint('scheduler', 'max-shown-tasks', 100000)
resources = config.getintdict('resources')
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(
retry_delay,
remove_delay,
worker_disconnect_delay,
state_path,
task_history_impl,
resources,
disable_persist,
disable_window,
disable_failures,
max_shown_tasks,
)
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0)
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, task_history_impl)
def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0,
state_path='/var/lib/luigi-server/state.pickle', task_history=None):
'''
(all arguments are in seconds)
Keyword Arguments:
retry_delay -- How long after a Task fails to try it again, or -1 to never retry
remove_delay -- How long after a Task finishes to remove it from the scheduler
state_path -- Path to state file (tasks and active workers)
worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers
'''
self._state_path = state_path
self._tasks = {}
self._retry_delay = retry_delay
self._remove_delay = remove_delay
self._worker_disconnect_delay = worker_disconnect_delay
self._active_workers = {} # map from id to timestamp (last updated)
self._task_history = task_history or history.NopHistory()
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler',
'worker-disconnect-delay', 60.0)
state_path = config.get('scheduler', 'state-path',
'/var/lib/luigi-server/state.pickle')
resources = config.getintdict('resources')
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(retry_delay, remove_delay,
worker_disconnect_delay,
state_path, task_history_impl,
resources)
