def __init__(self,
config=None,
resources=None,
task_history_impl=None,
**kwargs):
"""
Keyword Arguments:
:param config: an object of class "scheduler" or None (in which the global instance will be used)
:param resources: a dict of str->int constraints
:param task_history_override: ignore config and use this object as the task history
"""
self._config = config or scheduler(**kwargs)
self._state = SimpleTaskState(self._config.state_path)
if task_history_impl:
self._task_history = task_history_impl
elif self._config.record_task_history:
import db_task_history # Needs sqlalchemy, thus imported here
self._task_history = db_task_history.DbTaskHistory()
else:
self._task_history = history.NopHistory()
self._resources = resources or configuration.get_config().getintdict(
'resources') # TODO: Can we make this a Parameter?
self._make_task = functools.partial(
Task,
disable_failures=self._config.disable_failures,
disable_window=self._config.disable_window)
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler',
'worker-disconnect-delay', 60.0)
state_path = config.get('scheduler', 'state-path',
'/var/lib/luigi-server/state.pickle')
# Jobs are disabled if we see more than disable_failures failures in disable_window seconds.
# These disables last for disable_persist seconds.
disable_window = config.getint('scheduler', 'disable-window-seconds', 3600)
disable_failures = config.getint('scheduler', 'disable-num-failures', None)
disable_persist = config.getint('scheduler', 'disable-persist-seconds',
86400)
max_shown_tasks = config.getint('scheduler', 'max-shown-tasks', 100000)
resources = config.getintdict('resources')
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(
retry_delay,
remove_delay,
worker_disconnect_delay,
state_path,
task_history_impl,
resources,
disable_persist,
disable_window,
disable_failures,
max_shown_tasks,
)
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0)
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, task_history_impl)
def _create_scheduler():
config = configuration.get_config()
retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0)
remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0)
worker_disconnect_delay = config.getfloat('scheduler',
'worker-disconnect-delay', 60.0)
state_path = config.get('scheduler', 'state-path',
'/var/lib/luigi-server/state.pickle')
resources = config.getintdict('resources')
if config.getboolean('scheduler', 'record_task_history', False):
import db_task_history # Needs sqlalchemy, thus imported here
task_history_impl = db_task_history.DbTaskHistory()
else:
task_history_impl = task_history.NopHistory()
return scheduler.CentralPlannerScheduler(retry_delay, remove_delay,
worker_disconnect_delay,
state_path, task_history_impl,
resources)
