def __init__(self, config=None, resources=None, task_history_impl=None, **kwargs): """ Keyword Arguments: :param config: an object of class "scheduler" or None (in which the global instance will be used) :param resources: a dict of str->int constraints :param task_history_override: ignore config and use this object as the task history """ self._config = config or scheduler(**kwargs) self._state = SimpleTaskState(self._config.state_path) if task_history_impl: self._task_history = task_history_impl elif self._config.record_task_history: import db_task_history # Needs sqlalchemy, thus imported here self._task_history = db_task_history.DbTaskHistory() else: self._task_history = history.NopHistory() self._resources = resources or configuration.get_config().getintdict( 'resources') # TODO: Can we make this a Parameter? self._make_task = functools.partial( Task, disable_failures=self._config.disable_failures, disable_window=self._config.disable_window)
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle') # Jobs are disabled if we see more than disable_failures failures in disable_window seconds. # These disables last for disable_persist seconds. disable_window = config.getint('scheduler', 'disable-window-seconds', 3600) disable_failures = config.getint('scheduler', 'disable-num-failures', None) disable_persist = config.getint('scheduler', 'disable-persist-seconds', 86400) max_shown_tasks = config.getint('scheduler', 'max-shown-tasks', 100000) resources = config.getintdict('resources') if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler( retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl, resources, disable_persist, disable_window, disable_failures, max_shown_tasks, )
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, task_history_impl)
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle') resources = config.getintdict('resources') if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl, resources)