def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0, state_path='/var/lib/luigi-server/state.pickle', task_history=None, resources=None, disable_persist=0, disable_window=0, disable_failures=None): ''' (all arguments are in seconds) Keyword Arguments: retry_delay -- How long after a Task fails to try it again, or -1 to never retry remove_delay -- How long after a Task finishes to remove it from the scheduler state_path -- Path to state file (tasks and active workers) worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers ''' self._retry_delay = retry_delay self._remove_delay = remove_delay self._worker_disconnect_delay = worker_disconnect_delay self._task_history = task_history or history.NopHistory() self._state = SimpleTaskState(state_path) self._task_history = task_history or history.NopHistory() self._resources = resources self._disable_failures = disable_failures self._disable_window = disable_window self._make_task = functools.partial( Task, disable_failures=disable_failures, disable_window=datetime.timedelta(seconds=disable_window)) self._disable_persist = disable_persist self._disable_time = datetime.timedelta(seconds=disable_persist)
def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0, state_path='/var/lib/luigi-server/state.pickle', task_history=None, resources=None, disable_persist=0, disable_window=0, disable_failures=None, max_shown_tasks=100000): """ (all arguments are in seconds) Keyword Arguments: :param retry_delay: how long after a Task fails to try it again, or -1 to never retry. :param remove_delay: how long after a Task finishes to remove it from the scheduler. :param state_path: path to state file (tasks and active workers). :param worker_disconnect_delay: if a worker hasn't communicated for this long, remove it from active workers. """ self._config = SchedulerConfig( retry_delay=retry_delay, remove_delay=remove_delay, worker_disconnect_delay=worker_disconnect_delay, disable_failures=disable_failures, disable_window=disable_window, disable_persist=disable_persist, disable_time=disable_persist, max_shown_tasks=max_shown_tasks, ) self._state = SimpleTaskState(state_path) self._task_history = task_history or history.NopHistory() self._resources = resources self._make_task = functools.partial( Task, disable_failures=disable_failures, disable_window=disable_window)
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle') # Jobs are disabled if we see more than disable_failures failures in disable_window seconds. # These disables last for disable_persist seconds. disable_window = config.getint('scheduler', 'disable-window-seconds', 3600) disable_failures = config.getint('scheduler', 'disable-num-failures', None) disable_persist = config.getint('scheduler', 'disable-persist-seconds', 86400) max_shown_tasks = config.getint('scheduler', 'max-shown-tasks', 100000) resources = config.getintdict('resources') if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler( retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl, resources, disable_persist, disable_window, disable_failures, max_shown_tasks, )
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, task_history_impl)
def __init__(self, retry_delay=900.0, remove_delay=600.0, worker_disconnect_delay=60.0, state_path='/var/lib/luigi-server/state.pickle', task_history=None): ''' (all arguments are in seconds) Keyword Arguments: retry_delay -- How long after a Task fails to try it again, or -1 to never retry remove_delay -- How long after a Task finishes to remove it from the scheduler state_path -- Path to state file (tasks and active workers) worker_disconnect_delay -- If a worker hasn't communicated for this long, remove it from active workers ''' self._state_path = state_path self._tasks = {} self._retry_delay = retry_delay self._remove_delay = remove_delay self._worker_disconnect_delay = worker_disconnect_delay self._active_workers = {} # map from id to timestamp (last updated) self._task_history = task_history or history.NopHistory()
def _create_scheduler(): config = configuration.get_config() retry_delay = config.getfloat('scheduler', 'retry-delay', 900.0) remove_delay = config.getfloat('scheduler', 'remove-delay', 600.0) worker_disconnect_delay = config.getfloat('scheduler', 'worker-disconnect-delay', 60.0) state_path = config.get('scheduler', 'state-path', '/var/lib/luigi-server/state.pickle') resources = config.getintdict('resources') if config.getboolean('scheduler', 'record_task_history', False): import db_task_history # Needs sqlalchemy, thus imported here task_history_impl = db_task_history.DbTaskHistory() else: task_history_impl = task_history.NopHistory() return scheduler.CentralPlannerScheduler(retry_delay, remove_delay, worker_disconnect_delay, state_path, task_history_impl, resources)