def __init__(self, scheduler=CentralPlannerScheduler(), worker_id=None,
worker_processes=1, ping_interval=None, keep_alive=None,
wait_interval=None):
if not worker_id:
worker_id = 'worker-%09d' % random.randrange(0, 999999999)
config = configuration.get_config()
if ping_interval is None:
ping_interval = config.getfloat('core', 'worker-ping-interval', 1.0)
if keep_alive is None:
keep_alive = config.getboolean('core', 'worker-keep-alive', False)
self.__keep_alive = keep_alive
if keep_alive:
if wait_interval is None:
wait_interval = config.getint('core', 'worker-wait-interval', 1)
self.__wait_interval = wait_interval
self.__id = worker_id
self.__scheduler = scheduler
if (isinstance(scheduler, CentralPlannerScheduler)
and worker_processes != 1):
warnings.warn("Will only use one process when running with local in-process scheduler")
worker_processes = 1
self.worker_processes = worker_processes
self.host = socket.gethostname()
self.__scheduled_tasks = {}
# store the previous tasks executed by the same worker
# for debugging reasons
self._previous_tasks = []
class KeepAliveThread(threading.Thread):
""" Periodically tell the scheduler that the worker still lives """
def __init__(self):
super(KeepAliveThread, self).__init__()
self._should_stop = threading.Event()
def stop(self):
self._should_stop.set()
def run(self):
while True:
self._should_stop.wait(ping_interval)
if self._should_stop.is_set():
logger.info("Worker was stopped. Shutting down Keep-Alive thread")
break
try:
scheduler.ping(worker=worker_id)
except: # httplib.BadStatusLine:
logger.warning('Failed pinging scheduler')
self._keep_alive_thread = KeepAliveThread()
self._keep_alive_thread.daemon = True
self._keep_alive_thread.start()
def __init__(self,
scheduler=CentralPlannerScheduler(),
worker_id=None,
erroremail=None,
worker_processes=1):
if not worker_id:
worker_id = 'worker-%09d' % random.randrange(0, 999999999)
self.__id = worker_id
self.__erroremail = erroremail
self.__scheduler = scheduler
if isinstance(scheduler,
CentralPlannerScheduler) and worker_processes != 1:
warnings.warn(
"Will only use one process when running with local in-process scheduler"
)
worker_processes = 1
self.worker_processes = worker_processes
self.__scheduled_tasks = {}
self._previous_tasks = [
] # store the previous tasks executed by the same worker for debugging reasons
class KeepAliveThread(threading.Thread):
""" Peridiacally tell the scheduler that the worker still lives """
def run(self):
while True:
time.sleep(1.0)
try:
scheduler.ping(worker=worker_id)
except: # httplib.BadStatusLine:
print 'WARNING: could not ping!'
raise
k = KeepAliveThread()
k.daemon = True
k.start()
def __init__(self,
scheduler=CentralPlannerScheduler(),
worker_id=None,
worker_processes=1,
ping_interval=None,
keep_alive=None,
wait_interval=None,
max_reschedules=None):
self._worker_info = self._generate_worker_info()
if not worker_id:
worker_id = 'Worker(%s)' % ', '.join(
['%s=%s' % (k, v) for k, v in self._worker_info])
config = configuration.get_config()
if ping_interval is None:
ping_interval = config.getfloat('core', 'worker-ping-interval',
1.0)
if keep_alive is None:
keep_alive = config.getboolean('core', 'worker-keep-alive', False)
self.__keep_alive = keep_alive
if keep_alive:
if wait_interval is None:
wait_interval = config.getint('core', 'worker-wait-interval',
1)
self.__wait_interval = wait_interval
if max_reschedules is None:
max_reschedules = config.getint('core', 'max-reschedules', 1)
self.__max_reschedules = max_reschedules
self._id = worker_id
self._scheduler = scheduler
self.worker_processes = int(worker_processes)
self.host = socket.gethostname()
self._scheduled_tasks = {}
self.add_succeeded = True
self.run_succeeded = True
self.unfulfilled_counts = collections.defaultdict(int)
class KeepAliveThread(threading.Thread):
""" Periodically tell the scheduler that the worker still lives """
def __init__(self):
super(KeepAliveThread, self).__init__()
self._should_stop = threading.Event()
def stop(self):
self._should_stop.set()
def run(self):
while True:
self._should_stop.wait(ping_interval)
if self._should_stop.is_set():
logger.info(
"Worker %s was stopped. Shutting down Keep-Alive thread"
% worker_id)
break
try:
scheduler.ping(worker=worker_id)
except: # httplib.BadStatusLine:
logger.warning('Failed pinging scheduler')
self._keep_alive_thread = KeepAliveThread()
self._keep_alive_thread.daemon = True
self._keep_alive_thread.start()
# Keep info about what tasks are running (could be in other processes)
self._task_result_queue = multiprocessing.Queue()
self._running_tasks = {}