def run(self):
"""Run the worker"""
self._loop = True
while self._loop:
if self.input_queue is not None:
try:
data = self.input_queue.get(True, 1)
if self.prev_lock is not None:
self.logger.debug(
"TemplateDaemon acquires lock of "
"previous worker: %s", str(self.prev_lock))
acquire_lock(self.prev_lock)
self.queue.task_done()
except Queue.Empty:
continue
self.logger.info("New data received.")
res = do_stuff(data)
self.output_queue.put(res)
# After all the items have been processed, release the
# lock for the previous worker
if self.prev_lock is not None:
self.logger.debug(
"Writer releses lock of "
"previous worker: %s", str(self.prev_lock))
release_lock(self.prev_lock)
else:
time.sleep(1)
def find_dead_threads(workers, logger, config):
"""Check that all threads are alive, and try to reboot them if they've
died."""
for i, worker in enumerate(workers):
# Check if the worker is dead
try:
is_alive = worker.is_alive()
except AttributeError:
is_alive = True
if not is_alive:
workers[i] = restart_dead_worker(logger, config, worker, i)
release_lock(workers[i].prev_lock)
release_lock(workers[i].lock)
logger.info("Crashed worker restarted")
def find_dead_threads(workers, logger, config):
"""Check that all threads are alive, and try to reboot them if they've
died."""
for i, worker in enumerate(workers):
# Check if the worker is dead
try:
is_alive = worker.is_alive()
except AttributeError:
is_alive = True
if not is_alive:
workers[i] = restart_dead_worker(logger, config, worker, i)
release_lock(workers[i].prev_lock)
release_lock(workers[i].lock)
logger.info("Crashed worker restarted")
def release_locks(locks, log=None, log_msg=None):
"""Release locks and optionnally send log message to *log* function"""
if not isinstance(locks, list):
locks = [locks]
if log is not None and log_msg is not None:
log(log_msg)
ret_vals = []
for lock in locks:
ret_vals.append(release_lock(lock))
return max(ret_vals)
def invoke(self, context):
"""Invoke"""
# Set locking status, default to False
self.use_lock = context.get("use_lock", False)
self.logger.debug("Locking is used in TemplatePlugin: %s",
str(self.use_lock))
if self.use_lock:
self.logger.debug(
"TemplatePlugin acquires lock of previous "
"worker: %s", str(context["prev_lock"]))
acquire_lock(context["prev_lock"])
self.logger.info("Doing something.")
something = do_something_with_content(context["content"])
context["output_queue"].put(something)
if self.use_lock:
self.logger.debug("TemplatePlugin releases own lock %s",
str(context["lock"]))
release_lock(context["lock"])
# Wait 1 second to ensure next worker has time to acquire the
# lock
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
acquire_lock(context["lock"])
release_lock(context["lock"])
# After all the items have been processed, release the lock for
# the previous step
self.logger.debug("Scene loader releses lock of previous worker")
release_lock(context["prev_lock"])
def invoke(self, context):
"""Invoke."""
# Set locking status, default to False
self.use_lock = context.get("use_lock", False)
logger.debug("Locking is used in Fetcher: %s", str(self.use_lock))
if self.use_lock:
logger.debug("Fetcher acquires lock of previous "
"worker: %s", str(context["prev_lock"]))
acquire_lock(context["prev_lock"])
logger.info("Fetching files.")
message = fetch_files(context["content"],
context.get("destination", gettempdir()))
context["output_queue"].put(message)
if self.use_lock:
logger.debug("Fetcher releases own lock %s", str(context["lock"]))
release_lock(context["lock"])
# Wait 1 second to ensure next worker has time to acquire the
# lock
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
acquire_lock(context["lock"])
release_lock(context["lock"])
# After all the items have been processed, release the lock for
# the previous step
logger.debug("Fetcher releases lock of previous worker")
release_lock(context["prev_lock"])
def invoke(self, context):
"""Invoke"""
# Set locking status, default to False
self.use_lock = context.get("use_lock", False)
self.logger.debug("Locking is used in Satfire: %s",
str(self.use_lock))
if self.use_lock:
self.logger.debug("Satfire acquires lock of previous "
"worker: %s", str(context["prev_lock"]))
acquire_lock(context["prev_lock"])
self.logger.info("Finding forest fires.")
fff = ForestFire(context["config"])
try:
if fff.run(msg=context["content"]):
if "text_fname_pattern" in context["config"]:
fff.save_text()
if "hdf5_fname_pattern" in context["config"]:
fff.save_hdf5()
fff.clean()
finally:
if fff._pub is not None:
fff._pub.stop()
if self.use_lock:
self.logger.debug("Satfire releases own lock %s",
str(context["lock"]))
release_lock(context["lock"])
# Wait 1 second to ensure next worker has time to acquire the
# lock
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
acquire_lock(context["lock"])
release_lock(context["lock"])
# After all the items have been processed, release the lock for
# the previous step
self.logger.debug("Scene loader releses lock of previous worker")
release_lock(context["prev_lock"])
def test_release_lock(self):
self.assertTrue(self.lock.locked())
utils.release_lock(self.lock)
self.assertFalse(self.lock.locked())
def test_release_lock(self):
self.assertTrue(self.lock.locked())
utils.release_lock(self.lock)
self.assertFalse(self.lock.locked())