def start_tasks(self):
try:
logger.info('Starting CachingDataController')
# Add data controller to the queue first so that it
# can get the initial data and is not blocked
data_controller = CachingDataController(self.hydrodataset, self.common_variables, self.n_run, self.get_data, self.write_lock, self.has_write_lock, self.read_lock, self.read_count,
self.time_chunk, self.horiz_chunk, self.times, self.start, self.point_get, self.reference_location, cache_path=self.cache_path)
self.tasks.put(data_controller)
# Create CachingDataController worker
self.data_controller_process = Consumer(self.tasks, self.results, self.n_run, self.nproc_lock, self.active, self.get_data, name="CachingDataController")
self.data_controller_process.start()
logger.info('Adding %i particles as tasks' % len(self.particles))
for part in self.particles:
forcer = CachingForcer(self.cache_path,
particle=part,
common_variables=self.common_variables,
timevar=self.timevar,
times=self.times,
start_time=self.start,
models=self._models,
release_location_centroid=self.reference_location.point,
usebathy=self._use_bathymetry,
useshore=self._use_shoreline,
usesurface=self._use_seasurface,
reverse_distance=self.reverse_distance,
bathy_path=self.bathy_path,
shoreline_path=self.shoreline_path,
shoreline_feature=self.shoreline_feature,
time_method=self.time_method,
redis_url=self.redis_url,
redis_results_channel=self.redis_results_channel,
shoreline_index_buffer=self.shoreline_index_buffer,
get_data=self.get_data,
read_lock=self.read_lock,
has_read_lock=self.has_read_lock,
read_count=self.read_count,
point_get=self.point_get,
data_request_lock=self.data_request_lock,
has_data_request_lock=self.has_data_request_lock
)
self.tasks.put(forcer)
# Create workers for the particles.
self.procs = [Consumer(self.tasks, self.results, self.n_run, self.nproc_lock, self.active, self.get_data, name="CachingForcer-%d" % i)
for i in xrange(self.nproc - 1) ]
for w in self.procs:
w.start()
logger.info('Started %s' % w.name)
return True
except Exception:
logger.exception("Something didn't start correctly!")
return False
class CachingModelController(BaseModelController):
def __init__(self, **kwargs):
super(CachingModelController, self).__init__(**kwargs)
self.time_chunk = kwargs.get('time_chunk', 10)
self.horiz_chunk = kwargs.get('horiz_chunk', 5)
def get_number_of_tasks(self):
# Add the CachingDataController to the number of tasks
return len(self.particles) + 1
def start_tasks(self):
try:
logger.info('Starting CachingDataController')
# Add data controller to the queue first so that it
# can get the initial data and is not blocked
data_controller = CachingDataController(self.hydrodataset, self.common_variables, self.n_run, self.get_data, self.write_lock, self.has_write_lock, self.read_lock, self.read_count,
self.time_chunk, self.horiz_chunk, self.times, self.start, self.point_get, self.reference_location, cache_path=self.cache_path)
self.tasks.put(data_controller)
# Create CachingDataController worker
self.data_controller_process = Consumer(self.tasks, self.results, self.n_run, self.nproc_lock, self.active, self.get_data, name="CachingDataController")
self.data_controller_process.start()
logger.info('Adding %i particles as tasks' % len(self.particles))
for part in self.particles:
forcer = CachingForcer(self.cache_path,
particle=part,
common_variables=self.common_variables,
timevar=self.timevar,
times=self.times,
start_time=self.start,
models=self._models,
release_location_centroid=self.reference_location.point,
usebathy=self._use_bathymetry,
useshore=self._use_shoreline,
usesurface=self._use_seasurface,
reverse_distance=self.reverse_distance,
bathy_path=self.bathy_path,
shoreline_path=self.shoreline_path,
shoreline_feature=self.shoreline_feature,
time_method=self.time_method,
redis_url=self.redis_url,
redis_results_channel=self.redis_results_channel,
shoreline_index_buffer=self.shoreline_index_buffer,
get_data=self.get_data,
read_lock=self.read_lock,
has_read_lock=self.has_read_lock,
read_count=self.read_count,
point_get=self.point_get,
data_request_lock=self.data_request_lock,
has_data_request_lock=self.has_data_request_lock
)
self.tasks.put(forcer)
# Create workers for the particles.
self.procs = [ Consumer(self.tasks, self.results, self.n_run, self.nproc_lock, self.active, self.get_data, name="CachingForcer-%d" % i)
for i in xrange(self.nproc - 1) ]
for w in self.procs:
w.start()
logger.info('Started %s' % w.name)
return True
except Exception:
logger.exception("Something didn't start correctly!")
return False
def __str__(self):
return "*** CachingModelController ***"
def setup_run(self, **kwargs):
super(CachingModelController, self).setup_run(**kwargs)
# Should we remove the cache file at the end of the run?
self.remove_cache = kwargs.get("remove_cache", False)
self.cache_path = kwargs.get("cache_path", None)
# Create a temp file for the cache if nothing was passed in
if self.cache_path is None:
default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache")
temp_name = AsaRandom.filename(prefix=str(datetime.now().microsecond), suffix=".nc")
self.cache_path = os.path.join(default_cache_dir, temp_name)
# Be sure the cache directory exists
if not os.path.exists(os.path.dirname(self.cache_path)):
logger.info("Creating cache directory: %s" % self.cache_path)
os.makedirs(os.path.dirname(self.cache_path))
# Create the shared state objects
# Particles use this to tell the Data Controller to "get_data".
# The CachingDataController sets this to False when it is done writing to the cache file.
# Particles will wait for this to be False before reading from the cache file.
# If we are caching, this starts as True so the Particles don't take off. If we
# are not caching, this is False so the Particles can start immediatly.
self.get_data = self.mgr.Value('bool', True)
# Particles use this to tell the DataContoller which indices to 'get_data' for
self.point_get = self.mgr.Value('list', [0, 0, 0])
# This locks access to the 'has_data_request_lock' value
self.data_request_lock = self.mgr.Lock()
# This tracks which Particle PID is asking the CachingDataController for data
self.has_data_request_lock = self.mgr.Value('int', -1)
# The lock that controls access to modifying 'has_read_lock' and 'read_count'
self.read_lock = self.mgr.Lock()
# List of Particle PIDs that are reading from the cache
self.has_read_lock = self.mgr.list()
# The number of Particles that are reading from the cache
self.read_count = self.mgr.Value('int', 0)
# When something is writing to the cache file
self.write_lock = self.mgr.Lock()
# PID of process with lock
self.has_write_lock = self.mgr.Value('int', -1)
def listen_for_results(self):
try:
# Get results back from queue, test for failed particles
return_particles = []
retrieved = 0.
self.error_code = 0
logger.info("Waiting for %i particle results" % len(self.particles))
logger.progress((5, "Running model"))
while retrieved < self.number_of_tasks:
try:
# Returns a tuple of code, result
code, tempres = self.results.get(timeout=240)
except Queue.Empty:
# Poll the active processes to make sure they are all alive and then continue with loop
if not self.data_controller_process.is_alive() and self.data_controller_process.exitcode != 0:
# Data controller is zombied, kill off other processes.
self.get_data.value is False
self.results.put((-2, "CachingDataController"))
new_procs = []
old_procs = []
for p in self.procs:
if not p.is_alive() and p.exitcode != 0:
# Do what the Consumer would do if something finished.
# Add something to results queue
self.results.put((-3, "ZombieParticle"))
# Decrement nproc (CachingDataController exits when this is 0)
with self.nproc_lock:
self.n_run.value = self.n_run.value - 1
# Remove task from queue (so they can be joined later on)
self.tasks.task_done()
# Start a new Consumer. It will exit if there are no tasks available.
np = Consumer(self.tasks, self.results, self.n_run, self.nproc_lock, self.active, self.get_data, name=p.name)
new_procs.append(np)
old_procs.append(p)
# Release any locks the PID had
if p.pid in self.has_read_lock:
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(p.pid)
if self.has_data_request_lock.value == p.pid:
self.has_data_request_lock.value = -1
try:
self.data_request_lock.release()
except:
pass
if self.has_write_lock.value == p.pid:
self.has_write_lock.value = -1
try:
self.write_lock.release()
except:
pass
for p in old_procs:
try:
self.procs.remove(p)
except ValueError:
logger.warn("Did not find %s in the list of processes. Continuing on." % p.name)
for p in new_procs:
self.procs.append(p)
logger.warn("Started a new consumer (%s) to replace a zombie consumer" % p.name)
p.start()
else:
# We got one.
retrieved += 1
if code is None:
logger.warn("Got an unrecognized response from a task.")
elif code == -1:
logger.warn("Particle %s has FAILED!!" % tempres.uid)
elif code == -2:
self.error_code = code
logger.warn("CachingDataController has FAILED!! Removing cache file so the particles fail.")
try:
os.remove(self.cache_path)
except OSError:
logger.debug("Could not remove cache file, it probably never existed")
pass
elif code == -3:
self.error_code = code
logger.info("A zombie process was caught and task was removed from queue")
elif isinstance(tempres, Particle):
logger.info("Particle %d finished" % tempres.uid)
return_particles.append(tempres)
# We mulitply by 95 here to save 5% for the exporting
logger.progress((round((retrieved / self.number_of_tasks) * 90., 1), "Particle %d finished" % tempres.uid))
elif tempres == "CachingDataController":
logger.info("CachingDataController finished")
logger.progress((round((retrieved / self.number_of_tasks) * 90., 1), "CachingDataController finished"))
else:
logger.info("Got a strange result on results queue")
logger.info(str(tempres))
logger.info("Retrieved %i/%i results" % (int(retrieved), self.number_of_tasks))
if len(return_particles) != len(self.particles):
logger.warn("Some particles failed and are not included in the output")
# The results queue should be empty at this point
assert self.results.empty() is True
# Should be good to join on the tasks now that the queue is empty
logger.info("Joining the task queue")
self.tasks.join()
self.particles = return_particles
finally:
# Join all processes
logger.info("Joining the processes")
for w in self.procs + [self.data_controller_process]:
# Wait 20 seconds
w.join(20.)
if w.is_alive():
# Process is hanging, kill it.
logger.info("Terminating %s forcefully. This should have exited itself." % w.name)
w.terminate()
def cleanup(self):
super(CachingModelController, self).cleanup()
# Remove the cache file
if self.remove_cache is True:
try:
os.remove(self.cache_path)
except OSError:
logger.debug("Could not remove cache file, it probably never existed")
def start_tasks(self, **kwargs):
try:
logger.info('Starting CachingDataController')
# Add data controller to the queue first so that it
# can get the initial data and is not blocked
data_controller = CachingDataController(self.hydrodataset,
self.common_variables,
self.n_run,
self.get_data,
self.write_lock,
self.has_write_lock,
self.read_lock,
self.read_count,
self.time_chunk,
self.horiz_chunk,
self.times,
self.start,
self.point_get,
self.reference_location,
cache_path=self.cache_path)
self.tasks.put(data_controller)
# Create CachingDataController worker
self.data_controller_process = Consumer(
self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name="CachingDataController")
self.data_controller_process.start()
logger.info('Adding %i particles as tasks' %
self.total_particle_count())
for part in self.particles:
forcer = CachingForcer(
self.cache_path,
particle=part,
common_variables=self.common_variables,
timevar=self.timevar,
times=self.times,
start_time=self.start,
models=self._models,
release_location_centroid=self.reference_location.point,
usebathy=self._use_bathymetry,
useshore=self._use_shoreline,
usesurface=self._use_seasurface,
reverse_distance=self.reverse_distance,
bathy_path=self.bathy_path,
shoreline_path=self.shoreline_path,
shoreline_feature=self.shoreline_feature,
time_method=self.time_method,
shoreline_index_buffer=self.shoreline_index_buffer,
get_data=self.get_data,
read_lock=self.read_lock,
has_read_lock=self.has_read_lock,
read_count=self.read_count,
point_get=self.point_get,
data_request_lock=self.data_request_lock,
has_data_request_lock=self.has_data_request_lock)
self.tasks.put(forcer)
# Create workers for the particles.
self.procs = [
Consumer(self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name="CachingForcer-%d" % i)
for i in range(self.nproc - 1)
]
logger.progress((5, 'Running model'))
for w in self.procs:
w.start()
logger.info('Started %s' % w.name)
return True
except Exception:
logger.exception("Something didn't start correctly!")
return False
class CachingModelController(BaseModelController):
def __init__(self, **kwargs):
super(CachingModelController, self).__init__(**kwargs)
self.time_chunk = kwargs.get('time_chunk', 10)
self.horiz_chunk = kwargs.get('horiz_chunk', 5)
def total_task_count(self):
# Add the CachingDataController to the number of tasks
return self.total_particle_count() + 1
def start_tasks(self, **kwargs):
try:
logger.info('Starting CachingDataController')
# Add data controller to the queue first so that it
# can get the initial data and is not blocked
data_controller = CachingDataController(self.hydrodataset,
self.common_variables,
self.n_run,
self.get_data,
self.write_lock,
self.has_write_lock,
self.read_lock,
self.read_count,
self.time_chunk,
self.horiz_chunk,
self.times,
self.start,
self.point_get,
self.reference_location,
cache_path=self.cache_path)
self.tasks.put(data_controller)
# Create CachingDataController worker
self.data_controller_process = Consumer(
self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name="CachingDataController")
self.data_controller_process.start()
logger.info('Adding %i particles as tasks' %
self.total_particle_count())
for part in self.particles:
forcer = CachingForcer(
self.cache_path,
particle=part,
common_variables=self.common_variables,
timevar=self.timevar,
times=self.times,
start_time=self.start,
models=self._models,
release_location_centroid=self.reference_location.point,
usebathy=self._use_bathymetry,
useshore=self._use_shoreline,
usesurface=self._use_seasurface,
reverse_distance=self.reverse_distance,
bathy_path=self.bathy_path,
shoreline_path=self.shoreline_path,
shoreline_feature=self.shoreline_feature,
time_method=self.time_method,
shoreline_index_buffer=self.shoreline_index_buffer,
get_data=self.get_data,
read_lock=self.read_lock,
has_read_lock=self.has_read_lock,
read_count=self.read_count,
point_get=self.point_get,
data_request_lock=self.data_request_lock,
has_data_request_lock=self.has_data_request_lock)
self.tasks.put(forcer)
# Create workers for the particles.
self.procs = [
Consumer(self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name="CachingForcer-%d" % i)
for i in range(self.nproc - 1)
]
logger.progress((5, 'Running model'))
for w in self.procs:
w.start()
logger.info('Started %s' % w.name)
return True
except Exception:
logger.exception("Something didn't start correctly!")
return False
def __str__(self):
return "*** CachingModelController ***"
def setup_run(self, hydrodataset, **kwargs):
super(CachingModelController, self).setup_run(hydrodataset, **kwargs)
# Get the number of cores (may take some tuning) and create that
# many workers then pass particles into the queue for the workers
self.mgr = multiprocessing.Manager()
# This tracks if the system is 'alive'. Most looping whiles will check this
# and break out if it is False. This is True until something goes very wrong.
self.active = self.mgr.Value('bool', True)
# Either spin up the number of cores, or the number of tasks
self.nproc = min(multiprocessing.cpu_count() - 1,
self.total_task_count())
# Number of tasks that we need to run. This is decremented everytime something exits.
self.n_run = self.mgr.Value('int', self.total_task_count())
# The lock that controls access to the 'n_run' variable
self.nproc_lock = self.mgr.Lock()
# Create the task queue for all of the particles and the CachingDataController
self.tasks = multiprocessing.JoinableQueue(self.total_task_count())
# Create the result queue for all of the particles and the CachingDataController
self.results = self.mgr.Queue(self.total_task_count())
# Should we remove the cache file at the end of the run?
self.remove_cache = kwargs.get("remove_cache", False)
self.cache_path = kwargs.get("cache_path", None)
# Create a temp file for the cache if nothing was passed in
if self.cache_path is None:
default_cache_dir = os.path.join(os.path.dirname(__file__),
"_cache")
temp_name = AsaRandom.filename(prefix=str(
datetime.now().microsecond),
suffix=".nc")
self.cache_path = os.path.join(default_cache_dir, temp_name)
# Be sure the cache directory exists
if not os.path.exists(os.path.dirname(self.cache_path)):
logger.info("Creating cache directory: %s" % self.cache_path)
os.makedirs(os.path.dirname(self.cache_path))
# Create the shared state objects
# Particles use this to tell the Data Controller to "get_data".
# The CachingDataController sets this to False when it is done writing to the cache file.
# Particles will wait for this to be False before reading from the cache file.
# If we are caching, this starts as True so the Particles don't take off. If we
# are not caching, this is False so the Particles can start immediatly.
self.get_data = self.mgr.Value('bool', True)
# Particles use this to tell the DataContoller which indices to 'get_data' for
self.point_get = self.mgr.Value('list', [0, 0, 0])
# This locks access to the 'has_data_request_lock' value
self.data_request_lock = self.mgr.Lock()
# This tracks which Particle PID is asking the CachingDataController for data
self.has_data_request_lock = self.mgr.Value('int', -1)
# The lock that controls access to modifying 'has_read_lock' and 'read_count'
self.read_lock = self.mgr.Lock()
# List of Particle PIDs that are reading from the cache
self.has_read_lock = self.mgr.list()
# The number of Particles that are reading from the cache
self.read_count = self.mgr.Value('int', 0)
# When something is writing to the cache file
self.write_lock = self.mgr.Lock()
# PID of process with lock
self.has_write_lock = self.mgr.Value('int', -1)
def listen_for_results(self, output_h5_file, total_particles):
try:
# Get results back from queue, test for failed particles
return_particles = []
retrieved = 0.
self.error_code = 0
logger.info("Waiting for %i particle results" % total_particles)
while retrieved < self.total_task_count(
): # One for the CachingDataController
logger.info("looping in listen_for_results")
try:
# Returns a tuple of code, result
code, tempres = self.results.get(timeout=240)
except queue.Empty:
# Poll the active processes to make sure they are all alive and then continue with loop
if not self.data_controller_process.is_alive(
) and self.data_controller_process.exitcode != 0:
# Data controller is zombied, kill off other processes.
self.get_data.value is False
self.results.put((-2, "CachingDataController"))
new_procs = []
old_procs = []
for p in self.procs:
if not p.is_alive() and p.exitcode != 0:
# Do what the Consumer would do if something finished.
# Add something to results queue
self.results.put((-3, "ZombieParticle"))
# Decrement nproc (CachingDataController exits when this is 0)
with self.nproc_lock:
self.n_run.value = self.n_run.value - 1
# Remove task from queue (so they can be joined later on)
self.tasks.task_done()
# Start a new Consumer. It will exit if there are no tasks available.
np = Consumer(self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name=p.name)
new_procs.append(np)
old_procs.append(p)
# Release any locks the PID had
if p.pid in self.has_read_lock:
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(p.pid)
if self.has_data_request_lock.value == p.pid:
self.has_data_request_lock.value = -1
try:
self.data_request_lock.release()
except:
pass
if self.has_write_lock.value == p.pid:
self.has_write_lock.value = -1
try:
self.write_lock.release()
except:
pass
for p in old_procs:
try:
self.procs.remove(p)
except ValueError:
logger.warn(
"Did not find %s in the list of processes. Continuing on."
% p.name)
for p in new_procs:
self.procs.append(p)
logger.warn(
"Started a new consumer (%s) to replace a zombie consumer"
% p.name)
p.start()
else:
# We got one.
retrieved += 1
if code is None:
logger.warn(
"Got an unrecognized response from a task.")
elif code == -1:
logger.warn("Particle %s has FAILED!!" % tempres.uid)
elif code == -2:
self.error_code = code
logger.warn(
"CachingDataController has FAILED!! Removing cache file so the particles fail."
)
try:
os.remove(self.cache_path)
except OSError:
logger.debug(
"Could not remove cache file, it probably never existed"
)
pass
elif code == -3:
self.error_code = code
logger.info(
"A zombie process was caught and task was removed from queue"
)
elif isinstance(tempres, Particle):
logger.info("Particle %d finished" % tempres.uid)
return_particles.append(tempres)
# We mulitply by 95 here to save 5% for the exporting
logger.progress(
(round((retrieved / self.total_task_count()) * 90.,
1), "Particle %d finished" % tempres.uid))
elif tempres == "CachingDataController":
logger.info("CachingDataController finished")
logger.progress(
(round((retrieved / self.total_task_count()) * 90.,
1), "CachingDataController finished"))
else:
logger.info("Got a strange result on results queue")
logger.info(str(tempres))
logger.info("Retrieved %i/%i results" %
(int(retrieved), self.total_task_count()))
# Relax
time.sleep(1)
if len(return_particles) != total_particles:
logger.warn(
"Some particles failed and are not included in the output")
# The results queue should be empty at this point
assert self.results.empty() is True
# Should be good to join on the tasks now that the queue is empty
logger.info("Joining the task queue")
self.tasks.join()
self.tasks.close()
self.tasks.join_thread()
finally:
# Join all processes
logger.info("Joining the processes")
for w in self.procs + [self.data_controller_process]:
# Wait 20 seconds
w.join(20.)
if w.is_alive():
# Process is hanging, kill it.
logger.info(
"Terminating %s forcefully. This should have exited itself."
% w.name)
w.terminate()
if self.error_code == -2:
raise ValueError(
"Error in the BaseDataController (error_code was -2)")
results = ex.ResultsPyTable(output_h5_file)
for p in return_particles:
for x in range(len(p.locations)):
results.write(p.timestep_index_dump(x))
results.compute()
results.close()
return
def cleanup(self):
super(CachingModelController, self).cleanup()
# Remove Manager so it shuts down
del self.mgr
# Remove the cache file
if self.remove_cache is True:
try:
os.remove(self.cache_path)
except OSError:
logger.debug(
"Could not remove cache file, it probably never existed")
def listen_for_results(self, output_h5_file, total_particles):
try:
# Get results back from queue, test for failed particles
return_particles = []
retrieved = 0.
self.error_code = 0
logger.info("Waiting for %i particle results" % total_particles)
while retrieved < self.total_task_count(
): # One for the CachingDataController
logger.info("looping in listen_for_results")
try:
# Returns a tuple of code, result
code, tempres = self.results.get(timeout=240)
except queue.Empty:
# Poll the active processes to make sure they are all alive and then continue with loop
if not self.data_controller_process.is_alive(
) and self.data_controller_process.exitcode != 0:
# Data controller is zombied, kill off other processes.
self.get_data.value is False
self.results.put((-2, "CachingDataController"))
new_procs = []
old_procs = []
for p in self.procs:
if not p.is_alive() and p.exitcode != 0:
# Do what the Consumer would do if something finished.
# Add something to results queue
self.results.put((-3, "ZombieParticle"))
# Decrement nproc (CachingDataController exits when this is 0)
with self.nproc_lock:
self.n_run.value = self.n_run.value - 1
# Remove task from queue (so they can be joined later on)
self.tasks.task_done()
# Start a new Consumer. It will exit if there are no tasks available.
np = Consumer(self.tasks,
self.results,
self.n_run,
self.nproc_lock,
self.active,
self.get_data,
name=p.name)
new_procs.append(np)
old_procs.append(p)
# Release any locks the PID had
if p.pid in self.has_read_lock:
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(p.pid)
if self.has_data_request_lock.value == p.pid:
self.has_data_request_lock.value = -1
try:
self.data_request_lock.release()
except:
pass
if self.has_write_lock.value == p.pid:
self.has_write_lock.value = -1
try:
self.write_lock.release()
except:
pass
for p in old_procs:
try:
self.procs.remove(p)
except ValueError:
logger.warn(
"Did not find %s in the list of processes. Continuing on."
% p.name)
for p in new_procs:
self.procs.append(p)
logger.warn(
"Started a new consumer (%s) to replace a zombie consumer"
% p.name)
p.start()
else:
# We got one.
retrieved += 1
if code is None:
logger.warn(
"Got an unrecognized response from a task.")
elif code == -1:
logger.warn("Particle %s has FAILED!!" % tempres.uid)
elif code == -2:
self.error_code = code
logger.warn(
"CachingDataController has FAILED!! Removing cache file so the particles fail."
)
try:
os.remove(self.cache_path)
except OSError:
logger.debug(
"Could not remove cache file, it probably never existed"
)
pass
elif code == -3:
self.error_code = code
logger.info(
"A zombie process was caught and task was removed from queue"
)
elif isinstance(tempres, Particle):
logger.info("Particle %d finished" % tempres.uid)
return_particles.append(tempres)
# We mulitply by 95 here to save 5% for the exporting
logger.progress(
(round((retrieved / self.total_task_count()) * 90.,
1), "Particle %d finished" % tempres.uid))
elif tempres == "CachingDataController":
logger.info("CachingDataController finished")
logger.progress(
(round((retrieved / self.total_task_count()) * 90.,
1), "CachingDataController finished"))
else:
logger.info("Got a strange result on results queue")
logger.info(str(tempres))
logger.info("Retrieved %i/%i results" %
(int(retrieved), self.total_task_count()))
# Relax
time.sleep(1)
if len(return_particles) != total_particles:
logger.warn(
"Some particles failed and are not included in the output")
# The results queue should be empty at this point
assert self.results.empty() is True
# Should be good to join on the tasks now that the queue is empty
logger.info("Joining the task queue")
self.tasks.join()
self.tasks.close()
self.tasks.join_thread()
finally:
# Join all processes
logger.info("Joining the processes")
for w in self.procs + [self.data_controller_process]:
# Wait 20 seconds
w.join(20.)
if w.is_alive():
# Process is hanging, kill it.
logger.info(
"Terminating %s forcefully. This should have exited itself."
% w.name)
w.terminate()
if self.error_code == -2:
raise ValueError(
"Error in the BaseDataController (error_code was -2)")
results = ex.ResultsPyTable(output_h5_file)
for p in return_particles:
for x in range(len(p.locations)):
results.write(p.timestep_index_dump(x))
results.compute()
results.close()
return
