# set the axis labels and figure title
units = r"h$^{-1}$ [Mpc]"
plt.xlabel(LABELS[(AXIS+1)%3]+units)
plt.ylabel(LABELS[(AXIS-1)%3]+units)
cb.set_label(r"h$^2$ [M$_{\odot}$ Mpc$^{-3}$]")
slice_pos = INDEX*(cell_half_width[AXIS]*2.)+cell_half_width[AXIS]
plt.title(LABELS[AXIS]+" = {:03.2f}".format(slice_pos)+r"h$^{-1}$ Mpc")
# save the image or create a movie by recursively calling the `animate` function
if not MOVIE:
plt.draw()
plt.savefig("slice_"+LABELS[AXIS]+"_{:04d}.png".format(INDEX))
else:
print "Generating movie..."
try:
progressbar = ProgressBar(n_cell[AXIS]+1)
except:
pass
anim = animation.FuncAnimation(fig, animate, frames=n_cell[AXIS], blit=True)
# Save as mp4. This requires ffmpeg to be installed.
anim.save('slice_'+LABELS[AXIS]+'.mp4', fps=FPS, bitrate=BITRATE)
try:
progressbar.__exit__(None,None,None)
except:
pass
# set the axis labels and figure title
units = r"h$^{-1}$ [Mpc]"
plt.xlabel(LABELS[(AXIS + 1) % 3] + units)
plt.ylabel(LABELS[(AXIS - 1) % 3] + units)
cb.set_label(r"h$^2$ [M$_{\odot}$ Mpc$^{-3}$]")
slice_pos = INDEX * (cell_half_width[AXIS] * 2.) + cell_half_width[AXIS]
plt.title(LABELS[AXIS] + " = {:03.2f}".format(slice_pos) + r"h$^{-1}$ Mpc")
# save the image or create a movie by recursively calling the `animate` function
if not MOVIE:
plt.draw()
plt.savefig("slice_" + LABELS[AXIS] + "_{:04d}.png".format(INDEX))
else:
print "Generating movie..."
try:
progressbar = ProgressBar(n_cell[AXIS] + 1)
except:
pass
anim = animation.FuncAnimation(fig,
animate,
frames=n_cell[AXIS],
blit=True)
# Save as mp4. This requires ffmpeg to be installed.
anim.save('slice_' + LABELS[AXIS] + '.mp4', fps=FPS, bitrate=BITRATE)
try:
progressbar.__exit__(None, None, None)
except:
pass
class Pool(object):
"""
The main pool object. Manages a set of specified workers.
Usage::
commands = [
'ls -al',
'cd /tmp && mkdir foo',
'date',
'echo "Hello There."',
'sleep 2 && echo "Done."'
]
lil = Pool(workers=2)
lil.run(commands)
Optionally accepts a ``workers`` kwarg. Default is 1.
Optionally accepts a ``debug`` kwarg. Default is False.
Optionally accepts a ``wait_time`` kwarg. Default is 0.1.
"""
def __init__(self, workers=1, debug=False, wait_time=0.1):
if workers < 1:
raise NotEnoughWorkers("You need to use at least one worker.")
self.workers = workers
self.pool = {}
self.commands = []
self.callback = None
self.debug = debug
self.wait_time = wait_time
self.progressbar = None
def init_progressbar(self):
"""
Initialise the progress bar.
This only happens if run command is called with ``progressbar=True``.
"""
self.progressbar = ProgressBar(self.command_count())
def prepare_commands(self, commands):
"""
A hook to override how the commands are added.
By default, simply copies the provided command ``list`` to the
internal ``commands`` list.
"""
# Make a copy of the commands to run.
self.commands = commands[:]
def command_count(self):
"""
Returns the number of commands to be run.
Useful as a hook if you use a different structure for the commands.
"""
return len(self.commands)
def next_command(self):
"""
Fetches the next command for processing.
Will return ``None`` if there are no commands remaining (unless
``Pool.debug = True``).
"""
try:
return self.commands.pop(0)
except IndexError:
if self.debug:
raise
return None
def process_kwargs(self, command):
"""
A hook to alter the kwargs given to ``subprocess.Process``.
Takes a ``command`` argument, which is unused by default, but can be
used to switch the flags used.
By default, only specifies ``shell=True``.
"""
return {"shell": True}
def create_process(self, command):
"""
Given a provided command (string or list), creates a new process
to execute the command.
"""
logging.debug("Starting process to handle command '%s'." % command)
kwargs = self.process_kwargs(command)
return subprocess.Popen(command, **kwargs)
def set_callback(self, callback=None):
"""
Sets up a callback to be run whenever a process finishes.
If called with ``None`` or without any args, it will clear any
existing callback.
"""
self.callback = callback
def add_to_pool(self, proc):
"""
Adds a process to the pool.
"""
logging.debug("Adding %s to the pool." % proc.pid)
self.pool[proc.pid] = proc
def remove_from_pool(self, pid):
"""
Removes a process to the pool.
Fails silently if the process id is no longer present (unless
``Pool.debug = True``).
"""
try:
logging.debug("Removing %s from the pool" % pid)
del (self.pool[pid])
except KeyError:
if self.debug:
raise
def inspect_pool(self):
"""
A hook for inspecting the pool's current status.
By default, simply makes a log message and returns the length of
the pool.
"""
# Call ``len()`` just once.
pool_size = len(self.pool)
logging.debug("Current pool size: %s" % pool_size)
return pool_size
def busy_wait(self):
"""
A hook to control how often the busy-wait loop runs.
By default, sleeps for 0.1 seconds.
"""
time.sleep(self.wait_time)
def run(self, commands=None, callback=None, progressbar=False):
"""
The method to actually execute all the commands with the pool.
Optionally accepts a ``commands`` kwarg, as a shortcut not to have to
call ``Pool.prepare_commands``.
"""
if commands is not None:
self.prepare_commands(commands)
if callback is not None:
self.set_callback(callback)
if progressbar is True:
self.init_progressbar()
keep_running = True
while keep_running:
self.inspect_pool()
if len(self.pool) <= min(self.command_count(), self.workers):
command = self.next_command()
if not command:
self.busy_wait()
continue
proc = self.create_process(command)
self.add_to_pool(proc)
# Go in reverse order so offsets never get screwed up.
for pid in self.pool.keys():
logging.debug("Checking status on %s" % self.pool[pid].pid)
if self.pool[pid].poll() >= 0:
if self.callback:
self.callback(self.pool[pid])
if progressbar:
self.progressbar.update()
self.remove_from_pool(pid)
keep_running = self.command_count() or len(self.pool) > 0
self.busy_wait()
if progressbar:
self.progressbar.__exit__(None, None, None)
class Pool(object):
"""
The main pool object. Manages a set of specified workers.
Usage::
commands = [
'ls -al',
'cd /tmp && mkdir foo',
'date',
'echo "Hello There."',
'sleep 2 && echo "Done."'
]
lil = Pool(workers=2)
lil.run(commands)
Optionally accepts a ``workers`` kwarg. Default is 1.
Optionally accepts a ``debug`` kwarg. Default is False.
Optionally accepts a ``wait_time`` kwarg. Default is 0.1.
"""
def __init__(self, workers=1, debug=False, wait_time=0.1):
if workers < 1:
raise NotEnoughWorkers("You need to use at least one worker.")
self.workers = workers
self.pool = {}
self.commands = []
self.callback = None
self.debug = debug
self.wait_time = wait_time
self.progressbar = None
def init_progressbar(self):
"""
Initialise the progress bar.
This only happens if run command is called with ``progressbar=True``.
"""
self.progressbar = ProgressBar(self.command_count())
def prepare_commands(self, commands):
"""
A hook to override how the commands are added.
By default, simply copies the provided command ``list`` to the
internal ``commands`` list.
"""
# Make a copy of the commands to run.
self.commands = commands[:]
def command_count(self):
"""
Returns the number of commands to be run.
Useful as a hook if you use a different structure for the commands.
"""
return len(self.commands)
def next_command(self):
"""
Fetches the next command for processing.
Will return ``None`` if there are no commands remaining (unless
``Pool.debug = True``).
"""
try:
return self.commands.pop(0)
except IndexError:
if self.debug:
raise
return None
def process_kwargs(self, command):
"""
A hook to alter the kwargs given to ``subprocess.Process``.
Takes a ``command`` argument, which is unused by default, but can be
used to switch the flags used.
By default, only specifies ``shell=True``.
"""
return {
'shell': True,
}
def create_process(self, command):
"""
Given a provided command (string or list), creates a new process
to execute the command.
"""
logging.debug("Starting process to handle command '%s'." % command)
kwargs = self.process_kwargs(command)
return subprocess.Popen(command, **kwargs)
def set_callback(self, callback=None):
"""
Sets up a callback to be run whenever a process finishes.
If called with ``None`` or without any args, it will clear any
existing callback.
"""
self.callback = callback
def add_to_pool(self, proc):
"""
Adds a process to the pool.
"""
logging.debug("Adding %s to the pool." % proc.pid)
self.pool[proc.pid] = proc
def remove_from_pool(self, pid):
"""
Removes a process to the pool.
Fails silently if the process id is no longer present (unless
``Pool.debug = True``).
"""
try:
logging.debug("Removing %s from the pool" % pid)
del (self.pool[pid])
except KeyError:
if self.debug:
raise
def inspect_pool(self):
"""
A hook for inspecting the pool's current status.
By default, simply makes a log message and returns the length of
the pool.
"""
# Call ``len()`` just once.
pool_size = len(self.pool)
logging.debug("Current pool size: %s" % pool_size)
return pool_size
def busy_wait(self):
"""
A hook to control how often the busy-wait loop runs.
By default, sleeps for 0.1 seconds.
"""
time.sleep(self.wait_time)
def run(self, commands=None, callback=None, progressbar=False):
"""
The method to actually execute all the commands with the pool.
Optionally accepts a ``commands`` kwarg, as a shortcut not to have to
call ``Pool.prepare_commands``.
"""
if commands is not None:
self.prepare_commands(commands)
if callback is not None:
self.set_callback(callback)
if progressbar is True:
self.init_progressbar()
keep_running = True
while keep_running:
self.inspect_pool()
if len(self.pool) <= min(self.command_count(), self.workers):
command = self.next_command()
if not command:
self.busy_wait()
continue
proc = self.create_process(command)
self.add_to_pool(proc)
# Go in reverse order so offsets never get screwed up.
for pid in self.pool.keys():
logging.debug("Checking status on %s" % self.pool[pid].pid)
if self.pool[pid].poll() >= 0:
if self.callback:
self.callback(self.pool[pid])
if progressbar:
self.progressbar.update()
self.remove_from_pool(pid)
keep_running = self.command_count() or len(self.pool) > 0
self.busy_wait()
if progressbar:
self.progressbar.__exit__(None, None, None)
