class IPEngineTask(WinHPCTask):
task_name = Str('IPEngine', config=True)
engine_cmd = List(['ipengine.exe'], config=True)
engine_args = List(['--log-to-file', '--log-level', '40'], config=True)
# I don't want these to be configurable
std_out_file_path = CStr('', config=False)
std_err_file_path = CStr('', config=False)
min_cores = Int(1, config=False)
max_cores = Int(1, config=False)
min_sockets = Int(1, config=False)
max_sockets = Int(1, config=False)
min_nodes = Int(1, config=False)
max_nodes = Int(1, config=False)
unit_type = Str("Core", config=False)
work_directory = CStr('', config=False)
def __init__(self, config=None):
super(IPEngineTask, self).__init__(config=config)
the_uuid = uuid.uuid1()
self.std_out_file_path = os.path.join('log',
'ipengine-%s.out' % the_uuid)
self.std_err_file_path = os.path.join('log',
'ipengine-%s.err' % the_uuid)
@property
def command_line(self):
return ' '.join(self.engine_cmd + self.engine_args)
class WinHPCTask(Configurable):
task_id = Str('')
task_name = Str('')
version = Str("2.000")
min_cores = Int(1, config=True)
max_cores = Int(1, config=True)
min_sockets = Int(1, config=True)
max_sockets = Int(1, config=True)
min_nodes = Int(1, config=True)
max_nodes = Int(1, config=True)
unit_type = Str("Core", config=True)
command_line = CStr('', config=True)
work_directory = CStr('', config=True)
is_rerunnaable = Bool(True, config=True)
std_out_file_path = CStr('', config=True)
std_err_file_path = CStr('', config=True)
is_parametric = Bool(False, config=True)
environment_variables = Instance(dict, args=(), config=True)
def _write_attr(self, root, attr, key):
s = as_str(getattr(self, attr, ''))
if s:
root.set(key, s)
def as_element(self):
root = ET.Element('Task')
self._write_attr(root, 'version', '_A_Version')
self._write_attr(root, 'task_name', '_B_Name')
self._write_attr(root, 'min_cores', '_C_MinCores')
self._write_attr(root, 'max_cores', '_D_MaxCores')
self._write_attr(root, 'min_sockets', '_E_MinSockets')
self._write_attr(root, 'max_sockets', '_F_MaxSockets')
self._write_attr(root, 'min_nodes', '_G_MinNodes')
self._write_attr(root, 'max_nodes', '_H_MaxNodes')
self._write_attr(root, 'command_line', '_I_CommandLine')
self._write_attr(root, 'work_directory', '_J_WorkDirectory')
self._write_attr(root, 'is_rerunnaable', '_K_IsRerunnable')
self._write_attr(root, 'std_out_file_path', '_L_StdOutFilePath')
self._write_attr(root, 'std_err_file_path', '_M_StdErrFilePath')
self._write_attr(root, 'is_parametric', '_N_IsParametric')
self._write_attr(root, 'unit_type', '_O_UnitType')
root.append(self.get_env_vars())
return root
def get_env_vars(self):
env_vars = ET.Element('EnvironmentVariables')
for k, v in self.environment_variables.iteritems():
variable = ET.SubElement(env_vars, "Variable")
name = ET.SubElement(variable, "Name")
name.text = k
value = ET.SubElement(variable, "Value")
value.text = v
return env_vars
class SessionFactory(LoggingFactory):
"""The Base factory from which every factory in IPython.parallel inherits"""
packer = Str('', config=True)
unpacker = Str('', config=True)
ident = CStr('', config=True)
def _ident_default(self):
return str(uuid.uuid4())
username = CUnicode(os.environ.get('USER', 'username'), config=True)
exec_key = CUnicode('', config=True)
# not configurable:
context = Instance('zmq.Context', (), {})
session = Instance('IPython.parallel.streamsession.StreamSession')
loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
def _loop_default(self):
return IOLoop.instance()
def __init__(self, **kwargs):
super(SessionFactory, self).__init__(**kwargs)
exec_key = self.exec_key or None
# set the packers:
if not self.packer:
packer_f = unpacker_f = None
elif self.packer.lower() == 'json':
packer_f = ss.json_packer
unpacker_f = ss.json_unpacker
elif self.packer.lower() == 'pickle':
packer_f = ss.pickle_packer
unpacker_f = ss.pickle_unpacker
else:
packer_f = import_item(self.packer)
unpacker_f = import_item(self.unpacker)
# construct the session
self.session = ss.StreamSession(self.username,
self.ident,
packer=packer_f,
unpacker=unpacker_f,
key=exec_key)
class ControllerFactory(HubFactory):
"""Configurable for setting up a Hub and Schedulers."""
usethreads = Bool(False, config=True)
# pure-zmq downstream HWM
hwm = Int(0, config=True)
# internal
children = List()
mq_class = CStr('zmq.devices.ProcessMonitoredQueue')
def _usethreads_changed(self, name, old, new):
self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
def __init__(self, **kwargs):
super(ControllerFactory, self).__init__(**kwargs)
self.subconstructors.append(self.construct_schedulers)
def start(self):
super(ControllerFactory, self).start()
child_procs = []
for child in self.children:
child.start()
if isinstance(child, ProcessMonitoredQueue):
child_procs.append(child.launcher)
elif isinstance(child, Process):
child_procs.append(child)
if child_procs:
signal_children(child_procs)
def construct_schedulers(self):
children = self.children
mq = import_item(self.mq_class)
# maybe_inproc = 'inproc://monitor' if self.usethreads else self.monitor_url
# IOPub relay (in a Process)
q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
q.bind_in(self.client_info['iopub'])
q.bind_out(self.engine_info['iopub'])
q.setsockopt_out(zmq.SUBSCRIBE, '')
q.connect_mon(self.monitor_url)
q.daemon=True
children.append(q)
# Multiplexer Queue (in a Process)
q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
q.bind_in(self.client_info['mux'])
q.setsockopt_in(zmq.IDENTITY, 'mux')
q.bind_out(self.engine_info['mux'])
q.connect_mon(self.monitor_url)
q.daemon=True
children.append(q)
# Control Queue (in a Process)
q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
q.bind_in(self.client_info['control'])
q.setsockopt_in(zmq.IDENTITY, 'control')
q.bind_out(self.engine_info['control'])
q.connect_mon(self.monitor_url)
q.daemon=True
children.append(q)
# Task Queue (in a Process)
if self.scheme == 'pure':
self.log.warn("task::using pure XREQ Task scheduler")
q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
q.setsockopt_out(zmq.HWM, self.hwm)
q.bind_in(self.client_info['task'][1])
q.setsockopt_in(zmq.IDENTITY, 'task')
q.bind_out(self.engine_info['task'])
q.connect_mon(self.monitor_url)
q.daemon=True
children.append(q)
elif self.scheme == 'none':
self.log.warn("task::using no Task scheduler")
else:
self.log.info("task::using Python %s Task scheduler"%self.scheme)
sargs = (self.client_info['task'][1], self.engine_info['task'], self.monitor_url, self.client_info['notification'])
kwargs = dict(scheme=self.scheme,logname=self.log.name, loglevel=self.log.level, config=self.config)
q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
q.daemon=True
children.append(q)
class HubFactory(RegistrationFactory):
"""The Configurable for setting up a Hub."""
# name of a scheduler scheme
scheme = Str('leastload', config=True)
# port-pairs for monitoredqueues:
hb = Instance(list, config=True)
def _hb_default(self):
return util.select_random_ports(2)
mux = Instance(list, config=True)
def _mux_default(self):
return util.select_random_ports(2)
task = Instance(list, config=True)
def _task_default(self):
return util.select_random_ports(2)
control = Instance(list, config=True)
def _control_default(self):
return util.select_random_ports(2)
iopub = Instance(list, config=True)
def _iopub_default(self):
return util.select_random_ports(2)
# single ports:
mon_port = Instance(int, config=True)
def _mon_port_default(self):
return util.select_random_ports(1)[0]
notifier_port = Instance(int, config=True)
def _notifier_port_default(self):
return util.select_random_ports(1)[0]
ping = Int(1000, config=True) # ping frequency
engine_ip = CStr('127.0.0.1', config=True)
engine_transport = CStr('tcp', config=True)
client_ip = CStr('127.0.0.1', config=True)
client_transport = CStr('tcp', config=True)
monitor_ip = CStr('127.0.0.1', config=True)
monitor_transport = CStr('tcp', config=True)
monitor_url = CStr('')
db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
# not configurable
db = Instance('IPython.parallel.controller.dictdb.BaseDB')
heartmonitor = Instance(
'IPython.parallel.controller.heartmonitor.HeartMonitor')
subconstructors = List()
_constructed = Bool(False)
def _ip_changed(self, name, old, new):
self.engine_ip = new
self.client_ip = new
self.monitor_ip = new
self._update_monitor_url()
def _update_monitor_url(self):
self.monitor_url = "%s://%s:%i" % (self.monitor_transport,
self.monitor_ip, self.mon_port)
def _transport_changed(self, name, old, new):
self.engine_transport = new
self.client_transport = new
self.monitor_transport = new
self._update_monitor_url()
def __init__(self, **kwargs):
super(HubFactory, self).__init__(**kwargs)
self._update_monitor_url()
# self.on_trait_change(self._sync_ips, 'ip')
# self.on_trait_change(self._sync_transports, 'transport')
self.subconstructors.append(self.construct_hub)
def construct(self):
assert not self._constructed, "already constructed!"
for subc in self.subconstructors:
subc()
self._constructed = True
def start(self):
assert self._constructed, "must be constructed by self.construct() first!"
self.heartmonitor.start()
self.log.info("Heartmonitor started")
def construct_hub(self):
"""construct"""
client_iface = "%s://%s:" % (self.client_transport,
self.client_ip) + "%i"
engine_iface = "%s://%s:" % (self.engine_transport,
self.engine_ip) + "%i"
ctx = self.context
loop = self.loop
# Registrar socket
q = ZMQStream(ctx.socket(zmq.XREP), loop)
q.bind(client_iface % self.regport)
self.log.info("Hub listening on %s for registration." %
(client_iface % self.regport))
if self.client_ip != self.engine_ip:
q.bind(engine_iface % self.regport)
self.log.info("Hub listening on %s for registration." %
(engine_iface % self.regport))
### Engine connections ###
# heartbeat
hpub = ctx.socket(zmq.PUB)
hpub.bind(engine_iface % self.hb[0])
hrep = ctx.socket(zmq.XREP)
hrep.bind(engine_iface % self.hb[1])
self.heartmonitor = HeartMonitor(loop=loop,
pingstream=ZMQStream(hpub, loop),
pongstream=ZMQStream(hrep, loop),
period=self.ping,
logname=self.log.name)
### Client connections ###
# Notifier socket
n = ZMQStream(ctx.socket(zmq.PUB), loop)
n.bind(client_iface % self.notifier_port)
### build and launch the queues ###
# monitor socket
sub = ctx.socket(zmq.SUB)
sub.setsockopt(zmq.SUBSCRIBE, "")
sub.bind(self.monitor_url)
sub.bind('inproc://monitor')
sub = ZMQStream(sub, loop)
# connect the db
self.log.info('Hub using DB backend: %r' % (self.db_class.split()[-1]))
# cdir = self.config.Global.cluster_dir
self.db = import_item(self.db_class)(session=self.session.session,
config=self.config)
time.sleep(.25)
# build connection dicts
self.engine_info = {
'control': engine_iface % self.control[1],
'mux': engine_iface % self.mux[1],
'heartbeat':
(engine_iface % self.hb[0], engine_iface % self.hb[1]),
'task': engine_iface % self.task[1],
'iopub': engine_iface % self.iopub[1],
# 'monitor' : engine_iface%self.mon_port,
}
self.client_info = {
'control': client_iface % self.control[0],
'mux': client_iface % self.mux[0],
'task': (self.scheme, client_iface % self.task[0]),
'iopub': client_iface % self.iopub[0],
'notification': client_iface % self.notifier_port
}
self.log.debug("Hub engine addrs: %s" % self.engine_info)
self.log.debug("Hub client addrs: %s" % self.client_info)
self.hub = Hub(loop=loop,
session=self.session,
monitor=sub,
heartmonitor=self.heartmonitor,
query=q,
notifier=n,
db=self.db,
engine_info=self.engine_info,
client_info=self.client_info,
logname=self.log.name)
class PlainTextFormatter(BaseFormatter):
"""The default pretty-printer.
This uses :mod:`IPython.external.pretty` to compute the format data of
the object. If the object cannot be pretty printed, :func:`repr` is used.
See the documentation of :mod:`IPython.external.pretty` for details on
how to write pretty printers. Here is a simple example::
def dtype_pprinter(obj, p, cycle):
if cycle:
return p.text('dtype(...)')
if hasattr(obj, 'fields'):
if obj.fields is None:
p.text(repr(obj))
else:
p.begin_group(7, 'dtype([')
for i, field in enumerate(obj.descr):
if i > 0:
p.text(',')
p.breakable()
p.pretty(field)
p.end_group(7, '])')
"""
# The format type of data returned.
format_type = Str('text/plain')
# This subclass ignores this attribute as it always need to return
# something.
enabled = Bool(True, config=False)
# Look for a __pretty__ methods to use for pretty printing.
print_method = Str('__pretty__')
# Whether to pretty-print or not.
pprint = Bool(True, config=True)
# Whether to be verbose or not.
verbose = Bool(False, config=True)
# The maximum width.
max_width = Int(79, config=True)
# The newline character.
newline = Str('\n', config=True)
# format-string for pprinting floats
float_format = Str('%r')
# setter for float precision, either int or direct format-string
float_precision = CStr('', config=True)
def _float_precision_changed(self, name, old, new):
"""float_precision changed, set float_format accordingly.
float_precision can be set by int or str.
This will set float_format, after interpreting input.
If numpy has been imported, numpy print precision will also be set.
integer `n` sets format to '%.nf', otherwise, format set directly.
An empty string returns to defaults (repr for float, 8 for numpy).
This parameter can be set via the '%precision' magic.
"""
if '%' in new:
# got explicit format string
fmt = new
try:
fmt % 3.14159
except Exception:
raise ValueError(
"Precision must be int or format string, not %r" % new)
elif new:
# otherwise, should be an int
try:
i = int(new)
assert i >= 0
except ValueError:
raise ValueError(
"Precision must be int or format string, not %r" % new)
except AssertionError:
raise ValueError("int precision must be non-negative, not %r" %
i)
fmt = '%%.%if' % i
if 'numpy' in sys.modules:
# set numpy precision if it has been imported
import numpy
numpy.set_printoptions(precision=i)
else:
# default back to repr
fmt = '%r'
if 'numpy' in sys.modules:
import numpy
# numpy default is 8
numpy.set_printoptions(precision=8)
self.float_format = fmt
# Use the default pretty printers from IPython.external.pretty.
def _singleton_printers_default(self):
return pretty._singleton_pprinters.copy()
def _type_printers_default(self):
d = pretty._type_pprinters.copy()
d[float] = lambda obj, p, cycle: p.text(self.float_format % obj)
return d
def _deferred_printers_default(self):
return pretty._deferred_type_pprinters.copy()
#### FormatterABC interface ####
def __call__(self, obj):
"""Compute the pretty representation of the object."""
if not self.pprint:
try:
return repr(obj)
except TypeError:
return ''
else:
# This uses use StringIO, as cStringIO doesn't handle unicode.
stream = StringIO()
printer = pretty.RepresentationPrinter(
stream,
self.verbose,
self.max_width,
self.newline,
singleton_pprinters=self.singleton_printers,
type_pprinters=self.type_printers,
deferred_pprinters=self.deferred_printers)
printer.pretty(obj)
printer.flush()
return stream.getvalue()