def reg_std_version_check(version, modname):
def checkstdmod_version():
mod = ut.import_modname(modname)
return module_stdinfo_dict(mod)
ut.set_funcname(checkstdmod_version, '%s_version' % (modname,))
func = checkinfo(version)(checkstdmod_version)
return func
def make_queue_loop(iface1, iface2, name=None):
"""
Standard queue loop
Args:
iface1 (str): address for the client that deals
iface2 (str): address for the server that routes
name (None): (default = None)
"""
assert name is not None, 'must name queue'
queue_name = name + '_queue'
loop_name = queue_name + '_loop'
def queue_loop():
print = partial(ut.colorprint, color='green')
with ut.Indenter('[%s] ' % (queue_name,)):
if VERBOSE_JOBS:
print('Init make_queue_loop: name=%r' % (name,))
# bind the client dealer to the queue router
rout_sock = ctx.socket(zmq.ROUTER)
rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER')
rout_sock.bind(iface1)
if VERBOSE_JOBS:
print('bind %s_url2 = %r' % (name, iface1,))
# bind the server router to the queue dealer
deal_sock = ctx.socket(zmq.DEALER)
deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER')
deal_sock.bind(iface2)
if VERBOSE_JOBS:
print('bind %s_url2 = %r' % (name, iface2,))
if 1:
# the remainder of this function can be entirely replaced with
zmq.device(zmq.QUEUE, rout_sock, deal_sock)
else:
# but this shows what is really going on:
poller = zmq.Poller()
poller.register(rout_sock, zmq.POLLIN)
poller.register(deal_sock, zmq.POLLIN)
while True:
evts = dict(poller.poll())
# poll() returns a list of tuples [(socket, evt), (socket, evt)]
# dict(poll()) turns this into {socket:evt, socket:evt}
if rout_sock in evts:
msg = rout_sock.recv_multipart()
# ROUTER sockets prepend the identity of the jobiface,
# for routing replies
if VERBOSE_JOBS:
print('ROUTER relayed %r via DEALER' % (msg,))
deal_sock.send_multipart(msg)
if deal_sock in evts:
msg = deal_sock.recv_multipart()
if VERBOSE_JOBS:
print('DEALER relayed %r via ROUTER' % (msg,))
rout_sock.send_multipart(msg)
if VERBOSE_JOBS:
print('Exiting %s' % (loop_name,))
ut.set_funcname(queue_loop, loop_name)
return queue_loop
def _make_depc_getters(depc_name, attrname, tbl, col):
def _rowid_getter(self, rowids):
depc = getattr(self._ibs, depc_name)
data = depc.get(tbl, rowids, col, config=self._config)
if self._asarray:
data = np.array(data)
return data
ut.set_funcname(_rowid_getter, '_rowid_get_' + attrname)
_getter = _make_caching_getter(attrname, _rowid_getter)
return _rowid_getter, _getter
def _make_setters(objname, attrname):
ibs_funcname = 'set_%s_%s' % (objname, attrname)
def _rowid_setter(self, rowids, values, *args, **kwargs):
ibs_callable = getattr(self._ibs, ibs_funcname)
ibs_callable(rowids, values, *args, **kwargs)
ut.set_funcname(_rowid_setter, '_rowid_set_' + attrname)
_setter = _make_caching_setter(attrname, _rowid_setter)
return _rowid_setter, _setter
def _make_unflat_getter(objname, attrname):
ibs_funcname = 'get_%s_%s' % (objname, attrname)
def ibs_unflat_getter(self, *args, **kwargs):
ibs_callable = getattr(self._ibs, ibs_funcname)
rowids = self._rowids_list
ibs = self._ibs
return ibs.unflat_map(ibs_callable, rowids, *args, **kwargs)
ut.set_funcname(ibs_unflat_getter, 'unflat_' + ibs_funcname)
return ibs_unflat_getter
def _make_caching_setter(attrname, _rowid_setter):
def _setter(self, values, *args, **kwargs):
if self._ibs is None:
self._internal_attrs[attrname] = values
else:
if self._caching and attrname in self._internal_attrs:
self._internal_attrs[attrname] = values
_rowid_setter(self, self._rowids, values)
ut.set_funcname(_setter, '_set_' + attrname)
return _setter
def _make_cfg_getters(objname, attrname):
ibs_funcname = 'get_%s_%s' % (objname, attrname)
def _rowid_getter(self, rowids):
ibs_callable = getattr(self._ibs, ibs_funcname)
data = ibs_callable(rowids, config2_=self._config)
if self._asarray:
data = np.array(data)
return data
ut.set_funcname(_rowid_getter, '_rowid_get_' + attrname)
_getter = _make_caching_getter(attrname, _rowid_getter)
return _rowid_getter, _getter
def partial_imap_1to1(func, si_func):
import functools
@functools.wraps(si_func)
def wrapper(input_):
if not ut.isiterable(input_):
return func(si_func(input_))
else:
return list(map(func, si_func(input_)))
ut.set_funcname(wrapper, ut.get_callable_name(func) + '_mapper_' +
ut.get_funcname(si_func))
return wrapper
def _make_caching_getter(attrname, _rowid_getter):
def _getter(self):
if self._ibs is None or (self._caching and
attrname in self._internal_attrs):
data = self._internal_attrs[attrname]
else:
data = _rowid_getter(self, self._rowids)
if self._caching:
self._internal_attrs[attrname] = data
return data
ut.set_funcname(_getter, '_get_' + attrname)
return _getter
def make_queue_loop(name='collect'):
"""
Standard queue loop
Args:
name (None): (default = None)
"""
assert name is not None, 'must name queue'
queue_name = name + '_queue'
loop_name = queue_name + '_loop'
def queue_loop(port_dict):
iface1, iface2 = port_dict['collect_url1'], port_dict['collect_url2']
print = partial(ut.colorprint, color='green')
update_proctitle(queue_name)
with ut.Indenter('[%s] ' % (queue_name, )):
if VERBOSE_JOBS:
print('Init make_queue_loop: name=%r' % (name, ))
# bind the client dealer to the queue router
rout_sock = ctx.socket(zmq.ROUTER)
rout_sock.setsockopt_string(zmq.IDENTITY,
'queue.' + name + '.' + 'ROUTER')
rout_sock.bind(iface1)
if VERBOSE_JOBS:
print('bind %s_url1 = %r' % (
name,
iface1,
))
# bind the server router to the queue dealer
deal_sock = ctx.socket(zmq.DEALER)
deal_sock.setsockopt_string(zmq.IDENTITY,
'queue.' + name + '.' + 'DEALER')
deal_sock.bind(iface2)
if VERBOSE_JOBS:
print('bind %s_url2 = %r' % (
name,
iface2,
))
try:
if 1:
# the remainder of this function can be entirely replaced with
zmq.device(zmq.QUEUE, rout_sock, deal_sock)
else:
# but this shows what is really going on:
poller = zmq.Poller()
poller.register(rout_sock, zmq.POLLIN)
poller.register(deal_sock, zmq.POLLIN)
while True:
evts = dict(poller.poll())
# poll() returns a list of tuples [(socket, evt), (socket, evt)]
# dict(poll()) turns this into {socket:evt, socket:evt}
if rout_sock in evts:
msg = rout_sock.recv_multipart()
# ROUTER sockets prepend the identity of the jobiface,
# for routing replies
if VERBOSE_JOBS:
print('ROUTER relayed %r via DEALER' % (msg, ))
deal_sock.send_multipart(msg)
if deal_sock in evts:
msg = deal_sock.recv_multipart()
if VERBOSE_JOBS:
print('DEALER relayed %r via ROUTER' % (msg, ))
rout_sock.send_multipart(msg)
except KeyboardInterrupt:
print('Caught ctrl+c in collector loop. Gracefully exiting')
if VERBOSE_JOBS:
print('Exiting %s' % (loop_name, ))
ut.set_funcname(queue_loop, loop_name)
return queue_loop
