def reg_std_version_check(version, modname): def checkstdmod_version(): mod = ut.import_modname(modname) return module_stdinfo_dict(mod) ut.set_funcname(checkstdmod_version, '%s_version' % (modname,)) func = checkinfo(version)(checkstdmod_version) return func
def make_queue_loop(iface1, iface2, name=None): """ Standard queue loop Args: iface1 (str): address for the client that deals iface2 (str): address for the server that routes name (None): (default = None) """ assert name is not None, 'must name queue' queue_name = name + '_queue' loop_name = queue_name + '_loop' def queue_loop(): print = partial(ut.colorprint, color='green') with ut.Indenter('[%s] ' % (queue_name,)): if VERBOSE_JOBS: print('Init make_queue_loop: name=%r' % (name,)) # bind the client dealer to the queue router rout_sock = ctx.socket(zmq.ROUTER) rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER') rout_sock.bind(iface1) if VERBOSE_JOBS: print('bind %s_url2 = %r' % (name, iface1,)) # bind the server router to the queue dealer deal_sock = ctx.socket(zmq.DEALER) deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER') deal_sock.bind(iface2) if VERBOSE_JOBS: print('bind %s_url2 = %r' % (name, iface2,)) if 1: # the remainder of this function can be entirely replaced with zmq.device(zmq.QUEUE, rout_sock, deal_sock) else: # but this shows what is really going on: poller = zmq.Poller() poller.register(rout_sock, zmq.POLLIN) poller.register(deal_sock, zmq.POLLIN) while True: evts = dict(poller.poll()) # poll() returns a list of tuples [(socket, evt), (socket, evt)] # dict(poll()) turns this into {socket:evt, socket:evt} if rout_sock in evts: msg = rout_sock.recv_multipart() # ROUTER sockets prepend the identity of the jobiface, # for routing replies if VERBOSE_JOBS: print('ROUTER relayed %r via DEALER' % (msg,)) deal_sock.send_multipart(msg) if deal_sock in evts: msg = deal_sock.recv_multipart() if VERBOSE_JOBS: print('DEALER relayed %r via ROUTER' % (msg,)) rout_sock.send_multipart(msg) if VERBOSE_JOBS: print('Exiting %s' % (loop_name,)) ut.set_funcname(queue_loop, loop_name) return queue_loop
def _make_depc_getters(depc_name, attrname, tbl, col): def _rowid_getter(self, rowids): depc = getattr(self._ibs, depc_name) data = depc.get(tbl, rowids, col, config=self._config) if self._asarray: data = np.array(data) return data ut.set_funcname(_rowid_getter, '_rowid_get_' + attrname) _getter = _make_caching_getter(attrname, _rowid_getter) return _rowid_getter, _getter
def _make_setters(objname, attrname): ibs_funcname = 'set_%s_%s' % (objname, attrname) def _rowid_setter(self, rowids, values, *args, **kwargs): ibs_callable = getattr(self._ibs, ibs_funcname) ibs_callable(rowids, values, *args, **kwargs) ut.set_funcname(_rowid_setter, '_rowid_set_' + attrname) _setter = _make_caching_setter(attrname, _rowid_setter) return _rowid_setter, _setter
def _make_unflat_getter(objname, attrname): ibs_funcname = 'get_%s_%s' % (objname, attrname) def ibs_unflat_getter(self, *args, **kwargs): ibs_callable = getattr(self._ibs, ibs_funcname) rowids = self._rowids_list ibs = self._ibs return ibs.unflat_map(ibs_callable, rowids, *args, **kwargs) ut.set_funcname(ibs_unflat_getter, 'unflat_' + ibs_funcname) return ibs_unflat_getter
def _make_caching_setter(attrname, _rowid_setter): def _setter(self, values, *args, **kwargs): if self._ibs is None: self._internal_attrs[attrname] = values else: if self._caching and attrname in self._internal_attrs: self._internal_attrs[attrname] = values _rowid_setter(self, self._rowids, values) ut.set_funcname(_setter, '_set_' + attrname) return _setter
def _make_cfg_getters(objname, attrname): ibs_funcname = 'get_%s_%s' % (objname, attrname) def _rowid_getter(self, rowids): ibs_callable = getattr(self._ibs, ibs_funcname) data = ibs_callable(rowids, config2_=self._config) if self._asarray: data = np.array(data) return data ut.set_funcname(_rowid_getter, '_rowid_get_' + attrname) _getter = _make_caching_getter(attrname, _rowid_getter) return _rowid_getter, _getter
def partial_imap_1to1(func, si_func): import functools @functools.wraps(si_func) def wrapper(input_): if not ut.isiterable(input_): return func(si_func(input_)) else: return list(map(func, si_func(input_))) ut.set_funcname(wrapper, ut.get_callable_name(func) + '_mapper_' + ut.get_funcname(si_func)) return wrapper
def _make_caching_getter(attrname, _rowid_getter): def _getter(self): if self._ibs is None or (self._caching and attrname in self._internal_attrs): data = self._internal_attrs[attrname] else: data = _rowid_getter(self, self._rowids) if self._caching: self._internal_attrs[attrname] = data return data ut.set_funcname(_getter, '_get_' + attrname) return _getter
def make_queue_loop(name='collect'): """ Standard queue loop Args: name (None): (default = None) """ assert name is not None, 'must name queue' queue_name = name + '_queue' loop_name = queue_name + '_loop' def queue_loop(port_dict): iface1, iface2 = port_dict['collect_url1'], port_dict['collect_url2'] print = partial(ut.colorprint, color='green') update_proctitle(queue_name) with ut.Indenter('[%s] ' % (queue_name, )): if VERBOSE_JOBS: print('Init make_queue_loop: name=%r' % (name, )) # bind the client dealer to the queue router rout_sock = ctx.socket(zmq.ROUTER) rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER') rout_sock.bind(iface1) if VERBOSE_JOBS: print('bind %s_url1 = %r' % ( name, iface1, )) # bind the server router to the queue dealer deal_sock = ctx.socket(zmq.DEALER) deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER') deal_sock.bind(iface2) if VERBOSE_JOBS: print('bind %s_url2 = %r' % ( name, iface2, )) try: if 1: # the remainder of this function can be entirely replaced with zmq.device(zmq.QUEUE, rout_sock, deal_sock) else: # but this shows what is really going on: poller = zmq.Poller() poller.register(rout_sock, zmq.POLLIN) poller.register(deal_sock, zmq.POLLIN) while True: evts = dict(poller.poll()) # poll() returns a list of tuples [(socket, evt), (socket, evt)] # dict(poll()) turns this into {socket:evt, socket:evt} if rout_sock in evts: msg = rout_sock.recv_multipart() # ROUTER sockets prepend the identity of the jobiface, # for routing replies if VERBOSE_JOBS: print('ROUTER relayed %r via DEALER' % (msg, )) deal_sock.send_multipart(msg) if deal_sock in evts: msg = deal_sock.recv_multipart() if VERBOSE_JOBS: print('DEALER relayed %r via ROUTER' % (msg, )) rout_sock.send_multipart(msg) except KeyboardInterrupt: print('Caught ctrl+c in collector loop. Gracefully exiting') if VERBOSE_JOBS: print('Exiting %s' % (loop_name, )) ut.set_funcname(queue_loop, loop_name) return queue_loop