"Copy neighbors into the output matrix." used = np.zeros(dst.nrows, dtype=np.int32) for i in range(src.nrows): sp, ep = src.row_extent(i) for j in range(sp, ep): c = src.colinds[j] v = src.values[j] if c != i and v >= thresh: _insert(dst, used, limits, i, c, v) return used @njit(nogil=True, parallel=not is_mp_worker()) def _sort_nbrs(smat): for i in prange(smat.nrows): sp, ep = smat.row_extent(i) kvp_minheap_sort(sp, ep, smat.colinds, smat.values) @njit(nogil=True) def _make_sim_block(nitems, bsp, bitems, r_sp, r_ep, r_cs, r_vs, min_sim, max_nbrs): # pass 1: compute the size of each row sizes = np.zeros(bitems, np.int32) for i in range(nitems): for j in range(r_sp[i], r_ep[i]): # we accept the neighbor if it passes threshold and isn't a self-similarity r = r_cs[j] if i != bsp + r and r_vs[j] >= min_sim:
from lenskit import util, DataWarning, ConfigWarning from lenskit.data import sparse_ratings from lenskit.sharing import in_share_context from lenskit.util.parallel import is_mp_worker from lenskit.util.accum import kvp_minheap_insert, kvp_minheap_sort from . import Predictor _logger = logging.getLogger(__name__) def _make_blocks(n, size): "Create blocks for the range 0..n." return [(s, min(s + size, n)) for s in range(0, n, size)] @njit(parallel=not is_mp_worker()) def _sort_nbrs(smat: CSR): for i in prange(smat.nrows): sp, ep = smat.row_extent(i) kvp_minheap_sort(sp, ep, smat.colinds, smat.values) @njit def _trim_sim_block(nitems, bsp, bitems, block, min_sim, max_nbrs): # pass 1: compute the size of each row sizes = np.zeros(bitems, np.int32) for i in range(nitems): sp, ep = block.row_extent(i) for j in range(sp, ep): # we accept the neighbor if it passes threshold and isn't a self-similarity r = block.colinds[j]
""" TensorFlow-based algorithms. """ import logging from .util import have_usable_tensorflow from .biasedmf import BiasedMF # noqa: F401 from .ibmf import IntegratedBiasMF # noqa: F401 from .bpr import BPR # noqa: F401 from lenskit.util.parallel import is_mp_worker TF_AVAILABLE = have_usable_tensorflow() _log = logging.getLogger(__name__) if TF_AVAILABLE and is_mp_worker(): import tensorflow as _tf _log.info('disabling GPUs in worker process') _tf.config.set_visible_devices([], 'GPU') _log.info('disabling multithreading in worker') _tf.config.threading.set_inter_op_parallelism_threads(1) _tf.config.threading.set_intra_op_parallelism_threads(1)
""" TensorFlow-based algorithms. """ import logging from .biasedmf import BiasedMF # noqa: F401 from .ibmf import IntegratedBiasMF # noqa: F401 from .bpr import BPR # noqa: F401 from lenskit.util.parallel import is_mp_worker try: import tensorflow as _tf TF_AVAILABLE = True except ImportError: TF_AVAILABLE = False _log = logging.getLogger(__name__) if is_mp_worker(): _log.info('disabling GPUs in worker process') _tf.config.set_visible_devices([], 'GPU')
def _worker_status(blob, *args): _log.info('in worker %s', mp.current_process().name) return os.getpid(), is_worker(), is_mp_worker()
def _worker_status(blob, *args): return os.getpid(), is_worker(), is_mp_worker()