def get_mp(data, sublen_samp): proc_start = time.time() mp, mpind = pyscamp.selfjoin(data, sublen_samp, gpus=[0, 1, 2], pearson=True, precision="double") # mp, mpind = pyscamp.selfjoin(data, sublen_samp, gpus=[0, 1], pearson=True, precision="double") proc_end = time.time() Logger.info("Finished processing component in %f seconds" % (proc_end - proc_start)) return mp, mpind
def run_pyscamp(inputs, a, b, window, max_matches, thresh, ptype, rrows, rcols): args = {} args['pearson'] = True if thresh: args['threshold'] = thresh if rrows: args['mheight'] = rrows if rcols: args['mwidth'] = rcols if '--no_gpu' in extra_opts: args['gpus'] = [] if not max_matches: max_matches = 5 mp_columns_out = None mp_columns_out_index = None mp_rows_out = None mp_rows_out_index = None if ptype == "1NN_INDEX": if a == b: mp_columns_out, mp_columns_out_index = mp.selfjoin( inputs[a], window, **args) else: mp_columns_out, mp_columns_out_index = mp.abjoin( inputs[a], inputs[b], window, **args) elif ptype == "SUM_THRESH": if a == b: mp_columns_out = mp.selfjoin_sum(inputs[a], window, **args) else: mp_columns_out = mp.abjoin_sum(inputs[a], inputs[b], window, **args) elif ptype == "ALL_NEIGHBORS": if a == b: mp_columns_out = mp.selfjoin_knn(inputs[a], window, max_matches, **args) else: mp_columns_out = mp.abjoin_knn(inputs[a], inputs[b], window, max_matches, **args) else: raise ValueError( 'pyscamp does not support profile type {}'.format(ptype)) if mp_columns_out is not None: mp_columns_out = mp_columns_out.squeeze() if mp_columns_out_index is not None: mp_columns_out_index = mp_columns_out_index.squeeze() if mp_rows_out is not None: mp_rows_out = mp_rows_out.squeeze() if mp_rows_out_index is not None: mp_rows_out_index = mp_rows_out_index.squeeze() return mp_columns_out, mp_columns_out_index, mp_rows_out, mp_rows_out_index
return np.allclose(valid, check, equal_nan=True) failed = False arr = [] arr2 = [] num = 0 num2 = 0 arr = np.random.random(size=(8000, )) arr2 = np.random.random(size=(8000, )) dm_self = distance_matrix(arr, None, 1024) dm_ab = distance_matrix(arr, arr2, 1024) dist, index = mp.selfjoin(arr, 1024, pearson=True) dist = dist.reshape((len(dist), 1)) index = index.reshape((len(index), 1)) vdist, vindex = reduce_1nn_index(dm_self) if compare_vectors(vdist, dist) and compare_index(vindex, index): print("1NN INDEX Self join pass") else: failed = True print("1NN INDEX Self join fail") dist, index = mp.abjoin(arr, arr2, 1024, pearson=True) dist = dist.reshape((len(dist), 1)) index = index.reshape((len(index), 1)) vdist, vindex = reduce_1nn_index(dm_ab)