def write_sparse_output(train, test, fn, distance, dimension, count=100): from ann_benchmarks.algorithms.bruteforce import BruteForceBLAS f = h5py.File(fn, 'w') f.attrs['type'] = 'sparse' f.attrs['distance'] = distance f.attrs['dimension'] = dimension f.attrs['point_type'] = 'bit' print('train size: %9d * %4d' % (train.shape[0], dimension)) print('test size: %9d * %4d' % (test.shape[0], dimension)) # We ensure the sets are sorted train = numpy.array(list(map(sorted, train))) test = numpy.array(list(map(sorted, test))) flat_train = numpy.hstack(train.flatten()) flat_test = numpy.hstack(test.flatten()) f.create_dataset('train', (len(flat_train),), dtype=flat_train.dtype)[:] = flat_train f.create_dataset('test', (len(flat_test),), dtype=flat_test.dtype)[:] = flat_test neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i') distances = f.create_dataset('distances', (len(test), count), dtype='f') f.create_dataset('size_test', (len(test),), dtype='i')[:] = list(map(len, test)) f.create_dataset('size_train', (len(train),), dtype='i')[:] = list(map(len, train)) bf = BruteForceBLAS(distance, precision=train.dtype) bf.fit(train) for i, x in enumerate(test): if i % 1000 == 0: print('%d/%d...' % (i, len(test))) res = list(bf.query_with_distances(x, count)) res.sort(key=lambda t: t[-1]) neighbors[i] = [j for j, _ in res] distances[i] = [d for _, d in res] f.close()
def write_output(train, test, fn, distance, point_type='float', count=100): from ann_benchmarks.algorithms.bruteforce import BruteForceBLAS n = 0 f = h5py.File(fn, 'w') f.attrs['type'] = 'dense' f.attrs['distance'] = distance f.attrs['dimension'] = len(train[0]) f.attrs['point_type'] = point_type print('train size: %9d * %4d' % train.shape) print('test size: %9d * %4d' % test.shape) f.create_dataset('train', (len(train), len(train[0])), dtype=train.dtype)[:] = train f.create_dataset('test', (len(test), len(test[0])), dtype=test.dtype)[:] = test neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i') distances = f.create_dataset('distances', (len(test), count), dtype='f') bf = BruteForceBLAS(distance, precision=train.dtype) bf.fit(train) for i, x in enumerate(test): if i % 1000 == 0: print('%d/%d...' % (i, len(test))) res = list(bf.query_with_distances(x, count)) res.sort(key=lambda t: t[-1]) neighbors[i] = [j for j, _ in res] distances[i] = [d for _, d in res] f.close()
def my_write_output(train, test, out_fn, distance, point_type='float', count=100): from ann_benchmarks.algorithms.bruteforce import BruteForceBLAS n = 0 f = h5py.File(out_fn, 'w') f.attrs['distance'] = distance f.attrs['point_type'] = point_type print('train size: %9d * %4d' % train.shape) print('test size: %9d * %4d' % test.shape) # f.create_dataset('train', (len(train), len( # train[0])), dtype=train.dtype)[:] = train # f.create_dataset('test', (len(test), len( # test[0])), dtype=test.dtype)[:] = test # neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i') # distances = f.create_dataset('distances', (len(test), count), dtype='f') f.create_dataset('train', (TRAIN_SIZE, len( train[0])), dtype=train.dtype)[:] = train[:TRAIN_SIZE] f.create_dataset('test', (QUERY_NUM, len( test[0])), dtype=test.dtype)[:] = test[:QUERY_NUM] neighbors = f.create_dataset('neighbors', (QUERY_NUM, count), dtype='i') distances = f.create_dataset('distances', (QUERY_NUM, count), dtype='f') bf = BruteForceBLAS(distance, precision=train.dtype) train = datasets.dataset_transform[distance](train) test = datasets.dataset_transform[distance](test) bf.fit(train[:TRAIN_SIZE]) queries = [] for i, x in enumerate(test[:QUERY_NUM]): if i % 1000 == 0: print('%d/%d...' % (i, len(test[:QUERY_NUM]))) res = list(bf.query_with_distances(x, count)) res.sort(key=lambda t: t[-1]) neighbors[i] = [j for j, _ in res] distances[i] = [d for _, d in res] f.close()
def compute_distances(distance, count, X_train, X_test): print('computing max distances for queries...') bf = BruteForceBLAS(distance, precision=X_train.dtype) # Prepare queries bf.fit(X_train) queries = [] for x in X_test: correct = bf.query_with_distances(x, count) # disregard queries that don't have near neighbors. if len(correct) > 0: max_distance = max(correct, key=lambda (_, distance): distance)[1] queries.append((x, max_distance, correct)) if len(queries) % 100 == 0: print(len(queries), '...') return queries
def compute_distances(distance, count, X_train, X_test): print('computing max distances for queries...') bf = BruteForceBLAS(distance, precision=X_train.dtype) # Prepare queries bf.fit(X_train) queries = [] for x in X_test: correct = bf.query_with_distances(x, count) # disregard queries that don't have near neighbors. if len(correct) > 0: max_distance = max(correct, key=lambda (_, distance): distance)[1] queries.append((x, max_distance, correct)) if len(queries) % 100 == 0: print(len(queries), '...') return queries
def write_output(train, test, fn, distance, count=100): from ann_benchmarks.algorithms.bruteforce import BruteForceBLAS n = 0 f = h5py.File(fn, 'w') f.attrs['distance'] = distance print('train size: %9d * %4d' % train.shape) print('test size: %9d * %4d' % test.shape) f.create_dataset('train', (len(train), len(train[0])), dtype=train.dtype)[:] = train f.create_dataset('test', (len(test), len(test[0])), dtype=test.dtype)[:] = test neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i') distances = f.create_dataset('distances', (len(test), count), dtype='f') bf = BruteForceBLAS(distance, precision=numpy.float32) bf.fit(train) queries = [] for i, x in enumerate(test): if i % 1000 == 0: print('%d/%d...' % (i, test.shape[0])) res = list(bf.query_with_distances(x, count)) res.sort(key=lambda t: t[-1]) neighbors[i] = [j for j, _ in res] distances[i] = [d for _, d in res] f.close()
def write_output(train, test, fn, distance, count=3000): from ann_benchmarks.algorithms.bruteforce import BruteForceBLAS n = 0 f = h5py.File(fn, 'w') f.attrs['distance'] = distance print('train size: %9d * %4d' % train.shape) print('test size: %9d * %4d' % test.shape) f.create_dataset('train', (len(train), len(train[0])), dtype=train.dtype)[:] = train f.create_dataset('test', (len(test), len(test[0])), dtype=test.dtype)[:] = test neighbors = f.create_dataset('neighbors', (len(test), count), dtype='i') distances = f.create_dataset('distances', (len(test), count), dtype='f') bf = BruteForceBLAS(distance, precision=numpy.float32) bf.fit(train) queries = [] for i, x in enumerate(test): if i % 1000 == 0: print('%d/%d...' % (i, test.shape[0])) res = list(bf.query_with_distances(x, count)) res = res[::-1] neighbors[i] = [j for j, _ in res] distances[i] = [d for _, d in res] f.close()