def compute_metrics_all_runs(dataset, res, recompute=False):
true_nn_distances = list(dataset['distances'])
for i, (properties, run) in enumerate(res):
algo = properties['algo']
algo_name = properties['name']
# cache distances to avoid access to hdf5 file
# print('Load distances and times')
run_distances = numpy.array(run['distances'])
# print('... done')
if recompute and 'metrics' in run:
#print('Recomputing metrics, clearing cache')
del run['metrics']
metrics_cache = get_or_create_metrics(run)
dataset = properties['dataset']
try:
dataset = dataset.decode()
algo = algo.decode()
algo_name = algo_name.decode()
except:
pass
run_result = {
'algorithm': algo,
'parameters': algo_name,
'dataset': dataset,
'count': properties['count'],
'batch': properties['batch_mode'],
'filename': os.path.split(run.filename)[1],
}
for name, metric in metrics.items():
v = metric["function"](true_nn_distances, run_distances,
metrics_cache, properties)
run_result[name] = v
yield run_result
def compute_all_metrics(true_nn_distances, run, algo):
algo_name = run.attrs["name"]
print('--')
print(algo_name)
results = {}
# cache distances to avoid access to hdf5 file
run_distances = list(run["distances"])
run_attrs = dict(run.attrs)
for name, metric in metrics.items():
v = metric["function"](true_nn_distances, run_distances, run_attrs)
results[name] = v
if v:
print('%s: %g' % (name, v))
return (algo, algo_name, results)
def compute_all_metrics(true_nn_distances, run, properties):
algo = properties["algo"]
algo_name = properties["name"]
print('--')
print(algo_name)
results = {}
# cache distances to avoid access to hdf5 file
run_distances = list(run["distances"])
for name, metric in metrics.items():
v = metric["function"](true_nn_distances, run_distances, properties)
results[name] = v
if v:
print('%s: %g' % (name, v))
return (algo, algo_name, results)
def compute_all_metrics(true_nn_distances, run, properties, recompute=False):
algo = properties["algo"]
algo_name = properties["name"]
print('--')
print(algo_name)
results = {}
# cache distances to avoid access to hdf5 file
run_distances = numpy.array(run["distances"])
if recompute and 'metrics' in run:
del run['metrics']
metrics_cache = get_or_create_metrics(run)
for name, metric in metrics.items():
v = metric["function"](true_nn_distances, run_distances, metrics_cache, properties)
results[name] = v
if v:
print('%s: %g' % (name, v))
return (algo, algo_name, results)
def compute_metrics(qs, ds):
all_results = {}
all_algos = set()
for run in ds:
algo = run["library"]
algo_name = run["name"]
print "--"
print algo_name
results = {}
for name, metric in metrics.items():
v = metric["function"](qs, run)
results[name] = v
if v:
print "%s: %g" % (name, v)
all_algos.add(algo)
if not algo in all_results:
all_results[algo] = []
all_results[algo].append((algo, algo_name, results))
return (all_results, all_algos)
def compute_metrics(dataset, res):
all_results = {}
all_algos = set()
for run in res:
algo = run.attrs["library"]
algo_name = run.attrs["name"]
print('--')
print(algo_name)
results = {}
for name, metric in metrics.items():
v = metric["function"](dataset, run)
results[name] = v
if v:
print('%s: %g' % (name, v))
all_algos.add(algo)
if not algo in all_results:
all_results[algo] = []
all_results[algo].append((algo, algo_name, results))
return (all_results, all_algos)
def compute_metrics(qs, ds):
all_results = {}
all_algos = set()
for run in ds:
algo = run["library"]
algo_name = run["name"]
print "--"
print algo_name
results = {}
for name, metric in metrics.items():
v = metric["function"](qs, run)
results[name] = v
if v:
print "%s: %g" % (name, v)
all_algos.add(algo)
if not algo in all_results:
all_results[algo] = []
all_results[algo].append((algo, algo_name, results))
return (all_results, all_algos)
def compute_metrics(dataset, res):
all_results = {}
all_algos = set()
for definition, run in res:
algo = definition.algorithm
algo_name = definition.arguments # TODO(erikbern): stupid backwards compatibility thing
print('--')
print(algo_name)
results = {}
for name, metric in metrics.items():
v = metric["function"](dataset, run)
results[name] = v
if v:
print('%s: %g' % (name, v))
all_algos.add(algo)
if not algo in all_results:
all_results[algo] = []
all_results[algo].append((algo, algo_name, results))
return (all_results, all_algos)
def compute_metrics_all_runs(true_nn_distances, res, recompute=False):
for i, (properties, run) in enumerate(res):
algo = properties['algo']
algo_name = properties['name']
# cache distances to avoid access to hdf5 file
print('Load distances and times')
run_distances = np.array(run['distances'])
query_times = np.array(run['times'])
print('... done')
if recompute and 'metrics' in run:
print('Recomputing metrics, clearing cache')
del run['metrics']
metrics_cache = get_or_create_metrics(run)
run_result = {
'algorithm': algo,
'parameters': algo_name,
'dataset': properties['dataset'],
'count': properties['count']
}
for name, metric in metrics.items():
v = metric["function"](true_nn_distances, run_distances,
query_times, metrics_cache, properties)
run_result[name] = v
yield run_result