def benchmark(task_set, parallel=True):
task_set = task_sets[task_set]
trials = [
CM.get_data(collection, task_category, condition)
for collection, task_category, condition in task_set
]
def get_valid_properties(trials):
ips = []
rps = []
for trial_array in trials:
ips.append(set(trial_array.dtype.names) and set(image_properties))
rps.append(
set(trial_array.dtype.names) and set(response_properties))
ip = set.intersection(*ips)
rp = set.intersection(*rps)
return ip, rp
ips, rps = get_valid_properties(trials)
if parallel:
Parallel(verbose=700, n_jobs=10)\
(delayed(cache_composite_individual_self_consistency_all_metrics)\
(trials, image_property, response_property)\
for image_property, response_property in itertools.product(ips, rps))
else:
[
cache_composite_individual_self_consistency_all_metrics(
trials, image_property, response_property)
for image_property, response_property in itertools.product(
ips, rps)
]
def test_all_metrics():
trials = CM.get_data('hvm_basic_categorization_new', 'category')
results = all_metrics.composite_individual_self_consistency_all_metrics(
[trials], image_property='task_category', response_property='Response')
mr = results['metrics_results']
print[(metric, mr[metric]['self_consistency'],
mr[metric]['self_consistency_error']) for metric in mr.keys()]
def get_human_data_densely_sampled():
dataset = hvm.HvMWithDiscfade()
raw_data = confusion_matrices.get_data('hvm_dense_smp_v6_2rpw',
field='category')
# Add rep number to raw data, then clean
which_rep = {}
for worker in np.unique(raw_data['WorkerId']):
which_rep[worker] = {}
for filename in np.unique(raw_data['filename']):
which_rep[worker][filename] = 0
rep = np.zeros(raw_data['filename'].shape[0])
for i, trial in enumerate(raw_data):
filename = trial['filename']
worker = trial['WorkerId']
rep[i] = which_rep[worker][filename]
which_rep[worker][filename] += 1
raw_data_with_rep = raw_data.addcols([rep], names=['rep'])
# Get rid of everything but first two reps, get rid of learning reps (Images of V3 and V0)
data = raw_data_with_rep[raw_data_with_rep['rep'] < 2]
data = data[data['var'] == 'V6']
# Reformat to matrix
human_matrix = [] # images, reps, worker
canonical_order = dataset.meta['_id'][ImageSet1_inds]
workers = np.unique(data['WorkerId'])
n_workers = len(workers)
for worker in workers:
worker_data = data[data['WorkerId'] == worker]
rep0 = worker_data[worker_data['rep'] == 0]
rep1 = worker_data[worker_data['rep'] == 1]
c0 = []
c1 = []
for Imid in canonical_order:
c0.append(rep0[rep0['_id'] == Imid]['correct'])
c1.append(rep1[rep1['_id'] == Imid]['correct'])
X = np.column_stack([np.array(c0), np.array(c1)])
X = np.expand_dims(X, 2)
human_matrix.append(X)
assert set(np.unique(worker_data['_id'])) == set(canonical_order)
human_matrix = np.concatenate(human_matrix, 2)
assert human_matrix.shape == (128, 2, n_workers)
human_individuals = deepcopy(human_matrix)
human_reps = np.concatenate((human_matrix[:, 0, :], human_matrix[:, 1, :]),
1)
return human_reps, human_individuals, raw_data_with_rep
def benchmark(task_set, parallel=True):
task_set = task_sets[task_set]
trials = [CM.get_data(collection, task_category, condition) for collection, task_category, condition in task_set]
def get_valid_properties(trials):
ips = []
rps = []
for trial_array in trials:
ips.append(set(trial_array.dtype.names) and set(image_properties))
rps.append(set(trial_array.dtype.names) and set(response_properties))
ip = set.intersection(*ips)
rp = set.intersection(*rps)
return ip, rp
ips, rps = get_valid_properties(trials)
if parallel:
Parallel(verbose=700, n_jobs=10)\
(delayed(cache_composite_individual_self_consistency_all_metrics)\
(trials, image_property, response_property)\
for image_property, response_property in itertools.product(ips, rps))
else:
[cache_composite_individual_self_consistency_all_metrics(trials, image_property, response_property)
for image_property, response_property in itertools.product(ips, rps)]
def get_subordinate_human_data():
meta_field = 'obj'
data = cm.get_data('hvm_subordinate_2ways',
meta_field,
trial_data=['ImgData'])
return clean_and_two_way_type(data, meta_field)
def get_basic_human_data():
meta_field = 'category'
data = cm.get_data('hvm_basic_2ways', meta_field, trial_data=['ImgData'])
data = clean_and_two_way_type(data, meta_field)
return clean_and_two_way_type(data, meta_field)
def test_off_diagonal():
RM, _, _ = CM.get_response_matrix(
CM.get_data('hvm_basic_categorization_new', 'category'),
'task_category', 'Response')
print u.symmetrize_confusion_matrix(RM, take='off_diagonal')
def get_subordinate_human_data():
meta_field = 'obj'
data = cm.get_data('hvm_subordinate_2ways',
meta_field, trial_data=['ImgData'])
return clean_and_two_way_type(data, meta_field)
def get_basic_human_data():
meta_field = 'category'
data = cm.get_data('hvm_basic_2ways',
meta_field, trial_data=['ImgData'])
data = clean_and_two_way_type(data, meta_field)
return clean_and_two_way_type(data, meta_field)
def test_off_diagonal():
RM, _, _ = CM.get_response_matrix(CM.get_data('hvm_basic_categorization_new', 'category'),
'task_category', 'Response')
print u.symmetrize_confusion_matrix(RM, take='off_diagonal')
def test_all_metrics():
trials = CM.get_data('hvm_basic_categorization_new', 'category')
results = all_metrics.composite_individual_self_consistency_all_metrics([trials], image_property = 'task_category', response_property='Response')
mr = results['metrics_results']
print [(metric, mr[metric]['self_consistency'], mr[metric]['self_consistency_error']) for metric in mr.keys()]
