def get_exp_core(varlev, sandbox=True, partial=False,
selected_basic_objs=SELECTED_BASIC_OBJS):
meta, _, meta_cars = get_meta(varlev=varlev)
cars = np.unique(meta_cars['obj'])
combs = [(o1, o2) for o1 in selected_basic_objs for o2 in cars] + \
[e for e in itertools.combinations(cars, 2)]
urls = [get_url(e['obj'], e['id'], varlev) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1, varlev),
get_url_labeled_resp_img(o2, varlev)],
'meta': [{'obj': o} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/cars_subord.html',
htmldst='cars_subord_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.15,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="objectome_cars_subord. 30 cars and 30 non-cars", # noqa
collection_name='objectome_cars_subord_v0v3',
max_assignments=1,
bucket_name='objectome_cars_subord_v0v3',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir='tmp',
html_data=html_data,
frame_height_pix=1200,
othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa
set_destination=True,
)
if partial:
return exp
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
return exp
def get_exp(sandbox=True, dummy_upload=True):
dataset = lcs.LandoltCs()
meta = dataset.meta
preproc = dict(crop=None,
crop_rand=None,
dtype='float32',
mask=None,
mode='L',
normalize=False,
resize_to=(256, 256),
seed=0)
img_urls = dataset.publish_images(range(len(dataset.meta)),
preproc,
'acuity_tasks',
dummy_upload=dummy_upload)
response_im_inds = [29, 24, 19, 34, 14, 39, 4, 9]
response_urls = []
response_metas = []
response_labels = []
for ind in response_im_inds:
response_urls.append(img_urls[ind])
response_metas.append({k: meta[ind][k] for k in meta.dtype.names})
response_labels.append(None)
response_images = [{
'urls': response_urls,
'meta': response_metas,
'labels': response_labels
}]
combs = [[m['rotation'] for m in response_images[0]['meta']]]
additionalrules = [{
'old': 'LEARNINGPERIODNUMBER',
'new': str(10)
}, {
'old': 'OBJTYPE',
'new': 'Landolt C'
}]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 184,
'meta_field': 'rotation',
'meta': meta,
'urls': img_urls
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='landolt_c_task.html',
htmldst='landolt_c_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=
"***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="landolt c acuity", # noqa
collection_name='landolt_c',
max_assignments=20,
bucket_name='landolt_c',
trials_per_hit=184, # 150 + 8x4 repeats
html_data=html_data,
frame_height_pix=1200,
othersrc=[
'../../lib/dltk.js', '../../lib/dltkexpr.js',
'../../lib/dltkrsvp.js'
],
additionalrules=additionalrules)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
#n_total_trials = len(exp._trials['imgFiles'])
#assert n_total_trials == 40, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
#ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG
#rng = np.random.RandomState(0)
#rng.shuffle(ind_repeats)
# trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
# for e in exp._trials}
# offsets = np.arange(
# ACTUAL_TRIALS_PER_HIT - 3, -1,
# -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
# ).round().astype('int')
#
# n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
# n_applied_hits = 0
# for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
# -1, -ACTUAL_TRIALS_PER_HIT):
# for k in trials_qe:
# for i, offset in enumerate(offsets):
# exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# n_applied_hits += 1
#
# print '** n_applied_hits =', n_applied_hits
# print '** len for each in _trials =', \
# {e: len(exp._trials[e]) for e in exp._trials}
#
# # -- sanity check
# assert 38 == n_applied_hits, n_applied_hits
# assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles'])
# s_ref_labels = set([tuple(e) for e in trials_qe['labels']])
# print(s_ref_labels)
# offsets2 = np.arange(8 * 4)[::-1] + offsets
#
# ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit))
# assert all([set([tuple(e) for e in
# np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels
# for ib, ie in ibie[:-1]])
# print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, selected_basic_objs=SELECTED_BASIC_OBJS,
debug=False):
meta, _, meta_cars = get_meta()
cars = np.unique(meta_cars['obj'])
combs = [(o1, o2) for o1 in selected_basic_objs for o2 in cars] + \
[e for e in itertools.combinations(cars, 2)]
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)],
'meta': [{'obj': o} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/cars_subord.html',
htmldst='cars_subord_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.15,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="objectome_cars_subord. 30 cars and 30 non-cars", # noqa
collection_name='objectome_cars_subord_v3',
max_assignments=1,
bucket_name='objectome_cars_subord_v3',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir='tmp',
html_data=html_data,
frame_height_pix=1200,
othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa
set_destination=True,
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
if debug:
return exp
# repeat last 50 presentations twice to make the entire trials
# well aligned as multiples of 100
exp._trials['imgFiles'].extend(exp._trials['imgFiles'][-50:])
exp._trials['imgData'].extend(exp._trials['imgData'][-50:])
exp._trials['labels'].extend(exp._trials['labels'][-50:])
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (30 * 29 / 2 + 30 * 30) * 250 + 50
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
#
# car: 4 - car MB27534* v car MB27803
# car: 12 - train MB31405 v car MB27577*
# non-car: 21 - truck* v car MB29874
# non-car: 24 - book v MB28307*
ind_repeats = [4, 12, 21, 24] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['imgFiles'])
print '** n_qe =', n_qe
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 3338 == n_applied_hits
assert len(exp._trials['imgFiles']) == 3338 * (100 + 16)
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
print '**', s_ref_labels
offsets2 = np.arange(16)[::-1] + offsets
ibie = zip(range(0, 3338 * 116, 116), range(3339 * 116, 116))
assert all(
[[(e1, e2) for e1, e2 in
np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels
for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
del exp._trials['meta_field']
print '** Finished creating trials.'
return exp, html_data
def get_exp(category, sandbox=True, dummy_upload=True):
dataset = hvm.HvMWithDiscfade()
meta = dataset.meta
inds = (meta['category'] == category).nonzero()[0]
meta = meta[inds]
objs = np.unique(meta['obj'])
combs = [objs]
preproc = None
image_bucket_name = 'hvm_timing'
urls = dataset.publish_images(inds, preproc,
image_bucket_name,
dummy_upload=dummy_upload)
base_url = 'https://s3.amazonaws.com/hvm_timing/'
obj_resp_ids = [meta[meta['obj'] == o]['id'][0] for o in objs]
response_images = [{
'urls': [base_url + obj_id + '.png' for obj_id in obj_resp_ids],
'meta': [{'obj': obj, 'category': category} for obj in objs],
'labels': objs}]
mult = 2
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 90 * 8 * mult,
'meta_field': 'obj',
'meta': tb.tab_rowstack([meta] * mult),
'urls': urls * mult,
'shuffle_test': False,
}
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(LEARNING_PERIOD)},
{'old': 'OBJTYPE',
'new': category}]
trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32 + 16
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='hvm_subordinate.html',
htmldst='hvm_subordinate_' + category + '_n%05d.html',
tmpdir='tmp_subordinate_%s' % category,
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="hvm subordinate identification", # noqa
collection_name = 'hvm_subordinate_identification_%s' % category,
max_assignments=1,
bucket_name='hvm_subordinate_identification_test',
trials_per_hit=trials_per_hit, # 144 + 8x4 repeats + 16 training
html_data=html_data,
frame_height_pix=1200,
othersrc = ['objnames.js', '../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'],
additionalrules=additionalrules,
log_prefix='subordinate_' + category + '_'
)
# -- create trials
exp.createTrials(sampling='without-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == 90 * 8 * mult, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = repeat_inds[category] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
ind_learn = practice_inds[category]
goodids = [meta[i]['id'] for i in ind_learn]
trials_lrn = {}
for e in exp._trials:
trials_lrn[e] = []
got = []
for _ind, r in enumerate(exp._trials[e]):
if exp._trials['imgData'][_ind]['Sample']['id'] in goodids and exp._trials['imgData'][_ind]['Sample']['id'] not in got :
trials_lrn[e].append(copy.deepcopy(r))
got.append(exp._trials['imgData'][_ind]['Sample']['id'])
assert len(trials_lrn['imgData']) == len(goodids), len(trials_lrn['imgData'])
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
n_applied_hits += 1
for j in range(n_applied_hits):
for k in trials_lrn:
for i in range(len(ind_learn)):
exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i])
#shuffle test on a per-hit basis
for j in range(n_applied_hits):
rng = np.random.RandomState(seed=j)
perm = rng.permutation(8)
for i in range(trials_per_hit * j, min(trials_per_hit * (j+1), len(exp._trials['imgFiles']))):
f = copy.deepcopy(exp._trials['imgFiles'][i])
t = copy.deepcopy(exp._trials['imgData'][i])
f[1] = [f[1][_j] for _j in perm]
exp._trials['imgFiles'][i] = f
t['Test'] = [t['Test'][_j] for _j in perm]
exp._trials['imgData'][i] = t
l = copy.deepcopy(exp._trials['labels'][i])
exp._trials['labels'][i] = [l[_j] for _j in perm]
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert n_hits_floor == n_applied_hits == mult * 5, (n_hits_floor, n_applied_hits)
assert len(exp._trials['imgFiles']) == mult * (720 + 5 * (32 + 16)), len(exp._trials['imgFiles'])
"""
s_ref_labels = set([tuple(e) for e in trials_qe['labels']])
print(s_ref_labels)
offsets2 = np.arange(8 * 4)[::-1] + offsets
ibie = zip(range(0, 720 + 4 * 32, trials_per_hit), range(trials_per_hit, 720 + 4 * 32 + trials_per_hit, trials_per_hit))
assert all([set([tuple(e) for e in
np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels
for ib, ie in ibie[:-1]])
print '** Finished creating trials.'
"""
return exp, html_data
def get_exp(category, sandbox=True, dummy_upload=True):
dataset = hvm.HvMWithDiscfade()
meta = dataset.meta
inds = (meta['category'] == category).nonzero()[0]
meta = meta[inds]
objs = np.unique(meta['obj'])
combs = [objs]
preproc = None
image_bucket_name = 'hvm_timing'
urls = dataset.publish_images(inds,
preproc,
image_bucket_name,
dummy_upload=dummy_upload)
base_url = 'https://s3.amazonaws.com/hvm_timing/'
obj_resp_ids = [meta[meta['obj'] == o]['id'][0] for o in objs]
response_images = [{
'urls': [base_url + obj_id + '.png' for obj_id in obj_resp_ids],
'meta': [{
'obj': obj,
'category': category
} for obj in objs],
'labels':
objs
}]
mult = 2
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 90 * 8 * mult,
'meta_field': 'obj',
'meta': tb.tab_rowstack([meta] * mult),
'urls': urls * mult,
'shuffle_test': False,
}
additionalrules = [{
'old': 'LEARNINGPERIODNUMBER',
'new': str(LEARNING_PERIOD)
}, {
'old': 'OBJTYPE',
'new': category
}]
trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32 + 16
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='hvm_subordinate.html',
htmldst='hvm_subordinate_' + category + '_n%05d.html',
tmpdir='tmp_subordinate_%s' % category,
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=
"***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="hvm subordinate identification", # noqa
collection_name='hvm_subordinate_identification_%s' % category,
max_assignments=1,
bucket_name='hvm_subordinate_identification_test',
trials_per_hit=trials_per_hit, # 144 + 8x4 repeats + 16 training
html_data=html_data,
frame_height_pix=1200,
othersrc=[
'objnames.js', '../../lib/dltk.js', '../../lib/dltkexpr.js',
'../../lib/dltkrsvp.js'
],
additionalrules=additionalrules,
log_prefix='subordinate_' + category + '_')
# -- create trials
exp.createTrials(sampling='without-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == 90 * 8 * mult, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = repeat_inds[category] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {
e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials
}
ind_learn = practice_inds[category]
goodids = [meta[i]['id'] for i in ind_learn]
trials_lrn = {}
for e in exp._trials:
trials_lrn[e] = []
got = []
for _ind, r in enumerate(exp._trials[e]):
if exp._trials['imgData'][_ind]['Sample'][
'id'] in goodids and exp._trials['imgData'][_ind][
'Sample']['id'] not in got:
trials_lrn[e].append(copy.deepcopy(r))
got.append(exp._trials['imgData'][_ind]['Sample']['id'])
assert len(trials_lrn['imgData']) == len(goodids), len(
trials_lrn['imgData'])
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT /
float(len(ind_repeats))).round().astype('int')
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1,
-ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
n_applied_hits += 1
for j in range(n_applied_hits):
for k in trials_lrn:
for i in range(len(ind_learn)):
exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i])
#shuffle test on a per-hit basis
for j in range(n_applied_hits):
rng = np.random.RandomState(seed=j)
perm = rng.permutation(8)
for i in range(
trials_per_hit * j,
min(trials_per_hit * (j + 1), len(exp._trials['imgFiles']))):
f = copy.deepcopy(exp._trials['imgFiles'][i])
t = copy.deepcopy(exp._trials['imgData'][i])
f[1] = [f[1][_j] for _j in perm]
exp._trials['imgFiles'][i] = f
t['Test'] = [t['Test'][_j] for _j in perm]
exp._trials['imgData'][i] = t
l = copy.deepcopy(exp._trials['labels'][i])
exp._trials['labels'][i] = [l[_j] for _j in perm]
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert n_hits_floor == n_applied_hits == mult * 5, (n_hits_floor,
n_applied_hits)
assert len(exp._trials['imgFiles']) == mult * (720 + 5 * (32 + 16)), len(
exp._trials['imgFiles'])
"""
s_ref_labels = set([tuple(e) for e in trials_qe['labels']])
print(s_ref_labels)
offsets2 = np.arange(8 * 4)[::-1] + offsets
ibie = zip(range(0, 720 + 4 * 32, trials_per_hit), range(trials_per_hit, 720 + 4 * 32 + trials_per_hit, trials_per_hit))
assert all([set([tuple(e) for e in
np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels
for ib, ie in ibie[:-1]])
print '** Finished creating trials.'
"""
return exp, html_data
def get_exp(sandbox=True, debug=False, dummy_upload=True):
dataset = hvm.HvMWithDiscfade()
meta = dataset.meta
combs = []
response_images = []
for category in np.unique(meta['category']):
objs = np.unique(meta[meta['category']==category]['obj'])
obj_combs= [e for e in itertools.combinations(objs, 2)]
response_images.extend([{
'urls': [get_url_labeled_resp_img(o1, dataset), get_url_labeled_resp_img(o2, dataset)],
'meta': [{'obj': o, 'category': category} for o in [o1, o2]],
'labels': [hvm.OBJECT_NAMES[o1], hvm.OBJECT_NAMES[o2]]
} for o1, o2 in obj_combs])
combs.extend(obj_combs)
#urls = [get_url(e['obj'], e['id']) for e in meta]
urls = dataset.publish_images(range(len(dataset.meta)), None, 'hvm_timing',
dummy_upload=dummy_upload)
with open(path.join(path.dirname(__file__), 'tutorial_html'), 'r') as tutorial_html_file:
tutorial_html = tutorial_html_file.read()
label_func = lambda x: hvm.OBJECT_NAMES[x['obj']]
html_data = {
'combs': combs,
'response_images': response_images,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
'meta_query' : lambda x: x['var'] == 'V6',
'label_func': label_func
}
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(10)},
{'old': 'OBJTYPE',
'new': 'Object Recognition'},
{'old': 'TUTORIAL_HTML',
'new': tutorial_html},
{'old': 'CATDICT',
'new': json.dumps(hvm.OBJECT_NAMES)},
{'old': 'METAFIELD',
'new': "'obj'"}]
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/general_two_way.html',
htmldst='hvm_subordinate_2way_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see. Up to 50 cent performance based bonus',
reward=0.25,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment='hvm_subordinate_2ways',
collection_name='hvm_subordinate_2ways',
max_assignments=1,
bucket_name='hvm_2ways',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 140 + 6x4 repeats
html_data=html_data,
tmpdir='tmp',
frame_height_pix=1200,
othersrc=['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'],
additionalrules=additionalrules
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (8 * 7 / 2) * 250 * 8
if debug:
return exp, html_data
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['labels'])
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 400 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 50 * 164 * 8
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
offsets2 = np.arange(24)[::-1] + offsets
ibie = zip(range(0, 50 * 8 * 164, 164), range(164, 50 * 8 * 164, 164))
assert all(
[[(e1, e2) for e1, e2 in
np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels
for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
#del exp._trials['imgData']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, debug=False, dummy_upload=True):
dataset = hvm.HvMWithDiscfade()
meta = dataset.meta
response_images = []
categories = np.unique(meta['category'])
cat_combs = [e for e in itertools.combinations(categories, 2)]
response_images.extend([{
'urls': [get_url_labeled_resp_img(c1),
get_url_labeled_resp_img(c2)],
'meta': [{
'category': category
} for category in [c1, c2]],
'labels': [c1, c2]
} for c1, c2 in cat_combs])
combs = cat_combs
urls = dataset.publish_images(range(len(dataset.meta)),
None,
'hvm_timing',
dummy_upload=dummy_upload)
with open(path.join(path.dirname(__file__), 'tutorial_html_basic'),
'r') as tutorial_html_file:
tutorial_html = tutorial_html_file.read()
label_func = lambda x: hvm.OBJECT_NAMES[x['obj']]
html_data = {
'combs': combs,
'response_images': response_images,
'num_trials': 125 * 2,
'meta_field': 'category',
'meta': meta,
'urls': urls,
'shuffle_test': True,
'meta_query': lambda x: x['var'] == 'V6',
'label_func': label_func
}
cat_dict = {
'Animals': 'Animal',
'Boats': 'Boat',
'Cars': 'Car',
'Chairs': 'Chair',
'Faces': 'Face',
'Fruits': 'Fruit',
'Planes': 'Plane',
'Tables': 'Table'
}
additionalrules = [{
'old': 'LEARNINGPERIODNUMBER',
'new': str(10)
}, {
'old': 'OBJTYPE',
'new': 'Object Recognition'
}, {
'old': 'TUTORIAL_HTML',
'new': tutorial_html
}, {
'old': 'CATDICT',
'new': json.dumps(cat_dict)
}, {
'old': 'METAFIELD',
'new': "'category'"
}]
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/general_two_way.html',
htmldst='hvm_basic_2way_n%05d.html',
sandbox=sandbox,
title=
'Object recognition --- report what you see. Up to 50 cent performance based bonus',
reward=0.25,
duration=1600,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=
"***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment='hvm_basic_2ways',
collection_name='hvm_basic_2ways',
max_assignments=1,
bucket_name='hvm_2ways',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 140 + 6x4 repeats
html_data=html_data,
tmpdir='tmp',
frame_height_pix=1200,
othersrc=[
'../../lib/dltk.js', '../../lib/dltkexpr.js',
'../../lib/dltkrsvp.js'
],
additionalrules=additionalrules)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (8 * 7 / 2) * 250
if debug:
return exp, html_data
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {
e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials
}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['labels'])
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT /
float(len(ind_repeats))).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1,
-ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 50 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 50 * 164
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
offsets2 = np.arange(24)[::-1] + offsets
ibie = zip(range(0, 50 * 164, 164), range(164, 50 * 164, 164))
assert all([[(e1, e2)
for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]
] == s_ref_labels for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
#del exp._trials['imgData']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, dummy_upload=True):
dataset = hvm.HvMWithDiscfade()
meta = dataset.meta
categories = dataset.categories
combs = [categories]
inds = np.arange(len(meta))
preproc = None
image_bucket_name = 'hvm_timing'
urls = dataset.publish_images(inds, preproc,
image_bucket_name,
dummy_upload=dummy_upload)
base_url = 'https://canonical_images.s3.amazonaws.com/'
response_images = [{
'urls': [base_url + cat + '.png' for cat in categories],
'meta': [{'category': 'Animals'},
{'category': 'Boats'},
{'category': 'Cars'},
{'category': 'Chairs'},
{'category': 'Faces'},
{'category': 'Fruits'},
{'category': 'Planes'},
{'category': 'Tables'}],
'labels': categories}]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 90 * 64,
'meta_field': 'category',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(LEARNING_PERIOD)}]
trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='hvm_basic_categorization.html',
htmldst='hvm_basic_categorization_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="hvm basic categorization", # noqa
collection_name= 'hvm_basic_categorization',
max_assignments=1,
bucket_name='hvm_basic_categorization',
trials_per_hit=trials_per_hit, # 150 + 8x4 repeats
html_data=html_data,
frame_height_pix=1200,
othersrc = ['../../lib/dltk.js'],
additionalrules=additionalrules
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == 90 * 64, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 38 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles'])
s_ref_labels = set([tuple(e) for e in trials_qe['labels']])
print(s_ref_labels)
offsets2 = np.arange(8 * 4)[::-1] + offsets
ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit))
assert all([set([tuple(e) for e in
np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels
for ib, ie in ibie[:-1]])
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, debug=False):
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/objectome_64objs_v2a.html',
htmldst='objectome_64objs_v2a_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.15,
duration=1600,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=
"***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment='objectome_64objs_v2a',
collection_name='objectome_64objs_v2a',
max_assignments=1,
bucket_name='objectome_64objs_v2a',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir='tmp',
frame_height_pix=1200,
othersrc=[
'../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js',
'../../mturkutils/lib/dltkrsvp.js'
], # noqa
set_destination=True,
)
# -- create trials
# exp.createTrials(sampling='with-replacement', verbose=1) Don't use this
exp._trials, alldat, _ = createTrials()
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (64 * 63 / 2) * 250
if debug:
return exp
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
# 138334: dog v camel (difficult!)
# 139363: rhino v elephant (difficult!)
ind_repeats = [138334, 139363, 47, 9] * REPEATS_PER_QE_IMG
assert all(alldat[138334] == [701, 0, 33, 0, 33])
assert all(alldat[139363] == [44070, 2, 39, 39, 39])
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {
e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials
}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['imgFiles'])
print '** n_qe =', n_qe
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
# -- actual application
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT /
float(len(ind_repeats))).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1,
-ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 5040 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 5040 * (100 + 16)
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
print '**', s_ref_labels
offsets2 = np.arange(16)[::-1] + offsets
ibie = zip(range(0, 5040 * 116, 116), range(116, 5041 * 116, 116))
assert all([[(e1, e2)
for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]
] == s_ref_labels for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
# del exp._trials['imgData']
print '** Finished creating trials.'
return exp
def get_exp(sandbox=True, selected_basic_objs=SELECTED_BASIC_OBJS,
data_intercls=DATA_INTERCLS,
debug=False):
selected_intercls_pairs = np.array(data_intercls['CMinds_cars_tanks'])
selected_intercls_pairs = selected_intercls_pairs[data_intercls['si']]
selected_intercls_pairs[:, 0] -= 64 # subtract car base index
selected_intercls_pairs[:, 1] -= 64 + 90 # same for tanks
meta, _, meta_cars, meta_tanks = get_meta()
cars = np.unique(meta_cars['obj'])
tanks = np.unique(meta_tanks['obj'])
combs = [(cars[e1], tanks[e2]) for e1, e2 in selected_intercls_pairs]
combs_dummy1 = [(o1, o2) for o1 in selected_basic_objs for o2 in cars]
combs_dummy2 = [(o1, o2) for o1 in selected_basic_objs for o2 in tanks]
rng = np.random.RandomState(0)
rng.shuffle(combs_dummy1)
rng.shuffle(combs_dummy2)
combs = combs + combs_dummy1[:50] + combs_dummy2[:50]
assert len(combs) == 200
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)],
'meta': [{'obj': o} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/intercls_cars_tanks.html',
htmldst='intercls_cars_tanks_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.15,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="objectome_intercls_cars_tanks_v3. 100+100 selected pairs across 22 basic, 30 cars, and 30 tanks", # noqa
collection_name='objectome_intercls_cars_tanks_v3',
max_assignments=1,
bucket_name='objectome_intercls_cars_tanks_v3',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir='tmp',
html_data=html_data,
frame_height_pix=1200,
othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa
set_destination=True,
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
if debug:
return exp
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == 200 * 250
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
# the choice of v--- this array below is arbitrary
ind_repeats = [1, 9, 60, 27] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['imgFiles'])
print '** n_qe =', n_qe
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 500 == n_applied_hits
assert len(exp._trials['imgFiles']) == 500 * (100 + 16)
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
print '**', s_ref_labels
offsets2 = np.arange(16)[::-1] + offsets
ibie = zip(range(0, 500 * 116, 116), range(501 * 116, 116))
assert all(
[[(e1, e2) for e1, e2 in
np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels
for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
del exp._trials['meta_field']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True,
stimdur=100,
selected_basic_objs=SELECTED_BASIC_OBJS,
debug=False,
mode='default'):
meta = get_meta()
combs = [e for e in itertools.combinations(selected_basic_objs, 2)]
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1),
get_url_labeled_resp_img(o2)],
'meta': [{
'obj': o
} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
# few different options
htmlsrcdct = {}
descdct = {}
htmldstdct = {}
tmpdirdct = {}
addirules = {}
htmlsrcdct['default'] = 'web/objt_tcurve_o16s0.html'
descdct[
'default'] = "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['default'] = 'objt_tcurve_o16s0_%04d_n%%05d.html' % int(
stimdur) # noqa
tmpdirdct['default'] = 'tmp/t%04d' % int(stimdur)
addirules['default'] = []
htmlsrcdct['softnotice'] = 'web/objt_tcurve_o16s0_softnotice.html'
descdct['softnotice'] = descdct['default']
htmldstdct['softnotice'] = 'objt_tcurve_o16s0_soft_%04d_n%%05d.html' % int(
stimdur) # noqa
tmpdirdct['softnotice'] = 'tmp/t%04d_soft' % int(stimdur)
addirules['softnotice'] = []
htmlsrcdct['winonly'] = htmlsrcdct['default']
descdct[
'winonly'] = "***Chrome or Firefox on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['winonly'] = htmldstdct['default']
tmpdirdct['winonly'] = tmpdirdct['default']
addirules['winonly'] = [{
'old': "supportedOS: ['Windows', 'Mac', 'Linux']",
'new': "supportedOS: ['Windows']",
'n': 1,
}]
htmlsrcdct['winchromeonly'] = htmlsrcdct['default']
descdct[
'winchromeonly'] = "***Latest Chrome on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['winchromeonly'] = htmldstdct['default']
tmpdirdct['winchromeonly'] = tmpdirdct['default']
addirules['winchromeonly'] = [{
'old': "supportedBrowser: ['Chrome', 'Firefox']",
'new': "supportedBrowser: ['Chrome']",
'n': 1,
}] + addirules['winonly']
htmlsrcdct['winchromeonlymask'] = 'web/objt_tcurve_o16s0_postmask.html'
descdct['winchromeonlymask'] = descdct['winchromeonly']
htmldstdct[
'winchromeonlymask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(
stimdur) # noqa
tmpdirdct['winchromeonlymask'] = 'tmp/t%04d_mask' % int(stimdur)
addirules['winchromeonlymask'] = [{
'old': "supportedBrowser: ['Chrome', 'Firefox']",
'new': "supportedBrowser: ['Chrome']",
'n': 1,
}] + addirules['winonly']
htmlsrcdct['mask'] = 'web/objt_tcurve_o16s0_postmask.html'
descdct['mask'] = descdct['default']
htmldstdct['mask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(
stimdur) # noqa
tmpdirdct['mask'] = 'tmp/t%04d_mask' % int(stimdur)
addirules['mask'] = addirules['default']
exp = MatchToSampleFromDLDataExperiment(
htmlsrc=htmlsrcdct[mode],
htmldst=htmldstdct[mode],
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.25,
duration=1600,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=descdct[mode],
comment="objectome_time_curve_%dms. 0-th set of 16 objects" %
int(stimdur), # noqa
collection_name='objectome_tcurve_16objsset0',
max_assignments=1,
bucket_name='objectome_tcurve_16objsset0',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats
html_data=html_data,
tmpdir=tmpdirdct[mode],
frame_height_pix=1200,
othersrc=[
'../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js',
'../../mturkutils/lib/dltkrsvp.js'
], # noqa
additionalrules=[{
'old': 'stimduration = 100;',
'new': 'stimduration = %f;' % stimdur,
'n': 1,
}] + addirules[mode],
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (16 * 15 / 2) * 250
if debug:
return exp, html_data
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {
e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials
}
#print np.unique([e[0] for e in trials_qe['imgFiles']])
#print np.unique([tuple(e) for e in trials_qe['labels']])
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['labels'])
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT /
float(len(ind_repeats))).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1,
-ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
#exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 200 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 200 * 174
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
offsets2 = np.arange(24)[::-1] + offsets
ibie = zip(range(0, 200 * 174, 174), range(174, 201 * 174, 174))
assert all([[(e1, e2)
for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]
] == s_ref_labels for ib, ie in ibie])
#print set(s_ref_labels), len(set(s_ref_labels))
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, dummy_upload=True):
dataset = lcs.LandoltCs()
meta = dataset.meta
preproc = dict(crop=None, crop_rand=None, dtype='float32', mask=None, mode='L', normalize=False,
resize_to=(256, 256), seed=0)
img_urls = dataset.publish_images(range(len(dataset.meta)), preproc, 'acuity_tasks',
dummy_upload=dummy_upload)
response_im_inds = [29, 24, 19, 34, 14, 39, 4, 9]
response_urls = []
response_metas = []
response_labels = []
for ind in response_im_inds:
response_urls.append(img_urls[ind])
response_metas.append({k: meta[ind][k] for k in meta.dtype.names})
response_labels.append(None)
response_images = [{'urls': response_urls, 'meta': response_metas, 'labels': response_labels}]
combs = [[m['rotation'] for m in response_images[0]['meta']]]
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(10)},
{'old': 'OBJTYPE',
'new': 'Landolt C'}]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 184,
'meta_field': 'rotation',
'meta': meta,
'urls': img_urls
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='landolt_c_task.html',
htmldst='landolt_c_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="landolt c acuity", # noqa
collection_name= 'landolt_c',
max_assignments=20,
bucket_name='landolt_c',
trials_per_hit=184, # 150 + 8x4 repeats
html_data=html_data,
frame_height_pix=1200,
othersrc = ['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'],
additionalrules=additionalrules
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
#n_total_trials = len(exp._trials['imgFiles'])
#assert n_total_trials == 40, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
#ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG
#rng = np.random.RandomState(0)
#rng.shuffle(ind_repeats)
# trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
# for e in exp._trials}
# offsets = np.arange(
# ACTUAL_TRIALS_PER_HIT - 3, -1,
# -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
# ).round().astype('int')
#
# n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
# n_applied_hits = 0
# for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
# -1, -ACTUAL_TRIALS_PER_HIT):
# for k in trials_qe:
# for i, offset in enumerate(offsets):
# exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# n_applied_hits += 1
#
# print '** n_applied_hits =', n_applied_hits
# print '** len for each in _trials =', \
# {e: len(exp._trials[e]) for e in exp._trials}
#
# # -- sanity check
# assert 38 == n_applied_hits, n_applied_hits
# assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles'])
# s_ref_labels = set([tuple(e) for e in trials_qe['labels']])
# print(s_ref_labels)
# offsets2 = np.arange(8 * 4)[::-1] + offsets
#
# ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit))
# assert all([set([tuple(e) for e in
# np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels
# for ib, ie in ibie[:-1]])
# print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, debug=False):
exp = MatchToSampleFromDLDataExperiment(
htmlsrc="web/objectome_64objs_v2a.html",
htmldst="objectome_64objs_v2a_n%05d.html",
sandbox=sandbox,
title="Object recognition --- report what you see",
reward=0.15,
duration=1600,
keywords=["neuroscience", "psychology", "experiment", "object recognition"], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="objectome_64objs_v2a",
collection_name="objectome_64objs_v2a",
max_assignments=1,
bucket_name="objectome_64objs_v2a",
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir="tmp",
frame_height_pix=1200,
othersrc=["../../lib/dltk.js", "../../lib/dltkexpr.js", "../../lib/dltkrsvp.js"], # noqa
set_destination=True,
)
# -- create trials
# exp.createTrials(sampling='with-replacement', verbose=1) Don't use this
exp._trials, alldat, _ = createTrials()
n_total_trials = len(exp._trials["imgFiles"])
assert n_total_trials == (64 * 63 / 2) * 250
if debug:
return exp
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
# 138334: dog v camel (difficult!)
# 139363: rhino v elephant (difficult!)
ind_repeats = [138334, 139363, 47, 9] * REPEATS_PER_QE_IMG
assert all(alldat[138334] == [701, 0, 33, 0, 33])
assert all(alldat[139363] == [44070, 2, 39, 39, 39])
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe["imgFiles"])
print "** n_qe =", n_qe
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe["imgFiles"][i][1].reverse()
trials_qe["labels"][i].reverse()
# -- actual application
offsets = (
np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype("int")
)
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print "** n_applied_hits =", n_applied_hits
print "** len for each in _trials =", {e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 5040 == n_applied_hits, n_applied_hits
assert len(exp._trials["imgFiles"]) == 5040 * (100 + 16)
s_ref_labels = [tuple(e) for e in trials_qe["labels"]]
print "**", s_ref_labels
offsets2 = np.arange(16)[::-1] + offsets
ibie = zip(range(0, 5040 * 116, 116), range(116, 5041 * 116, 116))
assert all(
[[(e1, e2) for e1, e2 in np.array(exp._trials["labels"][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]
)
# -- drop unneeded, potentially abusable stuffs
# del exp._trials['imgData']
print "** Finished creating trials."
return exp
def get_exp(sandbox=True, stimdur=100,
selected_basic_objs=SELECTED_BASIC_OBJS, debug=False, mode='default'):
meta = get_meta()
combs = [e for e in itertools.combinations(selected_basic_objs, 2)]
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)],
'meta': [{'obj': o} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
# few different options
htmlsrcdct = {}
descdct = {}
htmldstdct = {}
tmpdirdct = {}
addirules = {}
htmlsrcdct['default'] = 'web/objt_tcurve_o16s0.html'
descdct['default'] = "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['default'] = 'objt_tcurve_o16s0_%04d_n%%05d.html' % int(stimdur) # noqa
tmpdirdct['default'] = 'tmp/t%04d' % int(stimdur)
addirules['default'] = []
htmlsrcdct['softnotice'] = 'web/objt_tcurve_o16s0_softnotice.html'
descdct['softnotice'] = descdct['default']
htmldstdct['softnotice'] = 'objt_tcurve_o16s0_soft_%04d_n%%05d.html' % int(stimdur) # noqa
tmpdirdct['softnotice'] = 'tmp/t%04d_soft' % int(stimdur)
addirules['softnotice'] = []
htmlsrcdct['winonly'] = htmlsrcdct['default']
descdct['winonly'] = "***Chrome or Firefox on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['winonly'] = htmldstdct['default']
tmpdirdct['winonly'] = tmpdirdct['default']
addirules['winonly'] = [{
'old': "supportedOS: ['Windows', 'Mac', 'Linux']",
'new': "supportedOS: ['Windows']",
'n': 1,
}]
htmlsrcdct['winchromeonly'] = htmlsrcdct['default']
descdct['winchromeonly'] = "***Latest Chrome on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa
htmldstdct['winchromeonly'] = htmldstdct['default']
tmpdirdct['winchromeonly'] = tmpdirdct['default']
addirules['winchromeonly'] = [{
'old': "supportedBrowser: ['Chrome', 'Firefox']",
'new': "supportedBrowser: ['Chrome']",
'n': 1,
}] + addirules['winonly']
htmlsrcdct['winchromeonlymask'] = 'web/objt_tcurve_o16s0_postmask.html'
descdct['winchromeonlymask'] = descdct['winchromeonly']
htmldstdct['winchromeonlymask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(stimdur) # noqa
tmpdirdct['winchromeonlymask'] = 'tmp/t%04d_mask' % int(stimdur)
addirules['winchromeonlymask'] = [{
'old': "supportedBrowser: ['Chrome', 'Firefox']",
'new': "supportedBrowser: ['Chrome']",
'n': 1,
}] + addirules['winonly']
htmlsrcdct['mask'] = 'web/objt_tcurve_o16s0_postmask.html'
descdct['mask'] = descdct['default']
htmldstdct['mask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(stimdur) # noqa
tmpdirdct['mask'] = 'tmp/t%04d_mask' % int(stimdur)
addirules['mask'] = addirules['default']
exp = MatchToSampleFromDLDataExperiment(
htmlsrc=htmlsrcdct[mode],
htmldst=htmldstdct[mode],
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.25,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description=descdct[mode],
comment="objectome_time_curve_%dms. 0-th set of 16 objects" % int(stimdur), # noqa
collection_name='objectome_tcurve_16objsset0',
max_assignments=1,
bucket_name='objectome_tcurve_16objsset0',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats
html_data=html_data,
tmpdir=tmpdirdct[mode],
frame_height_pix=1200,
othersrc=['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'], # noqa
additionalrules=[{
'old': 'stimduration = 100;',
'new': 'stimduration = %f;' % stimdur,
'n': 1,
}] + addirules[mode],
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (16 * 15 / 2) * 250
if debug:
return exp, html_data
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
#print np.unique([e[0] for e in trials_qe['imgFiles']])
#print np.unique([tuple(e) for e in trials_qe['labels']])
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['labels'])
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
#exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 200 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 200 * 174
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
offsets2 = np.arange(24)[::-1] + offsets
ibie = zip(range(0, 200 * 174, 174), range(174, 201 * 174, 174))
assert all([[(e1, e2) for e1, e2 in
np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels
for ib, ie in ibie])
#print set(s_ref_labels), len(set(s_ref_labels))
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, dummy_upload=True):
n_hits_from_data = len(meta) / ACTUAL_TRIALS_PER_HIT
categories = np.unique(dataset.meta['identity'])
combs = [categories]
inds = np.arange(len(meta))
preproc = dataset.default_preproc
#preproc['flip_tb'] = True
image_bucket_name = 'freiwald_tsao_2010'
urls = dataset.publish_images(inds, preproc,
image_bucket_name,
dummy_upload=dummy_upload)
base_url = 'https://canonical_images.s3.amazonaws.com/'
response_images = [{
'urls': [base_url + 'freiwald_tsao_2010_face%.2da.png' % i for i in range(1, 29)],
'meta': [{'identity': i} for i in range(1, 29)],
'labels': categories}]
mult = 10
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 220 * mult,
'meta_field': 'identity',
'meta': tb.tab_rowstack([meta] * mult),
'urls': urls * mult,
'shuffle_test': False,
}
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(LEARNING_PERIOD)}]
trials_per_hit = ACTUAL_TRIALS_PER_HIT + REPEATS_PER_QE_IMG * len(repeat_inds) + LEARNING_PERIOD
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='ft_identity.html',
htmldst='ft_identity_n%05d.html',
tmpdir='tmp_ft_identity',
sandbox=sandbox,
title='Face recognition --- report what you see',
reward=0.35,
duration=1500,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="freiwald_tsao_identification", # noqa
collection_name = 'freiwald_tsao_identification',
max_assignments=1,
bucket_name='freiwald_tsao_identification',
trials_per_hit=trials_per_hit,
html_data=html_data,
frame_height_pix=1200,
othersrc = ['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'],
additionalrules=additionalrules,
log_prefix='freiwald_tsao_identification_'
)
# -- create trials
exp.createTrials(sampling='without-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == 220 * mult, n_total_trials
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = repeat_inds * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
ind_learn = practice_inds
goodids = [meta[i]['id'] for i in ind_learn]
trials_lrn = {}
for e in exp._trials:
trials_lrn[e] = []
got = []
for _ind, r in enumerate(exp._trials[e]):
if exp._trials['imgData'][_ind]['Sample']['id'] in goodids and exp._trials['imgData'][_ind]['Sample']['id'] not in got :
trials_lrn[e].append(copy.deepcopy(r))
got.append(exp._trials['imgData'][_ind]['Sample']['id'])
assert len(trials_lrn['imgData']) == len(goodids), len(trials_lrn['imgData'])
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 1, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
print(len(offsets), offsets)
print('a', len(exp._trials['imgFiles']))
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
n_applied_hits += 1
print('b', len(exp._trials['imgFiles']))
for j in range(n_applied_hits):
for k in trials_lrn:
for i in range(len(ind_learn)):
exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i])
print('c', len(exp._trials['imgFiles']))
#shuffle test on a per-hit basis
for j in range(n_applied_hits):
rng = np.random.RandomState(seed=j)
perm = rng.permutation(28)
for i in range(trials_per_hit * j, min(trials_per_hit * (j+1), len(exp._trials['imgFiles']))):
f = copy.deepcopy(exp._trials['imgFiles'][i])
t = copy.deepcopy(exp._trials['imgData'][i])
f[1] = [f[1][_j] for _j in perm]
exp._trials['imgFiles'][i] = f
t['Test'] = [t['Test'][_j] for _j in perm]
exp._trials['imgData'][i] = t
l = copy.deepcopy(exp._trials['labels'][i])
exp._trials['labels'][i] = [l[_j] for _j in perm]
print('d', len(exp._trials['imgFiles']))
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
_K = LEARNING_PERIOD + REPEATS_PER_QE_IMG * len(repeat_inds)
# -- sanity check
assert n_hits_floor == n_applied_hits == mult * n_hits_from_data, (n_total_trials, ACTUAL_TRIALS_PER_HIT, n_hits_floor, n_applied_hits, mult, n_hits_from_data)
assert len(exp._trials['imgFiles']) == mult * (len(meta) + n_hits_from_data * _K), (len(exp._trials['imgFiles']), mult, (len(meta) + n_hits_from_data * _K), len(meta), n_hits_from_data, _K)
return exp, html_data
def get_exp(sandbox=True, dummy_upload=True):
dataset = hvm.HvMWithDot()
base_url = 'https://canonical_images.s3.amazonaws.com/'
fields = dataset.meta.dtype.names
meta = dataset.meta
names = ['RedPixelOnCanonical', 'RedPixelOffCanonical']
response_images = [{'urls': [base_url + name + '.png' for name in names],
'meta': [{field: meta[i][field] for field in fields} for i in [4020, 5060]],
'labels': ['The red pixel was on the object',
'The red pixel was not on the object']}]
preproc = {'crop': None,
'crop_rand': None,
'dtype': 'float32',
'mask': None,
'mode': 'RGB',
'normalize': False,
'resize_to': (256, 256),
'seed': 0}
cat_dict = "{true: 'Dot is on the object', false:'Dot is not on the object'}"
with open(path.join(path.dirname(__file__), 'tutorial_html'), 'r') as tutorial_html_file:
tutorial_html = tutorial_html_file.read()
additionalrules = [{'old': 'LEARNINGPERIODNUMBER',
'new': str(10)},
{'old': 'OBJTYPE',
'new': 'Figure/ground task'},
{'old': 'TUTORIAL_HTML',
'new': tutorial_html},
{'old': 'CATDICT',
'new': cat_dict},
{'old': 'METAFIELD',
'new': "'dot_on'"}]
variation_level = 'V6'
disallowed_images = [meta[i]['filename'] for i in [4020, 5060]]
query = lambda x: x['var'] == variation_level and (x['filename'] not in disallowed_images)
html_data = {
'response_images': response_images,
'combs': [[True, False]],
'num_trials': 1190,
'meta_field': 'dot_on',
'meta': meta,
'urls': dataset.publish_images(range(meta.shape[0]), preproc, 'hvm_figure_ground', dummy_upload),
'meta_query': query
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='general_two_way.html',
htmldst='hvm_figure_ground%05d.html',
sandbox=sandbox,
title='Report whether the dot is on the object',
reward=0.35,
duration=1500,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report dots are on an object or not. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment="hvm_figure_ground", # noqa
collection_name= 'hvm_figure_ground_2',
max_assignments=10,
bucket_name='hvm_figure_ground_2',
trials_per_hit=170, # 150 + 8x4 repeats
html_data=html_data,
frame_height_pix=1200,
othersrc = ['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'],
additionalrules=additionalrules
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
return exp, html_data
def get_exp(sandbox=True,
selected_basic_objs=SELECTED_BASIC_OBJS,
debug=False):
meta, _, meta_chairs = get_meta()
chairs = np.unique(meta_chairs['obj'])
combs = [(o1, o2) for o1 in selected_basic_objs for o2 in chairs] + \
[e for e in itertools.combinations(chairs, 2)]
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1),
get_url_labeled_resp_img(o2)],
'meta': [{
'obj': o
} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'response_images': response_images,
'combs': combs,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/chairs_subord.html',
htmldst='chairs_subord_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.15,
duration=1600,
keywords=[
'neuroscience', 'psychology', 'experiment', 'object recognition'
], # noqa
description=
"***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment=
"objectome_chairs_subord_v3. 30 chairs and 30 non-chairs", # noqa
collection_name='objectome_chairs_subord_v3',
max_assignments=1,
bucket_name='objectome_chairs_subord_v3',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats
tmpdir='tmp',
html_data=html_data,
frame_height_pix=1200,
othersrc=[
'../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js',
'../../mturkutils/lib/dltkrsvp.js'
], # noqa
set_destination=True,
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
if debug:
return exp
# repeat last 50 presentations twice to make the entire trials
# well aligned as multiples of 100
exp._trials['imgFiles'].extend(exp._trials['imgFiles'][-50:])
exp._trials['imgData'].extend(exp._trials['imgData'][-50:])
exp._trials['labels'].extend(exp._trials['labels'][-50:])
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (30 * 29 / 2 + 30 * 30) * 250 + 50
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
# the choice of v--- this array below is arbitrary
ind_repeats = [1, 10, 22, 24] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {
e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials
}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['imgFiles'])
print '** n_qe =', n_qe
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT /
float(len(ind_repeats))).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1,
-ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 3338 == n_applied_hits
assert len(exp._trials['imgFiles']) == 3338 * (100 + 16)
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
print '**', s_ref_labels
offsets2 = np.arange(16)[::-1] + offsets
ibie = zip(range(0, 3338 * 116, 116), range(3339 * 116, 116))
assert all([[(e1, e2)
for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]
] == s_ref_labels for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
del exp._trials['meta_field']
print '** Finished creating trials.'
return exp, html_data
def get_exp(sandbox=True, debug=False):
meta = get_meta()
combs = [e for e in itertools.combinations(np.unique(meta['obj']), 2)]
urls = [get_url(e['obj'], e['id']) for e in meta]
response_images = [{
'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)],
'meta': [{'obj': o} for o in [o1, o2]],
'labels': [o1, o2]
} for o1, o2 in combs]
html_data = {
'combs': combs,
'response_images': response_images,
'num_trials': 125 * 2,
'meta_field': 'obj',
'meta': meta,
'urls': urls,
'shuffle_test': True,
}
exp = MatchToSampleFromDLDataExperiment(
htmlsrc='web/objectome_64objs_v2.html',
htmldst='objectome_64objs_v2_n%05d.html',
sandbox=sandbox,
title='Object recognition --- report what you see',
reward=0.25,
duration=1600,
keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa
description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa
comment='objectome_64objs_v2',
collection_name='objectome_64objs_v2',
max_assignments=1,
bucket_name='objectome_64objs_v2',
trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats
html_data=html_data,
tmpdir='tmp',
frame_height_pix=1200,
othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa
)
# -- create trials
exp.createTrials(sampling='with-replacement', verbose=1)
n_total_trials = len(exp._trials['imgFiles'])
assert n_total_trials == (64 * 63 / 2) * 250
if debug:
return exp, html_data
# -- in each HIT, the followings will be repeated 4 times to
# estimate "quality" of data
ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG
rng = np.random.RandomState(0)
rng.shuffle(ind_repeats)
trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats]
for e in exp._trials}
# -- flip answer choices of some repeated images
n_qe = len(trials_qe['labels'])
# if True, flip
flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2)
assert len(flips) == n_qe
rng.shuffle(flips)
assert len(trials_qe.keys()) == 4
for i, flip in enumerate(flips):
if not flip:
continue
trials_qe['imgFiles'][i][1].reverse()
trials_qe['labels'][i].reverse()
trials_qe['imgData'][i]['Test'].reverse()
# -- actual application
offsets = np.arange(
ACTUAL_TRIALS_PER_HIT - 3, -1,
-ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))
).round().astype('int')
assert len(offsets) == len(offsets)
n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT
n_applied_hits = 0
for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT,
-1, -ACTUAL_TRIALS_PER_HIT):
for k in trials_qe:
for i, offset in enumerate(offsets):
exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i])
# exp._trials[k].insert(i_trial_begin + offset, 'test')
n_applied_hits += 1
print '** n_applied_hits =', n_applied_hits
print '** len for each in _trials =', \
{e: len(exp._trials[e]) for e in exp._trials}
# -- sanity check
assert 3360 == n_applied_hits, n_applied_hits
assert len(exp._trials['imgFiles']) == 3360 * 174
s_ref_labels = [tuple(e) for e in trials_qe['labels']]
offsets2 = np.arange(24)[::-1] + offsets
ibie = zip(range(0, 3360 * 174, 174), range(174, 3361 * 174, 174))
assert all(
[[(e1, e2) for e1, e2 in
np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels
for ib, ie in ibie])
# -- drop unneeded, potentially abusable stuffs
del exp._trials['imgData']
print '** Finished creating trials.'
return exp, html_data