def get_exp_core(varlev, sandbox=True, partial=False, selected_basic_objs=SELECTED_BASIC_OBJS): meta, _, meta_cars = get_meta(varlev=varlev) cars = np.unique(meta_cars['obj']) combs = [(o1, o2) for o1 in selected_basic_objs for o2 in cars] + \ [e for e in itertools.combinations(cars, 2)] urls = [get_url(e['obj'], e['id'], varlev) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1, varlev), get_url_labeled_resp_img(o2, varlev)], 'meta': [{'obj': o} for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/cars_subord.html', htmldst='cars_subord_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.15, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="objectome_cars_subord. 30 cars and 30 non-cars", # noqa collection_name='objectome_cars_subord_v0v3', max_assignments=1, bucket_name='objectome_cars_subord_v0v3', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir='tmp', html_data=html_data, frame_height_pix=1200, othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa set_destination=True, ) if partial: return exp # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) return exp
def get_exp(sandbox=True, dummy_upload=True): dataset = lcs.LandoltCs() meta = dataset.meta preproc = dict(crop=None, crop_rand=None, dtype='float32', mask=None, mode='L', normalize=False, resize_to=(256, 256), seed=0) img_urls = dataset.publish_images(range(len(dataset.meta)), preproc, 'acuity_tasks', dummy_upload=dummy_upload) response_im_inds = [29, 24, 19, 34, 14, 39, 4, 9] response_urls = [] response_metas = [] response_labels = [] for ind in response_im_inds: response_urls.append(img_urls[ind]) response_metas.append({k: meta[ind][k] for k in meta.dtype.names}) response_labels.append(None) response_images = [{ 'urls': response_urls, 'meta': response_metas, 'labels': response_labels }] combs = [[m['rotation'] for m in response_images[0]['meta']]] additionalrules = [{ 'old': 'LEARNINGPERIODNUMBER', 'new': str(10) }, { 'old': 'OBJTYPE', 'new': 'Landolt C' }] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 184, 'meta_field': 'rotation', 'meta': meta, 'urls': img_urls } exp = MatchToSampleFromDLDataExperiment( htmlsrc='landolt_c_task.html', htmldst='landolt_c_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.35, duration=1500, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description= "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="landolt c acuity", # noqa collection_name='landolt_c', max_assignments=20, bucket_name='landolt_c', trials_per_hit=184, # 150 + 8x4 repeats html_data=html_data, frame_height_pix=1200, othersrc=[ '../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js' ], additionalrules=additionalrules) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) #n_total_trials = len(exp._trials['imgFiles']) #assert n_total_trials == 40, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data #ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG #rng = np.random.RandomState(0) #rng.shuffle(ind_repeats) # trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] # for e in exp._trials} # offsets = np.arange( # ACTUAL_TRIALS_PER_HIT - 3, -1, # -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) # ).round().astype('int') # # n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT # n_applied_hits = 0 # for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, # -1, -ACTUAL_TRIALS_PER_HIT): # for k in trials_qe: # for i, offset in enumerate(offsets): # exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # n_applied_hits += 1 # # print '** n_applied_hits =', n_applied_hits # print '** len for each in _trials =', \ # {e: len(exp._trials[e]) for e in exp._trials} # # # -- sanity check # assert 38 == n_applied_hits, n_applied_hits # assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles']) # s_ref_labels = set([tuple(e) for e in trials_qe['labels']]) # print(s_ref_labels) # offsets2 = np.arange(8 * 4)[::-1] + offsets # # ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit)) # assert all([set([tuple(e) for e in # np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels # for ib, ie in ibie[:-1]]) # print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, selected_basic_objs=SELECTED_BASIC_OBJS, debug=False): meta, _, meta_cars = get_meta() cars = np.unique(meta_cars['obj']) combs = [(o1, o2) for o1 in selected_basic_objs for o2 in cars] + \ [e for e in itertools.combinations(cars, 2)] urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{'obj': o} for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/cars_subord.html', htmldst='cars_subord_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.15, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="objectome_cars_subord. 30 cars and 30 non-cars", # noqa collection_name='objectome_cars_subord_v3', max_assignments=1, bucket_name='objectome_cars_subord_v3', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir='tmp', html_data=html_data, frame_height_pix=1200, othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa set_destination=True, ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) if debug: return exp # repeat last 50 presentations twice to make the entire trials # well aligned as multiples of 100 exp._trials['imgFiles'].extend(exp._trials['imgFiles'][-50:]) exp._trials['imgData'].extend(exp._trials['imgData'][-50:]) exp._trials['labels'].extend(exp._trials['labels'][-50:]) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (30 * 29 / 2 + 30 * 30) * 250 + 50 # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data # # car: 4 - car MB27534* v car MB27803 # car: 12 - train MB31405 v car MB27577* # non-car: 21 - truck* v car MB29874 # non-car: 24 - book v MB28307* ind_repeats = [4, 12, 21, 24] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} # -- flip answer choices of some repeated images n_qe = len(trials_qe['imgFiles']) print '** n_qe =', n_qe # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 3338 == n_applied_hits assert len(exp._trials['imgFiles']) == 3338 * (100 + 16) s_ref_labels = [tuple(e) for e in trials_qe['labels']] print '**', s_ref_labels offsets2 = np.arange(16)[::-1] + offsets ibie = zip(range(0, 3338 * 116, 116), range(3339 * 116, 116)) assert all( [[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] del exp._trials['meta_field'] print '** Finished creating trials.' return exp, html_data
def get_exp(category, sandbox=True, dummy_upload=True): dataset = hvm.HvMWithDiscfade() meta = dataset.meta inds = (meta['category'] == category).nonzero()[0] meta = meta[inds] objs = np.unique(meta['obj']) combs = [objs] preproc = None image_bucket_name = 'hvm_timing' urls = dataset.publish_images(inds, preproc, image_bucket_name, dummy_upload=dummy_upload) base_url = 'https://s3.amazonaws.com/hvm_timing/' obj_resp_ids = [meta[meta['obj'] == o]['id'][0] for o in objs] response_images = [{ 'urls': [base_url + obj_id + '.png' for obj_id in obj_resp_ids], 'meta': [{'obj': obj, 'category': category} for obj in objs], 'labels': objs}] mult = 2 html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 90 * 8 * mult, 'meta_field': 'obj', 'meta': tb.tab_rowstack([meta] * mult), 'urls': urls * mult, 'shuffle_test': False, } additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(LEARNING_PERIOD)}, {'old': 'OBJTYPE', 'new': category}] trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32 + 16 exp = MatchToSampleFromDLDataExperiment( htmlsrc='hvm_subordinate.html', htmldst='hvm_subordinate_' + category + '_n%05d.html', tmpdir='tmp_subordinate_%s' % category, sandbox=sandbox, title='Object recognition --- report what you see', reward=0.35, duration=1500, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="hvm subordinate identification", # noqa collection_name = 'hvm_subordinate_identification_%s' % category, max_assignments=1, bucket_name='hvm_subordinate_identification_test', trials_per_hit=trials_per_hit, # 144 + 8x4 repeats + 16 training html_data=html_data, frame_height_pix=1200, othersrc = ['objnames.js', '../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'], additionalrules=additionalrules, log_prefix='subordinate_' + category + '_' ) # -- create trials exp.createTrials(sampling='without-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == 90 * 8 * mult, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = repeat_inds[category] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} ind_learn = practice_inds[category] goodids = [meta[i]['id'] for i in ind_learn] trials_lrn = {} for e in exp._trials: trials_lrn[e] = [] got = [] for _ind, r in enumerate(exp._trials[e]): if exp._trials['imgData'][_ind]['Sample']['id'] in goodids and exp._trials['imgData'][_ind]['Sample']['id'] not in got : trials_lrn[e].append(copy.deepcopy(r)) got.append(exp._trials['imgData'][_ind]['Sample']['id']) assert len(trials_lrn['imgData']) == len(goodids), len(trials_lrn['imgData']) offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) n_applied_hits += 1 for j in range(n_applied_hits): for k in trials_lrn: for i in range(len(ind_learn)): exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i]) #shuffle test on a per-hit basis for j in range(n_applied_hits): rng = np.random.RandomState(seed=j) perm = rng.permutation(8) for i in range(trials_per_hit * j, min(trials_per_hit * (j+1), len(exp._trials['imgFiles']))): f = copy.deepcopy(exp._trials['imgFiles'][i]) t = copy.deepcopy(exp._trials['imgData'][i]) f[1] = [f[1][_j] for _j in perm] exp._trials['imgFiles'][i] = f t['Test'] = [t['Test'][_j] for _j in perm] exp._trials['imgData'][i] = t l = copy.deepcopy(exp._trials['labels'][i]) exp._trials['labels'][i] = [l[_j] for _j in perm] print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert n_hits_floor == n_applied_hits == mult * 5, (n_hits_floor, n_applied_hits) assert len(exp._trials['imgFiles']) == mult * (720 + 5 * (32 + 16)), len(exp._trials['imgFiles']) """ s_ref_labels = set([tuple(e) for e in trials_qe['labels']]) print(s_ref_labels) offsets2 = np.arange(8 * 4)[::-1] + offsets ibie = zip(range(0, 720 + 4 * 32, trials_per_hit), range(trials_per_hit, 720 + 4 * 32 + trials_per_hit, trials_per_hit)) assert all([set([tuple(e) for e in np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels for ib, ie in ibie[:-1]]) print '** Finished creating trials.' """ return exp, html_data
def get_exp(category, sandbox=True, dummy_upload=True): dataset = hvm.HvMWithDiscfade() meta = dataset.meta inds = (meta['category'] == category).nonzero()[0] meta = meta[inds] objs = np.unique(meta['obj']) combs = [objs] preproc = None image_bucket_name = 'hvm_timing' urls = dataset.publish_images(inds, preproc, image_bucket_name, dummy_upload=dummy_upload) base_url = 'https://s3.amazonaws.com/hvm_timing/' obj_resp_ids = [meta[meta['obj'] == o]['id'][0] for o in objs] response_images = [{ 'urls': [base_url + obj_id + '.png' for obj_id in obj_resp_ids], 'meta': [{ 'obj': obj, 'category': category } for obj in objs], 'labels': objs }] mult = 2 html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 90 * 8 * mult, 'meta_field': 'obj', 'meta': tb.tab_rowstack([meta] * mult), 'urls': urls * mult, 'shuffle_test': False, } additionalrules = [{ 'old': 'LEARNINGPERIODNUMBER', 'new': str(LEARNING_PERIOD) }, { 'old': 'OBJTYPE', 'new': category }] trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32 + 16 exp = MatchToSampleFromDLDataExperiment( htmlsrc='hvm_subordinate.html', htmldst='hvm_subordinate_' + category + '_n%05d.html', tmpdir='tmp_subordinate_%s' % category, sandbox=sandbox, title='Object recognition --- report what you see', reward=0.35, duration=1500, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description= "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="hvm subordinate identification", # noqa collection_name='hvm_subordinate_identification_%s' % category, max_assignments=1, bucket_name='hvm_subordinate_identification_test', trials_per_hit=trials_per_hit, # 144 + 8x4 repeats + 16 training html_data=html_data, frame_height_pix=1200, othersrc=[ 'objnames.js', '../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js' ], additionalrules=additionalrules, log_prefix='subordinate_' + category + '_') # -- create trials exp.createTrials(sampling='without-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == 90 * 8 * mult, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = repeat_inds[category] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = { e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials } ind_learn = practice_inds[category] goodids = [meta[i]['id'] for i in ind_learn] trials_lrn = {} for e in exp._trials: trials_lrn[e] = [] got = [] for _ind, r in enumerate(exp._trials[e]): if exp._trials['imgData'][_ind]['Sample'][ 'id'] in goodids and exp._trials['imgData'][_ind][ 'Sample']['id'] not in got: trials_lrn[e].append(copy.deepcopy(r)) got.append(exp._trials['imgData'][_ind]['Sample']['id']) assert len(trials_lrn['imgData']) == len(goodids), len( trials_lrn['imgData']) offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype('int') n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) n_applied_hits += 1 for j in range(n_applied_hits): for k in trials_lrn: for i in range(len(ind_learn)): exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i]) #shuffle test on a per-hit basis for j in range(n_applied_hits): rng = np.random.RandomState(seed=j) perm = rng.permutation(8) for i in range( trials_per_hit * j, min(trials_per_hit * (j + 1), len(exp._trials['imgFiles']))): f = copy.deepcopy(exp._trials['imgFiles'][i]) t = copy.deepcopy(exp._trials['imgData'][i]) f[1] = [f[1][_j] for _j in perm] exp._trials['imgFiles'][i] = f t['Test'] = [t['Test'][_j] for _j in perm] exp._trials['imgData'][i] = t l = copy.deepcopy(exp._trials['labels'][i]) exp._trials['labels'][i] = [l[_j] for _j in perm] print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert n_hits_floor == n_applied_hits == mult * 5, (n_hits_floor, n_applied_hits) assert len(exp._trials['imgFiles']) == mult * (720 + 5 * (32 + 16)), len( exp._trials['imgFiles']) """ s_ref_labels = set([tuple(e) for e in trials_qe['labels']]) print(s_ref_labels) offsets2 = np.arange(8 * 4)[::-1] + offsets ibie = zip(range(0, 720 + 4 * 32, trials_per_hit), range(trials_per_hit, 720 + 4 * 32 + trials_per_hit, trials_per_hit)) assert all([set([tuple(e) for e in np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels for ib, ie in ibie[:-1]]) print '** Finished creating trials.' """ return exp, html_data
def get_exp(sandbox=True, debug=False, dummy_upload=True): dataset = hvm.HvMWithDiscfade() meta = dataset.meta combs = [] response_images = [] for category in np.unique(meta['category']): objs = np.unique(meta[meta['category']==category]['obj']) obj_combs= [e for e in itertools.combinations(objs, 2)] response_images.extend([{ 'urls': [get_url_labeled_resp_img(o1, dataset), get_url_labeled_resp_img(o2, dataset)], 'meta': [{'obj': o, 'category': category} for o in [o1, o2]], 'labels': [hvm.OBJECT_NAMES[o1], hvm.OBJECT_NAMES[o2]] } for o1, o2 in obj_combs]) combs.extend(obj_combs) #urls = [get_url(e['obj'], e['id']) for e in meta] urls = dataset.publish_images(range(len(dataset.meta)), None, 'hvm_timing', dummy_upload=dummy_upload) with open(path.join(path.dirname(__file__), 'tutorial_html'), 'r') as tutorial_html_file: tutorial_html = tutorial_html_file.read() label_func = lambda x: hvm.OBJECT_NAMES[x['obj']] html_data = { 'combs': combs, 'response_images': response_images, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, 'meta_query' : lambda x: x['var'] == 'V6', 'label_func': label_func } additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(10)}, {'old': 'OBJTYPE', 'new': 'Object Recognition'}, {'old': 'TUTORIAL_HTML', 'new': tutorial_html}, {'old': 'CATDICT', 'new': json.dumps(hvm.OBJECT_NAMES)}, {'old': 'METAFIELD', 'new': "'obj'"}] exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/general_two_way.html', htmldst='hvm_subordinate_2way_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see. Up to 50 cent performance based bonus', reward=0.25, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment='hvm_subordinate_2ways', collection_name='hvm_subordinate_2ways', max_assignments=1, bucket_name='hvm_2ways', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 140 + 6x4 repeats html_data=html_data, tmpdir='tmp', frame_height_pix=1200, othersrc=['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'], additionalrules=additionalrules ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (8 * 7 / 2) * 250 * 8 if debug: return exp, html_data # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} # -- flip answer choices of some repeated images n_qe = len(trials_qe['labels']) # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 400 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 50 * 164 * 8 s_ref_labels = [tuple(e) for e in trials_qe['labels']] offsets2 = np.arange(24)[::-1] + offsets ibie = zip(range(0, 50 * 8 * 164, 164), range(164, 50 * 8 * 164, 164)) assert all( [[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs #del exp._trials['imgData'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, debug=False, dummy_upload=True): dataset = hvm.HvMWithDiscfade() meta = dataset.meta response_images = [] categories = np.unique(meta['category']) cat_combs = [e for e in itertools.combinations(categories, 2)] response_images.extend([{ 'urls': [get_url_labeled_resp_img(c1), get_url_labeled_resp_img(c2)], 'meta': [{ 'category': category } for category in [c1, c2]], 'labels': [c1, c2] } for c1, c2 in cat_combs]) combs = cat_combs urls = dataset.publish_images(range(len(dataset.meta)), None, 'hvm_timing', dummy_upload=dummy_upload) with open(path.join(path.dirname(__file__), 'tutorial_html_basic'), 'r') as tutorial_html_file: tutorial_html = tutorial_html_file.read() label_func = lambda x: hvm.OBJECT_NAMES[x['obj']] html_data = { 'combs': combs, 'response_images': response_images, 'num_trials': 125 * 2, 'meta_field': 'category', 'meta': meta, 'urls': urls, 'shuffle_test': True, 'meta_query': lambda x: x['var'] == 'V6', 'label_func': label_func } cat_dict = { 'Animals': 'Animal', 'Boats': 'Boat', 'Cars': 'Car', 'Chairs': 'Chair', 'Faces': 'Face', 'Fruits': 'Fruit', 'Planes': 'Plane', 'Tables': 'Table' } additionalrules = [{ 'old': 'LEARNINGPERIODNUMBER', 'new': str(10) }, { 'old': 'OBJTYPE', 'new': 'Object Recognition' }, { 'old': 'TUTORIAL_HTML', 'new': tutorial_html }, { 'old': 'CATDICT', 'new': json.dumps(cat_dict) }, { 'old': 'METAFIELD', 'new': "'category'" }] exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/general_two_way.html', htmldst='hvm_basic_2way_n%05d.html', sandbox=sandbox, title= 'Object recognition --- report what you see. Up to 50 cent performance based bonus', reward=0.25, duration=1600, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description= "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment='hvm_basic_2ways', collection_name='hvm_basic_2ways', max_assignments=1, bucket_name='hvm_2ways', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 140 + 6x4 repeats html_data=html_data, tmpdir='tmp', frame_height_pix=1200, othersrc=[ '../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js' ], additionalrules=additionalrules) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (8 * 7 / 2) * 250 if debug: return exp, html_data # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = { e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials } # -- flip answer choices of some repeated images n_qe = len(trials_qe['labels']) # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 50 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 50 * 164 s_ref_labels = [tuple(e) for e in trials_qe['labels']] offsets2 = np.arange(24)[::-1] + offsets ibie = zip(range(0, 50 * 164, 164), range(164, 50 * 164, 164)) assert all([[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2] ] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs #del exp._trials['imgData'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, dummy_upload=True): dataset = hvm.HvMWithDiscfade() meta = dataset.meta categories = dataset.categories combs = [categories] inds = np.arange(len(meta)) preproc = None image_bucket_name = 'hvm_timing' urls = dataset.publish_images(inds, preproc, image_bucket_name, dummy_upload=dummy_upload) base_url = 'https://canonical_images.s3.amazonaws.com/' response_images = [{ 'urls': [base_url + cat + '.png' for cat in categories], 'meta': [{'category': 'Animals'}, {'category': 'Boats'}, {'category': 'Cars'}, {'category': 'Chairs'}, {'category': 'Faces'}, {'category': 'Fruits'}, {'category': 'Planes'}, {'category': 'Tables'}], 'labels': categories}] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 90 * 64, 'meta_field': 'category', 'meta': meta, 'urls': urls, 'shuffle_test': True, } additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(LEARNING_PERIOD)}] trials_per_hit = ACTUAL_TRIALS_PER_HIT + 32 exp = MatchToSampleFromDLDataExperiment( htmlsrc='hvm_basic_categorization.html', htmldst='hvm_basic_categorization_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.35, duration=1500, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="hvm basic categorization", # noqa collection_name= 'hvm_basic_categorization', max_assignments=1, bucket_name='hvm_basic_categorization', trials_per_hit=trials_per_hit, # 150 + 8x4 repeats html_data=html_data, frame_height_pix=1200, othersrc = ['../../lib/dltk.js'], additionalrules=additionalrules ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == 90 * 64, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 38 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles']) s_ref_labels = set([tuple(e) for e in trials_qe['labels']]) print(s_ref_labels) offsets2 = np.arange(8 * 4)[::-1] + offsets ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit)) assert all([set([tuple(e) for e in np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels for ib, ie in ibie[:-1]]) print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, debug=False): exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/objectome_64objs_v2a.html', htmldst='objectome_64objs_v2a_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.15, duration=1600, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description= "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment='objectome_64objs_v2a', collection_name='objectome_64objs_v2a', max_assignments=1, bucket_name='objectome_64objs_v2a', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir='tmp', frame_height_pix=1200, othersrc=[ '../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js' ], # noqa set_destination=True, ) # -- create trials # exp.createTrials(sampling='with-replacement', verbose=1) Don't use this exp._trials, alldat, _ = createTrials() n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (64 * 63 / 2) * 250 if debug: return exp # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data # 138334: dog v camel (difficult!) # 139363: rhino v elephant (difficult!) ind_repeats = [138334, 139363, 47, 9] * REPEATS_PER_QE_IMG assert all(alldat[138334] == [701, 0, 33, 0, 33]) assert all(alldat[139363] == [44070, 2, 39, 39, 39]) rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = { e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials } # -- flip answer choices of some repeated images n_qe = len(trials_qe['imgFiles']) print '** n_qe =', n_qe # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() # -- actual application offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 5040 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 5040 * (100 + 16) s_ref_labels = [tuple(e) for e in trials_qe['labels']] print '**', s_ref_labels offsets2 = np.arange(16)[::-1] + offsets ibie = zip(range(0, 5040 * 116, 116), range(116, 5041 * 116, 116)) assert all([[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2] ] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs # del exp._trials['imgData'] print '** Finished creating trials.' return exp
def get_exp(sandbox=True, selected_basic_objs=SELECTED_BASIC_OBJS, data_intercls=DATA_INTERCLS, debug=False): selected_intercls_pairs = np.array(data_intercls['CMinds_cars_tanks']) selected_intercls_pairs = selected_intercls_pairs[data_intercls['si']] selected_intercls_pairs[:, 0] -= 64 # subtract car base index selected_intercls_pairs[:, 1] -= 64 + 90 # same for tanks meta, _, meta_cars, meta_tanks = get_meta() cars = np.unique(meta_cars['obj']) tanks = np.unique(meta_tanks['obj']) combs = [(cars[e1], tanks[e2]) for e1, e2 in selected_intercls_pairs] combs_dummy1 = [(o1, o2) for o1 in selected_basic_objs for o2 in cars] combs_dummy2 = [(o1, o2) for o1 in selected_basic_objs for o2 in tanks] rng = np.random.RandomState(0) rng.shuffle(combs_dummy1) rng.shuffle(combs_dummy2) combs = combs + combs_dummy1[:50] + combs_dummy2[:50] assert len(combs) == 200 urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{'obj': o} for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/intercls_cars_tanks.html', htmldst='intercls_cars_tanks_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.15, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="objectome_intercls_cars_tanks_v3. 100+100 selected pairs across 22 basic, 30 cars, and 30 tanks", # noqa collection_name='objectome_intercls_cars_tanks_v3', max_assignments=1, bucket_name='objectome_intercls_cars_tanks_v3', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir='tmp', html_data=html_data, frame_height_pix=1200, othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa set_destination=True, ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) if debug: return exp n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == 200 * 250 # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data # the choice of v--- this array below is arbitrary ind_repeats = [1, 9, 60, 27] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} # -- flip answer choices of some repeated images n_qe = len(trials_qe['imgFiles']) print '** n_qe =', n_qe # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 500 == n_applied_hits assert len(exp._trials['imgFiles']) == 500 * (100 + 16) s_ref_labels = [tuple(e) for e in trials_qe['labels']] print '**', s_ref_labels offsets2 = np.arange(16)[::-1] + offsets ibie = zip(range(0, 500 * 116, 116), range(501 * 116, 116)) assert all( [[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] del exp._trials['meta_field'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, stimdur=100, selected_basic_objs=SELECTED_BASIC_OBJS, debug=False, mode='default'): meta = get_meta() combs = [e for e in itertools.combinations(selected_basic_objs, 2)] urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{ 'obj': o } for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } # few different options htmlsrcdct = {} descdct = {} htmldstdct = {} tmpdirdct = {} addirules = {} htmlsrcdct['default'] = 'web/objt_tcurve_o16s0.html' descdct[ 'default'] = "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['default'] = 'objt_tcurve_o16s0_%04d_n%%05d.html' % int( stimdur) # noqa tmpdirdct['default'] = 'tmp/t%04d' % int(stimdur) addirules['default'] = [] htmlsrcdct['softnotice'] = 'web/objt_tcurve_o16s0_softnotice.html' descdct['softnotice'] = descdct['default'] htmldstdct['softnotice'] = 'objt_tcurve_o16s0_soft_%04d_n%%05d.html' % int( stimdur) # noqa tmpdirdct['softnotice'] = 'tmp/t%04d_soft' % int(stimdur) addirules['softnotice'] = [] htmlsrcdct['winonly'] = htmlsrcdct['default'] descdct[ 'winonly'] = "***Chrome or Firefox on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['winonly'] = htmldstdct['default'] tmpdirdct['winonly'] = tmpdirdct['default'] addirules['winonly'] = [{ 'old': "supportedOS: ['Windows', 'Mac', 'Linux']", 'new': "supportedOS: ['Windows']", 'n': 1, }] htmlsrcdct['winchromeonly'] = htmlsrcdct['default'] descdct[ 'winchromeonly'] = "***Latest Chrome on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['winchromeonly'] = htmldstdct['default'] tmpdirdct['winchromeonly'] = tmpdirdct['default'] addirules['winchromeonly'] = [{ 'old': "supportedBrowser: ['Chrome', 'Firefox']", 'new': "supportedBrowser: ['Chrome']", 'n': 1, }] + addirules['winonly'] htmlsrcdct['winchromeonlymask'] = 'web/objt_tcurve_o16s0_postmask.html' descdct['winchromeonlymask'] = descdct['winchromeonly'] htmldstdct[ 'winchromeonlymask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int( stimdur) # noqa tmpdirdct['winchromeonlymask'] = 'tmp/t%04d_mask' % int(stimdur) addirules['winchromeonlymask'] = [{ 'old': "supportedBrowser: ['Chrome', 'Firefox']", 'new': "supportedBrowser: ['Chrome']", 'n': 1, }] + addirules['winonly'] htmlsrcdct['mask'] = 'web/objt_tcurve_o16s0_postmask.html' descdct['mask'] = descdct['default'] htmldstdct['mask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int( stimdur) # noqa tmpdirdct['mask'] = 'tmp/t%04d_mask' % int(stimdur) addirules['mask'] = addirules['default'] exp = MatchToSampleFromDLDataExperiment( htmlsrc=htmlsrcdct[mode], htmldst=htmldstdct[mode], sandbox=sandbox, title='Object recognition --- report what you see', reward=0.25, duration=1600, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description=descdct[mode], comment="objectome_time_curve_%dms. 0-th set of 16 objects" % int(stimdur), # noqa collection_name='objectome_tcurve_16objsset0', max_assignments=1, bucket_name='objectome_tcurve_16objsset0', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats html_data=html_data, tmpdir=tmpdirdct[mode], frame_height_pix=1200, othersrc=[ '../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js' ], # noqa additionalrules=[{ 'old': 'stimduration = 100;', 'new': 'stimduration = %f;' % stimdur, 'n': 1, }] + addirules[mode], ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (16 * 15 / 2) * 250 if debug: return exp, html_data # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = { e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials } #print np.unique([e[0] for e in trials_qe['imgFiles']]) #print np.unique([tuple(e) for e in trials_qe['labels']]) # -- flip answer choices of some repeated images n_qe = len(trials_qe['labels']) # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) #exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 200 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 200 * 174 s_ref_labels = [tuple(e) for e in trials_qe['labels']] offsets2 = np.arange(24)[::-1] + offsets ibie = zip(range(0, 200 * 174, 174), range(174, 201 * 174, 174)) assert all([[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2] ] == s_ref_labels for ib, ie in ibie]) #print set(s_ref_labels), len(set(s_ref_labels)) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, dummy_upload=True): dataset = lcs.LandoltCs() meta = dataset.meta preproc = dict(crop=None, crop_rand=None, dtype='float32', mask=None, mode='L', normalize=False, resize_to=(256, 256), seed=0) img_urls = dataset.publish_images(range(len(dataset.meta)), preproc, 'acuity_tasks', dummy_upload=dummy_upload) response_im_inds = [29, 24, 19, 34, 14, 39, 4, 9] response_urls = [] response_metas = [] response_labels = [] for ind in response_im_inds: response_urls.append(img_urls[ind]) response_metas.append({k: meta[ind][k] for k in meta.dtype.names}) response_labels.append(None) response_images = [{'urls': response_urls, 'meta': response_metas, 'labels': response_labels}] combs = [[m['rotation'] for m in response_images[0]['meta']]] additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(10)}, {'old': 'OBJTYPE', 'new': 'Landolt C'}] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 184, 'meta_field': 'rotation', 'meta': meta, 'urls': img_urls } exp = MatchToSampleFromDLDataExperiment( htmlsrc='landolt_c_task.html', htmldst='landolt_c_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.35, duration=1500, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="landolt c acuity", # noqa collection_name= 'landolt_c', max_assignments=20, bucket_name='landolt_c', trials_per_hit=184, # 150 + 8x4 repeats html_data=html_data, frame_height_pix=1200, othersrc = ['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], additionalrules=additionalrules ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) #n_total_trials = len(exp._trials['imgFiles']) #assert n_total_trials == 40, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data #ind_repeats = [3440, 3282, 3321, 3802, 5000, 3202, 4041, 4200] * REPEATS_PER_QE_IMG #rng = np.random.RandomState(0) #rng.shuffle(ind_repeats) # trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] # for e in exp._trials} # offsets = np.arange( # ACTUAL_TRIALS_PER_HIT - 3, -1, # -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) # ).round().astype('int') # # n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT # n_applied_hits = 0 # for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, # -1, -ACTUAL_TRIALS_PER_HIT): # for k in trials_qe: # for i, offset in enumerate(offsets): # exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # n_applied_hits += 1 # # print '** n_applied_hits =', n_applied_hits # print '** len for each in _trials =', \ # {e: len(exp._trials[e]) for e in exp._trials} # # # -- sanity check # assert 38 == n_applied_hits, n_applied_hits # assert len(exp._trials['imgFiles']) == 6976, len(exp._trials['imgFiles']) # s_ref_labels = set([tuple(e) for e in trials_qe['labels']]) # print(s_ref_labels) # offsets2 = np.arange(8 * 4)[::-1] + offsets # # ibie = zip(range(0, 6976, trials_per_hit), range(trials_per_hit, 6976 + trials_per_hit, trials_per_hit)) # assert all([set([tuple(e) for e in # np.array(exp._trials['labels'][ib:ie])[offsets2]]) == s_ref_labels # for ib, ie in ibie[:-1]]) # print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, debug=False): exp = MatchToSampleFromDLDataExperiment( htmlsrc="web/objectome_64objs_v2a.html", htmldst="objectome_64objs_v2a_n%05d.html", sandbox=sandbox, title="Object recognition --- report what you see", reward=0.15, duration=1600, keywords=["neuroscience", "psychology", "experiment", "object recognition"], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="objectome_64objs_v2a", collection_name="objectome_64objs_v2a", max_assignments=1, bucket_name="objectome_64objs_v2a", trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir="tmp", frame_height_pix=1200, othersrc=["../../lib/dltk.js", "../../lib/dltkexpr.js", "../../lib/dltkrsvp.js"], # noqa set_destination=True, ) # -- create trials # exp.createTrials(sampling='with-replacement', verbose=1) Don't use this exp._trials, alldat, _ = createTrials() n_total_trials = len(exp._trials["imgFiles"]) assert n_total_trials == (64 * 63 / 2) * 250 if debug: return exp # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data # 138334: dog v camel (difficult!) # 139363: rhino v elephant (difficult!) ind_repeats = [138334, 139363, 47, 9] * REPEATS_PER_QE_IMG assert all(alldat[138334] == [701, 0, 33, 0, 33]) assert all(alldat[139363] == [44070, 2, 39, 39, 39]) rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} # -- flip answer choices of some repeated images n_qe = len(trials_qe["imgFiles"]) print "** n_qe =", n_qe # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) for i, flip in enumerate(flips): if not flip: continue trials_qe["imgFiles"][i][1].reverse() trials_qe["labels"][i].reverse() # -- actual application offsets = ( np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype("int") ) assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print "** n_applied_hits =", n_applied_hits print "** len for each in _trials =", {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 5040 == n_applied_hits, n_applied_hits assert len(exp._trials["imgFiles"]) == 5040 * (100 + 16) s_ref_labels = [tuple(e) for e in trials_qe["labels"]] print "**", s_ref_labels offsets2 = np.arange(16)[::-1] + offsets ibie = zip(range(0, 5040 * 116, 116), range(116, 5041 * 116, 116)) assert all( [[(e1, e2) for e1, e2 in np.array(exp._trials["labels"][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie] ) # -- drop unneeded, potentially abusable stuffs # del exp._trials['imgData'] print "** Finished creating trials." return exp
def get_exp(sandbox=True, stimdur=100, selected_basic_objs=SELECTED_BASIC_OBJS, debug=False, mode='default'): meta = get_meta() combs = [e for e in itertools.combinations(selected_basic_objs, 2)] urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{'obj': o} for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } # few different options htmlsrcdct = {} descdct = {} htmldstdct = {} tmpdirdct = {} addirules = {} htmlsrcdct['default'] = 'web/objt_tcurve_o16s0.html' descdct['default'] = "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['default'] = 'objt_tcurve_o16s0_%04d_n%%05d.html' % int(stimdur) # noqa tmpdirdct['default'] = 'tmp/t%04d' % int(stimdur) addirules['default'] = [] htmlsrcdct['softnotice'] = 'web/objt_tcurve_o16s0_softnotice.html' descdct['softnotice'] = descdct['default'] htmldstdct['softnotice'] = 'objt_tcurve_o16s0_soft_%04d_n%%05d.html' % int(stimdur) # noqa tmpdirdct['softnotice'] = 'tmp/t%04d_soft' % int(stimdur) addirules['softnotice'] = [] htmlsrcdct['winonly'] = htmlsrcdct['default'] descdct['winonly'] = "***Chrome or Firefox on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['winonly'] = htmldstdct['default'] tmpdirdct['winonly'] = tmpdirdct['default'] addirules['winonly'] = [{ 'old': "supportedOS: ['Windows', 'Mac', 'Linux']", 'new': "supportedOS: ['Windows']", 'n': 1, }] htmlsrcdct['winchromeonly'] = htmlsrcdct['default'] descdct['winchromeonly'] = "***Latest Chrome on Windows only*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment." # noqa htmldstdct['winchromeonly'] = htmldstdct['default'] tmpdirdct['winchromeonly'] = tmpdirdct['default'] addirules['winchromeonly'] = [{ 'old': "supportedBrowser: ['Chrome', 'Firefox']", 'new': "supportedBrowser: ['Chrome']", 'n': 1, }] + addirules['winonly'] htmlsrcdct['winchromeonlymask'] = 'web/objt_tcurve_o16s0_postmask.html' descdct['winchromeonlymask'] = descdct['winchromeonly'] htmldstdct['winchromeonlymask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(stimdur) # noqa tmpdirdct['winchromeonlymask'] = 'tmp/t%04d_mask' % int(stimdur) addirules['winchromeonlymask'] = [{ 'old': "supportedBrowser: ['Chrome', 'Firefox']", 'new': "supportedBrowser: ['Chrome']", 'n': 1, }] + addirules['winonly'] htmlsrcdct['mask'] = 'web/objt_tcurve_o16s0_postmask.html' descdct['mask'] = descdct['default'] htmldstdct['mask'] = 'objt_tcurve_o16s0_mask_%04d_n%%05d.html' % int(stimdur) # noqa tmpdirdct['mask'] = 'tmp/t%04d_mask' % int(stimdur) addirules['mask'] = addirules['default'] exp = MatchToSampleFromDLDataExperiment( htmlsrc=htmlsrcdct[mode], htmldst=htmldstdct[mode], sandbox=sandbox, title='Object recognition --- report what you see', reward=0.25, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description=descdct[mode], comment="objectome_time_curve_%dms. 0-th set of 16 objects" % int(stimdur), # noqa collection_name='objectome_tcurve_16objsset0', max_assignments=1, bucket_name='objectome_tcurve_16objsset0', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats html_data=html_data, tmpdir=tmpdirdct[mode], frame_height_pix=1200, othersrc=['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'], # noqa additionalrules=[{ 'old': 'stimduration = 100;', 'new': 'stimduration = %f;' % stimdur, 'n': 1, }] + addirules[mode], ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (16 * 15 / 2) * 250 if debug: return exp, html_data # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} #print np.unique([e[0] for e in trials_qe['imgFiles']]) #print np.unique([tuple(e) for e in trials_qe['labels']]) # -- flip answer choices of some repeated images n_qe = len(trials_qe['labels']) # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) #exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 200 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 200 * 174 s_ref_labels = [tuple(e) for e in trials_qe['labels']] offsets2 = np.arange(24)[::-1] + offsets ibie = zip(range(0, 200 * 174, 174), range(174, 201 * 174, 174)) assert all([[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]) #print set(s_ref_labels), len(set(s_ref_labels)) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, dummy_upload=True): n_hits_from_data = len(meta) / ACTUAL_TRIALS_PER_HIT categories = np.unique(dataset.meta['identity']) combs = [categories] inds = np.arange(len(meta)) preproc = dataset.default_preproc #preproc['flip_tb'] = True image_bucket_name = 'freiwald_tsao_2010' urls = dataset.publish_images(inds, preproc, image_bucket_name, dummy_upload=dummy_upload) base_url = 'https://canonical_images.s3.amazonaws.com/' response_images = [{ 'urls': [base_url + 'freiwald_tsao_2010_face%.2da.png' % i for i in range(1, 29)], 'meta': [{'identity': i} for i in range(1, 29)], 'labels': categories}] mult = 10 html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 220 * mult, 'meta_field': 'identity', 'meta': tb.tab_rowstack([meta] * mult), 'urls': urls * mult, 'shuffle_test': False, } additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(LEARNING_PERIOD)}] trials_per_hit = ACTUAL_TRIALS_PER_HIT + REPEATS_PER_QE_IMG * len(repeat_inds) + LEARNING_PERIOD exp = MatchToSampleFromDLDataExperiment( htmlsrc='ft_identity.html', htmldst='ft_identity_n%05d.html', tmpdir='tmp_ft_identity', sandbox=sandbox, title='Face recognition --- report what you see', reward=0.35, duration=1500, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="freiwald_tsao_identification", # noqa collection_name = 'freiwald_tsao_identification', max_assignments=1, bucket_name='freiwald_tsao_identification', trials_per_hit=trials_per_hit, html_data=html_data, frame_height_pix=1200, othersrc = ['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], additionalrules=additionalrules, log_prefix='freiwald_tsao_identification_' ) # -- create trials exp.createTrials(sampling='without-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == 220 * mult, n_total_trials # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = repeat_inds * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} ind_learn = practice_inds goodids = [meta[i]['id'] for i in ind_learn] trials_lrn = {} for e in exp._trials: trials_lrn[e] = [] got = [] for _ind, r in enumerate(exp._trials[e]): if exp._trials['imgData'][_ind]['Sample']['id'] in goodids and exp._trials['imgData'][_ind]['Sample']['id'] not in got : trials_lrn[e].append(copy.deepcopy(r)) got.append(exp._trials['imgData'][_ind]['Sample']['id']) assert len(trials_lrn['imgData']) == len(goodids), len(trials_lrn['imgData']) offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 1, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') print(len(offsets), offsets) print('a', len(exp._trials['imgFiles'])) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) n_applied_hits += 1 print('b', len(exp._trials['imgFiles'])) for j in range(n_applied_hits): for k in trials_lrn: for i in range(len(ind_learn)): exp._trials[k].insert(trials_per_hit * j, trials_lrn[k][i]) print('c', len(exp._trials['imgFiles'])) #shuffle test on a per-hit basis for j in range(n_applied_hits): rng = np.random.RandomState(seed=j) perm = rng.permutation(28) for i in range(trials_per_hit * j, min(trials_per_hit * (j+1), len(exp._trials['imgFiles']))): f = copy.deepcopy(exp._trials['imgFiles'][i]) t = copy.deepcopy(exp._trials['imgData'][i]) f[1] = [f[1][_j] for _j in perm] exp._trials['imgFiles'][i] = f t['Test'] = [t['Test'][_j] for _j in perm] exp._trials['imgData'][i] = t l = copy.deepcopy(exp._trials['labels'][i]) exp._trials['labels'][i] = [l[_j] for _j in perm] print('d', len(exp._trials['imgFiles'])) print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} _K = LEARNING_PERIOD + REPEATS_PER_QE_IMG * len(repeat_inds) # -- sanity check assert n_hits_floor == n_applied_hits == mult * n_hits_from_data, (n_total_trials, ACTUAL_TRIALS_PER_HIT, n_hits_floor, n_applied_hits, mult, n_hits_from_data) assert len(exp._trials['imgFiles']) == mult * (len(meta) + n_hits_from_data * _K), (len(exp._trials['imgFiles']), mult, (len(meta) + n_hits_from_data * _K), len(meta), n_hits_from_data, _K) return exp, html_data
def get_exp(sandbox=True, dummy_upload=True): dataset = hvm.HvMWithDot() base_url = 'https://canonical_images.s3.amazonaws.com/' fields = dataset.meta.dtype.names meta = dataset.meta names = ['RedPixelOnCanonical', 'RedPixelOffCanonical'] response_images = [{'urls': [base_url + name + '.png' for name in names], 'meta': [{field: meta[i][field] for field in fields} for i in [4020, 5060]], 'labels': ['The red pixel was on the object', 'The red pixel was not on the object']}] preproc = {'crop': None, 'crop_rand': None, 'dtype': 'float32', 'mask': None, 'mode': 'RGB', 'normalize': False, 'resize_to': (256, 256), 'seed': 0} cat_dict = "{true: 'Dot is on the object', false:'Dot is not on the object'}" with open(path.join(path.dirname(__file__), 'tutorial_html'), 'r') as tutorial_html_file: tutorial_html = tutorial_html_file.read() additionalrules = [{'old': 'LEARNINGPERIODNUMBER', 'new': str(10)}, {'old': 'OBJTYPE', 'new': 'Figure/ground task'}, {'old': 'TUTORIAL_HTML', 'new': tutorial_html}, {'old': 'CATDICT', 'new': cat_dict}, {'old': 'METAFIELD', 'new': "'dot_on'"}] variation_level = 'V6' disallowed_images = [meta[i]['filename'] for i in [4020, 5060]] query = lambda x: x['var'] == variation_level and (x['filename'] not in disallowed_images) html_data = { 'response_images': response_images, 'combs': [[True, False]], 'num_trials': 1190, 'meta_field': 'dot_on', 'meta': meta, 'urls': dataset.publish_images(range(meta.shape[0]), preproc, 'hvm_figure_ground', dummy_upload), 'meta_query': query } exp = MatchToSampleFromDLDataExperiment( htmlsrc='general_two_way.html', htmldst='hvm_figure_ground%05d.html', sandbox=sandbox, title='Report whether the dot is on the object', reward=0.35, duration=1500, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report dots are on an object or not. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment="hvm_figure_ground", # noqa collection_name= 'hvm_figure_ground_2', max_assignments=10, bucket_name='hvm_figure_ground_2', trials_per_hit=170, # 150 + 8x4 repeats html_data=html_data, frame_height_pix=1200, othersrc = ['../../lib/dltk.js', '../../lib/dltkexpr.js', '../../lib/dltkrsvp.js'], additionalrules=additionalrules ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) return exp, html_data
def get_exp(sandbox=True, selected_basic_objs=SELECTED_BASIC_OBJS, debug=False): meta, _, meta_chairs = get_meta() chairs = np.unique(meta_chairs['obj']) combs = [(o1, o2) for o1 in selected_basic_objs for o2 in chairs] + \ [e for e in itertools.combinations(chairs, 2)] urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{ 'obj': o } for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'response_images': response_images, 'combs': combs, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/chairs_subord.html', htmldst='chairs_subord_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.15, duration=1600, keywords=[ 'neuroscience', 'psychology', 'experiment', 'object recognition' ], # noqa description= "***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 5 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment= "objectome_chairs_subord_v3. 30 chairs and 30 non-chairs", # noqa collection_name='objectome_chairs_subord_v3', max_assignments=1, bucket_name='objectome_chairs_subord_v3', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 16, # 100 + 4x4 repeats tmpdir='tmp', html_data=html_data, frame_height_pix=1200, othersrc=[ '../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js' ], # noqa set_destination=True, ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) if debug: return exp # repeat last 50 presentations twice to make the entire trials # well aligned as multiples of 100 exp._trials['imgFiles'].extend(exp._trials['imgFiles'][-50:]) exp._trials['imgData'].extend(exp._trials['imgData'][-50:]) exp._trials['labels'].extend(exp._trials['labels'][-50:]) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (30 * 29 / 2 + 30 * 30) * 250 + 50 # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data # the choice of v--- this array below is arbitrary ind_repeats = [1, 10, 22, 24] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = { e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials } # -- flip answer choices of some repeated images n_qe = len(trials_qe['imgFiles']) print '** n_qe =', n_qe # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange(ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats))).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 3338 == n_applied_hits assert len(exp._trials['imgFiles']) == 3338 * (100 + 16) s_ref_labels = [tuple(e) for e in trials_qe['labels']] print '**', s_ref_labels offsets2 = np.arange(16)[::-1] + offsets ibie = zip(range(0, 3338 * 116, 116), range(3339 * 116, 116)) assert all([[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2] ] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] del exp._trials['meta_field'] print '** Finished creating trials.' return exp, html_data
def get_exp(sandbox=True, debug=False): meta = get_meta() combs = [e for e in itertools.combinations(np.unique(meta['obj']), 2)] urls = [get_url(e['obj'], e['id']) for e in meta] response_images = [{ 'urls': [get_url_labeled_resp_img(o1), get_url_labeled_resp_img(o2)], 'meta': [{'obj': o} for o in [o1, o2]], 'labels': [o1, o2] } for o1, o2 in combs] html_data = { 'combs': combs, 'response_images': response_images, 'num_trials': 125 * 2, 'meta_field': 'obj', 'meta': meta, 'urls': urls, 'shuffle_test': True, } exp = MatchToSampleFromDLDataExperiment( htmlsrc='web/objectome_64objs_v2.html', htmldst='objectome_64objs_v2_n%05d.html', sandbox=sandbox, title='Object recognition --- report what you see', reward=0.25, duration=1600, keywords=['neuroscience', 'psychology', 'experiment', 'object recognition'], # noqa description="***You may complete as many HITs in this group as you want*** Complete a visual object recognition task where you report the identity of objects you see. We expect this HIT to take about 10 minutes or less, though you must finish in under 25 minutes. By completing this HIT, you understand that you are participating in an experiment for the Massachusetts Institute of Technology (MIT) Department of Brain and Cognitive Sciences. You may quit at any time, and you will remain anonymous. Contact the requester with questions or concerns about this experiment.", # noqa comment='objectome_64objs_v2', collection_name='objectome_64objs_v2', max_assignments=1, bucket_name='objectome_64objs_v2', trials_per_hit=ACTUAL_TRIALS_PER_HIT + 24, # 150 + 6x4 repeats html_data=html_data, tmpdir='tmp', frame_height_pix=1200, othersrc=['../../mturkutils/lib/dltk.js', '../../mturkutils/lib/dltkexpr.js', '../../mturkutils/lib/dltkrsvp.js'], # noqa ) # -- create trials exp.createTrials(sampling='with-replacement', verbose=1) n_total_trials = len(exp._trials['imgFiles']) assert n_total_trials == (64 * 63 / 2) * 250 if debug: return exp, html_data # -- in each HIT, the followings will be repeated 4 times to # estimate "quality" of data ind_repeats = [0, 4, 47, 9, 17, 18] * REPEATS_PER_QE_IMG rng = np.random.RandomState(0) rng.shuffle(ind_repeats) trials_qe = {e: [copy.deepcopy(exp._trials[e][r]) for r in ind_repeats] for e in exp._trials} # -- flip answer choices of some repeated images n_qe = len(trials_qe['labels']) # if True, flip flips = [True] * (n_qe / 2) + [False] * (n_qe - n_qe / 2) assert len(flips) == n_qe rng.shuffle(flips) assert len(trials_qe.keys()) == 4 for i, flip in enumerate(flips): if not flip: continue trials_qe['imgFiles'][i][1].reverse() trials_qe['labels'][i].reverse() trials_qe['imgData'][i]['Test'].reverse() # -- actual application offsets = np.arange( ACTUAL_TRIALS_PER_HIT - 3, -1, -ACTUAL_TRIALS_PER_HIT / float(len(ind_repeats)) ).round().astype('int') assert len(offsets) == len(offsets) n_hits_floor = n_total_trials / ACTUAL_TRIALS_PER_HIT n_applied_hits = 0 for i_trial_begin in xrange((n_hits_floor - 1) * ACTUAL_TRIALS_PER_HIT, -1, -ACTUAL_TRIALS_PER_HIT): for k in trials_qe: for i, offset in enumerate(offsets): exp._trials[k].insert(i_trial_begin + offset, trials_qe[k][i]) # exp._trials[k].insert(i_trial_begin + offset, 'test') n_applied_hits += 1 print '** n_applied_hits =', n_applied_hits print '** len for each in _trials =', \ {e: len(exp._trials[e]) for e in exp._trials} # -- sanity check assert 3360 == n_applied_hits, n_applied_hits assert len(exp._trials['imgFiles']) == 3360 * 174 s_ref_labels = [tuple(e) for e in trials_qe['labels']] offsets2 = np.arange(24)[::-1] + offsets ibie = zip(range(0, 3360 * 174, 174), range(174, 3361 * 174, 174)) assert all( [[(e1, e2) for e1, e2 in np.array(exp._trials['labels'][ib:ie])[offsets2]] == s_ref_labels for ib, ie in ibie]) # -- drop unneeded, potentially abusable stuffs del exp._trials['imgData'] print '** Finished creating trials.' return exp, html_data