def load_data(self,
key,
tier=None,
batch_size=1,
key_order=None,
exclude_from_normalization=None,
stimulus_types=None,
Sampler=None):
log.info('Loading {} dataset with tier={}'.format(
self._stimulus_type, tier))
datasets = StaticMultiDataset().fetch_data(key, key_order=key_order)
for k, dat in datasets.items():
if 'stats_source' in key:
log.info(
'Adding stats_source "{stats_source}" to dataset'.format(
**key))
dat.stats_source = key['stats_source']
log.info('Using statistics source ' + key['stats_source'])
datasets = self.add_transforms(key,
datasets,
exclude=exclude_from_normalization)
loaders = self.get_loaders(datasets, tier, batch_size, stimulus_types,
Sampler)
return datasets, loaders
def evaluate(self, key=None):
if key is None:
key = self.fetch1('KEY')
model = self.load_network(key)
model.eval()
model.cuda()
# get network configuration information
net_key = NetworkConfig().net_key(key)
train_key = TrainConfig().train_key(net_key)
testsets, testloaders = DataConfig().load_data(net_key, tier='test', cuda=True, **train_key)
scores, unit_scores = [], []
for readout_key, testloader in testloaders.items():
log.info('Computing test scores for ' + readout_key)
y, y_hat = compute_predictions(testloader, model, readout_key)
perf_scores = compute_scores(y, y_hat)
member_key = (StaticMultiDataset.Member() & key & dict(name=readout_key)).fetch1(dj.key)
member_key.update(key)
unit_ids = testloader.dataset.neurons.unit_ids
member_key['neurons'] = len(unit_ids)
member_key['pearson'] = perf_scores.pearson.mean()
scores.append(member_key)
unit_scores.extend(
[dict(member_key, unit_id=u, pearson=c) for u, c in zip(unit_ids, perf_scores.pearson)])
return scores, unit_scores
def load_data(self, key, tier=None, batch_size=1,
Sampler=None, t_first=False, cuda=False):
from .stats import BootstrapOracleTTest
assert tier in [None, 'train', 'validation', 'test']
datasets, loaders = super().load_data(
key, tier=tier, batch_size=batch_size, Sampler=Sampler,
cuda=cuda)
for rok, dataset in datasets.items():
member_key = (StaticMultiDataset.Member() & key &
dict(name=rok)).fetch1(dj.key)
all_units, all_pvals = (
BootstrapOracleTTest.UnitPValue & member_key).fetch(
'unit_id', 'unit_p_value')
assert len(all_pvals) > 0, \
'You forgot to populate BootstrapOracleTTest for group_id={}'.format(
member_key['group_id'])
units_mask = np.isin(all_units, dataset.neurons.unit_ids)
units, pvals = all_units[units_mask], all_pvals[units_mask]
assert np.all(
units == dataset.neurons.unit_ids), 'order of neurons has changed'
pval_thresh = np.power(10, float(key['p_val_power']))
selection = pvals < pval_thresh
log.info('Subsampling to {} neurons with BootstrapOracleTTest p-val < {:.0E}'.format(
selection.sum(), pval_thresh))
dataset.transforms.insert(
-1, Subsample(np.where(selection)[0]))
assert np.all(dataset.neurons.unit_ids ==
units[selection]), 'Units are inconsistent'
return datasets, loaders
def load_data(self, key, tier=None, batch_size=1, key_order=None, stimulus_types=None, Sampler=None):
from .stats import Oracle
datasets, loaders = super().load_data(
key, tier=tier, batch_size=batch_size, key_order=key_order,
stimulus_types=stimulus_types, Sampler=Sampler)
for rok, dataset in datasets.items():
member_key = (StaticMultiDataset.Member() & key &
dict(name=rok)).fetch1(dj.key)
okey = dict(key, **member_key)
okey['data_hash'] = okey.pop('oracle_source')
units, pearson = (Oracle.UnitScores() & okey).fetch(
'unit_id', 'pearson')
assert len(pearson) > 0, 'You forgot to populate oracle for data_hash="{}"'.format(
key['oracle_source'])
assert len(units) == len(
dataset.neurons.unit_ids), 'Number of neurons has changed'
assert np.all(
units == dataset.neurons.unit_ids), 'order of neurons has changed'
low, high = np.percentile(
pearson, [key['percent_low'], key['percent_high']])
selection = (pearson >= low) & (pearson <= high)
log.info(
'Subsampling to {} neurons above {:.2f} and below {} oracle'.format(selection.sum(), low, high))
dataset.transforms.insert(-1,
Subsample(np.where(selection)[0]))
assert np.all(dataset.neurons.unit_ids ==
units[selection]), 'Units are inconsistent'
return datasets, loaders
def compute_test_score_tuples(self, key, testloaders, model):
scores, unit_scores = [], []
for readout_key, testloader in testloaders.items():
log.info('Computing test scores for ' + readout_key)
y, y_hat = compute_predictions(testloader, model, readout_key)
perf_scores = compute_scores(y, y_hat)
member_key = (StaticMultiDataset.Member() & key & dict(name=readout_key)).fetch1(dj.key)
member_key.update(key)
unit_ids = testloader.dataset.neurons.unit_ids
member_key['neurons'] = len(unit_ids)
member_key['pearson'] = perf_scores.pearson.mean()
scores.append(member_key)
unit_scores.extend(
[dict(member_key, unit_id=u, pearson=c) for u, c in zip(unit_ids, perf_scores.pearson)])
return scores, unit_scores
def process_static_scans(target_scans):
"""
Function that goes and check for every table that needs to be populate as well as provide an option
to manaully populate AreaMembership and LayerMembership, assuming that all the neurons can be label the same Area and Layer
if not, they the user should manually do it.
Please refer to neuro_data/notebooks/pipeline_management notebook for an example
Args:
target_scans (list(dict(animal_id, session, scan_idx))): A list of dicts where each dicts contains the keys for a specific scan
Returns:
None
"""
for target_scan in target_scans:
print('[NeuroDataPipelineManagement]: Processing ' +
str(target_scan))
# Check if the scan has been processed completely
if pipeline_fuse.ScanDone() & target_scan:
print('[Preprocessing Check]: ScanDone Check Passed')
else:
print(
'[Preprocessing Check]: ' + str(target_scan) +
' Scan has not been processed yet, please look into pipeline for details'
)
return
# Check if neurons area are labeled
if pipeline_anatomy.AreaMembership() & target_scan:
print('[Preprocessing Check]: AreaMembership Check Passed')
else:
print('[Preprocessing Check]: ' + str(target_scan) +
" AreaMembership is not populated")
user_input = None
while user_input not in ['y', 'n']:
user_input = input(
'Should we manually insert if the area are known and all the same? [y/n]'
)
if user_input == 'y':
area = input(
'Input area to label neurons with [type exit for cancel]:'
)
while area not in [
'V1', 'LM', 'AL', 'RL', 'all-unknown'
]:
if area == 'exit':
return
print('Invalid Area!')
area = input('Input area to label neurons with:')
NeuroDataPipelineManagement.manually_insert_area_for_scan(
target_scan, area)
elif user_input == 'n':
return
# Check if neuron layers are labeled
if pipeline_anatomy.LayerMembership() & target_scan:
print('[Preprocessing Check]: LayerMembership Check Passed')
else:
print('[Preprocessing Check]: ' + str(target_scan) +
" LayerMembership is not populated")
user_input = None
while user_input not in ['y', 'n']:
user_input = input(
'Should we manually insert if the layer are known and all the same? [y/n]'
)
if user_input == 'y':
layer = input(
'Input layer to label neurons with [type exit to cancel]:'
)
while layer not in ['L1', 'L2/3', 'L4']:
if layer == 'exit':
return
print('Invalid Layer!')
layer = input('Input layer to label neurons with:')
NeuroDataPipelineManagement.manually_insert_layer_for_scan(
target_scan, layer)
elif user_input == 'n':
return
# Check pipeline_stimulus.Sync() table
if pipeline_stimulus.Sync() & target_scan:
print('[Preprocessing Check]: Sync Check Passed')
else:
print(
'[Preprocessing Check]: ' + str(target_scan) +
' pipeline_stimulus.Sync() table is not processed or failed to processed'
)
return
# All tables requirements are met, begin neurodata dataset population
print(
'[Preprocessing Check]: All table requirements passed, beginning neuro_data populating:'
)
# Get the ScanDone primary key reference
target_scan_done_key = (pipeline_fuse.ScanDone()
& target_scan).fetch1('KEY')
# Insert into StaticScanCandidate
if StaticScanCandidate & target_scan_done_key:
print(
'[NeuroData.Static Populate]: Scan has already been added to StaticScanCandidate'
)
else:
StaticScanCandidate.insert1(target_scan_done_key)
print(
'[NeuroData.Static Populate]: Successfully inserted Scan into StaticScanCandidate'
)
# Populating StaticScans
print("[NeuroData.Static Populate]: Populating StaticScan:")
StaticScan().populate(target_scan_done_key)
# Populating ImageNetSplit
print("[NeuroData.Static Populate]: Populating ImageNetSplit:")
ImageNetSplit().fill(target_scan_done_key)
# Populate ConditionTier
print("[NeuroData.Static Populate]: Populating ConditionTier:")
ConditionTier.populate(target_scan_done_key)
# Check for incorrect flip times
print(
"[NeuroData.Static Populate]: Checking for Incorrect Flip Times:"
)
trials = (pipeline_stimulus.Trial()
& target_scan).proj('flip_times').fetch(as_dict=True)
for trial in trials:
if trial['flip_times'].shape[
1] != 3: # correct number of flips, hardcoded
ExcludedTrial.insert1(trial, ignore_extra_fields=True)
# Populate Frame
print("[NeuroData.Static Populate]: Populating Frame:")
Frame.populate(dict(preproc_id=0))
# Populate InputResponse
print("[NeuroData.Static Populate]: Populating InputResponse:")
InputResponse().populate(target_scan_done_key, dict(preproc_id=0))
# Populate Eye
print("[NeuroData.Static Populate]: Populating Eye:")
Eye().populate(target_scan_done_key)
# Populate Treadmill
print("[NeuroData.Static Populate]: Populating Treadmill:")
Treadmill().populate(target_scan_done_key)
# Insert Scan into StaticMultiDatasetGroupAssignment with whatever is the next highest_group_id
print(
"[NeuroData.Static Populate]: Inserting Scan into StaticMultiDatasetGroupAssignment with next largest group_id:"
)
target_input_response_key = (InputResponse & target_scan
& dict(preproc_id=0)).fetch1('KEY')
if StaticMultiDatasetGroupAssignment & target_input_response_key:
print(
"[NeuroData.Static Populate]: Scan is already in StaticMultiDatasetGroupAssignment, skipping"
)
else:
target_input_response_key[
'group_id'] = StaticMultiDatasetGroupAssignment().fetch(
'group_id').max() + 1
target_input_response_key[
'description'] = 'Inserted from PipelineManagement'
StaticMultiDatasetGroupAssignment.insert1(
target_input_response_key)
# Fill StaticMultiDataset
print("[NeuroData.Static Populate]: Filling StaticMultiDataset:")
StaticMultiDataset().fill()
print('[NeuroData.Static Populate]: Generating HDF5 File')
InputResponse().get_filename(target_scan)
print('[PROCESSING COMPLETED FOR SCAN: ' + str(target_scan) +
']\n')