def get_teams_to_rectify(narps):
map_metadata_file = os.path.join(narps.dirs.dirs['orig'],
'narps_neurovault_images_details.csv')
map_metadata = get_map_metadata(map_metadata_file)
rectify_status = {}
# manual fix for one
rectify_status['R7D1'] = [5, 6]
for teamID in narps.teams:
for hyp in [5, 6]:
mdstring = map_metadata.query(
'teamID == "%s"' % teamID)['hyp%d_direction' % hyp].iloc[0]
rectify = mdstring.split()[0] == 'Negative'
if rectify:
if teamID not in rectify_status:
rectify_status[teamID] = []
rectify_status[teamID].append(hyp)
return (rectify_status)
def convert_to_zscores(self, map_metadata_file=None, overwrite=None):
"""
convert rectified images to z scores
- unthresholded images could be either t or z images
- if they are already z then just copy
- use metadata supplied by teams to determine image type
"""
log_to_file(self.dirs.logfile,
'\n\n%s' % sys._getframe().f_code.co_name)
func_args = inspect.getargvalues(inspect.currentframe()).locals
log_to_file(self.dirs.logfile, stringify_dict(func_args))
if overwrite is None:
overwrite = self.overwrite
if map_metadata_file is None:
map_metadata_file = os.path.join(
self.dirs.dirs['orig'], 'narps_neurovault_images_details.csv')
unthresh_stat_type = get_map_metadata(map_metadata_file)
metadata = get_metadata(self.metadata_file)
n_participants = metadata[['n_participants', 'NV_collection_string']]
n_participants.index = metadata.teamID
unthresh_stat_type = unthresh_stat_type.merge(n_participants,
left_index=True,
right_index=True)
for teamID in self.complete_image_sets:
if teamID not in unthresh_stat_type.index:
print('no map metadata for', teamID)
continue
# this is a bit of a kludge
# since some contrasts include all subjects
# but others only include some
# we don't have the number of participants in each
# group so we just use the entire number
n = unthresh_stat_type.loc[teamID, 'n_participants']
for hyp in range(1, 10):
infile = self.teams[teamID].images['unthresh']['rectified'][
hyp]
if not os.path.exists(infile):
print('skipping', infile)
continue
self.teams[teamID].images['unthresh']['zstat'][
hyp] = os.path.join(self.dirs.dirs['zstat'],
self.teams[teamID].datadir_label,
'hypo%d_unthresh.nii.gz' % hyp)
if not overwrite and os.path.exists(
self.teams[teamID].images['unthresh']['zstat'][hyp]):
continue
if unthresh_stat_type.loc[teamID,
'unthresh_type'].lower() == 't':
if not os.path.exists(
os.path.dirname(self.teams[teamID].
images['unthresh']['zstat'][hyp])):
os.mkdir(
os.path.dirname(
self.teams[teamID].images['unthresh']['zstat']
[hyp]))
print("converting %s (hyp %d) to z - %d participants" %
(teamID, hyp, n))
TtoZ(infile,
self.teams[teamID].images['unthresh']['zstat'][hyp],
n - 1)
elif unthresh_stat_type.loc[teamID, 'unthresh_type'] == 'z':
if not os.path.exists(
os.path.dirname(self.teams[teamID].
images['unthresh']['zstat'][hyp])):
os.mkdir(
os.path.dirname(
self.teams[teamID].images['unthresh']['zstat']
[hyp]))
if not os.path.exists(self.teams[teamID].images['unthresh']
['zstat'][hyp]):
print('copying', teamID)
shutil.copy(
infile,
os.path.dirname(
self.teams[teamID].images['unthresh']['zstat']
[hyp]))
else:
# if it's not T or Z then we skip it as it's not usable
print('skipping %s - other data type' % teamID)
def create_rectified_images(self, map_metadata_file=None, overwrite=None):
"""
create rectified images
- contrasts 5 and 6 were negative contrasts
some teams uploaded images where negative values
provided evidence in favor of the contrast
using metadata provided by teams, we identify these
images and flip their valence so that all maps
present positive evidence for each contrast
"""
log_to_file(self.dirs.logfile,
'\n\n%s' % sys._getframe().f_code.co_name)
func_args = inspect.getargvalues(inspect.currentframe()).locals
log_to_file(self.dirs.logfile, stringify_dict(func_args))
if map_metadata_file is None:
map_metadata_file = os.path.join(
self.dirs.dirs['orig'], 'narps_neurovault_images_details.csv')
map_metadata = get_map_metadata(map_metadata_file)
if overwrite is None:
overwrite = self.overwrite
for teamID in self.complete_image_sets:
for hyp in range(1, 10):
if hyp in [5, 6]:
mdstring = map_metadata.query('teamID == "%s"' %
teamID)['hyp%d_direction' %
hyp].iloc[0]
rectify = mdstring.split()[0] == 'Negative'
elif hyp == 9:
# manual fix for one team with reversed maps
if teamID in ['R7D1']:
mdstring = map_metadata.query(
'teamID == "%s"' % teamID)['hyp%d_direction' %
hyp].iloc[0]
rectify = True
else: # just copy the other hypotheses directly
rectify = False
# load data from unthresh map within
# positive voxels of thresholded mask
unthresh_file = self.teams[teamID].images['unthresh'][
'resampled'][hyp]
self.teams[teamID].images['unthresh']['rectified'][
hyp] = os.path.join(self.dirs.dirs['rectified'],
self.teams[teamID].datadir_label,
'hypo%d_unthresh.nii.gz' % hyp)
if not os.path.exists(
os.path.dirname(self.teams[teamID].images['unthresh']
['rectified'][hyp])):
os.mkdir(
os.path.dirname(self.teams[teamID].images['unthresh']
['rectified'][hyp]))
if overwrite or not os.path.exists(
self.teams[teamID].images['unthresh']['rectified']
[hyp]):
# if values were flipped for negative contrasts
if rectify:
print('rectifying hyp', hyp, 'for', teamID)
print(mdstring)
print('')
img = nibabel.load(unthresh_file)
img_rectified = nilearn.image.math_img('img*-1',
img=img)
img_rectified.to_filename(
self.teams[teamID].images['unthresh']['rectified']
[hyp])
self.rectified_list.append((teamID, hyp))
else: # just copy original
shutil.copy(
unthresh_file,
self.teams[teamID].images['unthresh']['rectified']
[hyp])
# write list of rectified teams to disk
if len(self.rectified_list) > 0:
with open(
os.path.join(self.dirs.dirs['metadata'],
'rectified_images_list.txt'), 'w') as f:
for l in self.rectified_list:
f.write('%s\t%s\n' % (l[0], l[1]))
def compare_thresh_unthresh_values(
dirs, collectionID, logfile,
unthresh_dataset='orig',
thresh_dataset='orig',
verbose=True,
error_thresh=.05,
create_histogram=False,
map_metadata_file=None):
"""examine unthresh values within thresholded map voxels
to check direction of maps
if more than error_thresh percent of voxels are
in opposite direction, then flag a problem
- we allow a few to bleed over due to interpolation"""
verbose = True
hyps = [i for i in range(1, 10)]
teamID = collectionID.split('_')[1]
diagnostic_data = pandas.DataFrame({
'collectionID': collectionID,
'teamID': teamID,
'hyp': hyps,
'autorectify': False,
'problem': numpy.nan,
'reverse_contrast': False,
'n_thresh_vox': numpy.nan,
'min_unthresh': numpy.nan,
'max_unthresh': numpy.nan,
'p_pos_unthresh': numpy.nan,
'p_neg_unthresh': numpy.nan})
teamdir_unthresh = os.path.join(
dirs.dirs[unthresh_dataset],
collectionID
)
teamdir_thresh = os.path.join(
dirs.dirs[thresh_dataset],
collectionID
)
print('using %s for thresh and %s for unthresh' %
(thresh_dataset, unthresh_dataset))
if not os.path.exists(teamdir_unthresh):
print('no unthresh %s for %s' % (unthresh_dataset, collectionID))
print(teamdir_unthresh)
return(None)
if not os.path.exists(teamdir_thresh):
print('no thresh %s for %s' % (thresh_dataset, collectionID))
print(teamdir_unthresh)
return(None)
for hyp in hyps:
autorectify = False
threshfile = os.path.join(
teamdir_thresh, 'hypo%d_thresh.nii.gz' % hyp)
if not os.path.exists(threshfile):
print('no thresh hyp %d for %s' % (hyp, collectionID))
continue
threshdata = nibabel.load(threshfile).get_data().flatten()
threshdata = numpy.nan_to_num(threshdata)
n_thresh_vox = numpy.sum(threshdata > 0)
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'n_thresh_vox'] = n_thresh_vox
if n_thresh_vox == 0:
log_to_file(
logfile,
'WARN: %s %d - empty mask' % (
collectionID, hyp
))
unthreshfile = os.path.join(
teamdir_thresh, 'hypo%d_unthresh.nii.gz' % hyp)
if not os.path.exists(unthreshfile):
print('no unthresh hyp %d for %s' % (hyp, collectionID))
continue
unthreshdata = nibabel.load(unthreshfile).get_data().flatten()
unthreshdata = numpy.nan_to_num(unthreshdata)
if not unthreshdata.shape == threshdata.shape:
log_to_file(
logfile,
'ERROR: thresh/unthresh size mismatch for %s hyp%d' %
(collectionID, hyp))
continue
if numpy.sum(threshdata > 0) > 0:
inmask_unthreshdata = unthreshdata[threshdata > 0]
min_val = numpy.min(inmask_unthreshdata)
max_val = numpy.max(inmask_unthreshdata)
p_pos_unthresh = numpy.mean(inmask_unthreshdata > 0)
p_neg_unthresh = numpy.mean(inmask_unthreshdata < 0)
else:
min_val = 0
max_val = 0
p_pos_unthresh = 0
p_neg_unthresh = 0
if max_val < 0: # need to rectify
autorectify = True
if verbose:
print('autorectify:', teamID, hyp)
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'autorectify'] = True
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'min_unthresh'] = min_val
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'max_unthresh'] = max_val
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'p_pos_unthresh'] = p_pos_unthresh
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'p_neg_unthresh'] = p_neg_unthresh
min_p_direction = numpy.min([p_pos_unthresh, p_neg_unthresh])
if min_p_direction > error_thresh:
log_to_file(
logfile,
'WARN: %s hyp%d invalid in-mask values (%f, %f)' % (
collectionID, hyp, p_neg_unthresh, p_pos_unthresh
))
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'problem'] = True
if create_histogram:
# also load their orig thresh map and create a histogram
orig_threshfile = os.path.join(
dirs.dirs['orig'],
collectionID,
'hypo%d_thresh.nii.gz' % hyp)
threshdata = nibabel.load(orig_threshfile).get_data()
threshdata = threshdata[numpy.abs(threshdata) > 1e-6]
plt.hist(threshdata, bins=50)
plt.savefig(
os.path.join(
dirs.dirs['diagnostics'],
'thresh_hist_%s_%d.pdf' % (
collectionID, hyp
)
)
)
plt.close()
# also get info from metadata file about direction
# of contrasts
if map_metadata_file is None:
map_metadata_file = os.path.join(
dirs.dirs['orig'],
'narps_neurovault_images_details_responses_corrected.csv')
map_metadata = get_map_metadata(map_metadata_file)
reverse_contrast = False
if hyp in [5, 6]:
mdstring = map_metadata.query(
'teamID == "%s"' % teamID
)['hyp%d_direction' % hyp].iloc[0]
reverse_contrast = mdstring.split()[0] == 'Negative'
if verbose:
print('manual rectify:', teamID, hyp)
elif hyp == 9 and teamID in ['R7D1']:
# manual fix for one team with reversed maps
reverse_contrast = True
diagnostic_data.loc[
diagnostic_data.hyp == hyp,
'reverse_contrast'] = reverse_contrast
if reverse_contrast != autorectify:
log_to_file(
logfile,
'WARN: %s %d rectification mismatch' %
(collectionID, hyp))
return(diagnostic_data)