def get_training_results():
X, Y = m.get_dataset(m.dataset1_filepath)
baseline = m.get_score(X, Y)
# print train_one(X, Y, 16, 0.1)
res = np.empty((len(error_rate), len(no_of_hidden_nodes)), dtype=float)
for i in range(len(error_rate)):
for j in range(len(no_of_hidden_nodes)):
print "Training for %d hidden nodes %f corruption level" % (
no_of_hidden_nodes[j], error_rate[i])
ari = train_one(X, Y, no_of_hidden_nodes[j], error_rate[i])
res[i][j] = ari
plot_result(res, baseline)
def __init__(self, input_path, name):
"""
Helper class for evaluating a model.
:param input_path: The path which was the output path for the model. Should contain the 'config.json' file.
:param name: The name of the experiment.
"""
self.model = None
self.name = name
self.input_path = input_path
self.output_path = input_path.parent.joinpath('eval')
self.metrics_path = self.output_path.joinpath('metrics')
self.output_path.mkdir(exist_ok=True, parents=True)
self.metrics_path.mkdir(exist_ok=True, parents=True)
self.config = self._config()
self.checkpoints = self._checkpoints()
self.database = self._load_database()
self.dataset = get_dataset(database=self.database, config=self.config)
if __name__ == '__main__':
from torch.utils import data
from main import get_argparser, get_dataset
opts = get_argparser().parse_args()
opts.path_txt = {
'images_train': '/raid/dataset/cvpr/images_train.txt',
'labels_train': '/raid/dataset/cvpr/labels_train.txt',
'images_val': '/raid/dataset/cvpr/images_val.txt',
'labels_val': '/raid/dataset/cvpr/labels_val.txt',
}
opts.num_classes = 18
_, val_dst = get_dataset(opts)
val_loader = data.DataLoader(val_dst,
batch_size=opts.val_batch_size,
shuffle=True,
num_workers=0,
drop_last=True)
model = network.deeplabv3plus_resnet101(num_classes=opts.num_classes,
output_stride=opts.output_stride)
para_dict = torch.load(
'./checkpoints/latest_deeplabv3plus_resnet101_cvpr_os16.pth')
model.load_state_dict(para_dict['model_state'])
# net_str = str(net)
# import pdb
#!/usr/bin/env python
# coding: utf-8
# this script is used to test whether everything is ok
import cPickle
import os.path
from main import get_dataset
from similarities import ItemSimilarity
from models import ItemPreferenceDataModel
from metrics import manhattan_distances
#from recommenders import ItemCFRecommender as Recommender
data = get_dataset()
model = ItemPreferenceDataModel(data)
sim = ItemSimilarity(model, manhattan_distances)
dump_file = 'cache_' + sim.__class__.__name__ + '_' + \
sim.metrix.__name__
def save_result(sim, filename):
with open(filename, 'w') as f :
cPickle.dump(sim.similarities, f)
def load_similarity(sim, filename):
if os.path.exists(filename) :
with open(filename, 'r') as f :
sim.similarities = cPickle.load(f)
else :
sim.similarities = {}
load_similarity(sim, dump_file)
def loop_main_and_plot(components, scoring, dataset, query_server=True,
force=False, sparsityThreshold=0.000005,
memory=Memory(cachedir='nilearn_cache'), **kwargs):
"""
Loop main.py to plot summaries of WB vs hemi ICA components
"""
out_dir = op.join('ica_imgs', dataset, 'analyses')
# Get the data once.
images, term_scores = get_dataset(dataset, max_images=200, # for testing
query_server=query_server)
# Initialize master DFs
(wb_master, R_master, L_master) = (pd.DataFrame() for i in range(3))
for c in components:
print("Running analysis with %d components" % c)
(wb_summary, R_sparsity, L_sparsity) = load_or_generate_summary(
images=images, term_scores=term_scores, n_components=c,
scoring=scoring, dataset=dataset, force=force,
sparsityThreshold=sparsityThreshold, memory=memory)
# Append them to master DFs
wb_master = wb_master.append(wb_summary)
R_master = R_master.append(R_sparsity)
L_master = L_master.append(L_sparsity)
### Generate component-specific plots ###
# Save component-specific images in the component dir
comp_outdir = op.join(out_dir, str(c))
# 1) Relationship between positive and negative HPI in wb components
out_path = op.join(comp_outdir, "1_PosNegHPI_%dcomponents.png" % c)
hpi_signs = ['pos', 'neg', 'abs']
# set color to be proportional to the symmetry in the sparsity (Pos-Neg/Abs),
# and set size to be proportional to the total sparsity (Abs)
color = (wb_summary['posTotal'] - wb_summary['negTotal']) / wb_summary['absTotal']
size = wb_summary['absTotal'] / 20.0
ax = wb_summary.plot.scatter(x='posHPI', y='negHPI', c=color, s=size,
xlim=(-1.1, 1.1), ylim=(-1.1, 1.1),
colormap='Reds', colorbar=True, figsize=(7, 6))
title = ax.set_title("\n".join(wrap("The relationship between HPI on "
"positive and negative side: "
"n_components = %d" % c, 60)))
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
ax.yaxis.set_ticks_position('left')
ax.yaxis.set_label_coords(-0.1, 0.5)
ax.spines['left'].set_position(('data', 0))
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
ticks = [-1.1, -1.0, -0.5, 0, 0.5, 1.0, 1.1]
labels = ['L', '-1.0', '-0.5', '0', '0.5', '1.0', 'R']
plt.setp(ax, xticks=ticks, xticklabels=labels, yticks=ticks, yticklabels=labels)
f = plt.gcf()
title.set_y(1.05)
f.subplots_adjust(top=0.8)
cax = f.get_axes()[1]
cax.set_ylabel('Balance between pos/neg(anti-correlated network)',
rotation=270, labelpad=20)
save_and_close(out_path)
# 2) Relationship between HPI and SAS in wb components
out_path = op.join(comp_outdir, "2_HPIvsSAS_%dcomponents.png" % c)
fh, axes = plt.subplots(1, 3, sharey=True, figsize=(18, 6))
fh.suptitle("The relationship between HPI values and SAS: "
"n_components = %d" % c, fontsize=16)
hpi_sign_colors = {'pos': 'r', 'neg': 'b', 'abs': 'g'}
for ax, sign in zip(axes, hpi_signs):
ax.scatter(wb_summary['%sHPI' % sign], wb_summary['wb_SAS'],
c=hpi_sign_colors[sign], s=wb_summary['%sTotal' % sign] / 20.0)
ax.set_xlabel("%s HPI" % sign)
ax.set_xlim(-1.1, 1.1)
ax.set_ylim(0, 1)
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
ax.yaxis.set_ticks_position('left')
ax.spines['left'].set_position(('data', 0))
ax.xaxis.set_ticks_position('bottom')
ax.spines['bottom'].set_position(('data', 0))
plt.setp(ax, xticks=ticks, xticklabels=labels)
fh.text(0.04, 0.5, "Spatial Asymmetry Score", va='center', rotation='vertical')
save_and_close(out_path)
### Generate plots over a range of specified n_components ###
# 1) HPI-for pos, neg, and abs in wb components
out_path = op.join(out_dir, '1_wb_HPI.png')
fh, axes = plt.subplots(1, 3, sharex=True, sharey=True, figsize=(18, 6))
fh.suptitle("Hemispheric Participation Index for each component", fontsize=16)
hpi_styles = {'pos': ['r', 'lightpink', 'above %d' % sparsityThreshold],
'neg': ['b', 'lightblue', 'below -%d' % sparsityThreshold],
'abs': ['g', 'lightgreen', 'with abs value above %d' % sparsityThreshold]}
by_comp = wb_master.groupby("n_comp")
for ax, sign in zip(axes, hpi_signs):
mean, sd = by_comp.mean()["%sHPI" % sign], by_comp.std()["%sHPI" % sign]
ax.fill_between(components, mean + sd, mean - sd, linewidth=0,
facecolor=hpi_styles[sign][1], alpha=0.5)
size = wb_master['%sTotal' % (sign)] / 20.0
ax.scatter(wb_master.n_comp, wb_master["%sHPI" % sign], label=sign,
c=hpi_styles[sign][0], s=size)
ax.plot(components, mean, c=hpi_styles[sign][0])
ax.set_xlim((0, components[-1] + 5))
ax.set_ylim((-1, 1))
ax.set_xticks(components)
ax.set_ylabel("HPI((R-L)/(R+L) for # of voxels %s" % (hpi_styles[sign][2]))
fh.text(0.5, 0.04, "# of components", ha="center")
save_and_close(out_path, fh=fh)
# 2) SAS for wb components
fh, ax = plt.subplots(1, 1, figsize=(18, 6))
fh.suptitle("Spatial Asymmetry Score for each component", fontsize=16)
sas_mean, sas_sd = by_comp.mean()["wb_SAS"], by_comp.std()["wb_SAS"]
ax.fill_between(components, sas_mean + sas_sd, sas_mean - sas_sd,
linewidth=0, facecolor='lightgrey', alpha=0.5)
size = wb_master["absTotal"] / 20.0
ax.scatter(wb_master.n_comp, wb_master["wb_SAS"], c='grey', s=size)
ax.plot(components, sas_mean, c='grey')
ax.set_xlim((0, components[-1] + 5))
ax.set_ylim((-1, 1))
ax.set_xticks(components)
ax.set_ylabel("SAS (higher values indicate asymmetry)")
out_path = op.join(out_dir, '2_wb_SAS.png')
save_and_close(out_path, fh=fh)
import warnings
from argparse import ArgumentParser
# Look for image computation errors
warnings.simplefilter('ignore', DeprecationWarning)
warnings.simplefilter('error', RuntimeWarning) # Detect bad NV images
# Arg parsing
parser = ArgumentParser(description="Really?")
parser.add_argument('check', nargs='?', default='data',
choices=('data', 'metadata'))
parser.add_argument('--offline', action='store_true', default=False)
parser.add_argument('--dataset', nargs='?', default='neurovault',
choices=['neurovault', 'abide', 'nyu'])
args = vars(parser.parse_args())
# Alias args
check = args.pop('check')
query_server = not args.pop('offline')
dataset = args.pop('dataset')
if dataset == 'neurovault':
args['fetch_terms'] = False
images = get_dataset(query_server=query_server, dataset=dataset, **args)[0]
if check == 'data':
qc_image_data(images=images, dataset=dataset)
else:
qc_image_metadata(images=images)
plt.show()
def main_ic_loop(components, scoring,
dataset, query_server=True, force=False,
memory=Memory(cachedir='nilearn_cache'), **kwargs):
# $FIX Test with just 'wb' and 'rl' matching until 'lr' matching is fixed
# match_methods = ['wb', 'rl', 'lr']
match_methods = ['wb', 'rl']
out_dir = op.join('ica_imgs', dataset)
mean_scores, unmatched = [], []
# Get the data once.
images, term_scores = get_dataset(
dataset, query_server=query_server)
for match_method in match_methods:
print("Plotting results for %s matching method" % match_method)
mean_score_d, num_unmatched_d = {}, {}
for c in components:
print("Running analysis with %d components" % c)
# main analysis is run for each component and match method:
# plotting for component comparisons are done only if force=True
img_d, score_mats_d, sign_mats_d = do_main_analysis(
dataset=dataset, images=images, term_scores=term_scores,
key=match_method, force=force, plot=force,
n_components=c, scoring=scoring, **kwargs)
# Get mean dissimilarity scores and number of unmatched for each comparisons
# in score_mats_d
for comp in score_mats_d:
score_mat, sign_mat = score_mats_d[comp], sign_mats_d[comp]
# For ("wb", "RL-forced") and ("wb", "RL-unforced")
if "forced" in comp[1]:
if "-forced" in comp[1]:
match, unmatch = get_match_idx_pair(score_mat, sign_mat, force=True)
elif "-unforced" in comp[1]:
match, unmatch = get_match_idx_pair(score_mat, sign_mat, force=False)
n_unmatched = unmatch["idx"].shape[1] if unmatch["idx"] is not None else 0
um_label = "unmatched RL"
mean_score = score_mat[[match["idx"][0], match["idx"][1]]].mean()
score_label = "%s" % (" vs ".join(comp))
# Store values in respective dict
if c == components[0]:
mean_score_d[score_label] = [mean_score]
if "-unforced" in comp[1]:
num_unmatched_d[um_label] = [n_unmatched]
else:
mean_score_d[score_label].append(mean_score)
if "-unforced" in comp[1]:
num_unmatched_d[um_label].append(n_unmatched)
# For ("wb", "R"), ("wb", "L") --wb matching or ("R", "L") --rl matching
else:
for force_match in [True, False]:
match, unmatch = get_match_idx_pair(score_mat, sign_mat, force=force_match)
mean_score = score_mat[[match["idx"][0], match["idx"][1]]].mean()
if force_match:
score_label = "%s%s" % (" vs ".join(comp), "-forced")
n_unmatched = None
else:
score_label = "%s%s" % (" vs ".join(comp), "-unforced")
n_unmatched = unmatch["idx"].shape[1] if unmatch["idx"] is not None else 0
um_label = "unmatched %s" % comp[1]
# Store values in respective dict
if c == components[0]:
mean_score_d[score_label] = [mean_score]
if not force_match:
num_unmatched_d[um_label] = [n_unmatched]
else:
mean_score_d[score_label].append(mean_score)
if not force_match:
num_unmatched_d[um_label].append(n_unmatched)
# Store vals as df
ms_df = pd.DataFrame(mean_score_d, index=components)
um_df = pd.DataFrame(num_unmatched_d, index=components)
mean_scores.append(ms_df)
unmatched.append(um_df)
# Save combined df
combined = pd.concat([ms_df, um_df], axis=1)
out = op.join(out_dir, '%s-matching_simscores.csv' % match_method)
combined.to_csv(out)
# We have all the scores for the matching method; now plot.
fh, axes = plt.subplots(1, len(match_methods), sharex=True, sharey=True, figsize=(18, 6))
fh.suptitle("Average dissimilarity scores for the best-match pairs", fontsize=16)
labels = ["wb vs R-unforced", "wb vs L-unforced", "R vs L-unforced", "wb vs RL-unforced",
"wb vs R-forced", "wb vs L-forced", "R vs L-forced", "wb vs RL-forced",
"unmatched R", "unmatched L", "unmatched RL"]
styles = ["r-", "b-", "m-", "g-",
"r:", "b:", "m:", "g:",
"r--", "b--", "m--"]
for i, ax in enumerate(axes):
ax2 = ax.twinx()
ms_df, um_df = mean_scores[i], unmatched[i]
for label, style in zip(labels, styles):
if label in ms_df.columns:
ms_df[label].plot(ax=ax, style=style)
elif label in um_df.columns:
um_df[label].plot(ax=ax2, style=style)
ax.set_title("%s-matching" % (match_methods[i]))
# Shrink current axis by 30%
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.75, box.height])
ax2.set_position([box.x0, box.y0, box.width * 0.75, box.height])
# Put the legends to the right of the current axis
ax.legend(loc='lower left', bbox_to_anchor=(1.3, 0.5))
ax2.legend(loc='upper left', bbox_to_anchor=(1.3, 0.5))
fh.text(0.5, 0.04, "# of components", ha="center")
fh.text(0.05, 0.5, "mean %s scores" % scoring, va='center', rotation='vertical')
fh.text(0.95, 0.5, "# of unmatched R- or L- components", va='center', rotation=-90)
out_path = op.join(out_dir, '%s_simscores.png' % scoring)
save_and_close(out_path, fh=fh)
