def worker(kernel_mode_log10T,learning_rate=1e1,training_epochs=1001,display_step=200):
kernel, mode, log10T = kernel_mode_log10T
url = 'https://s3-eu-west-1.amazonaws.com/nphc-data/{}_{}_log10T{}_with_params_without_N.pkl.gz'.format(kernel, mode, log10T)
from utils.loader import load_data
cumulants = load_data(url)[0]
d = cumulants.dim
# Starting point
sqrt_C = sqrtm(cumulants.C)
sqrt_L = np.sqrt(cumulants.L)
initial = np.dot(sqrt_C,np.diag(1./sqrt_L))
R = NPHC(cumulants,initial,alpha=0.01,training_epochs=training_epochs,\
display_step=display_step,learning_rate=learning_rate,optimizer='adam') #,l_l1=0.,l_l2=0.)
G = np.eye(d) - inv(R)
file_to_write = 'results/results_nphc_{}_{}.pkl.gz'.format(kernel,mode)
import gzip, pickle
f = gzip.open(file_to_write, 'wb')
pickle.dump(G,f,protocol=2)
f.close()
def main():
parser = argparse.ArgumentParser(description="Anomaly detection.",
epilog="Written by Maxime Chamberland.")
parser.add_argument("--i", metavar="data_path", dest="data_file",
help="Measures (.csv)", required=True,
type=abspath)
parser.add_argument("--demog", metavar="demog_path", dest="demog_file",
help="file containing demographics (.csv)", required=True,
type=abspath)
args = parser.parse_args()
#Load datasheets
#############################################
df_demog = loader.load_csv(args.demog_file)
df_data = loader.load_csv(args.data_file)
filename = args.data_file.rsplit('\\', 1)[-1]
st.sidebar.subheader("File Uploader")
up_demog = st.sidebar.file_uploader("Upload demographics", type="csv")
if up_demog:
df_demog = pd.read_csv(up_demog)
up_data = st.sidebar.file_uploader("Upload measures", type="csv")
if up_data:
datasheet = loader.load_data(up_data)
title = st.sidebar.text_input('Savename label', 'MY_ANALYSIS')
options = ("Spearman", "Pearson")
method = st.sidebar.radio("Method", options, 0)
y_choices = df_demog[df_demog.columns.difference(["Group", "ID"])].columns
y_axis = st.selectbox("Choose a clinical correlate below", y_choices, 0)
stats.correlate(method, df_demog, df_data, y_axis, title, filename)
def post_train_examples(files, model, thresh=0.8, dset=None):
"""
Show some example images with classifications.
"""
# Load some demo images
files = loader.load_data(files)
yes = [cv2.imread(random.choice(files[0])) for _ in range(2)]
no = [cv2.imread(random.choice(files[1])) for _ in range(2)]
# Resize
yes = [cv2.resize(img, (img_height, img_width)) for img in yes]
no = [cv2.resize(img, (img_height, img_width)) for img in no]
images = {"Cancerous": yes, "Healthy": no}
index = 1
plt.figure(figsize=[6, 6])
plt.title(f"ANN {dset} Set")
for (k, v) in images.items():
for i in v:
plt.subplot(2, 2, index)
data = tf.cast(i, tf.float32)
pred = model.predict(data[np.newaxis, ...])
pred = "Cancerous" if pred == 1 else "Healthy"
plt.title(f"Pred: {pred}, Actual {k}")
plt.imshow(i, cmap="gray")
index += 1
yes = files[0]
no = files[1]
yes = [cv2.resize(cv2.imread(img), (img_height, img_width)) for img in yes]
no = [cv2.resize(cv2.imread(img), (img_height, img_width)) for img in no]
X = yes + no
X = [tf.cast(i, tf.float32)[np.newaxis, ...] for i in X]
predict = np.asarray([model.predict(i) for i in X])
predict = predict.flatten()
predict = [step(i, thresh) for i in predict]
y = np.asarray([1 for _ in yes] + [0 for _ in no])
num_correct = np.sum(y == predict)
num_samples = len(y)
percentage_correct = round(num_correct / num_samples * 100, 2)
cm = np.array(confusion_matrix(y, predict, labels=[0, 1]))
confusion_df = pd.DataFrame(
cm,
index=["is_cancer", "is_healthy"],
columns=["predicted_cancer", "predicted_healthy"],
)
plt.figure(figsize=[8, 4])
plt.xticks(rotation="horizontal")
plt.title(
f"ANN {dset} Set: {num_samples} Samples\n{num_correct}/{num_samples} {percentage_correct}% Accuracy"
)
sns.heatmap(confusion_df, annot=True, fmt="g")
plt.show()
action='store_true',
default=True,
help='save log')
# TO ADD: save_result
args = parser.parse_args()
device = 'cuda:0'
randomized = False
args.work_dir = os.path.join(model_path, args.dataset + '_new')
if not os.path.exists(args.work_dir):
os.mkdir(args.work_dir)
data_dict, tag_categories, text_length, num_frames = loader.load_data(
data_path,
args.dataset,
frame_drop=args.frame_drop,
add_mirrored=args.add_mirrored)
data_dict_train, data_dict_eval = loader.split_data_dict(data_dict,
randomized=False,
fill=6)
any_dict_key = list(data_dict)[0]
affs_dim = data_dict[any_dict_key]['affective_features'].shape[-1]
num_joints = data_dict[any_dict_key]['positions'].shape[1]
coords = data_dict[any_dict_key]['positions'].shape[2]
joint_names = data_dict[any_dict_key]['joints_dict']['joints_names']
joint_parents = data_dict[any_dict_key]['joints_dict']['joints_parents']
data_loader = dict(train=data_dict_train, test=data_dict_eval)
prefix_length = int(0.3 * num_frames)
target_length = int(num_frames - prefix_length)
rots_dim = data_dict[any_dict_key]['rotations'].shape[-1]
parser.add_argument('--save-log',
action='store_true',
default=True,
help='save log')
parser.add_argument('--work-dir',
type=str,
default=model_path,
metavar='WD',
help='path to save')
# TO ADD: save_result
args = parser.parse_args()
device = 'cuda:0'
data, labels, data_train, labels_train, data_test, labels_test =\
loader.load_data(data_path, ftype_real, ftype_synth, coords, joints, cycles=cycles)
num_classes = np.unique(labels_train).shape[0]
data_loader = {
'train':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_train, labels_train, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True),
'test':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_test, labels_test, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
}
help='disables CUDA training')
parser.add_argument('--pavi-log', action='store_true', default=False,
help='pavi log')
parser.add_argument('--print-log', action='store_true', default=True,
help='print log')
parser.add_argument('--save-log', action='store_true', default=True,
help='save log')
args = parser.parse_args()
device = 'cuda:0'
datasets = ['BML', 'CMU', 'Human3.6M', 'ICT', 'SIG', 'UNC_RGB']
data, labels, data_train_all_folds, labels_train_all_folds,\
data_test_all_folds, labels_test_all_folds = \
loader.load_data(data_path, ftype, joints, coords, processed_data_path, cycles=cycles, num_folds=num_folds)
metrics_file_full_path = 'metrics.txt'
if not os.path.exists(metrics_file_full_path):
for init_idx in range(num_inits):
for fold_idx, (data_train, labels_train, data_test, labels_test) in enumerate(zip(data_train_all_folds, labels_train_all_folds,
data_test_all_folds, labels_test_all_folds)):
print('Running init {:02d}, fold {:02d}'.format(init_idx, fold_idx))
# saving trained models for each init and split in separate folders
model_path = os.path.join(base_path, 'model_classifier_combined_lstm_init_{:02d}_fold_{:02d}/features'.format(init_idx, fold_idx) + ftype)
args.work_dir = model_path
os.makedirs(model_path, exist_ok=True)
aff_features = len(data_train[0][0])
num_classes = np.unique(labels_train).shape[0]
data_loader = list()
data_loader.append(torch.utils.data.DataLoader(
def main():
parser = argparse.ArgumentParser(description="Anomaly detection.",
epilog="Written by Maxime Chamberland.")
parser.add_argument("--i",
metavar="data_path",
dest="data_file",
help="tract profiles (.xlsx)",
required=True,
type=abspath)
parser.add_argument("--demog",
metavar="demog_path",
dest="demog_file",
help="file containing demographics (.csv)",
required=True,
type=abspath)
args = parser.parse_args()
#Load datasheets
#############################################
df_demog = loader.load_csv(args.demog_file)
datasheet = loader.load_data(args.data_file)
st.sidebar.subheader("File Uploader")
up_demog = st.sidebar.file_uploader("Upload demographics", type="csv")
if up_demog:
df_demog = pd.read_csv(up_demog)
up_data = st.sidebar.file_uploader("Upload profiles", type="xlsx")
if up_data:
datasheet = loader.load_data(up_data)
title = st.sidebar.text_input('Savename', 'MY_ANALYSIS')
#Default metrics and groups to work with
metric = st.sidebar.selectbox("Choose a metric below",
list(datasheet.keys()), 0)
df_data = loader.combine_demog_and_data(df_demog, datasheet, metric)
group_ids = df_demog.Group[df_demog.Group != 0].unique()
group = st.sidebar.selectbox("Choose a patient group below", group_ids, 0)
#Display datasheets
#############################################
st.header("1. Visualisation section")
sns.set(font_scale=1.3)
if st.checkbox('Show demographics', False):
st.write("Demographics datasheet")
explorer.display_demog(df_demog)
if st.checkbox('Show dataset', False):
st.write("Tract-profiles datasheet")
explorer.display_data(df_data)
#Data exploration section
#############################################
#TODO automatically detect tracts
tract_list = [
'AF', 'ATR', 'CA', 'CC_1', 'CC_2', 'CC_3', 'CC_4', 'CC_5', 'CC_6',
'CC_7', 'CG', 'CST', 'FX', 'IFO', 'ILF', 'OR', 'SLF_I', 'SLF_II',
'SLF_III', 'UF', 'All'
]
if st.checkbox('Show tract profiles'):
plot_controls = st.checkbox('Plot Controls', True)
plot_patients = st.checkbox('Plot Patients', True)
show_indiv = st.checkbox('Show Individuals', False)
tract_selection = st.selectbox("Choose a tract to visualize:",
tract_list, 0)
#plot profiles
if (tract_selection.startswith('All')):
tract_selection = tract_list[:-1]
for idx, t in enumerate(tract_selection):
fig = explorer.plot_profile(df_data, df_demog, t, metric,
plot_controls, plot_patients,
group, show_indiv)
st.write(fig)
else:
fig = explorer.plot_profile(df_data, df_demog, tract_selection,
metric, plot_controls, plot_patients,
group, show_indiv)
st.write(fig)
#Anomaly detection section
#############################################
st.header("2. Analysis section")
options = ("Z-score", "PCA", "AutoEncoder") #, "SVM")
method = st.sidebar.radio("Method", options, 0)
hemi = "Both"
if method == "Z-score":
use_all = st.checkbox("Use all", True)
hemi_list = ("left", "right")
if not use_all:
hemi = st.radio("Hemisphere (if applicable))", hemi_list, 0)
tract_init = []
if method == "Z-score":
if not use_all:
uni_selection = st.selectbox(
"Choose a tract (univariate approach):", tract_list[:-1], 0)
tract_profile = [uni_selection]
else:
tract_profile = tract_list[:-1]
else:
if (st.checkbox('Clear all')):
tract_init = []
else:
tract_init = tract_list[:-1]
tract_profile = st.multiselect("Choose tracts:", tract_list[:-1],
tract_init)
rep_value = 10
if method == "SVM":
rep_value = 10
reps = st.sidebar.number_input('Iterations',
min_value=1,
max_value=100,
value=rep_value)
regress = False
if st.sidebar.checkbox('Regress confound?', True):
regress = not regress
#input_threshold = st.sidebar.number_input("Anomaly threshold", 0.0, 10.0, input_threshold)
st.write("Using ", method, "and the following tracts: ",
", ".join(tract_profile))
#Analysis section
#############################################
result = "No results to display."
if st.sidebar.button("Run"):
if method == "SVM":
AUC = launcher.svmachine(method, df_data, df_demog, regress,
tract_profile, group, hemi, metric, reps)
np.savetxt("tests/auc" + metric + method + title + ".csv",
AUC,
delimiter=",")
else:
AUC, result, fpr, tpr = launcher.run(method, df_data, df_demog,
regress, tract_profile, group,
hemi, metric, reps)
#Report section
#############################################
st.header("3. Report section")
reporter.final_report(AUC, result, fpr, tpr, method, metric, group,
title)
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--pavi-log', action='store_true', default=False,
help='pavi log')
parser.add_argument('--print-log', action='store_true', default=True,
help='print log')
parser.add_argument('--save-log', action='store_true', default=True,
help='save log')
parser.add_argument('--work-dir', type=str, default=model_path, metavar='WD',
help='path to save')
# TO ADD: save_result
args = parser.parse_args()
device = 'cuda:0'
data, labels = loader.load_data(data_path, ftype, coords, joints, cycles=cycles)
num_classes = np.unique(labels).shape[0]
graph_dict = {'strategy': 'spatial'}
emotions = ['Angry', 'Neutral', 'Happy', 'Sad']
if args.train:
test_size = 0.1
data, labels, data_train, labels_train, data_test, labels_test = loader.split_data(data, labels, test_size=test_size)
data_loader_train_test = list()
data_loader_train_test.append(torch.utils.data.DataLoader(
dataset=loader.TrainTestLoader(data_train, labels_train, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True))
data_loader_train_test.append(torch.utils.data.DataLoader(
dataset=loader.TrainTestLoader(data_test, labels_test, joints, coords, num_classes),
def main():
#Load datasheets
#############################################
rel_path = "../ressources/demogB.csv"
demog = join(script_dir, rel_path)
df_demog = loader.load_csv(demog)
rel_path = "../ressources/featuresB.xlsx"
features = join(script_dir, rel_path)
datasheet = loader.load_data(features)
st.sidebar.subheader("File Uploader")
up_demog = st.sidebar.file_uploader("Upload demographics", type="csv")
if up_demog:
df_demog = pd.read_csv(up_demog)
up_data = st.sidebar.file_uploader("Upload profiles", type="xlsx")
if up_data:
datasheet = loader.load_data(up_data)
title = st.sidebar.text_input('Savename', 'MY_ANALYSIS')
#Default metrics and groups to work with
metric = st.sidebar.selectbox("Choose a metric below",
list(datasheet.keys()), 0)
df_data = loader.combine_demog_and_data(df_demog, datasheet, metric)
group_ids = df_demog.Group[df_demog.Group != 0].unique()
group = st.sidebar.selectbox("Choose a patient group below", group_ids, 0)
subjs_ids = df_demog.ID
subject = st.sidebar.selectbox("Choose a subject below", subjs_ids, 1)
#Display datasheets
#############################################
st.header("1. Visualisation section")
sns.set(font_scale=1.3)
if st.checkbox('Show demographics', False):
st.write("Demographics datasheet")
explorer.display_demog(df_demog)
if st.checkbox('Show dataset', False):
st.write("Tract-profiles datasheet")
explorer.display_data(df_data)
#Data exploration section
#############################################
#TODO automatically detect tracts
tract_list = [
'AF_left', 'AF_right', 'ATR_left', 'ATR_right', 'CA', 'CC_1', 'CC_2',
'CC_3', 'CC_4', 'CC_5', 'CC_6', 'CC_7', 'CG_left', 'CG_right',
'CST_left', 'CST_right', 'FX_left', 'FX_right', 'IFO_left',
'IFO_right', 'ILF_left', 'ILF_right', 'OR_left', 'OR_right',
'SLF_I_left', 'SLF_II_left', 'SLF_III_left', 'SLF_I_right',
'SLF_II_right', 'SLF_III_right', 'UF_left', 'UF_right'
]
tract_list_uni = [
'AF', 'ATR', 'CA', 'CC_1', 'CC_2', 'CC_3', 'CC_4', 'CC_5', 'CC_6',
'CC_7', 'CG', 'CST', 'FX', 'IFO', 'ILF', 'OR', 'SLF_I', 'SLF_II',
'SLF_III', 'UF', 'All'
]
if st.checkbox('Show tract profiles'):
plot_controls = st.checkbox('Plot Controls', True)
plot_patients = st.checkbox('Plot Patients', True)
show_indiv = st.checkbox('Show Individuals', False)
tract_selection = st.selectbox("Choose a tract to visualize:",
tract_list_uni, 0)
#plot profiles
if (tract_selection.startswith('All')):
tract_selection = tract_list_uni[:-1]
for idx, t in enumerate(tract_selection):
fig = explorer.plot_profile(df_data, df_demog, t, metric,
plot_controls, plot_patients,
group, show_indiv)
st.write(fig)
else:
fig = explorer.plot_profile(df_data, df_demog, tract_selection,
metric, plot_controls, plot_patients,
group, show_indiv)
st.write(fig)
#Anomaly detection section
#############################################
st.header("2. Analysis section")
hemi = "Both"
tract_init = []
tract_list_demo = [
'AF_left', 'CC_4', 'CC_5', 'CST_right', 'IFO_left', 'ILF_left',
'OR_left', 'SLF_I_right', 'UF_left'
]
if (st.checkbox('Clear all')):
tract_init = []
tract_profile = st.multiselect("Choose tracts:", tract_list,
tract_init)
else:
tract_init = tract_list_demo
tract_profile = st.multiselect("Choose tracts:", tract_list,
tract_init)
regress = False
if st.sidebar.checkbox('Regress confound?', True):
regress = not regress
#input_threshold = st.sidebar.number_input("Anomaly threshold", 0.0, 10.0, input_threshold)
st.write("Using the following tracts: ", ", ".join(tract_profile))
LOOCV = False
#if st.sidebar.checkbox('Run all subjects?',False):
#LOOCV = not LOOCV
pop = [subject]
#if LOOCV:
#pop = subjs_ids
#Analysis section
#############################################
result = "No results to display."
finalpval = pd.DataFrame()
finalvector = pd.DataFrame()
if st.sidebar.button("Run"):
once = True
for s in pop:
x, x_hat, mae, p_along, p_overall, p_div = inspector.run(
s, df_data, df_demog, regress, tract_profile, hemi, metric)
cur_group = df_demog.loc[df_data['ID'] == s, 'Group']
#Report section
#############################################
st.header("3. Report section")
X = df_data.loc[:,
df_data.columns.str.startswith('Group')
| df_data.columns.str.startswith('ID') | df_data.
columns.str.contains('|'.join(tract_profile))]
dfpval, dfvector = reporter.plot_features(x, x_hat, mae, p_along,
p_overall, p_div, s,
metric, cur_group, title,
X.columns, once)
if once:
finalpval = dfpval
finalvector = dfvector
finalpval.append(dfpval)
finalvector.append(dfvector)
once = False
name = 'tests/p-val' + '_' + metric + '_' + title + '.csv'
st.markdown(reporter.get_csv_link(finalpval, name),
unsafe_allow_html=True)
default=True,
help='save log')
parser.add_argument('--work-dir',
type=str,
default=model_path,
metavar='WD',
help='path to save')
# TO ADD: save_result
args = parser.parse_args()
device = 'cuda:0'
test_size = 0.1
data, labels,\
data_train, labels_train,\
data_test, labels_test = loader.load_data(data_path, ftype, coords, joints,
cycles=cycles, test_size=test_size)
num_classes = np.unique(labels).shape[0]
graph_dict = {'strategy': 'spatial'}
emotions = ['Angry', 'Neutral', 'Happy', 'Sad']
if args.train:
data_loader = {
'train':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_train, labels_train, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True),
'test':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_test, labels_test, joints, coords, num_classes),
help='print log')
parser.add_argument('--save-log',
action='store_true',
default=True,
help='save log')
parser.add_argument('--work-dir',
type=str,
default=model_path,
metavar='WD',
help='path to save')
# TO ADD: save_result
args = parser.parse_args()
device = 'cuda:0'
data_train, data_test, labels_train, labels_test = loader.load_data(
data_path, ftype, joints, coords, cycles=cycles)
aff_features = len(data_train[0][0])
num_classes = np.unique(labels_train).shape[0]
data_loader = {
'train':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_train, labels_train, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True),
'test':
torch.utils.data.DataLoader(dataset=loader.TrainTestLoader(
data_test, labels_test, joints, coords, num_classes),
batch_size=args.batch_size,
shuffle=True,
drop_last=True)