def hidden_to_visible(self, data, tir=False):
visible = sigmoid_np((data @ self.w.transpose()) +
np.tile(self.a, (data.shape[0], 1)))
if tir:
aux = np.random.rand(visible.shape[0], visible.shape[1])
visible = 1 * (aux <= visible) + 0 * (aux > visible)
return visible
def visible_to_hidden(self, data, tir=False):
hidden = sigmoid_np((data @ self.w) +
np.tile(self.b, (data.shape[0], 1)))
if tir:
aux = np.random.rand(hidden.shape[0], hidden.shape[1])
hidden = 1 * (aux <= hidden) + 0 * (aux > hidden)
return hidden
def generate_signal32(length, mu, sigma, varparm=0.1):
hidden = np.zeros(length, dtype=int)
visible = np.zeros((length, mu.shape[-1]))
delta = np.zeros(length, dtype=float)
delta[0] = np.random.uniform(0, 1)
p0 = np.array([delta[0], 1 - delta[0]])
test = np.random.multinomial(1, p0)
hidden[0] = np.argmax(test)
visible[0] = multivariate_normal.rvs(mu[hidden[0]], sigma[hidden[0]])
for i in range(1, length):
delta[i] = sigmoid_np(multivariate_normal.rvs(delta[i - 1], varparm))
T = np.array([[delta[i], 1 - delta[i]], [1 - delta[i], delta[i]]])
test = np.random.multinomial(1, T[hidden[i - 1], :])
hidden[i] = np.argmax(test)
visible[i] = multivariate_normal.rvs(mu[hidden[i]], sigma[hidden[i]])
return hidden, visible
def get_val_threshold(learner):
preds, y = learner.TTA(n_aug=4)
preds = np.stack(preds, axis=-1)
print(preds.shape)
preds = sigmoid_np(preds)
pred = preds.max(axis=-1)
th = fit_val(pred, y)
th[th < 0.1] = 0.1
print('Thresholds: ', th)
print('F1 macro: ', f1_score(y, pred > th, average='macro'))
print('F1 macro (th = 0.5): ', f1_score(y, pred > 0.5, average='macro'))
print('F1 micro: ', f1_score(y, pred > th, average='micro'))
print('Fractions: ', (pred > th).mean(axis=0))
print('Fractions (true): ', (y > th).mean(axis=0))
return th
def main():
warnings.filterwarnings('ignore')
train_val_names = list({f[:36] for f in os.listdir(cfg.train_dir)})
test_names = list({f[:36] for f in os.listdir(cfg.test_dir)})
train_names, val_names = train_test_split(train_val_names, test_size=0.1, random_state=42)
batch_size = cfg.batch_size
target_size = 512
img_ds = get_data(train_names, val_names, test_names, target_size, batch_size, n_workers=5)
learner = ConvLearner.pretrained(dpn92, img_ds, ps=[0.5]) # use dropout 50%
learner.opt_fn = optim.Adam
learner.clip = 1.0
learner.crit = FocalLoss()
learner.metrics = [acc]
# print(learner.summary())
train(learner, lr=5e-4, save_name='base_dpn')
val_th = get_val_threshold(learner)
# TTA
preds_t, y_t = learner.TTA(n_aug=4, is_test=True)
preds_t = np.stack(preds_t, axis=-1)
preds_t = sigmoid_np(preds_t)
pred_t = preds_t.max(axis=-1) # max works better for F1 macro score
test_names = learner.data.test_ds.fnames
save_pred(pred_t, test_names, val_th, 'protein_classification_v.csv')
man_th = np.array([0.565, 0.39, 0.55, 0.345, 0.33, 0.39, 0.33, 0.45, 0.38, 0.39,
0.34, 0.42, 0.31, 0.38, 0.49, 0.50, 0.38, 0.43, 0.46, 0.40,
0.39, 0.505, 0.37, 0.47, 0.41, 0.545, 0.32, 0.1])
print('Fractions: ', (pred_t > man_th).mean(axis=0))
save_pred(pred_t, test_names, man_th, 'protein_classification.csv')
lb_prob = [0.362397820, 0.043841336, 0.075268817, 0.059322034, 0.075268817,
0.075268817, 0.043841336, 0.075268817, 0.010000000, 0.010000000,
0.010000000, 0.043841336, 0.043841336, 0.014198783, 0.043841336,
0.010000000, 0.028806584, 0.014198783, 0.028806584, 0.059322034,
0.010000000, 0.126126126, 0.028806584, 0.075268817, 0.010000000,
0.222493880, 0.028806584, 0.010000000]
test_th = get_test_threshold(pred_t, lb_prob, min_th=0.1)
save_pred(pred_t, test_names, test_th, 'protein_classification_f.csv')
save_pred(pred_t, test_names, 0.5, 'protein_classification_05.csv')
label_count, label_fraction = get_dataset_fraction(pd.read_csv(cfg.train_csv).set_index('Id'))
train_th = get_test_threshold(pred_t, label_fraction, min_th=0.05)
save_pred(pred_t, test_names, train_th, 'protein_classification_t.csv')
brute_th = get_brute_threshold(pred_t)
save_pred(pred_t, test_names, brute_th, 'protein_classification_b.csv')
noise['corr_param'][0],
noise['corr_param'][1]) + moving_average(
(img == 1) * np.random.normal(
noise['mu2'], noise['sig2'], img.shape),
noise['corr_param'][0], noise['corr_param'][1])
corr = 'corr'
corr_param = str(noise['corr_param'][0]) + '_' + str(
noise['corr_param'][1])
noise_param = '(' + str(noise['mu1']) + ',' + str(
noise['sig1']) + ')' + '_' + '(' + str(
noise['mu2']) + ',' + str(noise['sig2']) + ')'
cv.imwrite(
resfolder + '/' + imgf + '/' + str(resolution[0]) + '_' +
str(resolution[1]) + '_' + corr + '_' + corr_param +
noise_param + '.bmp',
sigmoid_np(img_noisy) * max_val)
data = img_noisy[test[0], test[1]].reshape(-1, 1)
kmeans = KMeans(n_clusters=kmeans_clusters).fit(data)
seg_kmeans = np.zeros((img.shape[0], img.shape[1]))
seg_kmeans[test[0], test[1]] = kmeans.labels_
cv.imwrite(
resfolder + '/' + imgf + '/' + str(resolution[0]) + '_' +
str(resolution[1]) + '_' + corr + '_' + corr_param +
noise_param + '_seg_kmeans' + '.bmp', seg_kmeans * max_val)
for model in models:
model['model'].give_param(*model['params'][idx])
seg = np.zeros(
(img.shape[0], img.shape[1])
) # Création d'une matrice vide qui va recevoir l'image segmentée
signal_length,
mu=np.array([[noise['mu1']], [noise['mu2']]]),
sigma=np.array([[[noise['sig1']]], [[noise['sig2']]]]))
img = np.zeros(resolution)
img[test[0], test[1]] = true_signal
cv.imwrite(
resfolder + '/' + signal['name'] + '_' + corr + '_' + corr_param +
noise_param + '_true_signal.bmp', img * max_val)
img = np.zeros(resolution)
img[test[0], test[1]] = signal_noisy.reshape((signal_noisy.shape[0], ))
cv.imwrite(
resfolder + '/' + signal['name'] + '_' + corr + '_' + corr_param +
noise_param + '.bmp',
sigmoid_np(img) * max_val)
data = signal_noisy.reshape(-1, 1)
kmeans = KMeans(n_clusters=kmeans_clusters).fit(data)
seg = kmeans.labels_
# terr = {signal['name'] + '_' + corr + '_' + corr_param + noise_param + '_seg_kmeans':calc_err(seg, true_signal)}
with open(os.path.join(resfolder, 'terr.txt'), 'r') as f:
content = json.load(f)
with open(os.path.join(resfolder, 'terr.txt'), 'w') as f:
content[signal['name'] + '_' + corr + '_' + corr_param +
noise_param + '_seg_kmeans'] = calc_err(seg, true_signal)
json.dump(content, f, ensure_ascii=False, indent=2)
img = np.zeros(resolution)
img[test[0], test[1]] = seg
cv.imwrite(
model['model'].give_param(*params)
sample_hidden, sample_visible = model['model'].generate_sample(
sample_length)
img_save = np.zeros(resolution)
img_save[test[0], test[1]] = sample_hidden
cv.imwrite(
resfolder + '/' + 'gen_' + model['name'] + '_' + corr + '_' +
corr_param + '_' + noise_param + '.bmp', img_save * max_val)
img_save = np.zeros(resolution)
img_save[test[0], test[1]] = sample_visible.reshape(
(sample_visible.shape[0], ))
cv.imwrite(
resfolder + '/' + 'gen_' + model['name'] + '_' + corr + '_' +
corr_param + '_' + noise_param + 'noisy.bmp',
sigmoid_np(img_save) * max_val)
sample_visible = sample_visible.reshape(-1, 1)
param_s = {
'p': model['model'].p.tolist(),
't': model['model'].t.tolist(),
'mu': model['model'].mu.tolist(),
'sig': model['model'].sigma.tolist()
}
with open(
os.path.join(
resfolder, 'gen_' + model['name'] + '_' + corr + '_' +
corr_param + '_' + noise_param + '_param.txt'), 'w') as f:
json.dump(param_s, f, ensure_ascii=False)
for m in models:
if m['name'] != model['name']:
img.shape)
corr = ''
corr_param = ''
else:
img_noisy = moving_average((img == 0) * np.random.normal(noise['mu1'], noise['sig1'], img.shape),
noise['corr_param'][0], noise['corr_param'][1]) + moving_average((
img == 1) * np.random.normal(
noise['mu2'], noise['sig2'],
img.shape), noise['corr_param'][0], noise['corr_param'][1])
corr = 'corr'
corr_param = str(noise['corr_param'][0]) + '_' + str(noise['corr_param'][1])
noise_param = '(' + str(noise['mu1']) + ',' + str(noise['sig1']) + ')' + '_' + '(' + str(
noise['mu2']) + ',' + str(noise['sig2']) + ')'
cv.imwrite(
resfolder + '/' + imgf + '/' + str(resolution[0]) + '_' + str(
resolution[1]) + '_' + corr + '_' + corr_param + noise_param + '.bmp', sigmoid_np(img_noisy) * max_val)
data = img_noisy[test[0], test[1]].reshape(-1, 1)
kmeans = KMeans(n_clusters=kmeans_clusters).fit(data)
seg_kmeans = np.zeros(
(img.shape[0], img.shape[1]))
seg_kmeans[test[0], test[1]] = kmeans.labels_
cv.imwrite(resfolder + '/' + imgf + '/' + str(resolution[0]) + '_' + str(
resolution[1]) + '_' + corr + '_' + corr_param + noise_param + '_seg_kmeans' + '.bmp', seg_kmeans * int(max_val/(kmeans_clusters-1)))
for model in models:
if not model['params']:
# if not model['name']=='hmc':
# model['model'].init_from_markov_chain(data)
# else:
# model['model'].init_data_prior(data)