def main():
params = Params()
model = RegressionModel(params)
# Use functions of the model to build the graph
out, states = model.inference(model.data_placeholder)
loss = model.loss(out, model.labels_placeholder)
train_op = model.train(loss, params.step_size)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
saver = tf.train.Saver(tf.all_variables())
for i in range(params.train_steps + 1):
data, labels = get_batch(params.batch_size, params.sequence_length, params.input_channels)
feed_dict = {
model.data_placeholder: data,
model.labels_placeholder: labels
}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
if i % params.print_every == 0:
print i, loss_value
if i % params.save_every == 0:
name = "model_{0}.ckpt".format(params.get_id)
checkpoint_path = os.path.join('./save', name)
# TODO: If we restore a model for further training, we should
# add the number of training steps it had completed to our global step here
saver.save(sess, checkpoint_path, global_step=i)
print "model saved to {0}-{1}".format(checkpoint_path, i)
with open('./save/{0}.model_param'.format(params.get_id), 'w') as f:
pickle.dump(params,
f,
protocol=2 # pickle.HIGHEST_PROTOCOL as of writing
)
data, labels = get_batch(params.batch_size, params.sequence_length, params.input_channels)
feed_dict = {
model.data_placeholder: data,
model.labels_placeholder: labels
}
vars = sess.run(out + states, feed_dict)
out_ = vars[0:len(out)]
states_ = vars[len(out)+1:]
d = data[0,0,:]
o = np.array(out_)[:, 0, 0]
l = labels[0,0,:]
x1 = range(d.shape[0])
x2 = range(1, d.shape[0] + 1)
# TODO: output graph every 100 steps
plt.scatter(x1, d, c='r')
plt.scatter(x2, o, c='g')
plt.scatter(x2, l, c='b', alpha=0.5)
plt.show()
print "data third dim", d
print "out", o
# print "states", np.array(states_)
print "labels third dim", l
def main():
params = Params()
model = RegressionModel(params)
# Use functions of the model to build the graph
out, states = model.inference(model.data_placeholder)
loss = model.loss(out, model.labels_placeholder)
train_op = model.train(loss, params.step_size)
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
saver = tf.train.Saver(tf.all_variables())
for i in range(params.train_steps + 1):
data, labels = get_batch(params.batch_size, params.sequence_length,
params.input_channels)
feed_dict = {
model.data_placeholder: data,
model.labels_placeholder: labels
}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
if i % params.print_every == 0:
print i, loss_value
if i % params.save_every == 0:
name = "model_{0}.ckpt".format(params.get_id)
checkpoint_path = os.path.join('./save', name)
# TODO: If we restore a model for further training, we should
# add the number of training steps it had completed to our global step here
saver.save(sess, checkpoint_path, global_step=i)
print "model saved to {0}-{1}".format(checkpoint_path, i)
with open('./save/{0}.model_param'.format(params.get_id),
'w') as f:
pickle.dump(
params,
f,
protocol=2 # pickle.HIGHEST_PROTOCOL as of writing
)
data, labels = get_batch(params.batch_size, params.sequence_length,
params.input_channels)
feed_dict = {
model.data_placeholder: data,
model.labels_placeholder: labels
}
vars = sess.run(out + states, feed_dict)
out_ = vars[0:len(out)]
states_ = vars[len(out) + 1:]
d = data[0, 0, :]
o = np.array(out_)[:, 0, 0]
l = labels[0, 0, :]
x1 = range(d.shape[0])
x2 = range(1, d.shape[0] + 1)
# TODO: output graph every 100 steps
plt.scatter(x1, d, c='r')
plt.scatter(x2, o, c='g')
plt.scatter(x2, l, c='b', alpha=0.5)
plt.show()
print "data third dim", d
print "out", o
# print "states", np.array(states_)
print "labels third dim", l
def test_student(args, student_train_loader, student_labels, student_test_loader, test_size, cat_emb_size, num_conts, device, sensitive_idx):
student_model = RegressionModel(emb_szs=cat_emb_size,
n_cont=num_conts,
emb_drop=0.04,
out_sz=1,
szs=[1000, 500, 250],
drops=[0.001, 0.01, 0.01],
y_range=(0, 1)).to(device)
criterion = nn.BCELoss()
optimizer = optim.SGD(student_model.parameters(), lr=args.lr, momentum=0)
steps = 0
running_loss = 0
correct = 0
print("========== Testing Student Model ==========")
for epoch in range(args.epochs):
student_model.train()
train_loader = student_loader(student_train_loader, student_labels)
for (cats, conts) , labels in train_loader:
#for _batch_idx, (data, target) in enumerate(tqdm(train_loader)):
#cats = data[0]
#conts = data[1]
steps += 1
optimizer.zero_grad()
output = student_model(cats, conts).view(-1)
labels = labels.to(torch.float32)
loss = criterion(output, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
# if steps % 50 == 0:
student_model.eval()
test_loss = 0
correct = 0
i = 0
avg_recall = 0
avg_precision = 0
overall_results = []
avg_eq_odds = 0
avg_dem_par = 0
avg_tpr = 0
avg_tp = 0
avg_tn = 0
avg_fp = 0
avg_fn = 0
with torch.no_grad():
for batch_idx, (cats, conts, target) in enumerate(student_test_loader):
print("target\n", sum(target))
i+=1
output = student_model(cats, conts)
loss += criterion(output, target).item()
test_loss = test_loss + ((1 / (batch_idx + 1)) * (loss.data - test_loss))
pred = (output > 0.5).float()
print("pred\n", sum(pred))
correct += pred.eq(target.view_as(pred)).sum().item()
curr_datetime = datetime.now()
curr_hour = curr_datetime.hour
curr_min = curr_datetime.minute
pred_df = pd.DataFrame(pred.numpy())
pred_df.to_csv(f"pred_results/{args.run_name}_{curr_hour}-{curr_min}.csv")
#print(pred, np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy()))
#correct += np.sum(np.squeeze(pred.eq(target.data.view_as(pred))).cpu().numpy())
#total += cats.size(0)
# confusion matrixç
tn, fp, fn, tp = confusion_matrix(target, pred).ravel()
avg_tn += tn
avg_fp += fp
avg_fn += fn
avg_tp += tp
# position of col for sensitive values
sensitive = [i[sensitive_idx].item() for i in cats]
cat_len = max(sensitive)
#exit()
sub_cm = []
# print(cat_len)
for j in range(cat_len+1):
try:
idx = list(locate(sensitive, lambda x: x == j))
sub_tar = target[idx]
sub_pred = pred[idx]
sub_tn, sub_fp, sub_fn, sub_tp = confusion_matrix(sub_tar, sub_pred).ravel()
except:
# when only one value to predict
print("----WHAT?")
temp_tar = int(sub_tar.numpy()[0])
temp_pred = int(sub_pred.numpy()[0])
# print(tar, pred)
if temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 1
elif temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 1, 0
elif not temp_tar and not temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 1, 0, 0, 0
elif not temp_tar and temp_pred:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 1, 0, 0
else:
sub_tn, sub_fp, sub_fn, sub_tp = 0, 0, 0, 0
total = mysum(sub_tn, sub_fp, sub_fn, sub_tp)
print("??", total)
sub_cm.append((sub_tn / total, sub_fp / total, sub_fn / total, sub_tp / total))
# Fairness metrics
group_metrics = MetricFrame({'precision': skm.precision_score, 'recall': skm.recall_score},
target, pred,
sensitive_features=sensitive)
demographic_parity = flm.demographic_parity_difference(target, pred,
sensitive_features=sensitive)
eq_odds = flm.equalized_odds_difference(target, pred,
sensitive_features=sensitive)
# metric_fns = {'true_positive_rate': true_positive_rate}
tpr = MetricFrame(true_positive_rate,
target, pred,
sensitive_features=sensitive)
# tpr = flm.true_positive_rate(target, pred,sample_weight=sensitive)
sub_results = group_metrics.overall.to_dict()
sub_results_by_group = group_metrics.by_group.to_dict()
# print("\n", group_metrics.by_group, "\n")
avg_precision += sub_results['precision']
avg_recall += sub_results['recall']
print("pre_rec", sub_results)
overall_results.append(sub_results_by_group)
avg_eq_odds += eq_odds
print("eqo", eq_odds)
avg_dem_par += demographic_parity
print("dempar", demographic_parity)
avg_tpr += tpr.difference(method='between_groups')
print("tpr", tpr.difference(method='between_groups'))
total = mysum(avg_tn, avg_fp, avg_fn, avg_tp)
print("!!", total)
cm = (avg_tn / total, avg_fp / total, avg_fn / total, avg_tp / total)
test_loss /= test_size
accuracy = correct / test_size
avg_loss = test_loss
return accuracy, avg_loss, avg_precision, avg_recall, avg_eq_odds, avg_tpr, avg_dem_par, cm, sub_cm, overall_results
