eval_horizon = int(t_time)
if eval_horizon >= num_Category:
print('ERROR: evaluation horizon is out of range')
result1[:, t] = result2[:, t] = -1
else:
risk = pred[:, :, t] #risk score until evalTime
risk_br = pred_br[:, :, t]
for k in range(num_Event):
result1[k, t] = weighted_c_index(
tr_time, (tr_label[:, 0] == k + 1).astype(int), risk[:, k],
te_time, (te_label[:, 0] == k + 1).astype(int),
eval_horizon) #-1 for no event (not comparable)
result2[k, t] = weighted_brier_score(
tr_time, (tr_label[:, 0] == k + 1).astype(int),
risk_br[:, k], te_time, (te_label[:,
0] == k + 1).astype(int),
eval_horizon) #-1 for no event (not comparable)
WEIGHTED_C_INDEX[:, :, out_itr] = result1
WEIGHTED_BRIER_SCORE[:, :, out_itr] = result2
### SAVE RESULTS
row_header = []
for t in range(num_Event):
row_header.append('Event_' + str(t + 1))
col_header1 = []
col_header2 = []
for t in evalTime:
col_header1.append(str(t) + 'months')
def get_valid_performance(DATA,
MASK,
in_parser,
out_itr,
eval_time=None,
MAX_VALUE=-99,
OUT_ITERATION=5,
seed=1234):
##### DATA & MASK
#pdb.set_trace()
(data, time, label) = DATA
(mask1, mask2) = MASK
x_dim = np.shape(data)[1]
_, num_Event, num_Category = np.shape(
mask1) # dim of mask1: [subj, Num_Event, Num_Category]
ACTIVATION_FN = {'relu': tf.nn.relu, 'elu': tf.nn.elu, 'tanh': tf.nn.tanh}
##### HYPER-PARAMETERS
mb_size = in_parser['mb_size'] #batch size
iteration = in_parser['iteration'] # 50000???
keep_prob = in_parser['keep_prob']
lr_train = in_parser['lr_train'] #what is this
alpha = in_parser['alpha'] #for log-likelihood loss
beta = in_parser['beta'] #for ranking loss
gamma = in_parser['gamma'] #for RNN-prediction loss
parameter_name = 'a' + str('%02.0f' % (10 * alpha)) + 'b' + str(
'%02.0f' % (10 * beta)) + 'c' + str('%02.0f' % (10 * gamma))
# something like 'a10b01c00'
initial_W = tf.contrib.layers.xavier_initializer(
) # set initial weights at good position
##### MAKE DICTIONARIES
# INPUT DIMENSIONS
input_dims = {
'x_dim': x_dim,
'num_Event': num_Event,
'num_Category': num_Category
}
# NETWORK HYPER-PARMETERS
network_settings = {
'h_dim_shared': in_parser['h_dim_shared'],
'num_layers_shared': in_parser['num_layers_shared'],
'h_dim_CS': in_parser['h_dim_CS'],
'num_layers_CS': in_parser['num_layers_CS'],
'active_fn': ACTIVATION_FN[in_parser['active_fn']],
'initial_W': initial_W
}
file_path_final = in_parser['out_path'] + '/itr_' + str(out_itr)
#change parameters...
if not os.path.exists(file_path_final + '/models/'):
os.makedirs(file_path_final + '/models/')
print(file_path_final + ' (a:' + str(alpha) + ' b:' + str(beta) + ' c:' +
str(gamma) + ')')
##### CREATE DEEPFHT NETWORK
tf.reset_default_graph()
config = tf.ConfigProto() # configure session
config.gpu_options.allow_growth = True # allow more gpu
sess = tf.Session(config=config) # session
model = Model_DeepHit(sess, "DeepHit", input_dims, network_settings)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
### TRAINING-TESTING SPLIT # also validation
(tr_data, te_data, tr_time, te_time, tr_label, te_label, tr_mask1,
te_mask1, tr_mask2, te_mask2) = train_test_split(data,
time,
label,
mask1,
mask2,
test_size=0.20,
random_state=seed)
(tr_data, va_data, tr_time, va_time, tr_label, va_label, tr_mask1,
va_mask1, tr_mask2, va_mask2) = train_test_split(tr_data,
tr_time,
tr_label,
tr_mask1,
tr_mask2,
test_size=0.20,
random_state=seed)
max_valid = -99
max_valid_scores = None
max_brier = -99
stop_flag = 0
if eval_time is None:
eval_time = [
int(np.percentile(tr_time, 25)),
int(np.percentile(tr_time, 50)),
int(np.percentile(tr_time, 75))
]
### TRAINING - MAIN
print("MAIN TRAINING ...")
print("EVALUATION TIMES: " + str(eval_time))
avg_loss = 0
for itr in range(iteration):
if stop_flag > 5: #for faster early stopping
break
else:
x_mb, k_mb, t_mb, m1_mb, m2_mb = f_get_minibatch(
mb_size, tr_data, tr_label, tr_time, tr_mask1, tr_mask2)
DATA = (x_mb, k_mb, t_mb)
MASK = (m1_mb, m2_mb)
PARAMETERS = (alpha, beta, gamma)
_, loss_curr = model.train(DATA, MASK, PARAMETERS, keep_prob,
lr_train)
avg_loss += loss_curr / 1000
if (itr + 1) % 1000 == 0:
print(
'|| ITR: ' + str('%04d' % (itr + 1)) + ' | Loss: ' +
colored(str('%.4f' %
(avg_loss)), 'yellow', attrs=['bold']))
avg_loss = 0
### VALIDATION (based on average C-index of our interest)
if (itr + 1) % 1000 == 0:
# pdb.set_trace()
### PREDICTION
pred = model.predict(va_data)
### EVALUATION
va_result1 = np.zeros([num_Event, len(eval_time)])
va_result2 = np.zeros([num_Event, len(eval_time)])
for t, t_time in enumerate(eval_time):
eval_horizon = int(t_time)
if eval_horizon >= num_Category:
print('ERROR: evaluation horizon is out of range')
va_result1[:, t] = va_result2[:, t] = -1
else:
risk = np.sum(pred[:, :, :(eval_horizon + 1)],
axis=2) #risk score until eval_time
for k in (range(num_Event)):
# va_result1[k, t] = c_index(risk[:,k], va_time, (va_label[:,0] == k+1).astype(int), eval_horizon) #-1 for no event (not comparable)
va_result1[k, t] = weighted_c_index(
tr_time, (tr_label[:, 0] == k + 1).astype(int),
risk[:, k], va_time,
(va_label[:, 0] == k + 1).astype(int),
eval_horizon)
va_result2[k, t] = weighted_brier_score(
tr_time, (tr_label[:, 0] == k + 1).astype(int),
risk[:, k], va_time,
(va_label[:, 0] == k + 1).astype(int),
eval_horizon)
tmp_valid = np.mean(va_result1[0, :]) # mean over all causes?
tmp_scores = va_result1[0, :]
brier_valid = np.mean(va_result2[0, :])
if tmp_valid > max_valid:
stop_flag = 0
max_valid = tmp_valid
max_brier = brier_valid
max_valid_scores = tmp_scores
print('updated.... average c-index = ' + str('%.4f' %
(tmp_valid)))
print('scores = ' + np.array_str(max_valid_scores))
if max_valid > MAX_VALUE:
saver.save(
sess, file_path_final + '/models/model_itr_' +
str(out_itr))
else:
stop_flag += 1
return max_valid, max_brier