def train_model(design,
filename,
columns,
targets,
comsize_third=20,
separator='\t',
**train_kwargs):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes=design[0],
a_func=design[1],
time=time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
comsize = 3 * comsize_third #Make sure it is divisible by three (3*X will create X jobs)
print('Number of members in the committee: ' + str(comsize))
print('Design used (size, function): ' + str(design))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 200
#print("Population size: " + str(train_kwargs['population_size']))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#print("Mutation rate: " + str(train_kwargs['mutation_chance']))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
#errorfunc = weighted_c_index_error
errorfunc = c_index_error
print("\nError function: " + errorfunc.__name__)
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
#trn_set = {}
trn_idx = {}
master_com = None
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
#Lambda times
for _time in xrange(1):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(
P, T, 1, binary_column=1, return_indices=True)
super_zip = zip(super_set, super_indices)
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
_t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
for com_num in xrange(comsize / 3):
count += 1
all_counts.append(count)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
trn_idx[count] = TRN_IDX
(netsize, hidden_func) = design
com = build_feedforward_committee(3,
len(P[0]),
netsize,
1,
hidden_function=hidden_func,
output_function='linear')
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee,
com,
train_evolutionary,
TRN_INPUTS,
TRN_TARGETS,
binary_target=1,
error_function=errorfunc,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
#TIME TO RECEIVE THE RESULTS
while (count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print(
'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
the server after this session.'.format(fastest_done,
all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets,
internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([
_set[0][0] for _set in internal_sets
]) #first 0 gives training sets, second 0 gives inputs.
if master_com is None:
master_com = com
else:
master_com.nets.extend(com.nets) #Add this batch of networks
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# patvals will be order-consistent with P and T
#for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
# for i in val_set_indices:
# patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))
for ((trn_in, trn_tar),
(val_in, val_tar)), idx, net in zip(internal_sets,
internal_sets_indices,
com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert ((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net=net))
#for pat, i in zip(allpats, xrange(len(patvals))):
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
# for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
# _C_ = -1
# for valpat in val_in:
# _C_ += 1
# if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
#print("Facit: \n" + str(valpat))
#print("Allpats-index = " + str(i))
#print("_C_ = " + str(_C_))
#print("idx_val[_C_]: " + str(idx[1][_C_]))
#print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
#print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
# patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
# break #Done with this data_set
avg_vals = numpy.array([
[numpy.mean(patval)] for patval in patvals
]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average com-validation C-Index so far : {0}'.format(
avg_val_c_index))
print('Saving committee so far in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(master_com, FILE)
return savefile
def train_model(design, filename, columns, targets, comsize_third = 20, separator = '\t', **train_kwargs):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes = design[0], a_func = design[1], time = time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator, use_header = True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
comsize = 3 * comsize_third #Make sure it is divisible by three (3*X will create X jobs)
print('Number of members in the committee: ' + str(comsize))
print('Design used (size, function): ' + str(design))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 200
#print("Population size: " + str(train_kwargs['population_size']))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#print("Mutation rate: " + str(train_kwargs['mutation_chance']))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
#errorfunc = weighted_c_index_error
errorfunc = c_index_error
print("\nError function: " + errorfunc.__name__)
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
#trn_set = {}
trn_idx = {}
master_com = None
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
#Lambda times
for _time in xrange(1):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(P, T, 1, binary_column = 1, return_indices = True)
super_zip = zip(super_set, super_indices)
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
for com_num in xrange(comsize / 3):
count += 1
all_counts.append(count)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
trn_idx[count] = TRN_IDX
(netsize, hidden_func) = design
com = build_feedforward_committee(3, len(P[0]), netsize, 1, hidden_function = hidden_func, output_function = 'linear')
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee, com, train_evolutionary, TRN_INPUTS,
TRN_TARGETS, binary_target = 1, error_function = errorfunc,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
#TIME TO RECEIVE THE RESULTS
while(count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
the server after this session.'.format(fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
if master_com is None:
master_com = com
else:
master_com.nets.extend(com.nets) #Add this batch of networks
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# patvals will be order-consistent with P and T
#for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
# for i in val_set_indices:
# patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))
for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net = net))
#for pat, i in zip(allpats, xrange(len(patvals))):
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
# for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
# _C_ = -1
# for valpat in val_in:
# _C_ += 1
# if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
#print("Facit: \n" + str(valpat))
#print("Allpats-index = " + str(i))
#print("_C_ = " + str(_C_))
#print("idx_val[_C_]: " + str(idx[1][_C_]))
#print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
#print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
# patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
# break #Done with this data_set
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average com-validation C-Index so far : {0}'.format(avg_val_c_index))
print('Saving committee so far in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(master_com, FILE)
return savefile
def model_contest(filename, columns, targets, designs, comsize_third = 5, repeat_times = 20, testfilename = None, separator = '\t', **train_kwargs):
'''
model_contest(filename, columns, targets, designs)
You must use column names! Here are example values for the input arguments:
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
targets = ['time', 'event']
Writes the results to '.winningdesigns_time.csv' and returns the filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
print('\nIncluding columns: ' + str(columns))
print('\nTarget columns: ' + str(targets))
P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
use_header = True)
if testfilename is not None:
Ptest, Ttest = parse_file(testfilename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
use_header = True)
else:
Ptest, Ttest = None, None
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print("T:" + str(T.shape))
print("P:" + str(P.shape))
if (Ptest is not None and Ttest is not None):
print("\nExternal Test Data set:")
print("Number of patients with events: " + str(Ttest[:, 1].sum()))
print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
print("Ttest:" + str(Ttest.shape))
print("Ptest:" + str(Ptest.shape))
comsize = 3 * comsize_third #Make sure it is divisible by three
print('\nNumber of members in each committee: ' + str(comsize))
print('Designs used in testing (size, function): ' + str(designs))
# We can generate a test set from the data set, but usually we don't want that
# Leave at 1 for no test set.
val_pieces = 1
print('Cross-test pieces: ' + str(val_pieces))
cross_times = repeat_times
print('Number of times to repeat procedure: ' + str(cross_times))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 50
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
tests = {}
#trn_set = {}
trn_idx = {}
all_best = []
all_best_com_val = []
all_best_avg_trn = []
all_best_avg_val = []
all_best_design = []
all_best_test = []
#Lambda times
for _time in xrange(cross_times):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(P, T, val_pieces , binary_column = 1, return_indices = True)
super_zip = zip(super_set, super_indices)
all_best.append({})
all_best_com_val.append({})
all_best_avg_trn.append({})
all_best_avg_val.append({})
all_best_design.append({})
all_best_test.append({})
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
TEST_INPUTS = TEST[0]
TEST_TARGETS = TEST[1]
#run each architecture design on a separate machine
best[_t] = None
best_com_val[_t] = 0
best_avg_trn[_t] = 0
best_avg_val[_t] = 0
best_design[_t] = None
best_test[_t] = None
for design in designs:
count += 1
all_counts.append(count)
(netsize, hidden_func) = design
com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, hidden_function = hidden_func,
output_function = 'linear')
tests[count] = (TEST_INPUTS, TEST_TARGETS)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
#print("TRN_IDX" + str(TRN_IDX))
#print("TEST_IDX" + str(TEST_IDX))
trn_idx[count] = TRN_IDX
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee, com, train_evolutionary, TRN_INPUTS,
TRN_TARGETS, binary_target = 1, error_function = c_index_error, **train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
while(count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
You should restart the server after this session.'.format(fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
TEST_INPUTS, TEST_TARGETS = tests[_c]
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
allpats = []
allpats.extend(internal_sets[0][0][0]) #Extend with training inputs
allpats.extend(internal_sets[0][1][0]) #Extend with validation inputs
allpats_targets = []
allpats_targets.extend(internal_sets[0][0][1]) #training targets
allpats_targets.extend(internal_sets[0][1][1]) #validation targets
allpats_targets = numpy.array(allpats_targets)
patvals = [[] for bah in xrange(len(allpats))]
#print(len(patvals))
#print(len(internal_sets_indices))
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# Will be order consistent with P and T
for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net = net))
#Need double brackets for dimensions to fit C-module
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(T, avg_vals)
trn_errors = numpy.array(trn_errors.values(), dtype = numpy.float64) ** -1
vald_errors = numpy.array(vald_errors.values(), dtype = numpy.float64) ** -1
avg_trn = numpy.mean(trn_errors)
avg_val = numpy.mean(vald_errors)
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
if avg_val_c_index > best_com_val[_t]:
best[_t] = com
best_com_val[_t] = avg_val_c_index
best_avg_trn[_t] = avg_trn
best_avg_val[_t] = avg_val
best_design[_t] = design
best_test[_t] = tests[_c]
print('\nWinning designs')
winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
with open(winnerfilename, 'w') as F:
print('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:')
F.write('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n')
for _time in xrange(len(all_best)):
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
for _t in best.keys():
TEST_INPUTS, TEST_TARGETS = best_test[_t]
com = best[_t]
if len(TEST_INPUTS) > 0:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TEST_INPUTS])
test_c_index = get_C_index(TEST_TARGETS, outputs)
elif Ptest is not None and Ttest is not None:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in Ptest])
test_c_index = get_C_index(Ttest, outputs)
else:
test_c_index = 0
print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))
F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))
return winnerfilename
def committee_test():
try:
netsize = input('Number of hidden nodes? [1]: ')
except SyntaxError as e:
netsize = 1
try:
comsize = input('Committee size? [1]: ')
except SyntaxError as e:
comsize = 1
try:
pop_size = input('Population size? [100]: ')
except SyntaxError as e:
pop_size = 100
try:
mutation_rate = input('Please input a mutation rate (0.05): ')
except SyntaxError as e:
mutation_rate = 0.05
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset.txt"
try:
columns = input("Which columns to include? (Do NOT forget trailing comma if only one column is used, e.g. '3,'\nAvailable columns are: 2, -4, -3, -2, -1. Just press ENTER for all columns.\n")
except SyntaxError:
columns = (2, -4, -3, -2, -1)
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
try:
cutoff = input('Cutoff for censored data? [9999 years]: ')
except SyntaxError as e:
cutoff = 9999
P, T = copy_without_censored(P, T, cutoff)
#Divide into validation sets
try:
test_size = float(input('Size of test set (not used in training)? Input in fractions. Default is [0.0]: '))
except:
test_size = 0.0
((TP, TT), (VP, VT)) = get_validation_set(P, T, validation_size = test_size, binary_column = 1)
print("Length of training set: " + str(len(TP)))
print("Length of test set: " + str(len(VP)))
try:
epochs = input("\nNumber of generations (1): ")
except SyntaxError as e:
epochs = 1
com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, output_function = 'linear')
#1 is the column in the target array which holds the binary censoring information
test_errors, vald_errors, data_sets = train_committee(com, train_evolutionary, P, T, 1, epochs, error_function = c_index_error, population_size = pop_size, mutation_chance = mutation_rate)
com.set_training_sets([set[0][0] for set in data_sets]) #first 0 gives training sets, second 0 gives inputs.
print('\nTest C_indices, Validation C_indices:')
for terr, verr in zip(test_errors.values(), vald_errors.values()):
print(str(1 / terr) + ", " + str(1 / verr))
if plt:
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TP]) #Need double brackets for dimensions to be right for numpy
kaplanmeier(time_array = TT[:, 0], event_array = TT[:, 1], output_array = outputs[:, 0], threshold = 0.5)
train_c_index = get_C_index(TT, outputs)
print("\nC-index on the training set: " + str(train_c_index))
if len(VP) > 0:
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in VP]) #Need double brackets for dimensions to be right for numpy
test_c_index = get_C_index(VT, outputs)
kaplanmeier(time_array = VT[:, 0], event_array = VT[:, 1], output_array = outputs[:, 0], threshold = 0.5)
print("C-index on the test set: " + str(test_c_index))
#raw_input("\nPress enter to show plots...")
plt.show()
try:
answer = input("\nDo you wish to print committee risk output? ['n']: ")
except (SyntaxError, NameError):
answer = 'n'
if answer != 'n' and answer != 'no':
inputs = read_data_file(filename)
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
outputs = [[com.risk_eval(patient)] for patient in P]
while len(inputs) > len(outputs):
outputs.insert(0, ["net_output"])
print("\n")
for rawline in zip(inputs, outputs):
line = ''
for col in rawline[0]:
line += str(col)
line += ','
for col in rawline[1]:
line += str(col)
print(line)
def com_cross():
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset.txt"
#try:
# columns = input("Which columns to include? (Do NOT forget trailing comma if only one column is used, e.g. '3,'\nAvailable columns are: 2, -4, -3, -2, -1. Just press ENTER for all columns.\n")
#except SyntaxError:
#if len(sys.argv) < 3:
columns = (2, -4, -3, -2, -1)
#else:
# columns = [int(col) for col in sys.argv[2:]]
print('\nIncluding columns: ' + str(columns))
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
#print('\nRemoving tail censored...')
#P, T = copy_without_censored(P, T)
#Divide into validation sets
#test_size = 0.33
#print('Size of test set (not used in training): ' + str(test_size))
#((TP, TT), (VP, VT)) = get_validation_set(P, T, validation_size = test_size, binary_column = 1)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
#print("Length of training set: " + str(len(TP)))
#print("Length of test set: " + str(len(VP)))
#try:
# comsize = input("Number of networks to cross-validate [10]: ")
#except SyntaxError:
if len(sys.argv) < 2:
netsize = 1
else:
netsize = int(sys.argv[1])
print("\nNumber of hidden nodes: " + str(netsize))
comsize = 4
print('Number of members in each committee: ' + str(comsize))
comnum = 5
print('Number of committees to cross-validate: ' + str(comnum))
times_to_cross = 3
print('Number of times to repeat cross-validation: ' + str(times_to_cross))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
pop_size = 100
print("Population size: " + str(pop_size))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
mutation_rate = 0.05
print("Mutation rate: " + str(mutation_rate))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
epochs = 100
print("Epochs: " + str(epochs))
for _cross_time in xrange(times_to_cross):
data_sets = get_cross_validation_sets(P, T, comnum , binary_column = 1)
print('\nTest Errors, Validation Errors:')
for _com_num, (TS, VS) in zip(xrange(comnum), data_sets):
com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, output_function = 'linear')
#1 is the column in the target array which holds the binary censoring information
test_errors, vald_errors, internal_sets = train_committee(com, train_evolutionary, TS[0], TS[1], 1, epochs, error_function = c_index_error, population_size = pop_size, mutation_chance = mutation_rate)
com.set_training_sets([set[0][0] for set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TS[0]]) #Need double brackets for dimensions to be right for numpy
train_c_index = get_C_index(TS[1], outputs)
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in VS[0]]) #Need double brackets for dimensions to be right for numpy
val_c_index = get_C_index(VS[1], outputs)
print(str(1.0 / train_c_index) + ", " + str(1.0 / val_c_index))
def model_contest(filename,
columns,
targets,
designs,
comsize_third=5,
repeat_times=20,
testfilename=None,
separator='\t',
**train_kwargs):
'''
model_contest(filename, columns, targets, designs)
You must use column names! Here are example values for the input arguments:
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
targets = ['time', 'event']
Writes the results to '.winningdesigns_time.csv' and returns the filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
print('\nIncluding columns: ' + str(columns))
print('\nTarget columns: ' + str(targets))
P, T = parse_file(filename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
if testfilename is not None:
Ptest, Ttest = parse_file(testfilename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
else:
Ptest, Ttest = None, None
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print("T:" + str(T.shape))
print("P:" + str(P.shape))
if (Ptest is not None and Ttest is not None):
print("\nExternal Test Data set:")
print("Number of patients with events: " + str(Ttest[:, 1].sum()))
print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
print("Ttest:" + str(Ttest.shape))
print("Ptest:" + str(Ptest.shape))
comsize = 3 * comsize_third #Make sure it is divisible by three
print('\nNumber of members in each committee: ' + str(comsize))
print('Designs used in testing (size, function): ' + str(designs))
# We can generate a test set from the data set, but usually we don't want that
# Leave at 1 for no test set.
val_pieces = 1
print('Cross-test pieces: ' + str(val_pieces))
cross_times = repeat_times
print('Number of times to repeat procedure: ' + str(cross_times))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 50
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
tests = {}
#trn_set = {}
trn_idx = {}
all_best = []
all_best_com_val = []
all_best_avg_trn = []
all_best_avg_val = []
all_best_design = []
all_best_test = []
#Lambda times
for _time in xrange(cross_times):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(
P, T, val_pieces, binary_column=1, return_indices=True)
super_zip = zip(super_set, super_indices)
all_best.append({})
all_best_com_val.append({})
all_best_avg_trn.append({})
all_best_avg_val.append({})
all_best_design.append({})
all_best_test.append({})
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
_t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
TEST_INPUTS = TEST[0]
TEST_TARGETS = TEST[1]
#run each architecture design on a separate machine
best[_t] = None
best_com_val[_t] = 0
best_avg_trn[_t] = 0
best_avg_val[_t] = 0
best_design[_t] = None
best_test[_t] = None
for design in designs:
count += 1
all_counts.append(count)
(netsize, hidden_func) = design
com = build_feedforward_committee(comsize,
len(P[0]),
netsize,
1,
hidden_function=hidden_func,
output_function='linear')
tests[count] = (TEST_INPUTS, TEST_TARGETS)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
#print("TRN_IDX" + str(TRN_IDX))
#print("TEST_IDX" + str(TEST_IDX))
trn_idx[count] = TRN_IDX
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee,
com,
train_evolutionary,
TRN_INPUTS,
TRN_TARGETS,
binary_target=1,
error_function=c_index_error,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
while (count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print(
'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
You should restart the server after this session.'.format(
fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets,
internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
TEST_INPUTS, TEST_TARGETS = tests[_c]
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([
_set[0][0] for _set in internal_sets
]) #first 0 gives training sets, second 0 gives inputs.
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
allpats = []
allpats.extend(internal_sets[0][0][0]) #Extend with training inputs
allpats.extend(internal_sets[0][1][0]) #Extend with validation inputs
allpats_targets = []
allpats_targets.extend(internal_sets[0][0][1]) #training targets
allpats_targets.extend(internal_sets[0][1][1]) #validation targets
allpats_targets = numpy.array(allpats_targets)
patvals = [[] for bah in xrange(len(allpats))]
#print(len(patvals))
#print(len(internal_sets_indices))
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# Will be order consistent with P and T
for ((trn_in, trn_tar),
(val_in, val_tar)), idx, net in zip(internal_sets,
internal_sets_indices,
com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert ((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net=net))
#Need double brackets for dimensions to fit C-module
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(T, avg_vals)
trn_errors = numpy.array(trn_errors.values(), dtype=numpy.float64)**-1
vald_errors = numpy.array(vald_errors.values(),
dtype=numpy.float64)**-1
avg_trn = numpy.mean(trn_errors)
avg_val = numpy.mean(vald_errors)
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
if avg_val_c_index > best_com_val[_t]:
best[_t] = com
best_com_val[_t] = avg_val_c_index
best_avg_trn[_t] = avg_trn
best_avg_val[_t] = avg_val
best_design[_t] = design
best_test[_t] = tests[_c]
print('\nWinning designs')
winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
with open(winnerfilename, 'w') as F:
print(
'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:'
)
F.write(
'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n'
)
for _time in xrange(len(all_best)):
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
for _t in best.keys():
TEST_INPUTS, TEST_TARGETS = best_test[_t]
com = best[_t]
if len(TEST_INPUTS) > 0:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)]
for inputs in TEST_INPUTS])
test_c_index = get_C_index(TEST_TARGETS, outputs)
elif Ptest is not None and Ttest is not None:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)]
for inputs in Ptest])
test_c_index = get_C_index(Ttest, outputs)
else:
test_c_index = 0
print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(
trn=best_avg_trn[_t],
val=best_avg_val[_t],
com_val=best_com_val[_t],
test=test_c_index,
dsn=best_design[_t]))
F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(
trn=best_avg_trn[_t],
val=best_avg_val[_t],
com_val=best_com_val[_t],
test=test_c_index,
dsn=best_design[_t]))
return winnerfilename