def testCrossValidationSplitting(self):
print ("Testing crossvalidation splitting")
import numpy
inputs = numpy.array([[x] for x in xrange(100)])
targets = inputs + 0.5
data_sets = get_cross_validation_sets(inputs, targets, 9)
for (ti, tt), (vi, vt) in data_sets:
# Verify that validation does not occur in test and vice versa
for i, t in zip(ti, tt):
assert i[0] not in vi
assert t[0] not in vt
for set in xrange(len(data_sets)):
(ti1, tt1), (vi1, vt1) = data_sets[set]
for set2 in xrange(len(data_sets)):
if set == set2:
continue
(ti2, tt2), (vi2, vt2) = data_sets[set2]
# Verify that validation does occur in test set and so on
for i, t in zip(vi1, vt1):
assert i[0] not in vi2
assert t[0] not in vt2
# shuffling test, make sure we don't scramble the data
print ("Testing cross shuffling")
inputs = numpy.zeros((1000, 2), dtype=numpy.float64)
inputs[:, 0] = numpy.linspace(100, 400, 1000)
inputs[:, 1] = numpy.linspace(0, 200, 1000)
def formula1(a, b):
return a * b
def formula2(a, b):
return a + b
targets = numpy.zeros((1000, 2), dtype=numpy.float64)
targets[:, 0] = formula1(inputs[:, 0], inputs[:, 1])
targets[:, 1] = formula2(inputs[:, 0], inputs[:, 1])
data_sets = get_cross_validation_sets(inputs, targets, 4)
for (ti, tt), (vi, vt) in data_sets:
# Verify length
assert len(ti) == 3.0 / 4.0 * len(targets)
assert len(tt) == 3.0 / 4.0 * len(targets)
assert len(vi) == 1.0 / 4.0 * len(targets)
assert len(vt) == 1.0 / 4.0 * len(targets)
# Verify that formulas are still correct
for i, t in zip(ti, tt):
assert t[0] == formula1(i[0], i[1])
assert t[1] == formula2(i[0], i[1])
for i, t in zip(vi, vt):
assert t[0] == formula1(i[0], i[1])
assert t[1] == formula2(i[0], i[1])
def train_model(design,
filename,
columns,
targets,
comsize_third=20,
separator='\t',
**train_kwargs):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes=design[0],
a_func=design[1],
time=time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
comsize = 3 * comsize_third #Make sure it is divisible by three (3*X will create X jobs)
print('Number of members in the committee: ' + str(comsize))
print('Design used (size, function): ' + str(design))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 200
#print("Population size: " + str(train_kwargs['population_size']))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#print("Mutation rate: " + str(train_kwargs['mutation_chance']))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
#errorfunc = weighted_c_index_error
errorfunc = c_index_error
print("\nError function: " + errorfunc.__name__)
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
#trn_set = {}
trn_idx = {}
master_com = None
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
#Lambda times
for _time in xrange(1):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(
P, T, 1, binary_column=1, return_indices=True)
super_zip = zip(super_set, super_indices)
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
_t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
for com_num in xrange(comsize / 3):
count += 1
all_counts.append(count)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
trn_idx[count] = TRN_IDX
(netsize, hidden_func) = design
com = build_feedforward_committee(3,
len(P[0]),
netsize,
1,
hidden_function=hidden_func,
output_function='linear')
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee,
com,
train_evolutionary,
TRN_INPUTS,
TRN_TARGETS,
binary_target=1,
error_function=errorfunc,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
#TIME TO RECEIVE THE RESULTS
while (count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print(
'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
the server after this session.'.format(fastest_done,
all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets,
internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([
_set[0][0] for _set in internal_sets
]) #first 0 gives training sets, second 0 gives inputs.
if master_com is None:
master_com = com
else:
master_com.nets.extend(com.nets) #Add this batch of networks
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# patvals will be order-consistent with P and T
#for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
# for i in val_set_indices:
# patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))
for ((trn_in, trn_tar),
(val_in, val_tar)), idx, net in zip(internal_sets,
internal_sets_indices,
com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert ((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net=net))
#for pat, i in zip(allpats, xrange(len(patvals))):
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
# for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
# _C_ = -1
# for valpat in val_in:
# _C_ += 1
# if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
#print("Facit: \n" + str(valpat))
#print("Allpats-index = " + str(i))
#print("_C_ = " + str(_C_))
#print("idx_val[_C_]: " + str(idx[1][_C_]))
#print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
#print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
# patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
# break #Done with this data_set
avg_vals = numpy.array([
[numpy.mean(patval)] for patval in patvals
]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average com-validation C-Index so far : {0}'.format(
avg_val_c_index))
print('Saving committee so far in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(master_com, FILE)
return savefile
def train_model(design, filename, columns, targets, comsize_third = 20, separator = '\t', **train_kwargs):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes = design[0], a_func = design[1], time = time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator, use_header = True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
comsize = 3 * comsize_third #Make sure it is divisible by three (3*X will create X jobs)
print('Number of members in the committee: ' + str(comsize))
print('Design used (size, function): ' + str(design))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 200
#print("Population size: " + str(train_kwargs['population_size']))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#print("Mutation rate: " + str(train_kwargs['mutation_chance']))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
#errorfunc = weighted_c_index_error
errorfunc = c_index_error
print("\nError function: " + errorfunc.__name__)
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
#trn_set = {}
trn_idx = {}
master_com = None
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
#Lambda times
for _time in xrange(1):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(P, T, 1, binary_column = 1, return_indices = True)
super_zip = zip(super_set, super_indices)
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
for com_num in xrange(comsize / 3):
count += 1
all_counts.append(count)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
trn_idx[count] = TRN_IDX
(netsize, hidden_func) = design
com = build_feedforward_committee(3, len(P[0]), netsize, 1, hidden_function = hidden_func, output_function = 'linear')
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee, com, train_evolutionary, TRN_INPUTS,
TRN_TARGETS, binary_target = 1, error_function = errorfunc,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
#TIME TO RECEIVE THE RESULTS
while(count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
the server after this session.'.format(fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
if master_com is None:
master_com = com
else:
master_com.nets.extend(com.nets) #Add this batch of networks
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# patvals will be order-consistent with P and T
#for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
# for i in val_set_indices:
# patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))
for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net = net))
#for pat, i in zip(allpats, xrange(len(patvals))):
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
# for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
# _C_ = -1
# for valpat in val_in:
# _C_ += 1
# if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
#print("Facit: \n" + str(valpat))
#print("Allpats-index = " + str(i))
#print("_C_ = " + str(_C_))
#print("idx_val[_C_]: " + str(idx[1][_C_]))
#print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
#print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
# patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
# break #Done with this data_set
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average com-validation C-Index so far : {0}'.format(avg_val_c_index))
print('Saving committee so far in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(master_com, FILE)
return savefile
def train_model(filename, columns, targets, separator = '\t', comsize=1):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
headers = []
headers.extend(columns)
headers.extend(targets) #Add targets to the end
targetcol = targets[0]
eventcol = targets[1]
savefile = ".cox_{time:.0f}.pcom".format(time = time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = False, separator = separator, use_header = True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print('Number of members in the committee: ' + str(comsize))
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
cox_committee = None
#Get an independant test set, 1/tau of the total.
super_set = get_cross_validation_sets(P, T, 1, binary_column = 1)
#For every blind test group
for ((TRN, TEST), _t) in zip(super_set, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
#Modulo expressions mean we can deal with any number of committees, not only multiples of three
_res = 1 if comsize == 1 else 0
for com_num in xrange(int(comsize / 3) + int((comsize % 3) / 2) + _res):
#Every time in the loop, create new validations sets of size 1/3. 3 everytime
_tmp_val_sets = get_cross_validation_sets(TRN_INPUTS, TRN_TARGETS, 3, binary_column = 1)
val_sets = []
if int(comsize / 3) > 0:
_max = 3
else:
_max = int((comsize % 3) / 2) * 2 + _res
for _tmp_val_set in _tmp_val_sets[:_max]:
((trn_in, trn_tar), (val_in, val_tar)) = _tmp_val_set
#Add target columns to the end
_trn = np.append(trn_in, trn_tar, axis = 1)
_val = np.append(val_in, val_tar, axis = 1)
val_sets.append((_trn, _val))
#And create 3 cox models, one for each validation
tmp_com = committee(val_sets, targetcol, eventcol, headers)
print("Adding this many members: " + str(len(tmp_com)))
if cox_committee is None:
cox_committee = tmp_com
else:
#Extend the big committee
cox_committee.members.extend(tmp_com.members)
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
print("Validating cox committee, this might take a little while...")
_count = 0
if len(cox_committee) < 3:
allpats_targets = np.empty((0, 2)) #All patients won't be in the target set in this case
for pat, i in zip(allpats, xrange(len(patvals))):
if _count % 50 == 0:
print("{0} / {1}".format(_count, len(patvals)))
_count += 1
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
for cox in cox_committee.members:
(_trn, _val) = cox.internal_set
trn_in = _trn[:, :-2] #Last two columns are targets
val_in = _val[:, :-2]
val_tar = _val[:, -2:]
for valpat, valtar in zip(val_in, val_tar):
if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
patvals[i].append(cox_committee.risk_eval(pat, cox = cox)) #Just to have something to count
if len(cox_committee) < 3:
allpats_targets = np.append(allpats_targets, [valtar], axis = 0)
#print cox_committee.risk_eval(pat, cox = cox)
break #Done with this data_set
avg_vals = []
for patval in patvals:
if len(patval) > 0:
avg_vals.append([np.mean(patval)])
avg_vals = np.array(avg_vals)
#avg_vals = np.array([[np.mean(patval)] for patval in patvals]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average validation C-Index: {0}'.format(avg_val_c_index))
print('Saving committee in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(cox_committee, FILE)
return savefile
def model_contest(filename, columns, targets, designs, comsize_third = 5, repeat_times = 20, testfilename = None, separator = '\t', **train_kwargs):
'''
model_contest(filename, columns, targets, designs)
You must use column names! Here are example values for the input arguments:
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
targets = ['time', 'event']
Writes the results to '.winningdesigns_time.csv' and returns the filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
print('\nIncluding columns: ' + str(columns))
print('\nTarget columns: ' + str(targets))
P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
use_header = True)
if testfilename is not None:
Ptest, Ttest = parse_file(testfilename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
use_header = True)
else:
Ptest, Ttest = None, None
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print("T:" + str(T.shape))
print("P:" + str(P.shape))
if (Ptest is not None and Ttest is not None):
print("\nExternal Test Data set:")
print("Number of patients with events: " + str(Ttest[:, 1].sum()))
print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
print("Ttest:" + str(Ttest.shape))
print("Ptest:" + str(Ptest.shape))
comsize = 3 * comsize_third #Make sure it is divisible by three
print('\nNumber of members in each committee: ' + str(comsize))
print('Designs used in testing (size, function): ' + str(designs))
# We can generate a test set from the data set, but usually we don't want that
# Leave at 1 for no test set.
val_pieces = 1
print('Cross-test pieces: ' + str(val_pieces))
cross_times = repeat_times
print('Number of times to repeat procedure: ' + str(cross_times))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 50
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
tests = {}
#trn_set = {}
trn_idx = {}
all_best = []
all_best_com_val = []
all_best_avg_trn = []
all_best_avg_val = []
all_best_design = []
all_best_test = []
#Lambda times
for _time in xrange(cross_times):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(P, T, val_pieces , binary_column = 1, return_indices = True)
super_zip = zip(super_set, super_indices)
all_best.append({})
all_best_com_val.append({})
all_best_avg_trn.append({})
all_best_avg_val.append({})
all_best_design.append({})
all_best_test.append({})
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
TEST_INPUTS = TEST[0]
TEST_TARGETS = TEST[1]
#run each architecture design on a separate machine
best[_t] = None
best_com_val[_t] = 0
best_avg_trn[_t] = 0
best_avg_val[_t] = 0
best_design[_t] = None
best_test[_t] = None
for design in designs:
count += 1
all_counts.append(count)
(netsize, hidden_func) = design
com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, hidden_function = hidden_func,
output_function = 'linear')
tests[count] = (TEST_INPUTS, TEST_TARGETS)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
#print("TRN_IDX" + str(TRN_IDX))
#print("TEST_IDX" + str(TEST_IDX))
trn_idx[count] = TRN_IDX
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee, com, train_evolutionary, TRN_INPUTS,
TRN_TARGETS, binary_target = 1, error_function = c_index_error, **train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
while(count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
You should restart the server after this session.'.format(fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
TEST_INPUTS, TEST_TARGETS = tests[_c]
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
allpats = []
allpats.extend(internal_sets[0][0][0]) #Extend with training inputs
allpats.extend(internal_sets[0][1][0]) #Extend with validation inputs
allpats_targets = []
allpats_targets.extend(internal_sets[0][0][1]) #training targets
allpats_targets.extend(internal_sets[0][1][1]) #validation targets
allpats_targets = numpy.array(allpats_targets)
patvals = [[] for bah in xrange(len(allpats))]
#print(len(patvals))
#print(len(internal_sets_indices))
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# Will be order consistent with P and T
for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net = net))
#Need double brackets for dimensions to fit C-module
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(T, avg_vals)
trn_errors = numpy.array(trn_errors.values(), dtype = numpy.float64) ** -1
vald_errors = numpy.array(vald_errors.values(), dtype = numpy.float64) ** -1
avg_trn = numpy.mean(trn_errors)
avg_val = numpy.mean(vald_errors)
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
if avg_val_c_index > best_com_val[_t]:
best[_t] = com
best_com_val[_t] = avg_val_c_index
best_avg_trn[_t] = avg_trn
best_avg_val[_t] = avg_val
best_design[_t] = design
best_test[_t] = tests[_c]
print('\nWinning designs')
winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
with open(winnerfilename, 'w') as F:
print('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:')
F.write('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n')
for _time in xrange(len(all_best)):
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
for _t in best.keys():
TEST_INPUTS, TEST_TARGETS = best_test[_t]
com = best[_t]
if len(TEST_INPUTS) > 0:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TEST_INPUTS])
test_c_index = get_C_index(TEST_TARGETS, outputs)
elif Ptest is not None and Ttest is not None:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in Ptest])
test_c_index = get_C_index(Ttest, outputs)
else:
test_c_index = 0
print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))
F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))
return winnerfilename
def train_model(filename, columns, targets, separator='\t', comsize=1):
'''
train_model(design, filename, columns, targets)
Given a design, will train a committee like that on the data specified. Will save the committee as
'.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
Returns this filename
'''
headers = []
headers.extend(columns)
headers.extend(targets) #Add targets to the end
targetcol = targets[0]
eventcol = targets[1]
savefile = ".cox_{time:.0f}.pcom".format(time=time.time())
print('\nIncluding columns: ' + str(columns))
print('Target columns: ' + str(targets))
P, T = parse_file(filename,
targetcols=targets,
inputcols=columns,
normalize=False,
separator=separator,
use_header=True)
#columns = (2, -6, -5, -4, -3, -2, -1)
#_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
#P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print('Number of members in the committee: ' + str(comsize))
allpats = P.copy()
#allpats[:, 1] = 1 #This is the event column
allpats_targets = T
patvals = [[] for bah in xrange(len(allpats))]
cox_committee = None
#Get an independant test set, 1/tau of the total.
super_set = get_cross_validation_sets(P, T, 1, binary_column=1)
#For every blind test group
for ((TRN, TEST), _t) in zip(super_set, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
#TEST_INPUTS = TEST[0]
#TEST_TARGETS = TEST[1]
#Modulo expressions mean we can deal with any number of committees, not only multiples of three
_res = 1 if comsize == 1 else 0
for com_num in xrange(
int(comsize / 3) + int((comsize % 3) / 2) + _res):
#Every time in the loop, create new validations sets of size 1/3. 3 everytime
_tmp_val_sets = get_cross_validation_sets(TRN_INPUTS,
TRN_TARGETS,
3,
binary_column=1)
val_sets = []
if int(comsize / 3) > 0:
_max = 3
else:
_max = int((comsize % 3) / 2) * 2 + _res
for _tmp_val_set in _tmp_val_sets[:_max]:
((trn_in, trn_tar), (val_in, val_tar)) = _tmp_val_set
#Add target columns to the end
_trn = np.append(trn_in, trn_tar, axis=1)
_val = np.append(val_in, val_tar, axis=1)
val_sets.append((_trn, _val))
#And create 3 cox models, one for each validation
tmp_com = committee(val_sets, targetcol, eventcol, headers)
print("Adding this many members: " + str(len(tmp_com)))
if cox_committee is None:
cox_committee = tmp_com
else:
#Extend the big committee
cox_committee.members.extend(tmp_com.members)
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
print("Validating cox committee, this might take a little while...")
_count = 0
if len(cox_committee) < 3:
allpats_targets = np.empty(
(0, 2)) #All patients won't be in the target set in this case
for pat, i in zip(allpats, xrange(len(patvals))):
if _count % 50 == 0:
print("{0} / {1}".format(_count, len(patvals)))
_count += 1
#We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
for cox in cox_committee.members:
(_trn, _val) = cox.internal_set
trn_in = _trn[:, :-2] #Last two columns are targets
val_in = _val[:, :-2]
val_tar = _val[:, -2:]
for valpat, valtar in zip(val_in, val_tar):
if (pat == valpat).all(
): #Checks each variable individually, all() does a boolean and between the results
patvals[i].append(cox_committee.risk_eval(
pat, cox=cox)) #Just to have something to count
if len(cox_committee) < 3:
allpats_targets = np.append(allpats_targets, [valtar],
axis=0)
#print cox_committee.risk_eval(pat, cox = cox)
break #Done with this data_set
avg_vals = []
for patval in patvals:
if len(patval) > 0:
avg_vals.append([np.mean(patval)])
avg_vals = np.array(avg_vals)
#avg_vals = np.array([[np.mean(patval)] for patval in patvals]) #Need double brackets for dimensions to fit C-module
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(allpats_targets, avg_vals)
print('Average validation C-Index: {0}'.format(avg_val_c_index))
print('Saving committee in {0}'.format(savefile))
with open(savefile, 'w') as FILE:
pickle.dump(cox_committee, FILE)
return savefile
def model_contest(filename,
columns,
targets,
designs,
comsize_third=5,
repeat_times=20,
testfilename=None,
separator='\t',
**train_kwargs):
'''
model_contest(filename, columns, targets, designs)
You must use column names! Here are example values for the input arguments:
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
targets = ['time', 'event']
Writes the results to '.winningdesigns_time.csv' and returns the filename
'''
starting_time = time.time()
fastest_done = None
m = Master()
#m.connect('gibson.thep.lu.se', 'science')
m.connect('130.235.189.249', 'science')
print('Connected to server')
m.clear_queues()
print('\nIncluding columns: ' + str(columns))
print('\nTarget columns: ' + str(targets))
P, T = parse_file(filename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
if testfilename is not None:
Ptest, Ttest = parse_file(testfilename,
targetcols=targets,
inputcols=columns,
normalize=True,
separator=separator,
use_header=True)
else:
Ptest, Ttest = None, None
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
print("T:" + str(T.shape))
print("P:" + str(P.shape))
if (Ptest is not None and Ttest is not None):
print("\nExternal Test Data set:")
print("Number of patients with events: " + str(Ttest[:, 1].sum()))
print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
print("Ttest:" + str(Ttest.shape))
print("Ptest:" + str(Ptest.shape))
comsize = 3 * comsize_third #Make sure it is divisible by three
print('\nNumber of members in each committee: ' + str(comsize))
print('Designs used in testing (size, function): ' + str(designs))
# We can generate a test set from the data set, but usually we don't want that
# Leave at 1 for no test set.
val_pieces = 1
print('Cross-test pieces: ' + str(val_pieces))
cross_times = repeat_times
print('Number of times to repeat procedure: ' + str(cross_times))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
if 'population_size' not in train_kwargs:
train_kwargs['population_size'] = 50
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
if 'mutation_chance' not in train_kwargs:
train_kwargs['mutation_chance'] = 0.25
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
if 'epochs' not in train_kwargs:
train_kwargs['epochs'] = 100
for k, v in train_kwargs.iteritems():
print(str(k) + ": " + str(v))
print('\n Job status:\n')
count = 0
all_counts = []
all_jobs = {}
tests = {}
#trn_set = {}
trn_idx = {}
all_best = []
all_best_com_val = []
all_best_avg_trn = []
all_best_avg_val = []
all_best_design = []
all_best_test = []
#Lambda times
for _time in xrange(cross_times):
#Get an independant test set, 1/tau of the total.
super_set, super_indices = get_cross_validation_sets(
P, T, val_pieces, binary_column=1, return_indices=True)
super_zip = zip(super_set, super_indices)
all_best.append({})
all_best_com_val.append({})
all_best_avg_trn.append({})
all_best_avg_val.append({})
all_best_design.append({})
all_best_test.append({})
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
#For every blind test group
for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
_t) in zip(super_zip, xrange(len(super_set))):
TRN_INPUTS = TRN[0]
TRN_TARGETS = TRN[1]
TEST_INPUTS = TEST[0]
TEST_TARGETS = TEST[1]
#run each architecture design on a separate machine
best[_t] = None
best_com_val[_t] = 0
best_avg_trn[_t] = 0
best_avg_val[_t] = 0
best_design[_t] = None
best_test[_t] = None
for design in designs:
count += 1
all_counts.append(count)
(netsize, hidden_func) = design
com = build_feedforward_committee(comsize,
len(P[0]),
netsize,
1,
hidden_function=hidden_func,
output_function='linear')
tests[count] = (TEST_INPUTS, TEST_TARGETS)
#trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
#print("TRN_IDX" + str(TRN_IDX))
#print("TEST_IDX" + str(TEST_IDX))
trn_idx[count] = TRN_IDX
#1 is the column in the target array which holds the binary censoring information
job = m.assemblejob((count, _time, _t, design),
train_committee,
com,
train_evolutionary,
TRN_INPUTS,
TRN_TARGETS,
binary_target=1,
error_function=c_index_error,
**train_kwargs)
all_jobs[count] = job
m.sendjob(job[0], job[1], *job[2], **job[3])
while (count > 0):
print('Remaining jobs: {0}'.format(all_counts))
if fastest_done is None:
ID, RESULT = m.getresult() #Blocks
fastest_done = time.time() - starting_time
else:
RETURNVALUE = m.get_waiting_result(2 * fastest_done)
if RETURNVALUE is not None:
ID, RESULT = RETURNVALUE
else:
print(
'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
You should restart the server after this session.'.format(
fastest_done, all_counts))
for _c in all_counts:
job = all_jobs[_c]
m.sendjob(job[0], job[1], *job[2], **job[3])
continue #Jump to next iteration
print('Result received! Processing...')
_c, _time, _t, design = ID
(com, trn_errors, vald_errors, internal_sets,
internal_sets_indices) = RESULT
if _c not in all_counts:
print('This result [{0}] has already been processed.'.format(_c))
continue
count -= 1
TEST_INPUTS, TEST_TARGETS = tests[_c]
#TRN_INPUTS, TRN_TARGETS = trn_set[_c]
TRN_IDX = trn_idx[_c]
all_counts.remove(_c)
com.set_training_sets([
_set[0][0] for _set in internal_sets
]) #first 0 gives training sets, second 0 gives inputs.
#Now what we'd like to do is get the value for each patient in the
#validation set, for all validation sets. Then I'd like to average the
#result for each such patient, over the different validation sets.
allpats = []
allpats.extend(internal_sets[0][0][0]) #Extend with training inputs
allpats.extend(internal_sets[0][1][0]) #Extend with validation inputs
allpats_targets = []
allpats_targets.extend(internal_sets[0][0][1]) #training targets
allpats_targets.extend(internal_sets[0][1][1]) #validation targets
allpats_targets = numpy.array(allpats_targets)
patvals = [[] for bah in xrange(len(allpats))]
#print(len(patvals))
#print(len(internal_sets_indices))
#1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
# Will be order consistent with P and T
for ((trn_in, trn_tar),
(val_in, val_tar)), idx, net in zip(internal_sets,
internal_sets_indices,
com.nets):
_C_ = -1
for valpat in val_in:
_C_ += 1
i = TRN_IDX[idx[1][_C_]]
pat = P[i]
#print("Facit: \n" + str(valpat))
#print("_C_ = " + str(_C_))
#print("i: " + str(i))
#print("P[TRN_IDX[i]] : " + str(pat))
assert ((pat == valpat).all())
patvals[i].append(com.risk_eval(pat, net=net))
#Need double brackets for dimensions to fit C-module
avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
#Now we have average validation ranks. do C-index on this
avg_val_c_index = get_C_index(T, avg_vals)
trn_errors = numpy.array(trn_errors.values(), dtype=numpy.float64)**-1
vald_errors = numpy.array(vald_errors.values(),
dtype=numpy.float64)**-1
avg_trn = numpy.mean(trn_errors)
avg_val = numpy.mean(vald_errors)
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
if avg_val_c_index > best_com_val[_t]:
best[_t] = com
best_com_val[_t] = avg_val_c_index
best_avg_trn[_t] = avg_trn
best_avg_val[_t] = avg_val
best_design[_t] = design
best_test[_t] = tests[_c]
print('\nWinning designs')
winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
with open(winnerfilename, 'w') as F:
print(
'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:'
)
F.write(
'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n'
)
for _time in xrange(len(all_best)):
best = all_best[_time]
best_com_val = all_best_com_val[_time]
best_avg_trn = all_best_avg_trn[_time]
best_avg_val = all_best_avg_val[_time]
best_design = all_best_design[_time]
best_test = all_best_test[_time]
for _t in best.keys():
TEST_INPUTS, TEST_TARGETS = best_test[_t]
com = best[_t]
if len(TEST_INPUTS) > 0:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)]
for inputs in TEST_INPUTS])
test_c_index = get_C_index(TEST_TARGETS, outputs)
elif Ptest is not None and Ttest is not None:
#Need double brackets for dimensions to be right for numpy
outputs = numpy.array([[com.risk_eval(inputs)]
for inputs in Ptest])
test_c_index = get_C_index(Ttest, outputs)
else:
test_c_index = 0
print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(
trn=best_avg_trn[_t],
val=best_avg_val[_t],
com_val=best_com_val[_t],
test=test_c_index,
dsn=best_design[_t]))
F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(
trn=best_avg_trn[_t],
val=best_avg_val[_t],
com_val=best_com_val[_t],
test=test_c_index,
dsn=best_design[_t]))
return winnerfilename
