def com_cross():
filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset.txt"
#try:
# columns = input("Which columns to include? (Do NOT forget trailing comma if only one column is used, e.g. '3,'\nAvailable columns are: 2, -4, -3, -2, -1. Just press ENTER for all columns.\n")
#except SyntaxError:
#if len(sys.argv) < 3:
columns = (2, -4, -3, -2, -1)
#else:
# columns = [int(col) for col in sys.argv[2:]]
print('\nIncluding columns: ' + str(columns))
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
#print('\nRemoving tail censored...')
#P, T = copy_without_censored(P, T)
#Divide into validation sets
#test_size = 0.33
#print('Size of test set (not used in training): ' + str(test_size))
#((TP, TT), (VP, VT)) = get_validation_set(P, T, validation_size = test_size, binary_column = 1)
print("\nData set:")
print("Number of patients with events: " + str(T[:, 1].sum()))
print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
#print("Length of training set: " + str(len(TP)))
#print("Length of test set: " + str(len(VP)))
#try:
# comsize = input("Number of networks to cross-validate [10]: ")
#except SyntaxError:
if len(sys.argv) < 2:
netsize = 1
else:
netsize = int(sys.argv[1])
print("\nNumber of hidden nodes: " + str(netsize))
comsize = 4
print('Number of members in each committee: ' + str(comsize))
comnum = 5
print('Number of committees to cross-validate: ' + str(comnum))
times_to_cross = 3
print('Number of times to repeat cross-validation: ' + str(times_to_cross))
#try:
# pop_size = input('Population size [50]: ')
#except SyntaxError as e:
pop_size = 100
print("Population size: " + str(pop_size))
#try:
# mutation_rate = input('Please input a mutation rate (0.25): ')
#except SyntaxError as e:
mutation_rate = 0.05
print("Mutation rate: " + str(mutation_rate))
#try:
# epochs = input("Number of generations (200): ")
#except SyntaxError as e:
epochs = 100
print("Epochs: " + str(epochs))
for _cross_time in xrange(times_to_cross):
data_sets = get_cross_validation_sets(P, T, comnum , binary_column = 1)
print('\nTest Errors, Validation Errors:')
for _com_num, (TS, VS) in zip(xrange(comnum), data_sets):
com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, output_function = 'linear')
#1 is the column in the target array which holds the binary censoring information
test_errors, vald_errors, internal_sets = train_committee(com, train_evolutionary, TS[0], TS[1], 1, epochs, error_function = c_index_error, population_size = pop_size, mutation_chance = mutation_rate)
com.set_training_sets([set[0][0] for set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TS[0]]) #Need double brackets for dimensions to be right for numpy
train_c_index = get_C_index(TS[1], outputs)
outputs = numpy.array([[com.risk_eval(inputs)] for inputs in VS[0]]) #Need double brackets for dimensions to be right for numpy
val_c_index = get_C_index(VS[1], outputs)
print(str(1.0 / train_c_index) + ", " + str(1.0 / val_c_index))
def train_single():
try:
netsize = input('Number of hidden nodes? [3]: ')
except SyntaxError as e:
netsize = 3
try:
pop_size = input('Population size? [50]: ')
except SyntaxError as e:
pop_size = 50
try:
mutation_rate = input('Please input a mutation rate (0.25): ')
except SyntaxError as e:
mutation_rate = 0.25
SB22 = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_SB22.txt"
Benmargskohorten = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_Benmargskohorten.txt"
SB91b = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_SB91b.txt"
all_studies = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset.txt"
#Real data
print("Studies to choose from:")
print("1: SB22")
print("2: Benmargskohorten")
print("3: SB91b")
print("0: All combined (default)")
try:
study = input("Which study to train on? [0]: ")
except SyntaxError as e:
study = 0
if study == 1:
filename = SB22
elif study == 2:
filename = Benmargskohorten
elif study == 3:
filename = SB91b
else:
filename = all_studies
try:
columns = input("Which columns to include? (Do NOT forget trailing comma if only one column is used, e.g. '3,'\nAvailable columns are: 2, -4, -3, -2, -1. Just press ENTER for all columns.\n")
except SyntaxError:
columns = (2, -4, -3, -2, -1)
#P, T = parse_file(filename, targetcols = [4, 5], inputcols = [2, -4, -3, -2, -1], ignorerows = [0], normalize = True)
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#Used for output comparison
studies = {}
studies[SB22] = parse_file(SB22, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[Benmargskohorten] = parse_file(Benmargskohorten, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[SB91b] = parse_file(SB91b, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[all_studies] = parse_file(all_studies, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
#P, T = copy_without_tailcensored(P, T)
#Divide into validation sets
#((tP, tT), (vP, vT)) = get_validation_set(P, T, validation_size = 0.25, binary_column = 1)
TandV = get_cross_validation_sets(P, T, 2 , binary_column = 1)
#Network part
p = len(P[0]) #number of input covariates
net = build_feedforward(p, netsize, 1, output_function = 'linear')
#net = build_feedforward_multilayered(p, [7, 10], 1, output_function = 'linear')
try:
epochs = input("Number of generations (200): ")
except SyntaxError as e:
epochs = 200
for times, ((tP, tT), (vP, vT)) in zip(xrange(2), TandV):
#train
net = test(net, tP, tT, vP, vT, filename, epochs, population_size = pop_size, mutation_rate = mutation_rate)
raw_input("Press enter to show plots...")
glogger.show()
#except SyntaxError:
columns = (2, -4, -3, -2, -1)
print('\nIncluding columns: ' + str(columns))
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
#print('\nRemoving tail censored...')
#P, T = copy_without_tailcensored(P, T)
try:
pieces = input('Number of crossvalidation pieces? [1]: ')
except SyntaxError as e:
pieces = 1
#Divide into validation sets
TandV = get_cross_validation_sets(P, T, pieces , binary_column = 1)
for set, ((tP, tT), (vP, vT)) in zip(range(pieces), TandV):
print("\nCross validation set " + str(set))
print("Training")
print("Number of patients with events: " + str(tT[:, 1].sum()))
print("Number of censored patients: " + str((1 - tT[:, 1]).sum()))
print("Validation")
print("Number of patients with events: " + str(vT[:, 1].sum()))
print("Number of censored patients: " + str((1 - vT[:, 1]).sum()))
try:
netsize = input('\nNumber of hidden nodes? [3]: ')
except SyntaxError as e:
netsize = 3
def train_single():
try:
netsize = input('Number of hidden nodes? [1]: ')
except SyntaxError as e:
netsize = 1
try:
pop_size = input('Population size? [100]: ')
except SyntaxError as e:
pop_size = 100
try:
mutation_rate = input('Please input a mutation rate (0.05): ')
except SyntaxError as e:
mutation_rate = 0.05
SB22 = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_SB22.txt"
Benmargskohorten = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_Benmargskohorten.txt"
SB91b = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset_SB91b.txt"
all_studies = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_SA_1889_dataset.txt"
#Real data
print("Studies to choose from:")
print("1: SB22")
print("2: Benmargskohorten")
print("3: SB91b")
print("0: All combined (default)")
try:
study = input("Which study to train on? [0]: ")
except SyntaxError as e:
study = 0
if study == 1:
filename = SB22
elif study == 2:
filename = Benmargskohorten
elif study == 3:
filename = SB91b
else:
filename = all_studies
try:
columns = input("Which columns to include? (Do NOT forget trailing comma if only one column is used, e.g. '3,'\nAvailable columns are: 2, -4, -3, -2, -1. Just press ENTER for all columns.\n")
except SyntaxError:
columns = (2, -4, -3, -2, -1)
#P, T = parse_file(filename, targetcols = [4, 5], inputcols = [2, -4, -3, -2, -1], ignorerows = [0], normalize = True)
P, T = parse_file(filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#Used for output comparison
studies = {}
studies[SB22] = parse_file(SB22, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[Benmargskohorten] = parse_file(Benmargskohorten, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[SB91b] = parse_file(SB91b, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
studies[all_studies] = parse_file(all_studies, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
#remove tail censored
try:
cutoff = input('Cutoff for censored data? [9999 years]: ')
except SyntaxError as e:
cutoff = 9999
P, T = copy_without_censored(P, T, cutoff)
#Divide into validation sets
try:
pieces = input('Size of validation set? Input denominator (1 for no validation set). Default is 1/[1] parts: ')
except:
pieces = 1
TandV = get_cross_validation_sets(P, T, pieces , binary_column = 1)
#Network part
p = len(P[0]) #number of input covariates
net = build_feedforward(p, netsize, 1, output_function = 'linear')
#net = build_feedforward_multilayered(p, [7, 10], 1, output_function = 'linear')
#Initial state
#outputs = net.sim(tP)
#orderscatter(outputs, tT, filename, 's')
try:
epochs = input("Number of generations (1): ")
except SyntaxError as e:
epochs = 1
for ((tP, tT), (vP, vT)) in TandV:
#train
net = test(net, tP, tT, vP, vT, filename, epochs, population_size = pop_size, mutation_rate = mutation_rate)
if plt:
outputs = net.sim(tP)
threshold = kaplanmeier(time_array = tT[:, 0], event_array = tT[:, 1], output_array = outputs[:, 0])
if len(vP) > 0:
outputs = net.sim(vP)
kaplanmeier(time_array = vT[:, 0], event_array = vT[:, 1], output_array = outputs[:, 0], threshold = threshold)
print("\nThreshold dividing the training set in two equal pieces: " + str(threshold))
raw_input("\nPress enter to show plots...")
plt.show()
try:
answer = input("Do you wish to print network output? Enter filename, or 'no' / 'n'. ['n']: ")
except (SyntaxError, NameError):
answer = 'n'
if os.path.exists(answer):
print("File exists. Will add random number to front")
answer = str(random.randint(0, 123456)) + answer
if answer != 'n' and answer != 'no':
print_output(answer, net, filename, targetcols = [4, 5], inputcols = columns, ignorerows = [0], normalize = True)
