def train_model(design,
                filename,
                columns,
                targets,
                comsize_third=20,
                separator='\t',
                **train_kwargs):
    '''
    train_model(design, filename, columns, targets)

    Given a design, will train a committee like that on the data specified. Will save the committee as
    '.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
    Returns this filename
    '''
    starting_time = time.time()
    fastest_done = None
    m = Master()

    #m.connect('gibson.thep.lu.se', 'science')
    m.connect('130.235.189.249', 'science')
    print('Connected to server')
    m.clear_queues()

    savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes=design[0],
                                                          a_func=design[1],
                                                          time=time.time())

    print('\nIncluding columns: ' + str(columns))
    print('Target columns: ' + str(targets))

    P, T = parse_file(filename,
                      targetcols=targets,
                      inputcols=columns,
                      normalize=True,
                      separator=separator,
                      use_header=True)

    #columns = (2, -6, -5, -4, -3, -2, -1)
    #_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
    #P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)

    print("\nData set:")
    print("Number of patients with events: " + str(T[:, 1].sum()))
    print("Number of censored patients: " + str((1 - T[:, 1]).sum()))

    comsize = 3 * comsize_third  #Make sure it is divisible by three (3*X will create X jobs)
    print('Number of members in the committee: ' + str(comsize))

    print('Design used (size, function): ' + str(design))

    #try:
    #    pop_size = input('Population size [50]: ')
    #except SyntaxError as e:
    if 'population_size' not in train_kwargs:
        train_kwargs['population_size'] = 200
    #print("Population size: " + str(train_kwargs['population_size']))

    #try:
    #    mutation_rate = input('Please input a mutation rate (0.25): ')
    #except SyntaxError as e:
    if 'mutation_chance' not in train_kwargs:
        train_kwargs['mutation_chance'] = 0.25
    #print("Mutation rate: " + str(train_kwargs['mutation_chance']))

    #try:
    #    epochs = input("Number of generations (200): ")
    #except SyntaxError as e:
    if 'epochs' not in train_kwargs:
        train_kwargs['epochs'] = 100

    for k, v in train_kwargs.iteritems():
        print(str(k) + ": " + str(v))

    #errorfunc = weighted_c_index_error
    errorfunc = c_index_error

    print("\nError function: " + errorfunc.__name__)

    print('\n Job status:\n')

    count = 0
    all_counts = []
    all_jobs = {}
    #trn_set = {}
    trn_idx = {}

    master_com = None

    allpats = P.copy()
    #allpats[:, 1] = 1 #This is the event column

    allpats_targets = T

    patvals = [[] for bah in xrange(len(allpats))]

    #Lambda times
    for _time in xrange(1):
        #Get an independant test set, 1/tau of the total.
        super_set, super_indices = get_cross_validation_sets(
            P, T, 1, binary_column=1, return_indices=True)
        super_zip = zip(super_set, super_indices)
        #For every blind test group
        for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
             _t) in zip(super_zip, xrange(len(super_set))):
            TRN_INPUTS = TRN[0]
            TRN_TARGETS = TRN[1]
            #TEST_INPUTS = TEST[0]
            #TEST_TARGETS = TEST[1]

            for com_num in xrange(comsize / 3):

                count += 1
                all_counts.append(count)

                #trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
                trn_idx[count] = TRN_IDX

                (netsize, hidden_func) = design

                com = build_feedforward_committee(3,
                                                  len(P[0]),
                                                  netsize,
                                                  1,
                                                  hidden_function=hidden_func,
                                                  output_function='linear')

                #1 is the column in the target array which holds the binary censoring information

                job = m.assemblejob((count, _time, _t, design),
                                    train_committee,
                                    com,
                                    train_evolutionary,
                                    TRN_INPUTS,
                                    TRN_TARGETS,
                                    binary_target=1,
                                    error_function=errorfunc,
                                    **train_kwargs)

                all_jobs[count] = job

                m.sendjob(job[0], job[1], *job[2], **job[3])

    #TIME TO RECEIVE THE RESULTS
    while (count > 0):
        print('Remaining jobs: {0}'.format(all_counts))
        if fastest_done is None:
            ID, RESULT = m.getresult()  #Blocks
            fastest_done = time.time() - starting_time
        else:
            RETURNVALUE = m.get_waiting_result(2 * fastest_done)
            if RETURNVALUE is not None:
                ID, RESULT = RETURNVALUE
            else:
                print(
                    'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
                the server after this session.'.format(fastest_done,
                                                       all_counts))
                for _c in all_counts:
                    job = all_jobs[_c]
                    m.sendjob(job[0], job[1], *job[2], **job[3])
                continue  #Jump to next iteration

        print('Result received! Processing...')
        _c, _time, _t, design = ID

        (com, trn_errors, vald_errors, internal_sets,
         internal_sets_indices) = RESULT

        if _c not in all_counts:
            print('This result [{0}] has already been processed.'.format(_c))
            continue

        count -= 1

        #TRN_INPUTS, TRN_TARGETS = trn_set[_c]
        TRN_IDX = trn_idx[_c]

        all_counts.remove(_c)

        com.set_training_sets([
            _set[0][0] for _set in internal_sets
        ])  #first 0 gives training sets, second 0 gives inputs.

        if master_com is None:
            master_com = com
        else:
            master_com.nets.extend(com.nets)  #Add this batch of networks

        #Now what we'd like to do is get the value for each patient in the
        #validation set, for all validation sets. Then I'd like to average the
        #result for each such patient, over the different validation sets.

        #1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
        # patvals will be order-consistent with P and T
        #for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
        #    for i in val_set_indices:
        #        patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))

        for ((trn_in, trn_tar),
             (val_in, val_tar)), idx, net in zip(internal_sets,
                                                 internal_sets_indices,
                                                 com.nets):
            _C_ = -1
            for valpat in val_in:
                _C_ += 1
                i = TRN_IDX[idx[1][_C_]]
                pat = P[i]
                #print("Facit: \n" + str(valpat))
                #print("_C_ = " + str(_C_))
                #print("i: " + str(i))
                #print("P[TRN_IDX[i]] : " + str(pat))
                assert ((pat == valpat).all())
                patvals[i].append(com.risk_eval(pat, net=net))

        #for pat, i in zip(allpats, xrange(len(patvals))):
        #We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
        #    for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
        #        _C_ = -1
        #        for valpat in val_in:
        #            _C_ += 1
        #            if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
        #print("Facit: \n" + str(valpat))
        #print("Allpats-index = " + str(i))
        #print("_C_ = " + str(_C_))
        #print("idx_val[_C_]: " + str(idx[1][_C_]))
        #print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
        #print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
        #                patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
        #                break #Done with this data_set

        avg_vals = numpy.array([
            [numpy.mean(patval)] for patval in patvals
        ])  #Need  double brackets for dimensions to fit C-module
        #Now we have average validation ranks. do C-index on this
        avg_val_c_index = get_C_index(allpats_targets, avg_vals)
        print('Average com-validation C-Index so far      : {0}'.format(
            avg_val_c_index))
        print('Saving committee so far in {0}'.format(savefile))
        with open(savefile, 'w') as FILE:
            pickle.dump(master_com, FILE)

    return savefile
def model_contest(filename, columns, targets, designs, comsize_third = 5, repeat_times = 20, testfilename = None, separator = '\t', **train_kwargs):
    '''
    model_contest(filename, columns, targets, designs)
    
    You must use column names! Here are example values for the input arguments:
        
    filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
    columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
               'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
    targets = ['time', 'event']
    
    Writes the results to '.winningdesigns_time.csv' and returns the filename
    '''

    starting_time = time.time()
    fastest_done = None
    m = Master()

    #m.connect('gibson.thep.lu.se', 'science')
    m.connect('130.235.189.249', 'science')
    print('Connected to server')
    m.clear_queues()

    print('\nIncluding columns: ' + str(columns))
    print('\nTarget columns: ' + str(targets))

    P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
                      use_header = True)

    if testfilename is not None:
        Ptest, Ttest = parse_file(testfilename, targetcols = targets, inputcols = columns, normalize = True, separator = separator,
                      use_header = True)
    else:
        Ptest, Ttest = None, None

    print("\nData set:")
    print("Number of patients with events: " + str(T[:, 1].sum()))
    print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
    print("T:" + str(T.shape))
    print("P:" + str(P.shape))
    if (Ptest is not None and Ttest is not None):
        print("\nExternal Test Data set:")
        print("Number of patients with events: " + str(Ttest[:, 1].sum()))
        print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
        print("Ttest:" + str(Ttest.shape))
        print("Ptest:" + str(Ptest.shape))

    comsize = 3 * comsize_third #Make sure it is divisible by three
    print('\nNumber of members in each committee: ' + str(comsize))

    print('Designs used in testing (size, function): ' + str(designs))

    # We can generate a test set from the data set, but usually we don't want that
    # Leave at 1 for no test set.
    val_pieces = 1
    print('Cross-test pieces: ' + str(val_pieces))

    cross_times = repeat_times
    print('Number of times to repeat procedure: ' + str(cross_times))

    #try:
    #    pop_size = input('Population size [50]: ')
    #except SyntaxError as e:
    if 'population_size' not in train_kwargs:
        train_kwargs['population_size'] = 50

    #try:
    #    mutation_rate = input('Please input a mutation rate (0.25): ')
    #except SyntaxError as e:
    if 'mutation_chance' not in train_kwargs:
        train_kwargs['mutation_chance'] = 0.25

    #try:
    #    epochs = input("Number of generations (200): ")
    #except SyntaxError as e:
    if 'epochs' not in train_kwargs:
        train_kwargs['epochs'] = 100

    for k, v in train_kwargs.iteritems():
        print(str(k) + ": " + str(v))

    print('\n Job status:\n')

    count = 0
    all_counts = []
    all_jobs = {}

    tests = {}
    #trn_set = {}
    trn_idx = {}
    all_best = []
    all_best_com_val = []
    all_best_avg_trn = []
    all_best_avg_val = []
    all_best_design = []
    all_best_test = []

    #Lambda times
    for _time in xrange(cross_times):
        #Get an independant test set, 1/tau of the total.
        super_set, super_indices = get_cross_validation_sets(P, T, val_pieces , binary_column = 1, return_indices = True)
        super_zip = zip(super_set, super_indices)

        all_best.append({})
        all_best_com_val.append({})
        all_best_avg_trn.append({})
        all_best_avg_val.append({})
        all_best_design.append({})
        all_best_test.append({})

        best = all_best[_time]
        best_com_val = all_best_com_val[_time]
        best_avg_trn = all_best_avg_trn[_time]
        best_avg_val = all_best_avg_val[_time]
        best_design = all_best_design[_time]
        best_test = all_best_test[_time]


        #For every blind test group
        for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
            TRN_INPUTS = TRN[0]
            TRN_TARGETS = TRN[1]
            TEST_INPUTS = TEST[0]
            TEST_TARGETS = TEST[1]

            #run each architecture design on a separate machine
            best[_t] = None
            best_com_val[_t] = 0
            best_avg_trn[_t] = 0
            best_avg_val[_t] = 0
            best_design[_t] = None
            best_test[_t] = None

            for design in designs:
                count += 1
                all_counts.append(count)

                (netsize, hidden_func) = design

                com = build_feedforward_committee(comsize, len(P[0]), netsize, 1, hidden_function = hidden_func,
                                                  output_function = 'linear')

                tests[count] = (TEST_INPUTS, TEST_TARGETS)
                #trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
                #print("TRN_IDX" + str(TRN_IDX))
                #print("TEST_IDX" + str(TEST_IDX))
                trn_idx[count] = TRN_IDX

                #1 is the column in the target array which holds the binary censoring information

                job = m.assemblejob((count, _time, _t, design),
                        train_committee, com, train_evolutionary, TRN_INPUTS,
                        TRN_TARGETS, binary_target = 1, error_function = c_index_error, **train_kwargs)

                all_jobs[count] = job

                m.sendjob(job[0], job[1], *job[2], **job[3])

    while(count > 0):
        print('Remaining jobs: {0}'.format(all_counts))
        if fastest_done is None:
            ID, RESULT = m.getresult() #Blocks
            fastest_done = time.time() - starting_time
        else:
            RETURNVALUE = m.get_waiting_result(2 * fastest_done)
            if RETURNVALUE is not None:
                ID, RESULT = RETURNVALUE
            else:
                print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
                You should restart the server after this session.'.format(fastest_done, all_counts))
                for _c in all_counts:
                    job = all_jobs[_c]
                    m.sendjob(job[0], job[1], *job[2], **job[3])
                continue #Jump to next iteration

        print('Result received! Processing...')
        _c, _time, _t, design = ID

        (com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT

        if _c not in all_counts:
            print('This result [{0}] has already been processed.'.format(_c))
            continue

        count -= 1

        TEST_INPUTS, TEST_TARGETS = tests[_c]
        #TRN_INPUTS, TRN_TARGETS = trn_set[_c]
        TRN_IDX = trn_idx[_c]

        all_counts.remove(_c)

        com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.

        #Now what we'd like to do is get the value for each patient in the
        #validation set, for all validation sets. Then I'd like to average the
        #result for each such patient, over the different validation sets.

        allpats = []
        allpats.extend(internal_sets[0][0][0]) #Extend with training inputs
        allpats.extend(internal_sets[0][1][0]) #Extend with validation inputs

        allpats_targets = []
        allpats_targets.extend(internal_sets[0][0][1]) #training targets
        allpats_targets.extend(internal_sets[0][1][1]) #validation targets
        allpats_targets = numpy.array(allpats_targets)

        patvals = [[] for bah in xrange(len(allpats))]

        #print(len(patvals))
        #print(len(internal_sets_indices))
        #1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
        # Will be order consistent with P and T
        for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
            _C_ = -1
            for valpat in val_in:
                _C_ += 1
                i = TRN_IDX[idx[1][_C_]]
                pat = P[i]
                #print("Facit: \n" + str(valpat))
                #print("_C_ = " + str(_C_))
                #print("i: " + str(i))
                #print("P[TRN_IDX[i]] : " + str(pat))
                assert((pat == valpat).all())
                patvals[i].append(com.risk_eval(pat, net = net))

        #Need  double brackets for dimensions to fit C-module
        avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
        #Now we have average validation ranks. do C-index on this
        avg_val_c_index = get_C_index(T, avg_vals)

        trn_errors = numpy.array(trn_errors.values(), dtype = numpy.float64) ** -1
        vald_errors = numpy.array(vald_errors.values(), dtype = numpy.float64) ** -1
        avg_trn = numpy.mean(trn_errors)
        avg_val = numpy.mean(vald_errors)

        best = all_best[_time]
        best_com_val = all_best_com_val[_time]
        best_avg_trn = all_best_avg_trn[_time]
        best_avg_val = all_best_avg_val[_time]
        best_design = all_best_design[_time]
        best_test = all_best_test[_time]

        if avg_val_c_index > best_com_val[_t]:
            best[_t] = com
            best_com_val[_t] = avg_val_c_index
            best_avg_trn[_t] = avg_trn
            best_avg_val[_t] = avg_val
            best_design[_t] = design
            best_test[_t] = tests[_c]


    print('\nWinning designs')
    winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
    with open(winnerfilename, 'w') as F:
        print('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:')
        F.write('Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n')
        for _time in xrange(len(all_best)):
            best = all_best[_time]
            best_com_val = all_best_com_val[_time]
            best_avg_trn = all_best_avg_trn[_time]
            best_avg_val = all_best_avg_val[_time]
            best_design = all_best_design[_time]
            best_test = all_best_test[_time]
            for _t in best.keys():
                TEST_INPUTS, TEST_TARGETS = best_test[_t]
                com = best[_t]

                if len(TEST_INPUTS) > 0:
                    #Need double brackets for dimensions to be right for numpy
                    outputs = numpy.array([[com.risk_eval(inputs)] for inputs in TEST_INPUTS])
                    test_c_index = get_C_index(TEST_TARGETS, outputs)
                elif Ptest is not None and Ttest is not None:
                    #Need double brackets for dimensions to be right for numpy
                    outputs = numpy.array([[com.risk_eval(inputs)] for inputs in Ptest])
                    test_c_index = get_C_index(Ttest, outputs)
                else:
                    test_c_index = 0

                print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
                      com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))
                F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(trn = best_avg_trn[_t], val = best_avg_val[_t],
                        com_val = best_com_val[_t], test = test_c_index, dsn = best_design[_t]))

    return winnerfilename
def train_model(design, filename, columns, targets, comsize_third = 20, separator = '\t', **train_kwargs):
    '''
    train_model(design, filename, columns, targets)

    Given a design, will train a committee like that on the data specified. Will save the committee as
    '.design_time.pcom' where design is replaced by the design and time is replaced by a string of numbers from time()
    Returns this filename
    '''
    starting_time = time.time()
    fastest_done = None
    m = Master()

    #m.connect('gibson.thep.lu.se', 'science')
    m.connect('130.235.189.249', 'science')
    print('Connected to server')
    m.clear_queues()

    savefile = ".{nodes}_{a_func}_{time:.0f}.pcom".format(nodes = design[0], a_func = design[1], time = time.time())

    print('\nIncluding columns: ' + str(columns))
    print('Target columns: ' + str(targets))

    P, T = parse_file(filename, targetcols = targets, inputcols = columns, normalize = True, separator = separator, use_header = True)

    #columns = (2, -6, -5, -4, -3, -2, -1)
    #_P, T = parse_file(filename, targetcols = [4, 5], inputcols = (2, -4, -3, -2, -1), ignorerows = [0], normalize = True)
    #P, _T = parse_file(filename, targetcols = [4], inputcols = columns, ignorerows = [0], normalize = True)

    print("\nData set:")
    print("Number of patients with events: " + str(T[:, 1].sum()))
    print("Number of censored patients: " + str((1 - T[:, 1]).sum()))

    comsize = 3 * comsize_third #Make sure it is divisible by three (3*X will create X jobs)
    print('Number of members in the committee: ' + str(comsize))

    print('Design used (size, function): ' + str(design))

    #try:
    #    pop_size = input('Population size [50]: ')
    #except SyntaxError as e:
    if 'population_size' not in train_kwargs:
        train_kwargs['population_size'] = 200
    #print("Population size: " + str(train_kwargs['population_size']))

    #try:
    #    mutation_rate = input('Please input a mutation rate (0.25): ')
    #except SyntaxError as e:
    if 'mutation_chance' not in train_kwargs:
        train_kwargs['mutation_chance'] = 0.25
    #print("Mutation rate: " + str(train_kwargs['mutation_chance']))

    #try:
    #    epochs = input("Number of generations (200): ")
    #except SyntaxError as e:
    if 'epochs' not in train_kwargs:
        train_kwargs['epochs'] = 100

    for k, v in train_kwargs.iteritems():
        print(str(k) + ": " + str(v))

    #errorfunc = weighted_c_index_error
    errorfunc = c_index_error

    print("\nError function: " + errorfunc.__name__)

    print('\n Job status:\n')

    count = 0
    all_counts = []
    all_jobs = {}
    #trn_set = {}
    trn_idx = {}

    master_com = None

    allpats = P.copy()
    #allpats[:, 1] = 1 #This is the event column

    allpats_targets = T

    patvals = [[] for bah in xrange(len(allpats))]

    #Lambda times
    for _time in xrange(1):
        #Get an independant test set, 1/tau of the total.
        super_set, super_indices = get_cross_validation_sets(P, T, 1, binary_column = 1, return_indices = True)
        super_zip = zip(super_set, super_indices)
        #For every blind test group
        for (((TRN, TEST), (TRN_IDX, TEST_IDX)), _t) in zip(super_zip, xrange(len(super_set))):
            TRN_INPUTS = TRN[0]
            TRN_TARGETS = TRN[1]
            #TEST_INPUTS = TEST[0]
            #TEST_TARGETS = TEST[1]

            for com_num in xrange(comsize / 3):

                count += 1
                all_counts.append(count)

                #trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
                trn_idx[count] = TRN_IDX

                (netsize, hidden_func) = design

                com = build_feedforward_committee(3, len(P[0]), netsize, 1, hidden_function = hidden_func, output_function = 'linear')

                #1 is the column in the target array which holds the binary censoring information

                job = m.assemblejob((count, _time, _t, design),
                    train_committee, com, train_evolutionary, TRN_INPUTS,
                    TRN_TARGETS, binary_target = 1, error_function = errorfunc,
                    **train_kwargs)

                all_jobs[count] = job

                m.sendjob(job[0], job[1], *job[2], **job[3])

    #TIME TO RECEIVE THE RESULTS
    while(count > 0):
        print('Remaining jobs: {0}'.format(all_counts))
        if fastest_done is None:
            ID, RESULT = m.getresult() #Blocks
            fastest_done = time.time() - starting_time
        else:
            RETURNVALUE = m.get_waiting_result(2 * fastest_done)
            if RETURNVALUE is not None:
                ID, RESULT = RETURNVALUE
            else:
                print('Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\nYou should restart \
                the server after this session.'.format(fastest_done, all_counts))
                for _c in all_counts:
                    job = all_jobs[_c]
                    m.sendjob(job[0], job[1], *job[2], **job[3])
                continue #Jump to next iteration

        print('Result received! Processing...')
        _c, _time, _t, design = ID

        (com, trn_errors, vald_errors, internal_sets, internal_sets_indices) = RESULT

        if _c not in all_counts:
            print('This result [{0}] has already been processed.'.format(_c))
            continue

        count -= 1

        #TRN_INPUTS, TRN_TARGETS = trn_set[_c]
        TRN_IDX = trn_idx[_c]

        all_counts.remove(_c)

        com.set_training_sets([_set[0][0] for _set in internal_sets]) #first 0 gives training sets, second 0 gives inputs.

        if master_com is None:
            master_com = com
        else:
            master_com.nets.extend(com.nets) #Add this batch of networks

        #Now what we'd like to do is get the value for each patient in the
        #validation set, for all validation sets. Then I'd like to average the
        #result for each such patient, over the different validation sets.



        #1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
        # patvals will be order-consistent with P and T
        #for (_trn_set_indices, val_set_indices), net in zip(internal_sets_indices, com.nets):
        #    for i in val_set_indices:
        #        patvals_new[TRN_IDX[i]].append(com.risk_eval(P[TRN_IDX[i]], net = net))

        for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
            _C_ = -1
            for valpat in val_in:
                _C_ += 1
                i = TRN_IDX[idx[1][_C_]]
                pat = P[i]
                #print("Facit: \n" + str(valpat))
                #print("_C_ = " + str(_C_))
                #print("i: " + str(i))
                #print("P[TRN_IDX[i]] : " + str(pat))
                assert((pat == valpat).all())
                patvals[i].append(com.risk_eval(pat, net = net))

        #for pat, i in zip(allpats, xrange(len(patvals))):
            #We could speed this up by only reading every third dataset, but I'm not sure if they are ordered correctly...
        #    for ((trn_in, trn_tar), (val_in, val_tar)), idx, net in zip(internal_sets, internal_sets_indices, com.nets):
        #        _C_ = -1
        #        for valpat in val_in:
        #            _C_ += 1
        #            if (pat == valpat).all(): #Checks each variable individually, all() does a boolean and between the results
                        #print("Facit: \n" + str(valpat))
                        #print("Allpats-index = " + str(i))
                        #print("_C_ = " + str(_C_))
                        #print("idx_val[_C_]: " + str(idx[1][_C_]))
                        #print("TRN_IDX[i]: " + str(TRN_IDX[idx[1][_C_]]))
                        #print("P[TRN_IDX[i]] : " + str(P[TRN_IDX[idx[1][_C_]]]))
        #                patvals[i].append(com.risk_eval(pat, net = net)) #Just to have something to count
        #                break #Done with this data_set

        avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals]) #Need  double brackets for dimensions to fit C-module
        #Now we have average validation ranks. do C-index on this
        avg_val_c_index = get_C_index(allpats_targets, avg_vals)
        print('Average com-validation C-Index so far      : {0}'.format(avg_val_c_index))
        print('Saving committee so far in {0}'.format(savefile))
        with open(savefile, 'w') as FILE:
            pickle.dump(master_com, FILE)

    return savefile
Exemple #4
0
def model_contest(filename,
                  columns,
                  targets,
                  designs,
                  comsize_third=5,
                  repeat_times=20,
                  testfilename=None,
                  separator='\t',
                  **train_kwargs):
    '''
    model_contest(filename, columns, targets, designs)
    
    You must use column names! Here are example values for the input arguments:
        
    filename = "/home/gibson/jonask/Dropbox/Ann-Survival-Phd/Two_thirds_of_the_n4369_dataset_with_logs_lymf.txt"
    columns = ('age', 'log(1+lymfmet)', 'n_pos', 'tumsize', 'log(1+er_cyt)', 'log(1+pgr_cyt)', 'pgr_cyt_pos',
               'er_cyt_pos', 'size_gt_20', 'er_cyt_pos', 'pgr_cyt_pos')
    targets = ['time', 'event']
    
    Writes the results to '.winningdesigns_time.csv' and returns the filename
    '''

    starting_time = time.time()
    fastest_done = None
    m = Master()

    #m.connect('gibson.thep.lu.se', 'science')
    m.connect('130.235.189.249', 'science')
    print('Connected to server')
    m.clear_queues()

    print('\nIncluding columns: ' + str(columns))
    print('\nTarget columns: ' + str(targets))

    P, T = parse_file(filename,
                      targetcols=targets,
                      inputcols=columns,
                      normalize=True,
                      separator=separator,
                      use_header=True)

    if testfilename is not None:
        Ptest, Ttest = parse_file(testfilename,
                                  targetcols=targets,
                                  inputcols=columns,
                                  normalize=True,
                                  separator=separator,
                                  use_header=True)
    else:
        Ptest, Ttest = None, None

    print("\nData set:")
    print("Number of patients with events: " + str(T[:, 1].sum()))
    print("Number of censored patients: " + str((1 - T[:, 1]).sum()))
    print("T:" + str(T.shape))
    print("P:" + str(P.shape))
    if (Ptest is not None and Ttest is not None):
        print("\nExternal Test Data set:")
        print("Number of patients with events: " + str(Ttest[:, 1].sum()))
        print("Number of censored patients: " + str((1 - Ttest[:, 1]).sum()))
        print("Ttest:" + str(Ttest.shape))
        print("Ptest:" + str(Ptest.shape))

    comsize = 3 * comsize_third  #Make sure it is divisible by three
    print('\nNumber of members in each committee: ' + str(comsize))

    print('Designs used in testing (size, function): ' + str(designs))

    # We can generate a test set from the data set, but usually we don't want that
    # Leave at 1 for no test set.
    val_pieces = 1
    print('Cross-test pieces: ' + str(val_pieces))

    cross_times = repeat_times
    print('Number of times to repeat procedure: ' + str(cross_times))

    #try:
    #    pop_size = input('Population size [50]: ')
    #except SyntaxError as e:
    if 'population_size' not in train_kwargs:
        train_kwargs['population_size'] = 50

    #try:
    #    mutation_rate = input('Please input a mutation rate (0.25): ')
    #except SyntaxError as e:
    if 'mutation_chance' not in train_kwargs:
        train_kwargs['mutation_chance'] = 0.25

    #try:
    #    epochs = input("Number of generations (200): ")
    #except SyntaxError as e:
    if 'epochs' not in train_kwargs:
        train_kwargs['epochs'] = 100

    for k, v in train_kwargs.iteritems():
        print(str(k) + ": " + str(v))

    print('\n Job status:\n')

    count = 0
    all_counts = []
    all_jobs = {}

    tests = {}
    #trn_set = {}
    trn_idx = {}
    all_best = []
    all_best_com_val = []
    all_best_avg_trn = []
    all_best_avg_val = []
    all_best_design = []
    all_best_test = []

    #Lambda times
    for _time in xrange(cross_times):
        #Get an independant test set, 1/tau of the total.
        super_set, super_indices = get_cross_validation_sets(
            P, T, val_pieces, binary_column=1, return_indices=True)
        super_zip = zip(super_set, super_indices)

        all_best.append({})
        all_best_com_val.append({})
        all_best_avg_trn.append({})
        all_best_avg_val.append({})
        all_best_design.append({})
        all_best_test.append({})

        best = all_best[_time]
        best_com_val = all_best_com_val[_time]
        best_avg_trn = all_best_avg_trn[_time]
        best_avg_val = all_best_avg_val[_time]
        best_design = all_best_design[_time]
        best_test = all_best_test[_time]

        #For every blind test group
        for (((TRN, TEST), (TRN_IDX, TEST_IDX)),
             _t) in zip(super_zip, xrange(len(super_set))):
            TRN_INPUTS = TRN[0]
            TRN_TARGETS = TRN[1]
            TEST_INPUTS = TEST[0]
            TEST_TARGETS = TEST[1]

            #run each architecture design on a separate machine
            best[_t] = None
            best_com_val[_t] = 0
            best_avg_trn[_t] = 0
            best_avg_val[_t] = 0
            best_design[_t] = None
            best_test[_t] = None

            for design in designs:
                count += 1
                all_counts.append(count)

                (netsize, hidden_func) = design

                com = build_feedforward_committee(comsize,
                                                  len(P[0]),
                                                  netsize,
                                                  1,
                                                  hidden_function=hidden_func,
                                                  output_function='linear')

                tests[count] = (TEST_INPUTS, TEST_TARGETS)
                #trn_set[count] = (TRN_INPUTS, TRN_TARGETS)
                #print("TRN_IDX" + str(TRN_IDX))
                #print("TEST_IDX" + str(TEST_IDX))
                trn_idx[count] = TRN_IDX

                #1 is the column in the target array which holds the binary censoring information

                job = m.assemblejob((count, _time, _t, design),
                                    train_committee,
                                    com,
                                    train_evolutionary,
                                    TRN_INPUTS,
                                    TRN_TARGETS,
                                    binary_target=1,
                                    error_function=c_index_error,
                                    **train_kwargs)

                all_jobs[count] = job

                m.sendjob(job[0], job[1], *job[2], **job[3])

    while (count > 0):
        print('Remaining jobs: {0}'.format(all_counts))
        if fastest_done is None:
            ID, RESULT = m.getresult()  #Blocks
            fastest_done = time.time() - starting_time
        else:
            RETURNVALUE = m.get_waiting_result(2 * fastest_done)
            if RETURNVALUE is not None:
                ID, RESULT = RETURNVALUE
            else:
                print(
                    'Timed out after {0} seconds. Putting remaining jobs {1} back on the queue.\n \
                You should restart the server after this session.'.format(
                        fastest_done, all_counts))
                for _c in all_counts:
                    job = all_jobs[_c]
                    m.sendjob(job[0], job[1], *job[2], **job[3])
                continue  #Jump to next iteration

        print('Result received! Processing...')
        _c, _time, _t, design = ID

        (com, trn_errors, vald_errors, internal_sets,
         internal_sets_indices) = RESULT

        if _c not in all_counts:
            print('This result [{0}] has already been processed.'.format(_c))
            continue

        count -= 1

        TEST_INPUTS, TEST_TARGETS = tests[_c]
        #TRN_INPUTS, TRN_TARGETS = trn_set[_c]
        TRN_IDX = trn_idx[_c]

        all_counts.remove(_c)

        com.set_training_sets([
            _set[0][0] for _set in internal_sets
        ])  #first 0 gives training sets, second 0 gives inputs.

        #Now what we'd like to do is get the value for each patient in the
        #validation set, for all validation sets. Then I'd like to average the
        #result for each such patient, over the different validation sets.

        allpats = []
        allpats.extend(internal_sets[0][0][0])  #Extend with training inputs
        allpats.extend(internal_sets[0][1][0])  #Extend with validation inputs

        allpats_targets = []
        allpats_targets.extend(internal_sets[0][0][1])  #training targets
        allpats_targets.extend(internal_sets[0][1][1])  #validation targets
        allpats_targets = numpy.array(allpats_targets)

        patvals = [[] for bah in xrange(len(allpats))]

        #print(len(patvals))
        #print(len(internal_sets_indices))
        #1 for the validation set. Was given to the com.nets in the same type of iteration, so order is same
        # Will be order consistent with P and T
        for ((trn_in, trn_tar),
             (val_in, val_tar)), idx, net in zip(internal_sets,
                                                 internal_sets_indices,
                                                 com.nets):
            _C_ = -1
            for valpat in val_in:
                _C_ += 1
                i = TRN_IDX[idx[1][_C_]]
                pat = P[i]
                #print("Facit: \n" + str(valpat))
                #print("_C_ = " + str(_C_))
                #print("i: " + str(i))
                #print("P[TRN_IDX[i]] : " + str(pat))
                assert ((pat == valpat).all())
                patvals[i].append(com.risk_eval(pat, net=net))

        #Need  double brackets for dimensions to fit C-module
        avg_vals = numpy.array([[numpy.mean(patval)] for patval in patvals])
        #Now we have average validation ranks. do C-index on this
        avg_val_c_index = get_C_index(T, avg_vals)

        trn_errors = numpy.array(trn_errors.values(), dtype=numpy.float64)**-1
        vald_errors = numpy.array(vald_errors.values(),
                                  dtype=numpy.float64)**-1
        avg_trn = numpy.mean(trn_errors)
        avg_val = numpy.mean(vald_errors)

        best = all_best[_time]
        best_com_val = all_best_com_val[_time]
        best_avg_trn = all_best_avg_trn[_time]
        best_avg_val = all_best_avg_val[_time]
        best_design = all_best_design[_time]
        best_test = all_best_test[_time]

        if avg_val_c_index > best_com_val[_t]:
            best[_t] = com
            best_com_val[_t] = avg_val_c_index
            best_avg_trn[_t] = avg_trn
            best_avg_val[_t] = avg_val
            best_design[_t] = design
            best_test[_t] = tests[_c]

    print('\nWinning designs')
    winnerfilename = '.winningdesigns_{0:.0f}.csv'.format(time.time())
    with open(winnerfilename, 'w') as F:
        print(
            'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design:'
        )
        F.write(
            'Average Training Perf, Average Validation Perf, Average Committee Validation Perf, Test Perf, Design\n'
        )
        for _time in xrange(len(all_best)):
            best = all_best[_time]
            best_com_val = all_best_com_val[_time]
            best_avg_trn = all_best_avg_trn[_time]
            best_avg_val = all_best_avg_val[_time]
            best_design = all_best_design[_time]
            best_test = all_best_test[_time]
            for _t in best.keys():
                TEST_INPUTS, TEST_TARGETS = best_test[_t]
                com = best[_t]

                if len(TEST_INPUTS) > 0:
                    #Need double brackets for dimensions to be right for numpy
                    outputs = numpy.array([[com.risk_eval(inputs)]
                                           for inputs in TEST_INPUTS])
                    test_c_index = get_C_index(TEST_TARGETS, outputs)
                elif Ptest is not None and Ttest is not None:
                    #Need double brackets for dimensions to be right for numpy
                    outputs = numpy.array([[com.risk_eval(inputs)]
                                           for inputs in Ptest])
                    test_c_index = get_C_index(Ttest, outputs)
                else:
                    test_c_index = 0

                print('{trn}, {val}, {com_val}, {test}, {dsn}'.format(
                    trn=best_avg_trn[_t],
                    val=best_avg_val[_t],
                    com_val=best_com_val[_t],
                    test=test_c_index,
                    dsn=best_design[_t]))
                F.write('{trn}, {val}, {com_val}, {test}, {dsn}\n'.format(
                    trn=best_avg_trn[_t],
                    val=best_avg_val[_t],
                    com_val=best_com_val[_t],
                    test=test_c_index,
                    dsn=best_design[_t]))

    return winnerfilename