def classify_final_test(data, features, data_to_classify, cost, g, svm_kernel):
    ##make all data becoming part of the training
    train_set, _ = utils.cut_dataset(range(len(data)), [], data, features)
    _, test_set = utils.cut_dataset([], range(len(data_to_classify)), data_to_classify, features)
    
    if svm_kernel == "linear":
        clf = svm.LinearSVC(C=cost) #used for gissete and madelon
    else:
        clf = svm.SVC(C=cost, gamma=g, kernel='rbf')  
    #
    clf.fit(train_set.fts_values, train_set.fts_pred)
    predictions = clf.predict(test_set.fts_values)
    preds = []
    for i in predictions:
        #print i
        preds.append(int(i))
    return preds
def classify_final_test(data, features, data_to_classify, cost, g, svm_kernel):
    ##make all data becoming part of the training
    train_set, _ = utils.cut_dataset(range(len(data)), [], data, features)
    _, test_set = utils.cut_dataset([], range(len(data_to_classify)),
                                    data_to_classify, features)

    if svm_kernel == "linear":
        clf = svm.LinearSVC(C=cost)  #used for gissete and madelon
    else:
        clf = svm.SVC(C=cost, gamma=g, kernel='rbf')
    #
    clf.fit(train_set.fts_values, train_set.fts_pred)
    predictions = clf.predict(test_set.fts_values)
    preds = []
    for i in predictions:
        #print i
        preds.append(int(i))
    return preds
def classify(folds, data, features, cost, gamma, svm_kernel):
    ##train_set, test_set = utils.cut_dataset(n_train, n_test, data, test_subset.features) ##get the features for the subset
    scores = []
    for fold in folds: ##run a classification for each fold
        train, test = fold ##division of data on that fold
        train_set, test_set = utils.cut_dataset(train, test, data, features)
        ##print train_set.fts_pred, test_set.fts_pred
        score = classification_svm(train_set, test_set, cost, gamma, svm_kernel)
        if math.isnan(score):
            #print "nan score"
            score = 0
        scores.append(score)
    
     #print scores
    final_score = sum(scores) / float(len(scores))
    return final_score
def classify(folds, data, features, cost, gamma, svm_kernel):
    ##train_set, test_set = utils.cut_dataset(n_train, n_test, data, test_subset.features) ##get the features for the subset
    scores = []
    for fold in folds:  ##run a classification for each fold
        train, test = fold  ##division of data on that fold
        train_set, test_set = utils.cut_dataset(train, test, data, features)
        ##print train_set.fts_pred, test_set.fts_pred
        score = classification_svm(train_set, test_set, cost, gamma,
                                   svm_kernel)
        if math.isnan(score):
            #print "nan score"
            score = 0
        scores.append(score)

    #print scores
    final_score = sum(scores) / float(len(scores))
    return final_score
Exemple #5
0
def worker_classification(id, work_list, data, features, n_train, n_test, data_pipe, com_pipe):
    worker_time = time.time() ##to know how much time did the worker work
    best_acc = 0 
    lowest_acc = 0
    wl_dict = {}  ##to keep all the generated subsets on this slave
    best_sets = {}
    last_test_sets = {} ##new test set since last talk com with manager
    total_wasted = 0
    testing = True
    number_of_tests = 0
    comm_rate = 60
    counter_time = time.time()
    
    talk_counter = 0
    talk_avgs = []
    talk_time = time.time()
    #print "started ", id, " with work ", translate_wl(work_list)
    while (testing):        
        if work_list ==  []:
            ##the last test sets will be sent on next talk to the manager
            if number_of_tests < 30:
                print id, " waiting cause ran out of work too soon" 
                time.sleep(10)
            work_list, best_acc, wasted_time = ask_for_work(id, data_pipe, com_pipe)
            total_wasted += wasted_time
            counter = comm_rate#random.randint(50, 150) #Since we need to send the last test sents to manager lets set this to a lower value
            if work_list == []: ##didnt receive work
                testing = False
                break
        
        test_subset = work_list[len(work_list)-1]  ##get the subset to test
        del(work_list[len(work_list)-1]) ##delete the subset from the work list ##according to sources removing from the end oof the list is much faster than removing from the end
        cannonical_name = ','.join(str(e) for e in test_subset.features) ##add the canonical name to the tested_sets
        
        train_set, test_set = utils.cut_dataset(n_train, n_test, data, test_subset.features) ##get the features for the subset
        acc = classification_svm(train_set, test_set) ##train and test the dataset
        test_subset.parents_accuracy.append(acc)
        number_of_tests += 1 
        last_test_sets[cannonical_name] = True ##add to last sets
        
        #debug_data[cannonical_name] = (test_subset.features, test_subset.parents_accuracy)
        if checkExpand(test_subset, best_acc): 
            new_combs, r_counter = get_combinations(work_list, test_subset, features, wl_dict) ##check for new combinantions
            work_list = new_combs + work_list  ##add new combinations to the list

        if acc > lowest_acc:  ##check if the subset is better then the previous ones
            if acc > best_acc: ## if acc is the best one found so far
                best_acc = acc
            lowest_acc = update_best_sets(best_sets, acc, test_subset)
        #counter -= 1
        
        #rts.append(time.time() - round_st)
        if (time.time() - counter_time) > comm_rate and testing: ## try to talk to manager
            checked = False;
            work_list, checked, wasted_time, acc = manager_control(id, work_list, last_test_sets, data_pipe, com_pipe, best_acc)
            wasted_time = 0.0
            total_wasted += wasted_time
            #print "PROCESS ", id , " Number of tests: ", number_of_tests, " in ", round(time.time() - worker_time, 2)
                
            if checked: ##worker communicated with manager
                talk_avgs.append(time.time() - talk_time)
                talk_counter += 1
                #print "PROCESS", id, " WORK SIZE ", len(work_list), "round time: ", rt ," expand count ", expand_count, " best acc:", max(best_sets), " global acc: ", best_acc
                print "PROCESS ", id , " Number of tests: ", number_of_tests
                last_test_sets = {} ##reset the tested list so less information is passed to the manager next time
                talk_time = time.time()
                counter_time = time.time()
     
            
    total_working_time = time.time() - worker_time
    print "PROCESS", id, ",waste_t:", total_wasted, ",work_t:", total_working_time, ",n_test:", number_of_tests, ",best acc:", max(best_sets), "talk_counter:", talk_counter, ",talk_avg:", round(sum(talk_avgs) / float(len(talk_avgs)),2)
Exemple #6
0
def worker_classification(id, work_list, data, features, n_train, n_test,
                          data_pipe, com_pipe):
    worker_time = time.time()  ##to know how much time did the worker work
    best_acc = 0
    lowest_acc = 0
    wl_dict = {}  ##to keep all the generated subsets on this slave
    best_sets = {}
    last_test_sets = {}  ##new test set since last talk com with manager
    total_wasted = 0
    testing = True
    number_of_tests = 0
    comm_rate = 60
    counter_time = time.time()

    talk_counter = 0
    talk_avgs = []
    talk_time = time.time()
    #print "started ", id, " with work ", translate_wl(work_list)
    while (testing):
        if work_list == []:
            ##the last test sets will be sent on next talk to the manager
            if number_of_tests < 30:
                print(id, " waiting cause ran out of work too soon")
                time.sleep(10)
            work_list, best_acc, wasted_time = ask_for_work(
                id, data_pipe, com_pipe)
            total_wasted += wasted_time
            counter = comm_rate  #random.randint(50, 150) #Since we need to send the last test sents to manager lets set this to a lower value
            if work_list == []:  ##didnt receive work
                testing = False
                break

        test_subset = work_list[len(work_list) - 1]  ##get the subset to test
        del (
            work_list[len(work_list) - 1]
        )  ##delete the subset from the work list ##according to sources removing from the end oof the list is much faster than removing from the end
        cannonical_name = ','.join(
            str(e) for e in
            test_subset.features)  ##add the canonical name to the tested_sets

        train_set, test_set = utils.cut_dataset(
            n_train, n_test, data,
            test_subset.features)  ##get the features for the subset
        acc = classification_svm(train_set,
                                 test_set)  ##train and test the dataset
        test_subset.parents_accuracy.append(acc)
        number_of_tests += 1
        last_test_sets[cannonical_name] = True  ##add to last sets

        #debug_data[cannonical_name] = (test_subset.features, test_subset.parents_accuracy)
        if checkExpand(test_subset, best_acc):
            new_combs, r_counter = get_combinations(
                work_list, test_subset, features,
                wl_dict)  ##check for new combinantions
            work_list = new_combs + work_list  ##add new combinations to the list

        if acc > lowest_acc:  ##check if the subset is better then the previous ones
            if acc > best_acc:  ## if acc is the best one found so far
                best_acc = acc
            lowest_acc = update_best_sets(best_sets, acc, test_subset)
        #counter -= 1

        #rts.append(time.time() - round_st)
        if (time.time() - counter_time
            ) > comm_rate and testing:  ## try to talk to manager
            checked = False
            work_list, checked, wasted_time, acc = manager_control(
                id, work_list, last_test_sets, data_pipe, com_pipe, best_acc)
            wasted_time = 0.0
            total_wasted += wasted_time
            #print "PROCESS ", id , " Number of tests: ", number_of_tests, " in ", round(time.time() - worker_time, 2)

            if checked:  ##worker communicated with manager
                talk_avgs.append(time.time() - talk_time)
                talk_counter += 1
                #print "PROCESS", id, " WORK SIZE ", len(work_list), "round time: ", rt ," expand count ", expand_count, " best acc:", max(best_sets), " global acc: ", best_acc
                print("PROCESS ", id, " Number of tests: ", number_of_tests)
                last_test_sets = {
                }  ##reset the tested list so less information is passed to the manager next time
                talk_time = time.time()
                counter_time = time.time()

    total_working_time = time.time() - worker_time
    print("PROCESS", id, ",waste_t:", total_wasted, ",work_t:",
          total_working_time, ",n_test:", number_of_tests, ",best acc:",
          max(best_sets), "talk_counter:", talk_counter, ",talk_avg:",
          round(sum(talk_avgs) / float(len(talk_avgs)), 2))